wardowl 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wardowl/__init__.py +60 -0
- wardowl/authorizers/__init__.py +14 -0
- wardowl/authorizers/_http.py +420 -0
- wardowl/authorizers/casbin.py +119 -0
- wardowl/authorizers/cedar.py +267 -0
- wardowl/authorizers/cerbos.py +249 -0
- wardowl/authorizers/keto.py +133 -0
- wardowl/authorizers/opa.py +114 -0
- wardowl/authorizers/openfga.py +226 -0
- wardowl/authorizers/spicedb.py +235 -0
- wardowl/authz.py +274 -0
- wardowl/backends/__init__.py +6 -0
- wardowl/backends/_engines.py +270 -0
- wardowl/backends/_errors.py +30 -0
- wardowl/backends/_ids.py +27 -0
- wardowl/backends/_vectors.py +22 -0
- wardowl/backends/cassandra.py +257 -0
- wardowl/backends/chroma.py +210 -0
- wardowl/backends/elasticsearch.py +93 -0
- wardowl/backends/faiss.py +169 -0
- wardowl/backends/lancedb.py +272 -0
- wardowl/backends/memory.py +75 -0
- wardowl/backends/milvus.py +334 -0
- wardowl/backends/mongodb.py +459 -0
- wardowl/backends/opensearch.py +94 -0
- wardowl/backends/pgvector.py +367 -0
- wardowl/backends/pinecone.py +314 -0
- wardowl/backends/qdrant.py +386 -0
- wardowl/backends/redis.py +328 -0
- wardowl/backends/vespa.py +279 -0
- wardowl/backends/weaviate.py +351 -0
- wardowl/caching.py +750 -0
- wardowl/events.py +104 -0
- wardowl/filters.py +141 -0
- wardowl/fusion.py +114 -0
- wardowl/haystack.py +183 -0
- wardowl/langchain.py +118 -0
- wardowl/llamaindex.py +155 -0
- wardowl/mcp.py +147 -0
- wardowl/otel.py +161 -0
- wardowl/py.typed +0 -0
- wardowl/redaction.py +156 -0
- wardowl/retriever.py +542 -0
- wardowl/server.py +341 -0
- wardowl/sync/__init__.py +108 -0
- wardowl/sync/google_drive.py +277 -0
- wardowl/testing.py +591 -0
- wardowl/types.py +152 -0
- wardowl-2.0.0.dist-info/METADATA +709 -0
- wardowl-2.0.0.dist-info/RECORD +52 -0
- wardowl-2.0.0.dist-info/WHEEL +4 -0
- wardowl-2.0.0.dist-info/licenses/LICENSE +21 -0
wardowl/__init__.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""wardowl: permission-aware retrieval for RAG.
|
|
2
|
+
|
|
3
|
+
Every user gets only the chunks they are allowed to read, whatever vector store, permission system
|
|
4
|
+
or framework you use.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from wardowl.authz import (
|
|
8
|
+
AllOf,
|
|
9
|
+
AnyOf,
|
|
10
|
+
AuthorizationError,
|
|
11
|
+
Authorizer,
|
|
12
|
+
FunctionAuthorizer,
|
|
13
|
+
MetadataACLAuthorizer,
|
|
14
|
+
TenantAuthorizer,
|
|
15
|
+
)
|
|
16
|
+
from wardowl.backends.memory import InMemoryBackend
|
|
17
|
+
from wardowl.caching import CachingAuthorizer
|
|
18
|
+
from wardowl.events import JsonLinesAudit, RetrievalEvent
|
|
19
|
+
from wardowl.filters import And, Filter, In, Or, filter_from_dict, filter_to_dict, matches
|
|
20
|
+
from wardowl.fusion import RankFusion
|
|
21
|
+
from wardowl.redaction import PatternRedactor, Redactor, SpanRedactor
|
|
22
|
+
from wardowl.retriever import PermissionedRetriever, Reranker, RetrievalResult, SearchBackend, Strategy
|
|
23
|
+
from wardowl.types import Document, Hit, Query, Subject
|
|
24
|
+
|
|
25
|
+
__version__ = "2.0.0"
|
|
26
|
+
|
|
27
|
+
__all__ = [
|
|
28
|
+
"AllOf",
|
|
29
|
+
"And",
|
|
30
|
+
"AnyOf",
|
|
31
|
+
"AuthorizationError",
|
|
32
|
+
"Authorizer",
|
|
33
|
+
"CachingAuthorizer",
|
|
34
|
+
"Document",
|
|
35
|
+
"Filter",
|
|
36
|
+
"FunctionAuthorizer",
|
|
37
|
+
"Hit",
|
|
38
|
+
"In",
|
|
39
|
+
"InMemoryBackend",
|
|
40
|
+
"JsonLinesAudit",
|
|
41
|
+
"MetadataACLAuthorizer",
|
|
42
|
+
"Or",
|
|
43
|
+
"PatternRedactor",
|
|
44
|
+
"PermissionedRetriever",
|
|
45
|
+
"Query",
|
|
46
|
+
"RankFusion",
|
|
47
|
+
"Redactor",
|
|
48
|
+
"Reranker",
|
|
49
|
+
"RetrievalEvent",
|
|
50
|
+
"RetrievalResult",
|
|
51
|
+
"SearchBackend",
|
|
52
|
+
"SpanRedactor",
|
|
53
|
+
"Strategy",
|
|
54
|
+
"Subject",
|
|
55
|
+
"TenantAuthorizer",
|
|
56
|
+
"__version__",
|
|
57
|
+
"filter_from_dict",
|
|
58
|
+
"filter_to_dict",
|
|
59
|
+
"matches",
|
|
60
|
+
]
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Authorizers for external permission systems. Each lives in its own module.
|
|
2
|
+
|
|
3
|
+
The HTTP authorizers (`opa.OPAAuthorizer`, `openfga.OpenFGAAuthorizer`,
|
|
4
|
+
`spicedb.SpiceDBAuthorizer`, `cerbos.CerbosAuthorizer`, `keto.KetoAuthorizer`) share one client: it
|
|
5
|
+
reuses connections (never across `fork()`), bounds each call by `timeout` (except the DNS lookup
|
|
6
|
+
of the host) and each answer by `max_response_bytes`, retries transient failures once, never
|
|
7
|
+
follows redirects, honours the standard proxy variables, refuses to send an `Authorization` header
|
|
8
|
+
unencrypted over the network unless `allow_insecure=True`, and can pause requests to a failing
|
|
9
|
+
server with a `CircuitBreaker`.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from wardowl.authorizers._http import CircuitBreaker
|
|
13
|
+
|
|
14
|
+
__all__ = ["CircuitBreaker"]
|
|
@@ -0,0 +1,420 @@
|
|
|
1
|
+
"""HTTP for the authorizers, with the standard library only.
|
|
2
|
+
|
|
3
|
+
* Connections are kept alive and reused, one per thread, so a busy retriever does not open a TCP
|
|
4
|
+
(and TLS) connection for every permission check. A connection made before `fork()` is never used
|
|
5
|
+
in the child process: parent and child would share one socket and could read each other's answers.
|
|
6
|
+
* `timeout` bounds a whole call: connecting to each address of the host, the TLS handshake,
|
|
7
|
+
sending, every read of the answer, the pause before a retry and the retry itself. An answer
|
|
8
|
+
larger than `max_response_bytes` (10 MiB by default) is refused.
|
|
9
|
+
* Looking up the host's addresses is the one step `timeout` does not bound, though the time it takes
|
|
10
|
+
counts against it. The standard library has no timeout for `getaddrinfo`, and a lookup run in a
|
|
11
|
+
thread cannot be stopped: each one that hung would hold a thread until the system resolver gave
|
|
12
|
+
up, and during a DNS outage they would pile up. The resolver's own settings bound it (`timeout`
|
|
13
|
+
and `attempts` in /etc/resolv.conf), a url with an IP address needs no lookup, and connections
|
|
14
|
+
are reused, so a lookup happens only when a new one is opened.
|
|
15
|
+
* A request that fails in a way that is safe to repeat is retried `retries` times (default once):
|
|
16
|
+
a refused or reset connection, a response that never arrived, or HTTP 502, 503 or 504. Every
|
|
17
|
+
request is a query (OPA's data API, OpenFGA's checks and object lists), so repeating one has no
|
|
18
|
+
side effects. Timeouts are not retried, and neither are other HTTP errors or invalid answers,
|
|
19
|
+
which would only repeat.
|
|
20
|
+
* A connection the server closed while it sat idle is replaced and the request sent again, once,
|
|
21
|
+
without counting as a retry.
|
|
22
|
+
* An optional `CircuitBreaker` stops calling a service that keeps failing, for a cooldown.
|
|
23
|
+
* Redirects are never followed: a permission service has no reason to send one.
|
|
24
|
+
* When a proxy applies to the host (the standard `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY`
|
|
25
|
+
variables), requests go through urllib, which handles proxies, without connection reuse.
|
|
26
|
+
* `ssl_context` is refused for an http:// url, where it would not be used. An `Authorization`
|
|
27
|
+
header is refused over http:// unless the url names this machine (`localhost`, `127.0.0.1`,
|
|
28
|
+
`::1`) and no proxy applies, or `allow_insecure=True` is passed.
|
|
29
|
+
|
|
30
|
+
Whatever still fails raises `AuthorizationError`, so retrieval fails closed.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import http.client
|
|
36
|
+
import io
|
|
37
|
+
import ipaddress
|
|
38
|
+
import json
|
|
39
|
+
import os
|
|
40
|
+
import socket
|
|
41
|
+
import ssl
|
|
42
|
+
import threading
|
|
43
|
+
import time
|
|
44
|
+
import urllib.error
|
|
45
|
+
import urllib.request
|
|
46
|
+
from collections.abc import Callable, Mapping
|
|
47
|
+
from typing import Any
|
|
48
|
+
from urllib.parse import SplitResult, urlsplit
|
|
49
|
+
|
|
50
|
+
from wardowl.authz import AuthorizationError
|
|
51
|
+
|
|
52
|
+
__all__ = ["DEFAULT_MAX_RESPONSE_BYTES", "CircuitBreaker", "JsonClient"]
|
|
53
|
+
|
|
54
|
+
_RETRY_STATUS = frozenset({502, 503, 504})
|
|
55
|
+
|
|
56
|
+
DEFAULT_MAX_RESPONSE_BYTES = 10 * 1024 * 1024
|
|
57
|
+
"""The largest answer a permission service may send, unless `max_response_bytes` says otherwise."""
|
|
58
|
+
|
|
59
|
+
_deadline = threading.local() # `at`: when the request in flight on this thread must be answered
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class CircuitBreaker:
|
|
63
|
+
"""Pauses requests to a permission service that keeps failing.
|
|
64
|
+
|
|
65
|
+
After `failures` consecutive failed requests the breaker opens, and every call raises
|
|
66
|
+
`AuthorizationError` at once, without contacting the service, for `reset_after` seconds. Then
|
|
67
|
+
one request is let through as a probe: if it succeeds the breaker closes, if it fails the
|
|
68
|
+
breaker opens again. A failure is a request that got no HTTP response, or a 5xx one; any other
|
|
69
|
+
response shows that the service is up. Retrieval fails closed while the breaker is open, as it
|
|
70
|
+
would if every request failed, but a struggling service is not flooded with requests.
|
|
71
|
+
|
|
72
|
+
One breaker can be shared by several authorizers that call the same service.
|
|
73
|
+
"""
|
|
74
|
+
|
|
75
|
+
def __init__(
|
|
76
|
+
self, failures: int = 5, reset_after: float = 30.0, *, clock: Callable[[], float] = time.monotonic
|
|
77
|
+
) -> None:
|
|
78
|
+
if failures < 1 or reset_after <= 0:
|
|
79
|
+
raise ValueError("failures and reset_after must be positive")
|
|
80
|
+
self.failures = failures
|
|
81
|
+
self.reset_after = reset_after
|
|
82
|
+
self._clock = clock
|
|
83
|
+
self._lock = threading.Lock()
|
|
84
|
+
self._consecutive = 0
|
|
85
|
+
self._opened_at: float | None = None
|
|
86
|
+
self._probing = False
|
|
87
|
+
|
|
88
|
+
@property
|
|
89
|
+
def is_open(self) -> bool:
|
|
90
|
+
"""True while requests are paused, for monitoring."""
|
|
91
|
+
with self._lock:
|
|
92
|
+
return self._opened_at is not None
|
|
93
|
+
|
|
94
|
+
def allow(self) -> None:
|
|
95
|
+
"""Raise `AuthorizationError` while open; after the cooldown, let one probe through."""
|
|
96
|
+
with self._lock:
|
|
97
|
+
if self._opened_at is None:
|
|
98
|
+
return
|
|
99
|
+
if self._probing or self._clock() - self._opened_at < self.reset_after:
|
|
100
|
+
raise AuthorizationError("the permission service keeps failing; requests are paused (circuit open)")
|
|
101
|
+
self._probing = True
|
|
102
|
+
|
|
103
|
+
def record(self, ok: bool) -> None:
|
|
104
|
+
"""Record the outcome of a request that `allow` let through."""
|
|
105
|
+
with self._lock:
|
|
106
|
+
probing, self._probing = self._probing, False
|
|
107
|
+
if ok:
|
|
108
|
+
self._consecutive = 0
|
|
109
|
+
self._opened_at = None
|
|
110
|
+
return
|
|
111
|
+
self._consecutive += 1
|
|
112
|
+
if probing or self._consecutive >= self.failures:
|
|
113
|
+
self._opened_at = self._clock()
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class _TooLarge(Exception):
|
|
117
|
+
"""The answer is larger than `max_response_bytes`."""
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _left(deadline: float) -> float:
|
|
121
|
+
"""The seconds left before `deadline`; `TimeoutError` when none are."""
|
|
122
|
+
left = deadline - time.monotonic()
|
|
123
|
+
if left <= 0:
|
|
124
|
+
raise TimeoutError("the deadline passed")
|
|
125
|
+
return left
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
class _DeadlineIO(io.RawIOBase):
|
|
129
|
+
"""A response's socket file: every read waits only for the time left before the deadline, so a
|
|
130
|
+
server that sends its answer slowly cannot stretch a call past `timeout`."""
|
|
131
|
+
|
|
132
|
+
def __init__(self, sock: socket.socket, deadline: float) -> None:
|
|
133
|
+
super().__init__()
|
|
134
|
+
self._raw = sock.makefile("rb", 0) # counted among the socket's files, as a response's own is
|
|
135
|
+
self._socket = sock
|
|
136
|
+
self._deadline = deadline
|
|
137
|
+
|
|
138
|
+
def readable(self) -> bool:
|
|
139
|
+
return True
|
|
140
|
+
|
|
141
|
+
def readinto(self, b: Any) -> int | None:
|
|
142
|
+
self._socket.settimeout(_left(self._deadline))
|
|
143
|
+
return self._raw.readinto(b)
|
|
144
|
+
|
|
145
|
+
def close(self) -> None:
|
|
146
|
+
if not self.closed:
|
|
147
|
+
self._raw.close()
|
|
148
|
+
super().close()
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class _Response(http.client.HTTPResponse):
|
|
152
|
+
"""An HTTP response read under the deadline of the request in flight on this thread."""
|
|
153
|
+
|
|
154
|
+
def __init__(self, sock: socket.socket, *args: Any, **kwargs: Any) -> None:
|
|
155
|
+
super().__init__(sock, *args, **kwargs)
|
|
156
|
+
deadline: float | None = getattr(_deadline, "at", None)
|
|
157
|
+
if deadline is not None:
|
|
158
|
+
plain = self.fp # nothing has been read from it yet
|
|
159
|
+
self.fp = io.BufferedReader(_DeadlineIO(sock, deadline))
|
|
160
|
+
plain.close()
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _create_connection(
|
|
164
|
+
address: tuple[str, int], timeout: Any = None, source_address: tuple[str, int] | None = None
|
|
165
|
+
) -> socket.socket:
|
|
166
|
+
"""`socket.create_connection` under the deadline of the request in flight on this thread.
|
|
167
|
+
|
|
168
|
+
Each address gets only the time left, not a fresh `timeout`, and the connected socket is then
|
|
169
|
+
set to the time still left, which bounds the TLS handshake that follows. The lookup of the
|
|
170
|
+
addresses is not bounded (see the module docstring).
|
|
171
|
+
"""
|
|
172
|
+
deadline: float | None = getattr(_deadline, "at", None)
|
|
173
|
+
if deadline is None:
|
|
174
|
+
return socket.create_connection(address, timeout, source_address)
|
|
175
|
+
host, port = address
|
|
176
|
+
failure = OSError(f"no address found for {host!r}")
|
|
177
|
+
for family, kind, proto, _, sockaddr in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
|
|
178
|
+
left = _left(deadline) # TimeoutError once the lookup or earlier addresses used up the time
|
|
179
|
+
sock = socket.socket(family, kind, proto)
|
|
180
|
+
try:
|
|
181
|
+
sock.settimeout(left)
|
|
182
|
+
if source_address is not None:
|
|
183
|
+
sock.bind(source_address)
|
|
184
|
+
sock.connect(sockaddr)
|
|
185
|
+
sock.settimeout(_left(deadline)) # what remains is all the TLS handshake gets
|
|
186
|
+
except OSError as exc:
|
|
187
|
+
sock.close()
|
|
188
|
+
if isinstance(exc, TimeoutError):
|
|
189
|
+
raise # the deadline has passed: there is no time left for another address
|
|
190
|
+
failure = exc
|
|
191
|
+
except BaseException:
|
|
192
|
+
sock.close()
|
|
193
|
+
raise
|
|
194
|
+
else:
|
|
195
|
+
return sock
|
|
196
|
+
raise failure
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
class _HTTPConnection(http.client.HTTPConnection):
|
|
200
|
+
response_class = _Response
|
|
201
|
+
|
|
202
|
+
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
|
203
|
+
super().__init__(*args, **kwargs)
|
|
204
|
+
self._create_connection = _create_connection
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
class _HTTPSConnection(http.client.HTTPSConnection):
|
|
208
|
+
response_class = _Response
|
|
209
|
+
|
|
210
|
+
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
|
211
|
+
super().__init__(*args, **kwargs)
|
|
212
|
+
self._create_connection = _create_connection
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
class _HTTPHandler(urllib.request.HTTPHandler):
|
|
216
|
+
def http_open(self, req: urllib.request.Request) -> http.client.HTTPResponse:
|
|
217
|
+
return self.do_open(_HTTPConnection, req)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
class _HTTPSHandler(urllib.request.HTTPSHandler):
|
|
221
|
+
def __init__(self, context: ssl.SSLContext) -> None:
|
|
222
|
+
super().__init__(context=context)
|
|
223
|
+
self._tls = context
|
|
224
|
+
|
|
225
|
+
def https_open(self, req: urllib.request.Request) -> http.client.HTTPResponse:
|
|
226
|
+
return self.do_open(_HTTPSConnection, req, context=self._tls)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
230
|
+
def redirect_request(self, *args: Any, **kwargs: Any) -> None:
|
|
231
|
+
return None # urllib then raises HTTPError for the 3xx response
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _proxy_applies(parts: SplitResult) -> bool:
|
|
235
|
+
return parts.scheme in urllib.request.getproxies() and not urllib.request.proxy_bypass(parts.hostname or "")
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _loopback(host: str) -> bool:
|
|
239
|
+
"""True when `host` names this machine, so a request to it never crosses a network."""
|
|
240
|
+
if host == "localhost":
|
|
241
|
+
return True
|
|
242
|
+
try:
|
|
243
|
+
return ipaddress.ip_address(host).is_loopback
|
|
244
|
+
except ValueError:
|
|
245
|
+
return False
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
class JsonClient:
|
|
249
|
+
"""POSTs JSON to one service and returns the decoded answer of a 200 response."""
|
|
250
|
+
|
|
251
|
+
def __init__(
|
|
252
|
+
self,
|
|
253
|
+
url: str,
|
|
254
|
+
*,
|
|
255
|
+
service: str,
|
|
256
|
+
timeout: float,
|
|
257
|
+
headers: Mapping[str, str],
|
|
258
|
+
retries: int = 1,
|
|
259
|
+
breaker: CircuitBreaker | None = None,
|
|
260
|
+
ssl_context: ssl.SSLContext | None = None,
|
|
261
|
+
max_response_bytes: int = DEFAULT_MAX_RESPONSE_BYTES,
|
|
262
|
+
allow_insecure: bool = False,
|
|
263
|
+
) -> None:
|
|
264
|
+
parts = urlsplit(url)
|
|
265
|
+
if parts.scheme not in ("http", "https") or not parts.hostname:
|
|
266
|
+
raise ValueError(f"the {service} url must start with http:// or https:// and name a host")
|
|
267
|
+
if retries < 0:
|
|
268
|
+
raise ValueError("retries must be zero or more")
|
|
269
|
+
if timeout <= 0 or max_response_bytes < 1:
|
|
270
|
+
raise ValueError("timeout and max_response_bytes must be positive")
|
|
271
|
+
if ssl_context is not None and parts.scheme != "https":
|
|
272
|
+
raise ValueError(f"ssl_context is only used with https://, but the {service} url is {parts.scheme}://")
|
|
273
|
+
self._proxied = _proxy_applies(parts)
|
|
274
|
+
cleartext = parts.scheme == "http" and (self._proxied or not _loopback(parts.hostname or ""))
|
|
275
|
+
if cleartext and not allow_insecure and any(h.lower() == "authorization" for h in headers):
|
|
276
|
+
raise ValueError(
|
|
277
|
+
f"the {service} url is http://, so the Authorization header would cross the network unencrypted: "
|
|
278
|
+
"use https://, or pass allow_insecure=True if the network is trusted"
|
|
279
|
+
)
|
|
280
|
+
self.url = url.rstrip("/")
|
|
281
|
+
self.service = service
|
|
282
|
+
self.timeout = timeout
|
|
283
|
+
self.retries = retries
|
|
284
|
+
self.breaker = breaker
|
|
285
|
+
self.max_response_bytes = max_response_bytes
|
|
286
|
+
self.headers = {"Content-Type": "application/json", **headers}
|
|
287
|
+
self._parts = parts
|
|
288
|
+
self._prefix = parts.path.rstrip("/")
|
|
289
|
+
self._ssl = ssl_context or (ssl.create_default_context() if parts.scheme == "https" else None)
|
|
290
|
+
self._local = threading.local()
|
|
291
|
+
|
|
292
|
+
def post_json(self, path: str, payload: Any, what: str, *, lines: bool = False) -> Any:
|
|
293
|
+
"""POST `payload` and return the decoded answer; with `lines`, a list with one decoded
|
|
294
|
+
value per line of a newline-delimited JSON answer (a streamed response)."""
|
|
295
|
+
try:
|
|
296
|
+
body = json.dumps(payload).encode()
|
|
297
|
+
except (TypeError, ValueError) as exc:
|
|
298
|
+
raise AuthorizationError(f"{what}: the input is not JSON-serialisable: {exc}") from exc
|
|
299
|
+
if self.breaker is not None:
|
|
300
|
+
self.breaker.allow()
|
|
301
|
+
deadline = time.monotonic() + self.timeout
|
|
302
|
+
reachable = False # recorded on every way out, so a probe can never leave the breaker stuck
|
|
303
|
+
try:
|
|
304
|
+
attempt = 0
|
|
305
|
+
while True:
|
|
306
|
+
last: BaseException | None = None # the exception behind the failure, kept as its cause
|
|
307
|
+
try:
|
|
308
|
+
status, raw = self._send(path, body, deadline)
|
|
309
|
+
except TimeoutError as exc:
|
|
310
|
+
raise AuthorizationError(f"{what} timed out after {self.timeout} s") from exc
|
|
311
|
+
except ssl.SSLCertVerificationError as exc:
|
|
312
|
+
raise AuthorizationError(f"{what}: TLS certificate verification failed: {exc}") from exc
|
|
313
|
+
except _TooLarge as exc:
|
|
314
|
+
reachable = True # the service answered; the answer is what is wrong
|
|
315
|
+
raise AuthorizationError(f"{what} answered with more than {self.max_response_bytes} bytes") from exc
|
|
316
|
+
except (OSError, http.client.HTTPException) as exc:
|
|
317
|
+
failure, last = f"{type(exc).__name__}: {exc}", exc
|
|
318
|
+
else:
|
|
319
|
+
if status not in _RETRY_STATUS:
|
|
320
|
+
reachable = status < 500
|
|
321
|
+
return self._decode(status, raw, what, lines)
|
|
322
|
+
failure = f"HTTP {status}"
|
|
323
|
+
pause = min(0.05 * 2**attempt, 1.0)
|
|
324
|
+
if attempt >= self.retries or time.monotonic() + pause >= deadline:
|
|
325
|
+
raise AuthorizationError(f"{what} failed: {failure}") from last
|
|
326
|
+
attempt += 1
|
|
327
|
+
time.sleep(pause)
|
|
328
|
+
finally:
|
|
329
|
+
if self.breaker is not None:
|
|
330
|
+
self.breaker.record(reachable)
|
|
331
|
+
|
|
332
|
+
@staticmethod
|
|
333
|
+
def _decode(status: int, raw: bytes, what: str, lines: bool = False) -> Any:
|
|
334
|
+
if status != 200:
|
|
335
|
+
raise AuthorizationError(f"{what} returned HTTP {status}")
|
|
336
|
+
try:
|
|
337
|
+
if lines:
|
|
338
|
+
return [json.loads(line) for line in raw.splitlines() if line.strip()]
|
|
339
|
+
return json.loads(raw)
|
|
340
|
+
except ValueError as exc:
|
|
341
|
+
raise AuthorizationError(f"{what} returned invalid JSON") from exc
|
|
342
|
+
|
|
343
|
+
def _read(self, response: http.client.HTTPResponse) -> bytes:
|
|
344
|
+
"""The whole body; `_TooLarge` when it is over `max_response_bytes`, before reading it when
|
|
345
|
+
the length is announced."""
|
|
346
|
+
limit = self.max_response_bytes
|
|
347
|
+
if response.length is not None:
|
|
348
|
+
if response.length > limit:
|
|
349
|
+
raise _TooLarge(response.length)
|
|
350
|
+
return response.read() # raises IncompleteRead if the body is cut short
|
|
351
|
+
raw = response.read(limit + 1) # chunked, or ended by closing the connection
|
|
352
|
+
if len(raw) > limit:
|
|
353
|
+
raise _TooLarge(len(raw))
|
|
354
|
+
return raw
|
|
355
|
+
|
|
356
|
+
# ---- transports ------------------------------------------------------------------------
|
|
357
|
+
|
|
358
|
+
def _send(self, path: str, body: bytes, deadline: float) -> tuple[int, bytes]:
|
|
359
|
+
_deadline.at = deadline
|
|
360
|
+
try:
|
|
361
|
+
if self._proxied:
|
|
362
|
+
return self._send_urllib(path, body, deadline)
|
|
363
|
+
held: tuple[int, http.client.HTTPConnection] | None = getattr(self._local, "conn", None)
|
|
364
|
+
if held is not None:
|
|
365
|
+
self._local.conn = None
|
|
366
|
+
pid, conn = held
|
|
367
|
+
if pid != os.getpid():
|
|
368
|
+
conn.close() # made before fork(): this process's copy of the parent's socket
|
|
369
|
+
else:
|
|
370
|
+
try:
|
|
371
|
+
return self._exchange(conn, path, body, deadline)
|
|
372
|
+
except (ConnectionError, http.client.BadStatusLine):
|
|
373
|
+
pass # the server closed the idle connection; send once more on a new one
|
|
374
|
+
return self._exchange(self._connect(), path, body, deadline)
|
|
375
|
+
finally:
|
|
376
|
+
_deadline.at = None
|
|
377
|
+
|
|
378
|
+
def _connect(self) -> http.client.HTTPConnection:
|
|
379
|
+
host, port = self._parts.hostname or "", self._parts.port
|
|
380
|
+
if self._parts.scheme == "https":
|
|
381
|
+
return _HTTPSConnection(host, port, context=self._ssl)
|
|
382
|
+
return _HTTPConnection(host, port)
|
|
383
|
+
|
|
384
|
+
def _exchange(self, conn: http.client.HTTPConnection, path: str, body: bytes, deadline: float) -> tuple[int, bytes]:
|
|
385
|
+
try:
|
|
386
|
+
sock = conn.sock
|
|
387
|
+
if sock is None:
|
|
388
|
+
conn.timeout = _left(deadline)
|
|
389
|
+
conn.connect() # TCP and TLS within the deadline, by _create_connection
|
|
390
|
+
sock = conn.sock
|
|
391
|
+
sock.settimeout(_left(deadline))
|
|
392
|
+
conn.request("POST", self._prefix + path, body=body, headers=self.headers)
|
|
393
|
+
response = conn.getresponse() # read under the deadline, by _Response
|
|
394
|
+
raw = self._read(response)
|
|
395
|
+
except BaseException:
|
|
396
|
+
conn.close()
|
|
397
|
+
raise
|
|
398
|
+
if response.will_close:
|
|
399
|
+
conn.close()
|
|
400
|
+
else:
|
|
401
|
+
self._local.conn = (os.getpid(), conn)
|
|
402
|
+
return response.status, raw
|
|
403
|
+
|
|
404
|
+
def _send_urllib(self, path: str, body: bytes, deadline: float) -> tuple[int, bytes]:
|
|
405
|
+
handlers: list[Any] = [_NoRedirect(), _HTTPHandler()]
|
|
406
|
+
if self._ssl is not None:
|
|
407
|
+
handlers.append(_HTTPSHandler(self._ssl))
|
|
408
|
+
opener = urllib.request.build_opener(*handlers)
|
|
409
|
+
# The scheme is restricted to http(s) in __init__, so this cannot open file:// or others.
|
|
410
|
+
request = urllib.request.Request(self.url + path, data=body, headers=self.headers, method="POST") # noqa: S310
|
|
411
|
+
try:
|
|
412
|
+
with opener.open(request, timeout=_left(deadline)) as response:
|
|
413
|
+
return response.status, self._read(response)
|
|
414
|
+
except urllib.error.HTTPError as exc:
|
|
415
|
+
exc.close()
|
|
416
|
+
return exc.code, b"" # only a 200 answer's body is read
|
|
417
|
+
except urllib.error.URLError as exc:
|
|
418
|
+
if isinstance(exc.reason, TimeoutError):
|
|
419
|
+
raise TimeoutError(str(exc.reason)) from exc
|
|
420
|
+
raise
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""Casbin authorizer: decisions per chunk from a Casbin enforcer, in this process.
|
|
2
|
+
|
|
3
|
+
Pass the `casbin.Enforcer` you already use, built with your model, adapter and policies; the
|
|
4
|
+
`casbin` extra installs the library. Casbin evaluates in the process, so there is no service to
|
|
5
|
+
reach.
|
|
6
|
+
|
|
7
|
+
* `check` asks the enforcer about each chunk, with `batch_enforce` when it has one. The request is
|
|
8
|
+
`(subject.id, object, action)`, where the object is `hit.metadata[resource_field]` (or the chunk
|
|
9
|
+
id when `resource_field` is None) and the action is mapped through `actions`; a chunk without its
|
|
10
|
+
object is denied. `request` replaces that tuple, for ABAC models whose matchers read attributes.
|
|
11
|
+
* With `group_subjects=True`, the request is also made with each of the subject's groups in place of
|
|
12
|
+
its id, written `group_prefix` + the group (`"group:eng"` by default), and a chunk is allowed if
|
|
13
|
+
any request is: for allow-only models whose groups come from the verified token. Policies for
|
|
14
|
+
groups must name them with the prefix. Users and roles share Casbin's subject field, so without
|
|
15
|
+
it a group named like a user or a role (`"admin"`, `"ceo@example.com"`) would be granted that
|
|
16
|
+
user's or role's access; a subject id that starts with the prefix raises `AuthorizationError`
|
|
17
|
+
for the same reason. A model with deny effects must keep memberships in its own `g` policies
|
|
18
|
+
instead, since a deny for one group would not override an allow for another.
|
|
19
|
+
* `filter_for` returns None, so the retriever post-filters and every candidate is decided by Casbin.
|
|
20
|
+
|
|
21
|
+
An enforcer that raises, or answers with anything but one boolean per request, makes `check` raise
|
|
22
|
+
`AuthorizationError`.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
28
|
+
from typing import Any
|
|
29
|
+
|
|
30
|
+
from wardowl.authz import AuthorizationError
|
|
31
|
+
from wardowl.filters import Filter
|
|
32
|
+
from wardowl.types import Hit, Subject
|
|
33
|
+
|
|
34
|
+
__all__ = ["CasbinAuthorizer"]
|
|
35
|
+
|
|
36
|
+
Request = Callable[[Subject, Hit, str], Sequence[Any]]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class CasbinAuthorizer:
|
|
40
|
+
"""Decides each chunk with a Casbin enforcer.
|
|
41
|
+
|
|
42
|
+
`enforcer` is a `casbin.Enforcer` (or `SyncedEnforcer`, to share one across threads that also
|
|
43
|
+
change policies). `actions` maps wardowl actions to Casbin actions (default
|
|
44
|
+
`{"read": "read"}`). With `group_subjects=True`, groups are asked about as `group_prefix` + the
|
|
45
|
+
group, so policies for groups must use that prefix.
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
def __init__(
|
|
49
|
+
self,
|
|
50
|
+
enforcer: Any,
|
|
51
|
+
*,
|
|
52
|
+
actions: Mapping[str, str] | None = None,
|
|
53
|
+
resource_field: str | None = "doc_id",
|
|
54
|
+
group_subjects: bool = False,
|
|
55
|
+
group_prefix: str = "group:",
|
|
56
|
+
request: Request | None = None,
|
|
57
|
+
) -> None:
|
|
58
|
+
if not callable(getattr(enforcer, "enforce", None)):
|
|
59
|
+
raise TypeError("enforcer must be a casbin Enforcer (with an enforce method)")
|
|
60
|
+
if not isinstance(group_prefix, str) or not group_prefix:
|
|
61
|
+
raise ValueError("group_prefix must be a non-empty string, so groups cannot pass for users or roles")
|
|
62
|
+
self.enforcer = enforcer
|
|
63
|
+
self.actions = dict(actions or {"read": "read"})
|
|
64
|
+
self.resource_field = resource_field
|
|
65
|
+
self.group_subjects = group_subjects
|
|
66
|
+
self.group_prefix = group_prefix
|
|
67
|
+
self.request = request
|
|
68
|
+
|
|
69
|
+
def _action(self, action: str) -> str:
|
|
70
|
+
if action not in self.actions:
|
|
71
|
+
raise AuthorizationError(f"no Casbin action is configured for action {action!r}")
|
|
72
|
+
return self.actions[action]
|
|
73
|
+
|
|
74
|
+
def _object(self, hit: Hit) -> str | None:
|
|
75
|
+
if self.resource_field is None:
|
|
76
|
+
return hit.id
|
|
77
|
+
value = hit.metadata.get(self.resource_field)
|
|
78
|
+
return value if isinstance(value, str) and value else None
|
|
79
|
+
|
|
80
|
+
def _who(self, subject: Subject) -> list[str]:
|
|
81
|
+
"""The Casbin subjects to ask about: the subject's id, then its groups with their prefix."""
|
|
82
|
+
if not self.group_subjects:
|
|
83
|
+
return [subject.id]
|
|
84
|
+
if subject.id.startswith(self.group_prefix):
|
|
85
|
+
raise AuthorizationError(f"subject id {subject.id!r} starts with the group prefix {self.group_prefix!r}")
|
|
86
|
+
return [subject.id, *(self.group_prefix + g for g in sorted(subject.groups))]
|
|
87
|
+
|
|
88
|
+
def _requests(self, subject: Subject, hit: Hit, action: str, who: Sequence[str]) -> list[Sequence[Any]]:
|
|
89
|
+
if self.request is not None:
|
|
90
|
+
return [self.request(subject, hit, action)]
|
|
91
|
+
obj = self._object(hit)
|
|
92
|
+
if obj is None:
|
|
93
|
+
return []
|
|
94
|
+
return [(name, obj, action) for name in who]
|
|
95
|
+
|
|
96
|
+
def check(self, subject: Subject, hits: Sequence[Hit], action: str = "read") -> list[bool]:
|
|
97
|
+
casbin_action = self._action(action)
|
|
98
|
+
who = self._who(subject) if self.request is None else []
|
|
99
|
+
try:
|
|
100
|
+
per_hit = [self._requests(subject, hit, casbin_action, who) for hit in hits]
|
|
101
|
+
requests = [list(r) for rs in per_hit for r in rs]
|
|
102
|
+
if not requests:
|
|
103
|
+
return [False] * len(hits)
|
|
104
|
+
batch = getattr(self.enforcer, "batch_enforce", None)
|
|
105
|
+
answers = list(batch(requests)) if callable(batch) else [self.enforcer.enforce(*r) for r in requests]
|
|
106
|
+
except Exception as exc:
|
|
107
|
+
raise AuthorizationError(f"Casbin failed: {exc}") from exc
|
|
108
|
+
if len(answers) != len(requests) or not all(isinstance(a, bool) for a in answers):
|
|
109
|
+
raise AuthorizationError("Casbin did not answer each request with a boolean")
|
|
110
|
+
decisions: list[bool] = []
|
|
111
|
+
position = 0
|
|
112
|
+
for rs in per_hit:
|
|
113
|
+
decisions.append(any(answers[position : position + len(rs)]))
|
|
114
|
+
position += len(rs)
|
|
115
|
+
return decisions
|
|
116
|
+
|
|
117
|
+
def filter_for(self, subject: Subject, action: str = "read") -> Filter | None:
|
|
118
|
+
self._action(action)
|
|
119
|
+
return None
|