clue-api 1.7.0.dev446__tar.gz → 1.7.0.dev450__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/PKG-INFO +1 -1
  2. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/config.py +3 -0
  3. clue_api-1.7.0.dev450/clue/models/jwks.py +19 -0
  4. clue_api-1.7.0.dev450/clue/services/jwt_service.py +476 -0
  5. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/pyproject.toml +1 -1
  6. clue_api-1.7.0.dev446/clue/services/jwt_service.py +0 -254
  7. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/LICENSE +0 -0
  8. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/README.md +0 -0
  9. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/.gitignore +0 -0
  10. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/__init__.py +0 -0
  11. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/api/__init__.py +0 -0
  12. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/api/base.py +0 -0
  13. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/api/v1/__init__.py +0 -0
  14. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/api/v1/actions.py +0 -0
  15. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/api/v1/auth.py +0 -0
  16. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/api/v1/configs.py +0 -0
  17. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/api/v1/fetchers.py +0 -0
  18. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/api/v1/lookup.py +0 -0
  19. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/api/v1/registration.py +0 -0
  20. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/api/v1/static.py +0 -0
  21. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/api/v1/sync.py +0 -0
  22. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/app.py +0 -0
  23. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/cache/__init__.py +0 -0
  24. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/common/__init__.py +0 -0
  25. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/common/bytes_utils.py +0 -0
  26. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/common/classification.py +0 -0
  27. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/common/classification.yml +0 -0
  28. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/common/dict_utils.py +0 -0
  29. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/common/exceptions.py +0 -0
  30. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/common/forge.py +0 -0
  31. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/common/json_utils.py +0 -0
  32. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/common/list_utils.py +0 -0
  33. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/common/logging/__init__.py +0 -0
  34. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/common/logging/audit.py +0 -0
  35. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/common/logging/format.py +0 -0
  36. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/common/regex.py +0 -0
  37. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/common/str_utils.py +0 -0
  38. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/common/swagger.py +0 -0
  39. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/common/uid.py +0 -0
  40. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/config.py +0 -0
  41. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/constants/__init__.py +0 -0
  42. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/constants/env.py +0 -0
  43. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/constants/supported_types.py +0 -0
  44. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/cronjobs/__init__.py +0 -0
  45. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/cronjobs/plugins.py +0 -0
  46. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/error.py +0 -0
  47. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/extensions/__init__.py +0 -0
  48. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/extensions/config.py +0 -0
  49. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/gunicorn_config.py +0 -0
  50. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/healthz.py +0 -0
  51. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/helper/discover.py +0 -0
  52. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/helper/headers.py +0 -0
  53. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/helper/oauth.py +0 -0
  54. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/helper/obo.py +0 -0
  55. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/helper/plugin_requests.py +0 -0
  56. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/__init__.py +0 -0
  57. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/actions.py +0 -0
  58. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/auth_user.py +0 -0
  59. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/fetchers.py +0 -0
  60. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/graph.py +0 -0
  61. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/model_list.py +0 -0
  62. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/network.py +0 -0
  63. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/results/__init__.py +0 -0
  64. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/results/base.py +0 -0
  65. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/results/file.py +0 -0
  66. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/results/graph.py +0 -0
  67. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/results/image.py +0 -0
  68. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/results/status.py +0 -0
  69. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/results/validation.py +0 -0
  70. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/schema.py +0 -0
  71. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/selector.py +0 -0
  72. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/sync.py +0 -0
  73. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/models/validators.py +0 -0
  74. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/patched.py +0 -0
  75. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/plugin/__init__.py +0 -0
  76. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/plugin/celery_app.py +0 -0
  77. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/plugin/helpers/__init__.py +0 -0
  78. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/plugin/helpers/central_server.py +0 -0
  79. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/plugin/helpers/email_render.py +0 -0
  80. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/plugin/helpers/token.py +0 -0
  81. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/plugin/helpers/trino.py +0 -0
  82. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/plugin/models.py +0 -0
  83. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/plugin/utils.py +0 -0
  84. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/py.typed +0 -0
  85. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/remote/__init__.py +0 -0
  86. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/remote/datatypes/__init__.py +0 -0
  87. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/remote/datatypes/cache.py +0 -0
  88. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/remote/datatypes/events.py +0 -0
  89. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/remote/datatypes/hash.py +0 -0
  90. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/remote/datatypes/queues/__init__.py +0 -0
  91. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/remote/datatypes/queues/comms.py +0 -0
  92. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/remote/datatypes/set.py +0 -0
  93. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/remote/datatypes/user_quota_tracker.py +0 -0
  94. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/security/__init__.py +0 -0
  95. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/security/obo.py +0 -0
  96. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/security/utils.py +0 -0
  97. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/services/action_service.py +0 -0
  98. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/services/auth_service.py +0 -0
  99. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/services/config_service.py +0 -0
  100. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/services/fetcher_service.py +0 -0
  101. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/services/lookup_service.py +0 -0
  102. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/services/mongo_service.py +0 -0
  103. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/services/type_service.py +0 -0
  104. {clue_api-1.7.0.dev446 → clue_api-1.7.0.dev450}/clue/services/user_service.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: clue-api
3
- Version: 1.7.0.dev446
3
+ Version: 1.7.0.dev450
4
4
  Summary: Clue distributed enrichment service
5
5
  License: MIT
6
6
  License-File: LICENSE
@@ -125,6 +125,9 @@ class OAuthProvider(BaseModel):
125
125
  class OAuth(BaseModel):
126
126
  enabled: bool = Field(description="Enable use of OAuth?", default=False)
127
127
  gravatar_enabled: bool = Field(description="Enable gravatar?", default=False)
128
+ jwks_refresh_cooldown_seconds: int = Field(
129
+ default=60, ge=1, description="Minimum interval in seconds between outbound JWKS refresh attempts"
130
+ )
128
131
  providers: dict[str, OAuthProvider] = Field(default={}, description="OAuth provider configuration")
129
132
  other_audiences: list[str] | None = Field(
130
133
  default=None, description="What other audiences in JWT tokens should Clue accept?"
@@ -0,0 +1,19 @@
1
+ from typing import Any, NamedTuple
2
+
3
+
4
+ class JWKSSnapshot(NamedTuple):
5
+ """An immutable-by-convention generation of provider keys and the times they were fetched.
6
+
7
+ Keys and timestamps are bundled so a reader that holds one snapshot always judges key age by the
8
+ timestamps captured with those keys, never by a newer refresh. Instances must not be mutated after
9
+ they are published to the cache or the pinned fallback.
10
+
11
+ Attributes:
12
+ jwks: JWK dictionaries keyed by `kid`.
13
+ providers: The OAuth provider name that owns each `kid`.
14
+ timestamps: Monotonic time of each provider's last successful fetch, keyed by provider name.
15
+ """
16
+
17
+ jwks: dict[str, dict[str, Any]]
18
+ providers: dict[str, str]
19
+ timestamps: dict[str, float]
@@ -0,0 +1,476 @@
1
+ # implementation based on this stackoverflow post:
2
+ # https://stackoverflow.com/a/67943659
3
+
4
+
5
+ import time
6
+ from collections.abc import Iterable
7
+ from threading import Lock
8
+ from typing import Any, Optional, cast
9
+
10
+ import jwt
11
+ import requests
12
+ from jwt.api_jwk import PyJWK
13
+
14
+ from clue.common.exceptions import ClueKeyError, ClueValueError
15
+ from clue.common.logging import get_logger
16
+ from clue.config import cache, config
17
+ from clue.models.jwks import JWKSSnapshot
18
+ from clue.security.utils import decode_jwt_payload
19
+
20
+ logger = get_logger(__file__)
21
+
22
+
23
+ # Shared across users in this worker so only one JWKS fetch can run at a time.
24
+ _jwks_refresh_lock = Lock()
25
+ _last_jwks_refresh_time: float | None = None
26
+ # Last successfully fetched JWKS. Kept outside SimpleCache so cache pressure can never lock out known keys.
27
+ _pinned_jwks: JWKSSnapshot | None = None
28
+ # Keys whose provider was last fetched longer ago than this are never served, from the cache or the pin.
29
+ MAX_PINNED_JWKS_STALE_SECONDS = 12 * 60 * 60
30
+ JWKS_CACHE_TIMEOUT_SECONDS = 60 * 60 * 12
31
+
32
+
33
+ def get_jwk(access_token: str) -> PyJWK:
34
+ """Get the JSON Web Key associated with the given JWT"""
35
+ # "kid" is the JSON Web Key's identifier. It tells us which key was used to validate the token.
36
+ kid = jwt.get_unverified_header(access_token).get("kid")
37
+ if not kid or not isinstance(kid, str):
38
+ raise ClueValueError("Unexpected kid value in access token: %s", kid)
39
+
40
+ jwks, _ = get_jwks()
41
+
42
+ try:
43
+ # Check to see if we have it cached
44
+ key = PyJWK(jwks[kid])
45
+ except KeyError:
46
+ # The unverified kid may be attacker-controlled; refresh only if the cooldown permits it.
47
+ try:
48
+ jwks, _ = get_jwks(refresh=True)
49
+ key = PyJWK(jwks[kid])
50
+ except KeyError as e:
51
+ raise ClueKeyError("There is no valid JWK for this token.") from e
52
+
53
+ return key
54
+
55
+
56
+ def get_provider(access_token: str) -> str:
57
+ """Get the provider of a given access token
58
+
59
+ Args:
60
+ access_token (str): The access token to determine the provider of
61
+
62
+ Raises:
63
+ ClueValueError: The provider of this access token does not match any supported providers
64
+
65
+ Returns:
66
+ str: The provider of the token
67
+ """
68
+ # "kid" is the JSON Web Key's identifier. It tells us which key was used to validate the token.
69
+ kid = jwt.get_unverified_header(access_token).get("kid")
70
+ if not kid or not isinstance(kid, str):
71
+ raise ClueValueError("Unexpected kid value in access token: %s", kid)
72
+
73
+ _, providers = get_jwks()
74
+
75
+ try:
76
+ # Check to see if we have it cached
77
+ oauth_provider = providers[kid]
78
+ except KeyError:
79
+ # Provider lookups use the same refresh cooldown as key lookups.
80
+ try:
81
+ _, providers = get_jwks(refresh=True)
82
+ oauth_provider = providers[kid]
83
+ except KeyError as e:
84
+ raise ClueValueError("The provider of this access token does not match any supported providers") from e
85
+
86
+ return oauth_provider
87
+
88
+
89
+ def _fresh_only(snapshot: JWKSSnapshot, now: float | None = None) -> JWKSSnapshot:
90
+ """Filters out keys and providers whose last fetch is older than the stale TTL.
91
+
92
+ This is pure: freshness is judged only by the timestamps carried in `snapshot`, never by module state,
93
+ so a concurrent refresh cannot make an old snapshot look newer than it is.
94
+
95
+ Args:
96
+ snapshot: The snapshot of keys, kid owners and provider fetch times to filter.
97
+ now: Reference monotonic time. Defaults to `time.monotonic()` when omitted.
98
+
99
+ Returns:
100
+ A new snapshot containing only providers fetched less than `MAX_PINNED_JWKS_STALE_SECONDS` before
101
+ `now`, and the keys they own. Providers missing from `snapshot.timestamps` are treated as expired.
102
+ """
103
+ now = time.monotonic() if now is None else now
104
+ fresh_providers = {
105
+ name for name, fetched_at in snapshot.timestamps.items() if now - fetched_at < MAX_PINNED_JWKS_STALE_SECONDS
106
+ }
107
+ jwks = {kid: jwk for kid, jwk in snapshot.jwks.items() if snapshot.providers.get(kid) in fresh_providers}
108
+ return JWKSSnapshot(
109
+ jwks,
110
+ {kid: snapshot.providers[kid] for kid in jwks},
111
+ {name: fetched_at for name, fetched_at in snapshot.timestamps.items() if name in fresh_providers},
112
+ )
113
+
114
+
115
+ def _unexpired_pinned_jwks() -> JWKSSnapshot | None:
116
+ """Returns the pinned snapshot with expired providers removed, for use as a fallback.
117
+
118
+ Returns:
119
+ The unexpired part of the pinned snapshot, or None if there is no pinned snapshot or every provider has
120
+ expired. A fresh provider that returned no keys is authoritative, so the result may have no keys.
121
+ """
122
+ pinned = _pinned_jwks # single read: keys and timestamps travel together
123
+ if pinned is None:
124
+ return None
125
+
126
+ fresh = _fresh_only(pinned)
127
+ # Any fresh timestamp is authoritative, even with no keys; other providers' expired keys don't matter.
128
+ return fresh if fresh.timestamps else None
129
+
130
+
131
+ def _known_jwks(cached_jwks: JWKSSnapshot | None) -> JWKSSnapshot:
132
+ """Returns the best unexpired keys available without fetching.
133
+
134
+ Used on cooldowns and while another request holds the refresh lock, so valid tokens keep verifying
135
+ after cache evictions.
136
+
137
+ Args:
138
+ cached_jwks: The snapshot read from the cache, or None if it was evicted or never populated.
139
+
140
+ Returns:
141
+ The unexpired cached snapshot if there is one, otherwise the unexpired pinned snapshot, otherwise an
142
+ empty snapshot.
143
+ """
144
+ if cached_jwks is not None:
145
+ return _fresh_only(cached_jwks)
146
+ return _unexpired_pinned_jwks() or JWKSSnapshot({}, {}, {})
147
+
148
+
149
+ def _is_valid_jwk(jwk: Any) -> bool:
150
+ """Checks that an untrusted JWK entry is structurally usable.
151
+
152
+ Args:
153
+ jwk: A single entry from a provider's `keys` array.
154
+
155
+ Returns:
156
+ True if `jwk` is a dict with a non-empty string `kid`, a string `kty`, and key material that
157
+ `PyJWK` accepts; False otherwise.
158
+ """
159
+ if not (
160
+ isinstance(jwk, dict) and isinstance(jwk.get("kid"), str) and jwk["kid"] and isinstance(jwk.get("kty"), str)
161
+ ):
162
+ return False
163
+
164
+ try:
165
+ PyJWK(jwk)
166
+ except Exception: # Key material is untrusted; any parsing failure means the key is unusable.
167
+ return False
168
+ return True
169
+
170
+
171
+ def _fetch_provider_jwks(jwks_uri: str) -> list[dict[str, Any]]:
172
+ """Fetches and validates one provider's JWKS.
173
+
174
+ Args:
175
+ jwks_uri: The provider's JWKS endpoint.
176
+
177
+ Returns:
178
+ The valid keys from the response. Malformed entries are skipped with a warning; an empty list is an
179
+ authoritative "no keys" answer.
180
+
181
+ Raises:
182
+ requests.RequestException: The request failed or returned an HTTP error status.
183
+ ValueError: The payload is not a JSON object with a `keys` list, or every entry in a non-empty
184
+ `keys` list is malformed.
185
+ """
186
+ response = requests.get(jwks_uri, timeout=10)
187
+ response.raise_for_status()
188
+ payload = response.json()
189
+ if not isinstance(payload, dict) or not isinstance(payload.get("keys"), list):
190
+ raise ValueError(f"Malformed JWKS payload from {jwks_uri}") # noqa: TRY004
191
+
192
+ valid_keys = [jwk for jwk in payload["keys"] if _is_valid_jwk(jwk)]
193
+ skipped = len(payload["keys"]) - len(valid_keys)
194
+ if skipped:
195
+ logger.warning("Ignoring %d malformed JWK(s) from %s", skipped, jwks_uri)
196
+ if payload["keys"] and not valid_keys:
197
+ raise ValueError(f"JWKS from {jwks_uri} contains no valid keys")
198
+
199
+ return valid_keys
200
+
201
+
202
+ def _fetch_all_providers(now: float) -> tuple[JWKSSnapshot, dict[str, Exception]]:
203
+ """Fetches every configured provider independently so one failure cannot discard another's result.
204
+
205
+ Args:
206
+ now: Monotonic time recorded as the fetch time of each provider that succeeds.
207
+
208
+ Returns:
209
+ A snapshot of the providers that were fetched successfully (including those returning no keys), and
210
+ the exception for each provider that failed.
211
+ """
212
+ jwks: dict[str, dict[str, Any]] = {}
213
+ providers: dict[str, str] = {}
214
+ timestamps: dict[str, float] = {}
215
+ failures: dict[str, Exception] = {}
216
+ for provider_name, provider_data in config.auth.oauth.providers.items():
217
+ if not provider_data.jwks_uri:
218
+ continue
219
+
220
+ try:
221
+ provider_jwks = _fetch_provider_jwks(provider_data.jwks_uri)
222
+ except (requests.RequestException, ValueError, KeyError, TypeError, AttributeError) as e:
223
+ failures[provider_name] = e
224
+ continue
225
+
226
+ timestamps[provider_name] = now
227
+ jwks |= {jwk["kid"]: jwk for jwk in provider_jwks}
228
+ providers |= {jwk["kid"]: provider_name for jwk in provider_jwks}
229
+
230
+ return JWKSSnapshot(jwks, providers, timestamps), failures
231
+
232
+
233
+ def _carry_over_failed_providers(fresh: JWKSSnapshot, failed: Iterable[str]) -> JWKSSnapshot:
234
+ """Adds each failed provider's pinned keys, with their original fetch times, to a fresh snapshot.
235
+
236
+ Expiry is not checked here: the caller runs `_fresh_only` on the result, so carried-over providers
237
+ cannot outlive `MAX_PINNED_JWKS_STALE_SECONDS` however many partial refreshes happen.
238
+
239
+ Args:
240
+ fresh: Snapshot of the providers that were fetched successfully. Its entries win on a `kid` clash.
241
+ failed: Names of providers whose refresh failed.
242
+
243
+ Returns:
244
+ A new snapshot combining `fresh` with the pinned data of the failed providers.
245
+ """
246
+ old = _pinned_jwks
247
+ if old is None:
248
+ return fresh
249
+
250
+ carried = {kid for kid, owner in old.providers.items() if owner in failed}
251
+ return JWKSSnapshot(
252
+ {kid: old.jwks[kid] for kid in carried} | fresh.jwks,
253
+ {kid: old.providers[kid] for kid in carried} | fresh.providers,
254
+ {name: fetched_at for name, fetched_at in old.timestamps.items() if name in failed} | fresh.timestamps,
255
+ )
256
+
257
+
258
+ def _get_jwks_snapshot(refresh: bool) -> JWKSSnapshot:
259
+ global _last_jwks_refresh_time, _pinned_jwks
260
+
261
+ cached_jwks: JWKSSnapshot | None = cache.get("get_jwks")
262
+ if cached_jwks is not None and not refresh:
263
+ return _fresh_only(cached_jwks)
264
+
265
+ # Nonblocking: a waiting thread would stall behind a slow provider (up to the request timeout), so
266
+ # overlapping callers serve existing keys or fail fast instead.
267
+ if not _jwks_refresh_lock.acquire(blocking=False):
268
+ return _known_jwks(cached_jwks)
269
+
270
+ # The cache is only written inside this lock, and never with fallback data, so stale snapshots cannot roll it back.
271
+ try:
272
+ # Another request may have updated the cache before we acquired the lock.
273
+ cached_jwks = cache.get("get_jwks")
274
+ now = time.monotonic()
275
+ if (
276
+ _last_jwks_refresh_time is not None
277
+ and now - _last_jwks_refresh_time < config.auth.oauth.jwks_refresh_cooldown_seconds
278
+ ):
279
+ return _known_jwks(cached_jwks)
280
+
281
+ # Recorded before fetching so failed and timed-out attempts are rate-limited too; otherwise an attacker
282
+ # sending unknown kids while a provider is down could trigger an outbound request per call.
283
+ _last_jwks_refresh_time = now
284
+ fetched, failures = _fetch_all_providers(now)
285
+
286
+ if not fetched.timestamps:
287
+ if not failures:
288
+ return _known_jwks(cached_jwks)
289
+
290
+ first_error = next(iter(failures.values()))
291
+ fallback = _unexpired_pinned_jwks()
292
+ if fallback is None:
293
+ raise first_error
294
+ logger.warning("Failed to refresh JWKS, serving pinned keys: %s", first_error)
295
+ return fallback
296
+
297
+ # Successful providers are authoritative (even with no keys); failed ones keep their unexpired pinned keys.
298
+ if failures:
299
+ logger.warning("Failed to refresh JWKS for providers %s", ", ".join(failures))
300
+
301
+ # Re-checked with a fresh clock reading: slow requests may have pushed carried-over keys past the limit
302
+ # since `now` was taken, and the caller must never receive a key that expired during the fetch.
303
+ result = _fresh_only(_carry_over_failed_providers(fetched, failures), now=time.monotonic())
304
+ # One assignment publishes keys and timestamps together, so readers never see mismatched generations.
305
+ _pinned_jwks = result
306
+ cache.set("get_jwks", result, timeout=JWKS_CACHE_TIMEOUT_SECONDS)
307
+ return result
308
+ finally:
309
+ _jwks_refresh_lock.release()
310
+
311
+
312
+ def get_jwks(refresh: bool = False) -> tuple[dict[str, dict[str, Any]], dict[str, str]]:
313
+ """Get the JSON Web Key Set for all supported providers
314
+
315
+ Args:
316
+ refresh (bool): Request a refresh, subject to the outbound fetch cooldown.
317
+
318
+ Returns:
319
+ tuple[dict[str, str], dict[str, str]]: The JWKS and the providers that are included in it
320
+ """
321
+ snapshot = _get_jwks_snapshot(refresh)
322
+ return snapshot.jwks, snapshot.providers
323
+
324
+
325
+ def extract_audience(access_token: str) -> list[str]:
326
+ "Extract the audience from an encoded JWT."
327
+ audience: list[str] | str | None = decode_jwt_payload(access_token).get("aud", None)
328
+
329
+ if not audience:
330
+ return []
331
+
332
+ return [audience] if not isinstance(audience, list) else audience
333
+
334
+
335
+ def get_audience(oauth_provider: str) -> str:
336
+ """Get the audience for the specified OAuth provider
337
+
338
+ Args:
339
+ oauth_provider (str): The OAuth provider to retrieve the audience of
340
+
341
+ Raises:
342
+ ClueValueError: The provider is azure, and is improperly formatted
343
+
344
+ Returns:
345
+ str: The audience of the provider
346
+ """
347
+ audience: str = "clue"
348
+ provider_data = config.auth.oauth.providers[oauth_provider]
349
+ if provider_data.audience:
350
+ audience = provider_data.audience
351
+ elif provider_data.client_id:
352
+ audience = provider_data.client_id
353
+
354
+ if oauth_provider == "azure" and f"{audience}/.default" not in provider_data.scope:
355
+ raise ClueValueError("Azure scope must contain the <client_id>/.default claim!")
356
+
357
+ return audience
358
+
359
+
360
+ def decode(
361
+ access_token: str,
362
+ key: Optional[str] = None,
363
+ algorithms: Optional[list[str]] = None,
364
+ audience: Optional[str] = None,
365
+ validate_audience: bool = False,
366
+ **kwargs,
367
+ ) -> dict[str, Any]:
368
+ """Decode an access token into a JSON Web Token dict
369
+
370
+ Args:
371
+ access_token (str): The access token to decode
372
+ key (Optional[str], optional): The key used to sign the token. Defaults to None.
373
+ algorithms (Optional[list[str]], optional): The algorithm to use when decoding. Defaults to None.
374
+ audience (Optional[str], optional): The audience to check against, if validating the audience. Defaults to None.
375
+ validate_audience (bool, optional): Should we validate the audience? Defaults to False.
376
+
377
+ Returns:
378
+ dict[str, Any]: The decoded JWT, in dict format
379
+ """
380
+ if not key:
381
+ key = get_jwk(access_token).key
382
+
383
+ if not algorithms:
384
+ algorithms = [jwt.get_unverified_header(access_token).get("alg", "HS256")]
385
+
386
+ if validate_audience and not audience:
387
+ audience = get_audience(get_provider(access_token))
388
+
389
+ try:
390
+ return jwt.decode(
391
+ jwt=access_token,
392
+ key=cast(str, key),
393
+ algorithms=algorithms,
394
+ audience=audience,
395
+ options={"verify_aud": validate_audience},
396
+ **kwargs,
397
+ ) # type: ignore
398
+ except jwt.exceptions.InvalidAudienceError:
399
+ logger.debug("Default audience did not match - checking additional audiences")
400
+ if config.auth.oauth.other_audiences is not None:
401
+ # The main audience isn't valid, let's try the others
402
+ for audience in config.auth.oauth.other_audiences:
403
+ logger.debug("Checking audience %s", audience)
404
+ try:
405
+ return jwt.decode(
406
+ jwt=access_token,
407
+ key=cast(str, key),
408
+ algorithms=algorithms,
409
+ audience=audience,
410
+ options={"verify_aud": validate_audience},
411
+ **kwargs,
412
+ ) # type: ignore
413
+ except jwt.InvalidAudienceError:
414
+ continue
415
+
416
+ logger.warning(
417
+ "Default and additional audiences failed to validate. Expected: %s, Actual: %s",
418
+ audience,
419
+ ",".join(extract_audience(access_token)),
420
+ )
421
+ raise
422
+
423
+
424
+ def fetch_sa_token() -> Optional[str]:
425
+ """Use a service account to fetch a valid token, if service accounts are enabled"""
426
+ if not config.auth.service_account.enabled:
427
+ return None
428
+
429
+ # TODO: Eventually support multiple accounts
430
+ service_account = config.auth.service_account.accounts[0]
431
+ cache_key = f"sa_refresh_token_{service_account.username}"
432
+
433
+ provider = config.auth.oauth.providers[service_account.provider]
434
+
435
+ try:
436
+ # Eventually switch this to a redis cache (the rest of this file too)
437
+ refresh_token = cache.get(key=cache_key)
438
+ use_cache = True
439
+ except AttributeError:
440
+ refresh_token = None
441
+ use_cache = False
442
+
443
+ if refresh_token:
444
+ sa_jwt = requests.post(
445
+ provider.access_token_url,
446
+ data={
447
+ "client_id": provider.client_id,
448
+ "client_secret": provider.client_secret,
449
+ "grant_type": "refresh_token",
450
+ "refresh_token": refresh_token,
451
+ "scope": provider.scope,
452
+ },
453
+ timeout=30,
454
+ ).json()
455
+ else:
456
+ sa_jwt = requests.post(
457
+ provider.access_token_url,
458
+ data={
459
+ "client_id": provider.client_id,
460
+ "client_secret": provider.client_secret,
461
+ "grant_type": "password",
462
+ "username": service_account.username,
463
+ "password": service_account.password,
464
+ "scope": provider.scope,
465
+ },
466
+ timeout=30,
467
+ ).json()
468
+
469
+ if "error" in sa_jwt:
470
+ logger.critical("[%s]: %s", sa_jwt["error"], sa_jwt["error_description"])
471
+ return None
472
+
473
+ if "refresh_token" in sa_jwt and use_cache:
474
+ cache.set(cache_key, sa_jwt["refresh_token"], timeout=60 * 60 * 12)
475
+
476
+ return sa_jwt["access_token"]
@@ -141,7 +141,7 @@ log_cli_level = "WARN"
141
141
  [tool.poetry]
142
142
  package-mode = true
143
143
  name = "clue-api"
144
- version = "1.7.0.dev446"
144
+ version = "1.7.0.dev450"
145
145
  description = "Clue distributed enrichment service"
146
146
  authors = ["Canadian Centre for Cyber Security <contact@cyber.gc.ca>"]
147
147
  license = "MIT"
@@ -1,254 +0,0 @@
1
- # implementation based on this stackoverflow post:
2
- # https://stackoverflow.com/a/67943659
3
-
4
-
5
- from typing import Any, Optional, cast
6
-
7
- import jwt
8
- import requests
9
- from jwt.api_jwk import PyJWK
10
-
11
- from clue.common.exceptions import ClueKeyError, ClueValueError
12
- from clue.common.logging import get_logger
13
- from clue.config import cache, config
14
- from clue.security.utils import decode_jwt_payload
15
-
16
- logger = get_logger(__file__)
17
-
18
-
19
- def get_jwk(access_token: str) -> PyJWK:
20
- """Get the JSON Web Key associated with the given JWT"""
21
- # "kid" is the JSON Web Key's identifier. It tells us which key was used to validate the token.
22
- kid = jwt.get_unverified_header(access_token).get("kid")
23
- if not kid or not isinstance(kid, str):
24
- raise ClueValueError("Unexpected kid value in access token: %s", kid)
25
-
26
- jwks, _ = get_jwks()
27
-
28
- try:
29
- # Check to see if we have it cached
30
- key = PyJWK(jwks[kid])
31
- except KeyError:
32
- # We don't, so we need to refresh the key set
33
- cache.delete(key="get_jwks")
34
- try:
35
- jwks, _ = get_jwks()
36
- key = PyJWK(jwks[kid])
37
- except KeyError as e:
38
- raise ClueKeyError("There is no valid JWK for this token.") from e
39
-
40
- return key
41
-
42
-
43
- def get_provider(access_token: str) -> str:
44
- """Get the provider of a given access token
45
-
46
- Args:
47
- access_token (str): The access token to determine the provider of
48
-
49
- Raises:
50
- ClueValueError: The provider of this access token does not match any supported providers
51
-
52
- Returns:
53
- str: The provider of the token
54
- """
55
- # "kid" is the JSON Web Key's identifier. It tells us which key was used to validate the token.
56
- kid = jwt.get_unverified_header(access_token).get("kid")
57
- if not kid or not isinstance(kid, str):
58
- raise ClueValueError("Unexpected kid value in access token: %s", kid)
59
-
60
- _, providers = get_jwks()
61
-
62
- try:
63
- # Check to see if we have it cached
64
- oauth_provider = providers[kid]
65
- except KeyError:
66
- # We don't, so we need to refresh the key set
67
- cache.delete(key="get_jwks")
68
- try:
69
- _, providers = get_jwks()
70
- oauth_provider = providers[kid]
71
- except KeyError as e:
72
- raise ClueValueError("The provider of this access token does not match any supported providers") from e
73
-
74
- return oauth_provider
75
-
76
-
77
- @cache.cached(timeout=60 * 60 * 12, key_prefix="get_jwks") # Cached for 12hrs
78
- def get_jwks() -> tuple[dict[str, dict[str, Any]], dict[str, str]]:
79
- """Get the JSON Web Key Set for all supported providers
80
-
81
- Returns:
82
- tuple[dict[str, str], dict[str, str]]: The JWKS and the providers that are included in it
83
- """
84
- # JWKS = JSON Web Key Set. We merge the key set from all oauth providers
85
- jwks: dict[str, dict[str, Any]] = {}
86
- # Mapping of keys to their provider (i.e. azure, keycloak)
87
- providers: dict[str, str] = {}
88
-
89
- for (
90
- provider_name,
91
- provider_data,
92
- ) in config.auth.oauth.providers.items():
93
- # Fetch the JSON Web Key Set for each provider that supports them
94
- if provider_data.jwks_uri:
95
- provider_jwks = requests.get(provider_data.jwks_uri, timeout=10).json()["keys"]
96
- for jwk in provider_jwks:
97
- jwks[jwk["kid"]] = jwk
98
- providers[jwk["kid"]] = provider_name
99
-
100
- return (jwks, providers)
101
-
102
-
103
- def extract_audience(access_token: str) -> list[str]:
104
- "Extract the audience from an encoded JWT."
105
- audience: list[str] | str | None = decode_jwt_payload(access_token).get("aud", None)
106
-
107
- if not audience:
108
- return []
109
-
110
- return [audience] if not isinstance(audience, list) else audience
111
-
112
-
113
- def get_audience(oauth_provider: str) -> str:
114
- """Get the audience for the specified OAuth provider
115
-
116
- Args:
117
- oauth_provider (str): The OAuth provider to retrieve the audience of
118
-
119
- Raises:
120
- ClueValueError: The provider is azure, and is improperly formatted
121
-
122
- Returns:
123
- str: The audience of the provider
124
- """
125
- audience: str = "clue"
126
- provider_data = config.auth.oauth.providers[oauth_provider]
127
- if provider_data.audience:
128
- audience = provider_data.audience
129
- elif provider_data.client_id:
130
- audience = provider_data.client_id
131
-
132
- if oauth_provider == "azure" and f"{audience}/.default" not in provider_data.scope:
133
- raise ClueValueError("Azure scope must contain the <client_id>/.default claim!")
134
-
135
- return audience
136
-
137
-
138
- def decode(
139
- access_token: str,
140
- key: Optional[str] = None,
141
- algorithms: Optional[list[str]] = None,
142
- audience: Optional[str] = None,
143
- validate_audience: bool = False,
144
- **kwargs,
145
- ) -> dict[str, Any]:
146
- """Decode an access token into a JSON Web Token dict
147
-
148
- Args:
149
- access_token (str): The access token to decode
150
- key (Optional[str], optional): The key used to sign the token. Defaults to None.
151
- algorithms (Optional[list[str]], optional): The algorithm to use when decoding. Defaults to None.
152
- audience (Optional[str], optional): The audience to check against, if validating the audience. Defaults to None.
153
- validate_audience (bool, optional): Should we validate the audience? Defaults to False.
154
-
155
- Returns:
156
- dict[str, Any]: The decoded JWT, in dict format
157
- """
158
- if not key:
159
- key = get_jwk(access_token).key
160
-
161
- if not algorithms:
162
- algorithms = [jwt.get_unverified_header(access_token).get("alg", "HS256")]
163
-
164
- if validate_audience and not audience:
165
- audience = get_audience(get_provider(access_token))
166
-
167
- try:
168
- return jwt.decode(
169
- jwt=access_token,
170
- key=cast(str, key),
171
- algorithms=algorithms,
172
- audience=audience,
173
- options={"verify_aud": validate_audience},
174
- **kwargs,
175
- ) # type: ignore
176
- except jwt.exceptions.InvalidAudienceError:
177
- logger.debug("Default audience did not match - checking additional audiences")
178
- if config.auth.oauth.other_audiences is not None:
179
- # The main audience isn't valid, let's try the others
180
- for audience in config.auth.oauth.other_audiences:
181
- logger.debug("Checking audience %s", audience)
182
- try:
183
- return jwt.decode(
184
- jwt=access_token,
185
- key=cast(str, key),
186
- algorithms=algorithms,
187
- audience=audience,
188
- options={"verify_aud": validate_audience},
189
- **kwargs,
190
- ) # type: ignore
191
- except jwt.InvalidAudienceError:
192
- continue
193
-
194
- logger.warning(
195
- "Default and additional audiences failed to validate. Expected: %s, Actual: %s",
196
- audience,
197
- ",".join(extract_audience(access_token)),
198
- )
199
- raise
200
-
201
-
202
- def fetch_sa_token() -> Optional[str]:
203
- """Use a service account to fetch a valid token, if service accounts are enabled"""
204
- if not config.auth.service_account.enabled:
205
- return None
206
-
207
- # TODO: Eventually support multiple accounts
208
- service_account = config.auth.service_account.accounts[0]
209
- cache_key = f"sa_refresh_token_{service_account.username}"
210
-
211
- provider = config.auth.oauth.providers[service_account.provider]
212
-
213
- try:
214
- # Eventually switch this to a redis cache (the rest of this file too)
215
- refresh_token = cache.get(key=cache_key)
216
- use_cache = True
217
- except AttributeError:
218
- refresh_token = None
219
- use_cache = False
220
-
221
- if refresh_token:
222
- sa_jwt = requests.post(
223
- provider.access_token_url,
224
- data={
225
- "client_id": provider.client_id,
226
- "client_secret": provider.client_secret,
227
- "grant_type": "refresh_token",
228
- "refresh_token": refresh_token,
229
- "scope": provider.scope,
230
- },
231
- timeout=30,
232
- ).json()
233
- else:
234
- sa_jwt = requests.post(
235
- provider.access_token_url,
236
- data={
237
- "client_id": provider.client_id,
238
- "client_secret": provider.client_secret,
239
- "grant_type": "password",
240
- "username": service_account.username,
241
- "password": service_account.password,
242
- "scope": provider.scope,
243
- },
244
- timeout=30,
245
- ).json()
246
-
247
- if "error" in sa_jwt:
248
- logger.critical("[%s]: %s", sa_jwt["error"], sa_jwt["error_description"])
249
- return None
250
-
251
- if "refresh_token" in sa_jwt and use_cache:
252
- cache.set(cache_key, sa_jwt["refresh_token"], timeout=60 * 60 * 12)
253
-
254
- return sa_jwt["access_token"]
File without changes