clue-api 1.0.0.dev34__tar.gz → 1.0.0.dev45__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/PKG-INFO +1 -1
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/app.py +16 -3
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/forge.py +2 -2
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/logging/__init__.py +20 -15
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/logging/audit.py +9 -9
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/logging/format.py +6 -6
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/cronjobs/__init__.py +2 -1
- clue_api-1.0.0.dev45/clue/extensions/__init__.py +25 -0
- clue_api-1.0.0.dev45/clue/extensions/config.py +74 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/config.py +8 -3
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/plugin/__init__.py +2 -2
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/remote/datatypes/cache.py +1 -1
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/security/obo.py +41 -8
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/services/type_service.py +1 -1
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/pyproject.toml +2 -2
- clue_api-1.0.0.dev34/clue/models/utils.py +0 -52
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/LICENSE +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/README.md +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/.gitignore +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/__init__.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/__init__.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/base.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/v1/__init__.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/v1/actions.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/v1/auth.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/v1/configs.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/v1/fetchers.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/v1/lookup.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/v1/registration.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/v1/static.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/cache/__init__.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/__init__.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/classification.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/classification.yml +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/dict_utils.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/exceptions.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/json_utils.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/list_utils.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/regex.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/str_utils.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/swagger.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/uid.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/config.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/constants/__init__.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/constants/supported_types.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/cronjobs/plugins.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/error.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/gunicorn_config.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/healthz.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/helper/discover.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/helper/headers.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/helper/oauth.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/__init__.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/actions.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/fetchers.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/graph.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/model_list.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/network.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/results/__init__.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/results/base.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/results/graph.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/results/image.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/results/status.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/results/validation.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/selector.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/validators.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/patched.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/plugin/helpers/__init__.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/plugin/helpers/central_server.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/plugin/helpers/email_render.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/plugin/helpers/token.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/plugin/helpers/trino.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/plugin/interactive.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/plugin/models.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/plugin/utils.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/remote/__init__.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/remote/datatypes/__init__.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/remote/datatypes/events.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/remote/datatypes/hash.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/remote/datatypes/queues/__init__.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/remote/datatypes/queues/comms.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/remote/datatypes/set.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/remote/datatypes/user_quota_tracker.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/security/__init__.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/security/utils.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/services/action_service.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/services/auth_service.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/services/config_service.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/services/fetcher_service.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/services/jwt_service.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/services/lookup_service.py +0 -0
- {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/services/user_service.py +0 -0
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import os
|
|
3
3
|
import re
|
|
4
|
-
from typing import Any
|
|
4
|
+
from typing import Any, cast
|
|
5
5
|
|
|
6
6
|
import elasticapm
|
|
7
7
|
from authlib.integrations.flask_client import OAuth
|
|
8
8
|
from elasticapm.contrib.flask import ElasticAPM
|
|
9
9
|
from flasgger import Swagger
|
|
10
10
|
from flask import Flask
|
|
11
|
+
from flask.blueprints import Blueprint
|
|
11
12
|
from flask.logging import default_handler
|
|
12
13
|
from prometheus_client import make_wsgi_app
|
|
13
14
|
from werkzeug.middleware.dispatcher import DispatcherMiddleware
|
|
@@ -25,10 +26,11 @@ from clue.common.logging import get_logger
|
|
|
25
26
|
from clue.config import DEBUG, SECRET_KEY, cache, config
|
|
26
27
|
from clue.cronjobs import setup_jobs as setup_cron_jobs
|
|
27
28
|
from clue.error import errors
|
|
29
|
+
from clue.extensions import get_extensions
|
|
28
30
|
from clue.healthz import healthz
|
|
29
31
|
|
|
30
|
-
SESSION_COOKIE_SAMESITE = os.environ.get("
|
|
31
|
-
HSTS_MAX_AGE = os.environ.get("
|
|
32
|
+
SESSION_COOKIE_SAMESITE = os.environ.get("CLUE_SESSION_COOKIE_SAMESITE", None)
|
|
33
|
+
HSTS_MAX_AGE = os.environ.get("CLUE_HSTS_MAX_AGE", None)
|
|
32
34
|
|
|
33
35
|
logger = get_logger(__file__)
|
|
34
36
|
|
|
@@ -94,6 +96,17 @@ app.register_blueprint(fetchers_api)
|
|
|
94
96
|
app.register_blueprint(lookup_api)
|
|
95
97
|
app.register_blueprint(registration_api)
|
|
96
98
|
app.register_blueprint(static_api)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
logger.info("Checking plugins for additional routes")
|
|
102
|
+
for plugin in get_extensions():
|
|
103
|
+
if not plugin.modules.routes:
|
|
104
|
+
continue
|
|
105
|
+
|
|
106
|
+
for route in cast(list[Blueprint], plugin.modules.routes):
|
|
107
|
+
logger.info("Enabling additional endpoint: %s", route.url_prefix)
|
|
108
|
+
app.register_blueprint(route)
|
|
109
|
+
|
|
97
110
|
# Setup OAuth providers
|
|
98
111
|
if config.auth.oauth.enabled:
|
|
99
112
|
providers = []
|
|
@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING
|
|
|
10
10
|
from flask_caching import Cache
|
|
11
11
|
|
|
12
12
|
from clue.common.dict_utils import recursive_update
|
|
13
|
-
from clue.common.logging.format import
|
|
13
|
+
from clue.common.logging.format import CLUE_DATE_FORMAT, CLUE_LOG_FORMAT
|
|
14
14
|
from clue.common.str_utils import default_string_value
|
|
15
15
|
|
|
16
16
|
APP_NAME: str = default_string_value(env_name="APP_NAME", default="clue") # type: ignore[assignment]
|
|
@@ -27,7 +27,7 @@ logger = logging.getLogger(f"{APP_NAME}.common.forge")
|
|
|
27
27
|
logger.setLevel(logging.INFO)
|
|
28
28
|
console = logging.StreamHandler()
|
|
29
29
|
console.setLevel(logging.INFO)
|
|
30
|
-
console.setFormatter(logging.Formatter(
|
|
30
|
+
console.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
|
|
31
31
|
logger.addHandler(console)
|
|
32
32
|
|
|
33
33
|
|
|
@@ -10,10 +10,10 @@ from typing import Optional, Self, Union
|
|
|
10
10
|
from flask import request
|
|
11
11
|
|
|
12
12
|
from clue.common.logging.format import (
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
13
|
+
CLUE_DATE_FORMAT,
|
|
14
|
+
CLUE_JSON_FORMAT,
|
|
15
|
+
CLUE_LOG_FORMAT,
|
|
16
|
+
CLUE_SYSLOG_FORMAT,
|
|
17
17
|
)
|
|
18
18
|
from clue.common.str_utils import default_string_value
|
|
19
19
|
|
|
@@ -50,10 +50,15 @@ class JsonFormatter(logging.Formatter):
|
|
|
50
50
|
record.exc_info = None
|
|
51
51
|
|
|
52
52
|
if record.exc_text:
|
|
53
|
-
record.
|
|
53
|
+
record.msg += "\n" + record.exc_text
|
|
54
54
|
record.exc_text = None
|
|
55
55
|
|
|
56
|
-
record.
|
|
56
|
+
record.msg = json.dumps(record.msg)
|
|
57
|
+
|
|
58
|
+
record.asctime = self.formatTime(record, self.datefmt)
|
|
59
|
+
|
|
60
|
+
record.message = record.msg
|
|
61
|
+
|
|
57
62
|
return self._style.format(record)
|
|
58
63
|
|
|
59
64
|
def formatException(self, exc_info): # noqa: N802
|
|
@@ -120,9 +125,9 @@ def init_logging(name: str, log_level: Optional[int] = None): # noqa: C901
|
|
|
120
125
|
)
|
|
121
126
|
dbg_file_handler.setLevel(logging.DEBUG)
|
|
122
127
|
if config.logging.log_as_json:
|
|
123
|
-
dbg_file_handler.setFormatter(JsonFormatter(
|
|
128
|
+
dbg_file_handler.setFormatter(JsonFormatter(CLUE_JSON_FORMAT))
|
|
124
129
|
else:
|
|
125
|
-
dbg_file_handler.setFormatter(logging.Formatter(
|
|
130
|
+
dbg_file_handler.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
|
|
126
131
|
logger.addHandler(dbg_file_handler)
|
|
127
132
|
|
|
128
133
|
if log_level <= logging.INFO:
|
|
@@ -133,9 +138,9 @@ def init_logging(name: str, log_level: Optional[int] = None): # noqa: C901
|
|
|
133
138
|
)
|
|
134
139
|
op_file_handler.setLevel(logging.INFO)
|
|
135
140
|
if config.logging.log_as_json:
|
|
136
|
-
op_file_handler.setFormatter(JsonFormatter(
|
|
141
|
+
op_file_handler.setFormatter(JsonFormatter(CLUE_JSON_FORMAT))
|
|
137
142
|
else:
|
|
138
|
-
op_file_handler.setFormatter(logging.Formatter(
|
|
143
|
+
op_file_handler.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
|
|
139
144
|
logger.addHandler(op_file_handler)
|
|
140
145
|
|
|
141
146
|
if log_level <= logging.ERROR:
|
|
@@ -146,24 +151,24 @@ def init_logging(name: str, log_level: Optional[int] = None): # noqa: C901
|
|
|
146
151
|
)
|
|
147
152
|
err_file_handler.setLevel(logging.ERROR)
|
|
148
153
|
if config.logging.log_as_json:
|
|
149
|
-
err_file_handler.setFormatter(JsonFormatter(
|
|
154
|
+
err_file_handler.setFormatter(JsonFormatter(CLUE_JSON_FORMAT))
|
|
150
155
|
else:
|
|
151
|
-
err_file_handler.setFormatter(logging.Formatter(
|
|
156
|
+
err_file_handler.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
|
|
152
157
|
logger.addHandler(err_file_handler)
|
|
153
158
|
|
|
154
159
|
if config.logging.log_to_console:
|
|
155
160
|
console = logging.StreamHandler()
|
|
156
161
|
if config.logging.log_as_json:
|
|
157
|
-
console.setFormatter(JsonFormatter(
|
|
162
|
+
console.setFormatter(JsonFormatter(CLUE_JSON_FORMAT))
|
|
158
163
|
else:
|
|
159
|
-
console.setFormatter(logging.Formatter(
|
|
164
|
+
console.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
|
|
160
165
|
logger.addHandler(console)
|
|
161
166
|
|
|
162
167
|
if config.logging.log_to_syslog and config.logging.syslog_host and config.logging.syslog_port:
|
|
163
168
|
syslog_handler = logging.handlers.SysLogHandler(
|
|
164
169
|
address=(config.logging.syslog_host, config.logging.syslog_port)
|
|
165
170
|
)
|
|
166
|
-
syslog_handler.formatter = logging.Formatter(
|
|
171
|
+
syslog_handler.formatter = logging.Formatter(CLUE_SYSLOG_FORMAT)
|
|
167
172
|
logger.addHandler(syslog_handler)
|
|
168
173
|
|
|
169
174
|
return logger.getChild(name)
|
|
@@ -5,10 +5,10 @@ import sys
|
|
|
5
5
|
from flask import request
|
|
6
6
|
|
|
7
7
|
from clue.common.logging.format import (
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
8
|
+
CLUE_AUDIT_FORMAT,
|
|
9
|
+
CLUE_DATE_FORMAT,
|
|
10
|
+
CLUE_ISO_DATE_FORMAT,
|
|
11
|
+
CLUE_LOG_FORMAT,
|
|
12
12
|
)
|
|
13
13
|
from clue.config import DEBUG, config
|
|
14
14
|
|
|
@@ -63,12 +63,12 @@ if AUDIT:
|
|
|
63
63
|
if not os.path.exists(config.logging.log_directory):
|
|
64
64
|
os.makedirs(config.logging.log_directory)
|
|
65
65
|
|
|
66
|
-
fh = logging.FileHandler(os.path.join(config.logging.log_directory, "
|
|
66
|
+
fh = logging.FileHandler(os.path.join(config.logging.log_directory, "clue_audit.log"))
|
|
67
67
|
fh.setLevel(logging.DEBUG)
|
|
68
68
|
fh.setFormatter(
|
|
69
69
|
logging.Formatter(
|
|
70
|
-
|
|
71
|
-
|
|
70
|
+
CLUE_LOG_FORMAT if DEBUG else CLUE_AUDIT_FORMAT,
|
|
71
|
+
CLUE_DATE_FORMAT if DEBUG else CLUE_ISO_DATE_FORMAT,
|
|
72
72
|
)
|
|
73
73
|
)
|
|
74
74
|
AUDIT_LOG.addHandler(fh)
|
|
@@ -77,8 +77,8 @@ ch = logging.StreamHandler(sys.stdout)
|
|
|
77
77
|
ch.setLevel(logging.INFO)
|
|
78
78
|
ch.setFormatter(
|
|
79
79
|
logging.Formatter(
|
|
80
|
-
|
|
81
|
-
|
|
80
|
+
CLUE_LOG_FORMAT if DEBUG else CLUE_AUDIT_FORMAT,
|
|
81
|
+
CLUE_DATE_FORMAT if DEBUG else CLUE_ISO_DATE_FORMAT,
|
|
82
82
|
)
|
|
83
83
|
)
|
|
84
84
|
AUDIT_LOG.addHandler(ch)
|
|
@@ -13,10 +13,10 @@ except Exception: # noqa: S110
|
|
|
13
13
|
|
|
14
14
|
APP_NAME: str = default_string_value(env_name="APP_NAME", default="clue") # type: ignore[assignment]
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
16
|
+
CLUE_SYSLOG_FORMAT = f"HWL %(levelname)8s {hostname} %(process)5d %(name)40s | %(message)s"
|
|
17
|
+
CLUE_LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s | %(message)s"
|
|
18
|
+
CLUE_DATE_FORMAT = "%y/%m/%d %H:%M:%S"
|
|
19
|
+
CLUE_JSON_FORMAT = (
|
|
20
20
|
f"{{"
|
|
21
21
|
f'"@timestamp": "%(asctime)s", '
|
|
22
22
|
f'"event": {{ "module": "{APP_NAME}", "dataset": "%(name)s" }}, '
|
|
@@ -25,8 +25,8 @@ BRL_JSON_FORMAT = (
|
|
|
25
25
|
f'"process": {{ "pid": "%(process)d" }}, '
|
|
26
26
|
f'"message": %(message)s}}'
|
|
27
27
|
)
|
|
28
|
-
|
|
29
|
-
|
|
28
|
+
CLUE_ISO_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
|
|
29
|
+
CLUE_AUDIT_FORMAT = json.dumps(
|
|
30
30
|
{
|
|
31
31
|
"date": "%(asctime)s",
|
|
32
32
|
"type": "audit",
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import importlib
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
from clue.common.logging import get_logger
|
|
5
|
+
from clue.config import config as _config # Python gets BIG mad if we don't alias this
|
|
6
|
+
from clue.extensions.config import BaseExtensionConfig
|
|
7
|
+
|
|
8
|
+
logger = get_logger(__file__)
|
|
9
|
+
|
|
10
|
+
EXTENSIONS: dict[str, Optional[BaseExtensionConfig]] = {}
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def get_extensions() -> list[BaseExtensionConfig]:
|
|
14
|
+
"Get a set of extension configurations based on the clue settings."
|
|
15
|
+
for extension in _config.core.extensions:
|
|
16
|
+
if extension in EXTENSIONS:
|
|
17
|
+
continue
|
|
18
|
+
|
|
19
|
+
try:
|
|
20
|
+
EXTENSIONS[extension] = importlib.import_module(f"{extension}.config").config
|
|
21
|
+
except (ImportError, ModuleNotFoundError):
|
|
22
|
+
logger.exception("Exception when loading extension %s", extension)
|
|
23
|
+
EXTENSIONS[extension] = None
|
|
24
|
+
|
|
25
|
+
return [extension for extension in EXTENSIONS.values() if extension]
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
from pydantic import BaseModel, ImportString, model_validator
|
|
5
|
+
from pydantic_settings import (
|
|
6
|
+
BaseSettings,
|
|
7
|
+
PydanticBaseSettingsSource,
|
|
8
|
+
YamlConfigSettingsSource,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
from clue.common.logging import CLUE_DATE_FORMAT, CLUE_LOG_FORMAT
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger("clue.extensions.config")
|
|
14
|
+
logger.setLevel(logging.INFO)
|
|
15
|
+
console = logging.StreamHandler()
|
|
16
|
+
console.setLevel(logging.INFO)
|
|
17
|
+
console.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
|
|
18
|
+
logger.addHandler(console)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Modules(BaseModel):
|
|
22
|
+
"A list of components exposed for use in Clue by this plugin."
|
|
23
|
+
|
|
24
|
+
routes: list[ImportString] = []
|
|
25
|
+
obo_module: ImportString | None = None
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class BaseExtensionConfig(BaseSettings):
|
|
29
|
+
"Configuration File for Plugin"
|
|
30
|
+
|
|
31
|
+
name: str
|
|
32
|
+
features: dict[str, bool] = {}
|
|
33
|
+
|
|
34
|
+
modules: Modules = Modules()
|
|
35
|
+
|
|
36
|
+
@model_validator(mode="before")
|
|
37
|
+
@classmethod
|
|
38
|
+
def initialize_extension_configuration(cls, data: Any) -> Any: # noqa: C901
|
|
39
|
+
"Convert a raw yaml config into an object ready for validation by pydantic"
|
|
40
|
+
if not isinstance(data, dict):
|
|
41
|
+
return data
|
|
42
|
+
|
|
43
|
+
# Default mutation requires plugin name
|
|
44
|
+
if "name" not in data:
|
|
45
|
+
logger.warning("Name is missing from configuration")
|
|
46
|
+
return data
|
|
47
|
+
|
|
48
|
+
plugin_name = data["name"]
|
|
49
|
+
logger.debug("Beginning configuration parsing for plugin %s", plugin_name)
|
|
50
|
+
|
|
51
|
+
if "modules" not in data:
|
|
52
|
+
return data
|
|
53
|
+
|
|
54
|
+
if "routes" in data["modules"] and isinstance(data["modules"]["routes"], list):
|
|
55
|
+
new_routes: list[str] = []
|
|
56
|
+
for route in data["modules"]["routes"]:
|
|
57
|
+
new_routes.append(f"{plugin_name}.routes.{route}" if "." not in route else route)
|
|
58
|
+
|
|
59
|
+
data["modules"]["routes"] = new_routes
|
|
60
|
+
|
|
61
|
+
if "obo_module" in data["modules"]:
|
|
62
|
+
if isinstance(data["modules"]["obo_module"], bool):
|
|
63
|
+
data["modules"]["obo_module"] = f"{plugin_name}.obo:get_obo_token"
|
|
64
|
+
|
|
65
|
+
return data
|
|
66
|
+
|
|
67
|
+
@classmethod
|
|
68
|
+
def settings_customise_sources(
|
|
69
|
+
cls, # noqa: ANN102
|
|
70
|
+
*args, # noqa: ANN002
|
|
71
|
+
**kwargs, # noqa: ANN002, ANN102
|
|
72
|
+
) -> tuple[PydanticBaseSettingsSource, ...]:
|
|
73
|
+
"Adds a YamlConfigSettingsSource object at the end of the settings_customize_sources response."
|
|
74
|
+
return (*super().settings_customise_sources(*args, **kwargs), YamlConfigSettingsSource(cls))
|
|
@@ -4,6 +4,7 @@ import os
|
|
|
4
4
|
from email.utils import parseaddr
|
|
5
5
|
from enum import Enum
|
|
6
6
|
from pathlib import Path
|
|
7
|
+
from typing import Self
|
|
7
8
|
from uuid import uuid4
|
|
8
9
|
|
|
9
10
|
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
@@ -14,11 +15,10 @@ from pydantic_settings import (
|
|
|
14
15
|
SettingsConfigDict,
|
|
15
16
|
YamlConfigSettingsSource,
|
|
16
17
|
)
|
|
17
|
-
from typing_extensions import Self
|
|
18
18
|
|
|
19
19
|
from clue.common import forge
|
|
20
20
|
from clue.common.exceptions import ClueValueError
|
|
21
|
-
from clue.common.logging.format import
|
|
21
|
+
from clue.common.logging.format import CLUE_DATE_FORMAT, CLUE_LOG_FORMAT
|
|
22
22
|
from clue.common.str_utils import default_string_value
|
|
23
23
|
|
|
24
24
|
AUTO_PROPERTY_TYPE = ["access", "classification", "type", "role", "remove_role", "group"]
|
|
@@ -216,8 +216,13 @@ class Metrics(BaseModel):
|
|
|
216
216
|
|
|
217
217
|
|
|
218
218
|
class Core(BaseModel):
|
|
219
|
+
extensions: set[str] = Field(description="A list of extensions to load", default=set())
|
|
220
|
+
|
|
219
221
|
metrics: Metrics = Metrics()
|
|
222
|
+
"Configuration for Metrics Collection"
|
|
223
|
+
|
|
220
224
|
redis: RedisServer = RedisServer()
|
|
225
|
+
"Configuration for Redis instances"
|
|
221
226
|
|
|
222
227
|
|
|
223
228
|
class LogLevel(str, Enum):
|
|
@@ -394,7 +399,7 @@ if os.getenv("AZURE_TEST_CONFIG", None) is not None:
|
|
|
394
399
|
logger.setLevel(logging.INFO)
|
|
395
400
|
console = logging.StreamHandler()
|
|
396
401
|
console.setLevel(logging.INFO)
|
|
397
|
-
console.setFormatter(logging.Formatter(
|
|
402
|
+
console.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
|
|
398
403
|
logger.addHandler(console)
|
|
399
404
|
|
|
400
405
|
logger.info("Azure build environment detected, adding additional config path")
|
|
@@ -22,7 +22,7 @@ from clue.common.exceptions import (
|
|
|
22
22
|
TimeoutException,
|
|
23
23
|
UnprocessableException,
|
|
24
24
|
)
|
|
25
|
-
from clue.common.logging.format import
|
|
25
|
+
from clue.common.logging.format import CLUE_DATE_FORMAT, CLUE_LOG_FORMAT
|
|
26
26
|
from clue.models.actions import (
|
|
27
27
|
Action,
|
|
28
28
|
ActionBase,
|
|
@@ -66,7 +66,7 @@ def build_default_logger() -> logging.Logger:
|
|
|
66
66
|
logger.setLevel(logging.INFO)
|
|
67
67
|
console = logging.StreamHandler()
|
|
68
68
|
console.setLevel(logging.INFO)
|
|
69
|
-
console.setFormatter(logging.Formatter(
|
|
69
|
+
console.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
|
|
70
70
|
logger.addHandler(console)
|
|
71
71
|
|
|
72
72
|
return logger
|
|
@@ -9,7 +9,7 @@ DEFAULT_TTL = 60 * 60 # 1 Hour
|
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
class RedisCache(object):
|
|
12
|
-
def __init__(self, prefix="
|
|
12
|
+
def __init__(self, prefix="clue_cache", host=None, port=None, ttl=DEFAULT_TTL):
|
|
13
13
|
self.c = get_client(host, port, False)
|
|
14
14
|
self.prefix = prefix
|
|
15
15
|
self.ttl = ttl
|
|
@@ -4,6 +4,7 @@ from typing import Optional
|
|
|
4
4
|
from clue.common.exceptions import InvalidDataException
|
|
5
5
|
from clue.common.logging import get_logger
|
|
6
6
|
from clue.config import config, get_redis
|
|
7
|
+
from clue.extensions import get_extensions
|
|
7
8
|
from clue.remote.datatypes.set import ExpiringSet
|
|
8
9
|
from clue.security.utils import decode_jwt_payload
|
|
9
10
|
|
|
@@ -31,7 +32,34 @@ def _get_token_raw(service: str, user: str) -> Optional[str]:
|
|
|
31
32
|
return None
|
|
32
33
|
|
|
33
34
|
|
|
34
|
-
def
|
|
35
|
+
def try_validate_expiry(obo_access_token: str):
|
|
36
|
+
"""Validates the expiry of an OBO (On-Behalf-Of) access token.
|
|
37
|
+
|
|
38
|
+
Attempts to decode the JWT payload of the provided token and checks the 'exp' (expiry) field.
|
|
39
|
+
If the token has expired, logs a warning and returns None.
|
|
40
|
+
If the token is not a JWT or the 'exp' field is missing, logs a warning and skips expiry validation.
|
|
41
|
+
|
|
42
|
+
Args:
|
|
43
|
+
obo_access_token (str): The OBO access token to validate.
|
|
44
|
+
|
|
45
|
+
Returns:
|
|
46
|
+
str or None: The original token if valid or expiry cannot be determined, otherwise None if expired.
|
|
47
|
+
"""
|
|
48
|
+
try:
|
|
49
|
+
expiry = datetime.fromtimestamp(decode_jwt_payload(obo_access_token)["exp"])
|
|
50
|
+
|
|
51
|
+
if expiry < datetime.now():
|
|
52
|
+
logger.warning("Cached token has expired")
|
|
53
|
+
return None
|
|
54
|
+
except IndexError:
|
|
55
|
+
logger.warning("Token is not a JWT, skipping expiry validation")
|
|
56
|
+
except KeyError:
|
|
57
|
+
logger.warning("'exp' field is missing, skipping expiry validation")
|
|
58
|
+
|
|
59
|
+
return obo_access_token
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def get_obo_token(service: str, access_token: str, user: str, force_refresh: bool = False): # noqa: C901
|
|
35
63
|
"""Gets an On-Behalf-Of token from either the Redis cache or from the provided authentication plugin.
|
|
36
64
|
|
|
37
65
|
Args:
|
|
@@ -60,17 +88,22 @@ def get_obo_token(service: str, access_token: str, user: str, force_refresh: boo
|
|
|
60
88
|
obo_access_token = _get_token_raw(service, user)
|
|
61
89
|
|
|
62
90
|
if obo_access_token is not None:
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
if expiry < datetime.now():
|
|
66
|
-
logger.warning("Cached token has expired")
|
|
67
|
-
obo_access_token = None
|
|
91
|
+
obo_access_token = try_validate_expiry(obo_access_token)
|
|
68
92
|
|
|
69
93
|
if obo_access_token is None:
|
|
70
94
|
logger.info(f"Fetching OBO token for user {user} to service {service}")
|
|
71
95
|
|
|
72
|
-
|
|
73
|
-
|
|
96
|
+
extension_get_obo_token = None
|
|
97
|
+
for extension in get_extensions():
|
|
98
|
+
if extension.modules.obo_module:
|
|
99
|
+
extension_get_obo_token = extension.modules.obo_module
|
|
100
|
+
break
|
|
101
|
+
|
|
102
|
+
if extension_get_obo_token is None:
|
|
103
|
+
logger.info("No OBO function provided, returning provided access token")
|
|
104
|
+
return access_token
|
|
105
|
+
|
|
106
|
+
obo_access_token = extension_get_obo_token(service, access_token, user)
|
|
74
107
|
|
|
75
108
|
if obo_access_token:
|
|
76
109
|
service_token_store = _get_obo_token_store(service, user)
|
|
@@ -17,7 +17,7 @@ logger = get_logger(__file__)
|
|
|
17
17
|
|
|
18
18
|
# Either cache for one second in debug mode, or five minutes in production
|
|
19
19
|
CACHE_TIMEOUT: int = 1 if DEBUG else 5 * 60
|
|
20
|
-
CACHE = RedisCache(prefix="
|
|
20
|
+
CACHE = RedisCache(prefix="clue_types", ttl=CACHE_TIMEOUT)
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
def get_types_regular_expressions(user: dict[str, Any]):
|
|
@@ -156,7 +156,7 @@ suppress-none-returning = true
|
|
|
156
156
|
###################
|
|
157
157
|
[tool.pytest.ini_options]
|
|
158
158
|
log_cli = true
|
|
159
|
-
log_cli_level = "
|
|
159
|
+
log_cli_level = "WARN"
|
|
160
160
|
|
|
161
161
|
###################
|
|
162
162
|
# Poetry settings #
|
|
@@ -164,7 +164,7 @@ log_cli_level = "INFO"
|
|
|
164
164
|
[tool.poetry]
|
|
165
165
|
package-mode = true
|
|
166
166
|
name = "clue-api"
|
|
167
|
-
version = "1.0.0.
|
|
167
|
+
version = "1.0.0.dev45"
|
|
168
168
|
description = "Clue distributed enrichment service"
|
|
169
169
|
authors = ["Canadian Centre for Cyber Security <contact@cyber.gc.ca>"]
|
|
170
170
|
license = "MIT"
|
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
import random
|
|
2
|
-
from datetime import datetime
|
|
3
|
-
from typing import Union, get_args, get_origin
|
|
4
|
-
|
|
5
|
-
from pydantic import BaseModel
|
|
6
|
-
from pydantic_core import Url
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
def generate_example_model(model: type[BaseModel], as_list: bool = False): # noqa: C901
|
|
10
|
-
"""Populate fields with example values"""
|
|
11
|
-
result = {}
|
|
12
|
-
|
|
13
|
-
if as_list:
|
|
14
|
-
full_list = []
|
|
15
|
-
for _ in range(random.randint(1, 3)): # noqa: S311
|
|
16
|
-
full_list.append(generate_example_model(model))
|
|
17
|
-
|
|
18
|
-
return full_list
|
|
19
|
-
|
|
20
|
-
for name, field_info in model.model_fields.items():
|
|
21
|
-
field_type = field_info.annotation
|
|
22
|
-
if field_type:
|
|
23
|
-
if get_origin(field_type) is Union and type(None) in get_args(field_type):
|
|
24
|
-
field_type = (
|
|
25
|
-
next((_type for _type in get_args(field_type) if _type is not type(None)), None) or field_type
|
|
26
|
-
)
|
|
27
|
-
|
|
28
|
-
_as_list = False
|
|
29
|
-
if get_origin(field_type) is list:
|
|
30
|
-
_as_list = True
|
|
31
|
-
field_type = get_args(field_type)[0]
|
|
32
|
-
|
|
33
|
-
_issubclass = False
|
|
34
|
-
try:
|
|
35
|
-
if field_type:
|
|
36
|
-
_issubclass = issubclass(field_type, BaseModel)
|
|
37
|
-
except TypeError:
|
|
38
|
-
pass
|
|
39
|
-
|
|
40
|
-
if field_type and _issubclass:
|
|
41
|
-
result[name] = generate_example_model(field_type, as_list=_as_list)
|
|
42
|
-
elif field_info.examples:
|
|
43
|
-
result[name] = random.choice(field_info.examples) # noqa: S311
|
|
44
|
-
elif field_info.default:
|
|
45
|
-
result[name] = field_info.default
|
|
46
|
-
|
|
47
|
-
if isinstance(result[name], datetime):
|
|
48
|
-
result[name] = result[name].isoformat()
|
|
49
|
-
elif isinstance(result[name], Url):
|
|
50
|
-
result[name] = str(result[name])
|
|
51
|
-
|
|
52
|
-
return dict(result)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|