clue-api 1.0.0.dev34__tar.gz → 1.0.0.dev45__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/PKG-INFO +1 -1
  2. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/app.py +16 -3
  3. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/forge.py +2 -2
  4. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/logging/__init__.py +20 -15
  5. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/logging/audit.py +9 -9
  6. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/logging/format.py +6 -6
  7. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/cronjobs/__init__.py +2 -1
  8. clue_api-1.0.0.dev45/clue/extensions/__init__.py +25 -0
  9. clue_api-1.0.0.dev45/clue/extensions/config.py +74 -0
  10. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/config.py +8 -3
  11. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/plugin/__init__.py +2 -2
  12. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/remote/datatypes/cache.py +1 -1
  13. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/security/obo.py +41 -8
  14. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/services/type_service.py +1 -1
  15. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/pyproject.toml +2 -2
  16. clue_api-1.0.0.dev34/clue/models/utils.py +0 -52
  17. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/LICENSE +0 -0
  18. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/README.md +0 -0
  19. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/.gitignore +0 -0
  20. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/__init__.py +0 -0
  21. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/__init__.py +0 -0
  22. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/base.py +0 -0
  23. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/v1/__init__.py +0 -0
  24. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/v1/actions.py +0 -0
  25. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/v1/auth.py +0 -0
  26. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/v1/configs.py +0 -0
  27. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/v1/fetchers.py +0 -0
  28. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/v1/lookup.py +0 -0
  29. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/v1/registration.py +0 -0
  30. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/api/v1/static.py +0 -0
  31. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/cache/__init__.py +0 -0
  32. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/__init__.py +0 -0
  33. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/classification.py +0 -0
  34. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/classification.yml +0 -0
  35. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/dict_utils.py +0 -0
  36. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/exceptions.py +0 -0
  37. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/json_utils.py +0 -0
  38. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/list_utils.py +0 -0
  39. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/regex.py +0 -0
  40. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/str_utils.py +0 -0
  41. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/swagger.py +0 -0
  42. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/common/uid.py +0 -0
  43. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/config.py +0 -0
  44. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/constants/__init__.py +0 -0
  45. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/constants/supported_types.py +0 -0
  46. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/cronjobs/plugins.py +0 -0
  47. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/error.py +0 -0
  48. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/gunicorn_config.py +0 -0
  49. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/healthz.py +0 -0
  50. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/helper/discover.py +0 -0
  51. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/helper/headers.py +0 -0
  52. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/helper/oauth.py +0 -0
  53. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/__init__.py +0 -0
  54. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/actions.py +0 -0
  55. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/fetchers.py +0 -0
  56. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/graph.py +0 -0
  57. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/model_list.py +0 -0
  58. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/network.py +0 -0
  59. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/results/__init__.py +0 -0
  60. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/results/base.py +0 -0
  61. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/results/graph.py +0 -0
  62. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/results/image.py +0 -0
  63. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/results/status.py +0 -0
  64. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/results/validation.py +0 -0
  65. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/selector.py +0 -0
  66. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/models/validators.py +0 -0
  67. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/patched.py +0 -0
  68. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/plugin/helpers/__init__.py +0 -0
  69. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/plugin/helpers/central_server.py +0 -0
  70. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/plugin/helpers/email_render.py +0 -0
  71. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/plugin/helpers/token.py +0 -0
  72. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/plugin/helpers/trino.py +0 -0
  73. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/plugin/interactive.py +0 -0
  74. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/plugin/models.py +0 -0
  75. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/plugin/utils.py +0 -0
  76. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/remote/__init__.py +0 -0
  77. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/remote/datatypes/__init__.py +0 -0
  78. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/remote/datatypes/events.py +0 -0
  79. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/remote/datatypes/hash.py +0 -0
  80. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/remote/datatypes/queues/__init__.py +0 -0
  81. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/remote/datatypes/queues/comms.py +0 -0
  82. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/remote/datatypes/set.py +0 -0
  83. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/remote/datatypes/user_quota_tracker.py +0 -0
  84. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/security/__init__.py +0 -0
  85. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/security/utils.py +0 -0
  86. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/services/action_service.py +0 -0
  87. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/services/auth_service.py +0 -0
  88. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/services/config_service.py +0 -0
  89. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/services/fetcher_service.py +0 -0
  90. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/services/jwt_service.py +0 -0
  91. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/services/lookup_service.py +0 -0
  92. {clue_api-1.0.0.dev34 → clue_api-1.0.0.dev45}/clue/services/user_service.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: clue-api
3
- Version: 1.0.0.dev34
3
+ Version: 1.0.0.dev45
4
4
  Summary: Clue distributed enrichment service
5
5
  License: MIT
6
6
  License-File: LICENSE
@@ -1,13 +1,14 @@
1
1
  import logging
2
2
  import os
3
3
  import re
4
- from typing import Any
4
+ from typing import Any, cast
5
5
 
6
6
  import elasticapm
7
7
  from authlib.integrations.flask_client import OAuth
8
8
  from elasticapm.contrib.flask import ElasticAPM
9
9
  from flasgger import Swagger
10
10
  from flask import Flask
11
+ from flask.blueprints import Blueprint
11
12
  from flask.logging import default_handler
12
13
  from prometheus_client import make_wsgi_app
13
14
  from werkzeug.middleware.dispatcher import DispatcherMiddleware
@@ -25,10 +26,11 @@ from clue.common.logging import get_logger
25
26
  from clue.config import DEBUG, SECRET_KEY, cache, config
26
27
  from clue.cronjobs import setup_jobs as setup_cron_jobs
27
28
  from clue.error import errors
29
+ from clue.extensions import get_extensions
28
30
  from clue.healthz import healthz
29
31
 
30
- SESSION_COOKIE_SAMESITE = os.environ.get("BRL_SESSION_COOKIE_SAMESITE", None)
31
- HSTS_MAX_AGE = os.environ.get("BRL_HSTS_MAX_AGE", None)
32
+ SESSION_COOKIE_SAMESITE = os.environ.get("CLUE_SESSION_COOKIE_SAMESITE", None)
33
+ HSTS_MAX_AGE = os.environ.get("CLUE_HSTS_MAX_AGE", None)
32
34
 
33
35
  logger = get_logger(__file__)
34
36
 
@@ -94,6 +96,17 @@ app.register_blueprint(fetchers_api)
94
96
  app.register_blueprint(lookup_api)
95
97
  app.register_blueprint(registration_api)
96
98
  app.register_blueprint(static_api)
99
+
100
+
101
+ logger.info("Checking plugins for additional routes")
102
+ for plugin in get_extensions():
103
+ if not plugin.modules.routes:
104
+ continue
105
+
106
+ for route in cast(list[Blueprint], plugin.modules.routes):
107
+ logger.info("Enabling additional endpoint: %s", route.url_prefix)
108
+ app.register_blueprint(route)
109
+
97
110
  # Setup OAuth providers
98
111
  if config.auth.oauth.enabled:
99
112
  providers = []
@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING
10
10
  from flask_caching import Cache
11
11
 
12
12
  from clue.common.dict_utils import recursive_update
13
- from clue.common.logging.format import BRL_DATE_FORMAT, BRL_LOG_FORMAT
13
+ from clue.common.logging.format import CLUE_DATE_FORMAT, CLUE_LOG_FORMAT
14
14
  from clue.common.str_utils import default_string_value
15
15
 
16
16
  APP_NAME: str = default_string_value(env_name="APP_NAME", default="clue") # type: ignore[assignment]
@@ -27,7 +27,7 @@ logger = logging.getLogger(f"{APP_NAME}.common.forge")
27
27
  logger.setLevel(logging.INFO)
28
28
  console = logging.StreamHandler()
29
29
  console.setLevel(logging.INFO)
30
- console.setFormatter(logging.Formatter(BRL_LOG_FORMAT, BRL_DATE_FORMAT))
30
+ console.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
31
31
  logger.addHandler(console)
32
32
 
33
33
 
@@ -10,10 +10,10 @@ from typing import Optional, Self, Union
10
10
  from flask import request
11
11
 
12
12
  from clue.common.logging.format import (
13
- BRL_DATE_FORMAT,
14
- BRL_JSON_FORMAT,
15
- BRL_LOG_FORMAT,
16
- BRL_SYSLOG_FORMAT,
13
+ CLUE_DATE_FORMAT,
14
+ CLUE_JSON_FORMAT,
15
+ CLUE_LOG_FORMAT,
16
+ CLUE_SYSLOG_FORMAT,
17
17
  )
18
18
  from clue.common.str_utils import default_string_value
19
19
 
@@ -50,10 +50,15 @@ class JsonFormatter(logging.Formatter):
50
50
  record.exc_info = None
51
51
 
52
52
  if record.exc_text:
53
- record.message += "\n" + record.exc_text
53
+ record.msg += "\n" + record.exc_text
54
54
  record.exc_text = None
55
55
 
56
- record.message = json.dumps(record.message)
56
+ record.msg = json.dumps(record.msg)
57
+
58
+ record.asctime = self.formatTime(record, self.datefmt)
59
+
60
+ record.message = record.msg
61
+
57
62
  return self._style.format(record)
58
63
 
59
64
  def formatException(self, exc_info): # noqa: N802
@@ -120,9 +125,9 @@ def init_logging(name: str, log_level: Optional[int] = None): # noqa: C901
120
125
  )
121
126
  dbg_file_handler.setLevel(logging.DEBUG)
122
127
  if config.logging.log_as_json:
123
- dbg_file_handler.setFormatter(JsonFormatter(BRL_JSON_FORMAT))
128
+ dbg_file_handler.setFormatter(JsonFormatter(CLUE_JSON_FORMAT))
124
129
  else:
125
- dbg_file_handler.setFormatter(logging.Formatter(BRL_LOG_FORMAT, BRL_DATE_FORMAT))
130
+ dbg_file_handler.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
126
131
  logger.addHandler(dbg_file_handler)
127
132
 
128
133
  if log_level <= logging.INFO:
@@ -133,9 +138,9 @@ def init_logging(name: str, log_level: Optional[int] = None): # noqa: C901
133
138
  )
134
139
  op_file_handler.setLevel(logging.INFO)
135
140
  if config.logging.log_as_json:
136
- op_file_handler.setFormatter(JsonFormatter(BRL_JSON_FORMAT))
141
+ op_file_handler.setFormatter(JsonFormatter(CLUE_JSON_FORMAT))
137
142
  else:
138
- op_file_handler.setFormatter(logging.Formatter(BRL_LOG_FORMAT, BRL_DATE_FORMAT))
143
+ op_file_handler.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
139
144
  logger.addHandler(op_file_handler)
140
145
 
141
146
  if log_level <= logging.ERROR:
@@ -146,24 +151,24 @@ def init_logging(name: str, log_level: Optional[int] = None): # noqa: C901
146
151
  )
147
152
  err_file_handler.setLevel(logging.ERROR)
148
153
  if config.logging.log_as_json:
149
- err_file_handler.setFormatter(JsonFormatter(BRL_JSON_FORMAT))
154
+ err_file_handler.setFormatter(JsonFormatter(CLUE_JSON_FORMAT))
150
155
  else:
151
- err_file_handler.setFormatter(logging.Formatter(BRL_LOG_FORMAT, BRL_DATE_FORMAT))
156
+ err_file_handler.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
152
157
  logger.addHandler(err_file_handler)
153
158
 
154
159
  if config.logging.log_to_console:
155
160
  console = logging.StreamHandler()
156
161
  if config.logging.log_as_json:
157
- console.setFormatter(JsonFormatter(BRL_JSON_FORMAT))
162
+ console.setFormatter(JsonFormatter(CLUE_JSON_FORMAT))
158
163
  else:
159
- console.setFormatter(logging.Formatter(BRL_LOG_FORMAT, BRL_DATE_FORMAT))
164
+ console.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
160
165
  logger.addHandler(console)
161
166
 
162
167
  if config.logging.log_to_syslog and config.logging.syslog_host and config.logging.syslog_port:
163
168
  syslog_handler = logging.handlers.SysLogHandler(
164
169
  address=(config.logging.syslog_host, config.logging.syslog_port)
165
170
  )
166
- syslog_handler.formatter = logging.Formatter(BRL_SYSLOG_FORMAT)
171
+ syslog_handler.formatter = logging.Formatter(CLUE_SYSLOG_FORMAT)
167
172
  logger.addHandler(syslog_handler)
168
173
 
169
174
  return logger.getChild(name)
@@ -5,10 +5,10 @@ import sys
5
5
  from flask import request
6
6
 
7
7
  from clue.common.logging.format import (
8
- BRL_AUDIT_FORMAT,
9
- BRL_DATE_FORMAT,
10
- BRL_ISO_DATE_FORMAT,
11
- BRL_LOG_FORMAT,
8
+ CLUE_AUDIT_FORMAT,
9
+ CLUE_DATE_FORMAT,
10
+ CLUE_ISO_DATE_FORMAT,
11
+ CLUE_LOG_FORMAT,
12
12
  )
13
13
  from clue.config import DEBUG, config
14
14
 
@@ -63,12 +63,12 @@ if AUDIT:
63
63
  if not os.path.exists(config.logging.log_directory):
64
64
  os.makedirs(config.logging.log_directory)
65
65
 
66
- fh = logging.FileHandler(os.path.join(config.logging.log_directory, "brl_audit.log"))
66
+ fh = logging.FileHandler(os.path.join(config.logging.log_directory, "clue_audit.log"))
67
67
  fh.setLevel(logging.DEBUG)
68
68
  fh.setFormatter(
69
69
  logging.Formatter(
70
- BRL_LOG_FORMAT if DEBUG else BRL_AUDIT_FORMAT,
71
- BRL_DATE_FORMAT if DEBUG else BRL_ISO_DATE_FORMAT,
70
+ CLUE_LOG_FORMAT if DEBUG else CLUE_AUDIT_FORMAT,
71
+ CLUE_DATE_FORMAT if DEBUG else CLUE_ISO_DATE_FORMAT,
72
72
  )
73
73
  )
74
74
  AUDIT_LOG.addHandler(fh)
@@ -77,8 +77,8 @@ ch = logging.StreamHandler(sys.stdout)
77
77
  ch.setLevel(logging.INFO)
78
78
  ch.setFormatter(
79
79
  logging.Formatter(
80
- BRL_LOG_FORMAT if DEBUG else BRL_AUDIT_FORMAT,
81
- BRL_DATE_FORMAT if DEBUG else BRL_ISO_DATE_FORMAT,
80
+ CLUE_LOG_FORMAT if DEBUG else CLUE_AUDIT_FORMAT,
81
+ CLUE_DATE_FORMAT if DEBUG else CLUE_ISO_DATE_FORMAT,
82
82
  )
83
83
  )
84
84
  AUDIT_LOG.addHandler(ch)
@@ -13,10 +13,10 @@ except Exception: # noqa: S110
13
13
 
14
14
  APP_NAME: str = default_string_value(env_name="APP_NAME", default="clue") # type: ignore[assignment]
15
15
 
16
- BRL_SYSLOG_FORMAT = f"HWL %(levelname)8s {hostname} %(process)5d %(name)40s | %(message)s"
17
- BRL_LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s | %(message)s"
18
- BRL_DATE_FORMAT = "%y/%m/%d %H:%M:%S"
19
- BRL_JSON_FORMAT = (
16
+ CLUE_SYSLOG_FORMAT = f"HWL %(levelname)8s {hostname} %(process)5d %(name)40s | %(message)s"
17
+ CLUE_LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s | %(message)s"
18
+ CLUE_DATE_FORMAT = "%y/%m/%d %H:%M:%S"
19
+ CLUE_JSON_FORMAT = (
20
20
  f"{{"
21
21
  f'"@timestamp": "%(asctime)s", '
22
22
  f'"event": {{ "module": "{APP_NAME}", "dataset": "%(name)s" }}, '
@@ -25,8 +25,8 @@ BRL_JSON_FORMAT = (
25
25
  f'"process": {{ "pid": "%(process)d" }}, '
26
26
  f'"message": %(message)s}}'
27
27
  )
28
- BRL_ISO_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
29
- BRL_AUDIT_FORMAT = json.dumps(
28
+ CLUE_ISO_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
29
+ CLUE_AUDIT_FORMAT = json.dumps(
30
30
  {
31
31
  "date": "%(asctime)s",
32
32
  "type": "audit",
@@ -27,4 +27,5 @@ def setup_jobs():
27
27
  except Exception as e:
28
28
  logger.critical("Error when initializing %s - %s", module, e)
29
29
 
30
- scheduler.start()
30
+ if scheduler.state != 1:
31
+ scheduler.start()
@@ -0,0 +1,25 @@
1
+ import importlib
2
+ from typing import Optional
3
+
4
+ from clue.common.logging import get_logger
5
+ from clue.config import config as _config # Python gets BIG mad if we don't alias this
6
+ from clue.extensions.config import BaseExtensionConfig
7
+
8
+ logger = get_logger(__file__)
9
+
10
+ EXTENSIONS: dict[str, Optional[BaseExtensionConfig]] = {}
11
+
12
+
13
+ def get_extensions() -> list[BaseExtensionConfig]:
14
+ "Get a set of extension configurations based on the clue settings."
15
+ for extension in _config.core.extensions:
16
+ if extension in EXTENSIONS:
17
+ continue
18
+
19
+ try:
20
+ EXTENSIONS[extension] = importlib.import_module(f"{extension}.config").config
21
+ except (ImportError, ModuleNotFoundError):
22
+ logger.exception("Exception when loading extension %s", extension)
23
+ EXTENSIONS[extension] = None
24
+
25
+ return [extension for extension in EXTENSIONS.values() if extension]
@@ -0,0 +1,74 @@
1
+ import logging
2
+ from typing import Any
3
+
4
+ from pydantic import BaseModel, ImportString, model_validator
5
+ from pydantic_settings import (
6
+ BaseSettings,
7
+ PydanticBaseSettingsSource,
8
+ YamlConfigSettingsSource,
9
+ )
10
+
11
+ from clue.common.logging import CLUE_DATE_FORMAT, CLUE_LOG_FORMAT
12
+
13
+ logger = logging.getLogger("clue.extensions.config")
14
+ logger.setLevel(logging.INFO)
15
+ console = logging.StreamHandler()
16
+ console.setLevel(logging.INFO)
17
+ console.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
18
+ logger.addHandler(console)
19
+
20
+
21
+ class Modules(BaseModel):
22
+ "A list of components exposed for use in Clue by this plugin."
23
+
24
+ routes: list[ImportString] = []
25
+ obo_module: ImportString | None = None
26
+
27
+
28
+ class BaseExtensionConfig(BaseSettings):
29
+ "Configuration File for Plugin"
30
+
31
+ name: str
32
+ features: dict[str, bool] = {}
33
+
34
+ modules: Modules = Modules()
35
+
36
+ @model_validator(mode="before")
37
+ @classmethod
38
+ def initialize_extension_configuration(cls, data: Any) -> Any: # noqa: C901
39
+ "Convert a raw yaml config into an object ready for validation by pydantic"
40
+ if not isinstance(data, dict):
41
+ return data
42
+
43
+ # Default mutation requires plugin name
44
+ if "name" not in data:
45
+ logger.warning("Name is missing from configuration")
46
+ return data
47
+
48
+ plugin_name = data["name"]
49
+ logger.debug("Beginning configuration parsing for plugin %s", plugin_name)
50
+
51
+ if "modules" not in data:
52
+ return data
53
+
54
+ if "routes" in data["modules"] and isinstance(data["modules"]["routes"], list):
55
+ new_routes: list[str] = []
56
+ for route in data["modules"]["routes"]:
57
+ new_routes.append(f"{plugin_name}.routes.{route}" if "." not in route else route)
58
+
59
+ data["modules"]["routes"] = new_routes
60
+
61
+ if "obo_module" in data["modules"]:
62
+ if isinstance(data["modules"]["obo_module"], bool):
63
+ data["modules"]["obo_module"] = f"{plugin_name}.obo:get_obo_token"
64
+
65
+ return data
66
+
67
+ @classmethod
68
+ def settings_customise_sources(
69
+ cls, # noqa: ANN102
70
+ *args, # noqa: ANN002
71
+ **kwargs, # noqa: ANN002, ANN102
72
+ ) -> tuple[PydanticBaseSettingsSource, ...]:
73
+ "Adds a YamlConfigSettingsSource object at the end of the settings_customize_sources response."
74
+ return (*super().settings_customise_sources(*args, **kwargs), YamlConfigSettingsSource(cls))
@@ -4,6 +4,7 @@ import os
4
4
  from email.utils import parseaddr
5
5
  from enum import Enum
6
6
  from pathlib import Path
7
+ from typing import Self
7
8
  from uuid import uuid4
8
9
 
9
10
  from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
@@ -14,11 +15,10 @@ from pydantic_settings import (
14
15
  SettingsConfigDict,
15
16
  YamlConfigSettingsSource,
16
17
  )
17
- from typing_extensions import Self
18
18
 
19
19
  from clue.common import forge
20
20
  from clue.common.exceptions import ClueValueError
21
- from clue.common.logging.format import BRL_DATE_FORMAT, BRL_LOG_FORMAT
21
+ from clue.common.logging.format import CLUE_DATE_FORMAT, CLUE_LOG_FORMAT
22
22
  from clue.common.str_utils import default_string_value
23
23
 
24
24
  AUTO_PROPERTY_TYPE = ["access", "classification", "type", "role", "remove_role", "group"]
@@ -216,8 +216,13 @@ class Metrics(BaseModel):
216
216
 
217
217
 
218
218
  class Core(BaseModel):
219
+ extensions: set[str] = Field(description="A list of extensions to load", default=set())
220
+
219
221
  metrics: Metrics = Metrics()
222
+ "Configuration for Metrics Collection"
223
+
220
224
  redis: RedisServer = RedisServer()
225
+ "Configuration for Redis instances"
221
226
 
222
227
 
223
228
  class LogLevel(str, Enum):
@@ -394,7 +399,7 @@ if os.getenv("AZURE_TEST_CONFIG", None) is not None:
394
399
  logger.setLevel(logging.INFO)
395
400
  console = logging.StreamHandler()
396
401
  console.setLevel(logging.INFO)
397
- console.setFormatter(logging.Formatter(BRL_LOG_FORMAT, BRL_DATE_FORMAT))
402
+ console.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
398
403
  logger.addHandler(console)
399
404
 
400
405
  logger.info("Azure build environment detected, adding additional config path")
@@ -22,7 +22,7 @@ from clue.common.exceptions import (
22
22
  TimeoutException,
23
23
  UnprocessableException,
24
24
  )
25
- from clue.common.logging.format import BRL_DATE_FORMAT, BRL_LOG_FORMAT
25
+ from clue.common.logging.format import CLUE_DATE_FORMAT, CLUE_LOG_FORMAT
26
26
  from clue.models.actions import (
27
27
  Action,
28
28
  ActionBase,
@@ -66,7 +66,7 @@ def build_default_logger() -> logging.Logger:
66
66
  logger.setLevel(logging.INFO)
67
67
  console = logging.StreamHandler()
68
68
  console.setLevel(logging.INFO)
69
- console.setFormatter(logging.Formatter(BRL_LOG_FORMAT, BRL_DATE_FORMAT))
69
+ console.setFormatter(logging.Formatter(CLUE_LOG_FORMAT, CLUE_DATE_FORMAT))
70
70
  logger.addHandler(console)
71
71
 
72
72
  return logger
@@ -9,7 +9,7 @@ DEFAULT_TTL = 60 * 60 # 1 Hour
9
9
 
10
10
 
11
11
  class RedisCache(object):
12
- def __init__(self, prefix="brl_cache", host=None, port=None, ttl=DEFAULT_TTL):
12
+ def __init__(self, prefix="clue_cache", host=None, port=None, ttl=DEFAULT_TTL):
13
13
  self.c = get_client(host, port, False)
14
14
  self.prefix = prefix
15
15
  self.ttl = ttl
@@ -4,6 +4,7 @@ from typing import Optional
4
4
  from clue.common.exceptions import InvalidDataException
5
5
  from clue.common.logging import get_logger
6
6
  from clue.config import config, get_redis
7
+ from clue.extensions import get_extensions
7
8
  from clue.remote.datatypes.set import ExpiringSet
8
9
  from clue.security.utils import decode_jwt_payload
9
10
 
@@ -31,7 +32,34 @@ def _get_token_raw(service: str, user: str) -> Optional[str]:
31
32
  return None
32
33
 
33
34
 
34
- def get_obo_token(service: str, access_token: str, user: str, force_refresh: bool = False):
35
+ def try_validate_expiry(obo_access_token: str):
36
+ """Validates the expiry of an OBO (On-Behalf-Of) access token.
37
+
38
+ Attempts to decode the JWT payload of the provided token and checks the 'exp' (expiry) field.
39
+ If the token has expired, logs a warning and returns None.
40
+ If the token is not a JWT or the 'exp' field is missing, logs a warning and skips expiry validation.
41
+
42
+ Args:
43
+ obo_access_token (str): The OBO access token to validate.
44
+
45
+ Returns:
46
+ str or None: The original token if valid or expiry cannot be determined, otherwise None if expired.
47
+ """
48
+ try:
49
+ expiry = datetime.fromtimestamp(decode_jwt_payload(obo_access_token)["exp"])
50
+
51
+ if expiry < datetime.now():
52
+ logger.warning("Cached token has expired")
53
+ return None
54
+ except IndexError:
55
+ logger.warning("Token is not a JWT, skipping expiry validation")
56
+ except KeyError:
57
+ logger.warning("'exp' field is missing, skipping expiry validation")
58
+
59
+ return obo_access_token
60
+
61
+
62
+ def get_obo_token(service: str, access_token: str, user: str, force_refresh: bool = False): # noqa: C901
35
63
  """Gets an On-Behalf-Of token from either the Redis cache or from the provided authentication plugin.
36
64
 
37
65
  Args:
@@ -60,17 +88,22 @@ def get_obo_token(service: str, access_token: str, user: str, force_refresh: boo
60
88
  obo_access_token = _get_token_raw(service, user)
61
89
 
62
90
  if obo_access_token is not None:
63
- expiry = datetime.fromtimestamp(decode_jwt_payload(obo_access_token)["exp"])
64
-
65
- if expiry < datetime.now():
66
- logger.warning("Cached token has expired")
67
- obo_access_token = None
91
+ obo_access_token = try_validate_expiry(obo_access_token)
68
92
 
69
93
  if obo_access_token is None:
70
94
  logger.info(f"Fetching OBO token for user {user} to service {service}")
71
95
 
72
- # TODO: figure out plugin-based OBO system
73
- obo_access_token = None
96
+ extension_get_obo_token = None
97
+ for extension in get_extensions():
98
+ if extension.modules.obo_module:
99
+ extension_get_obo_token = extension.modules.obo_module
100
+ break
101
+
102
+ if extension_get_obo_token is None:
103
+ logger.info("No OBO function provided, returning provided access token")
104
+ return access_token
105
+
106
+ obo_access_token = extension_get_obo_token(service, access_token, user)
74
107
 
75
108
  if obo_access_token:
76
109
  service_token_store = _get_obo_token_store(service, user)
@@ -17,7 +17,7 @@ logger = get_logger(__file__)
17
17
 
18
18
  # Either cache for one second in debug mode, or five minutes in production
19
19
  CACHE_TIMEOUT: int = 1 if DEBUG else 5 * 60
20
- CACHE = RedisCache(prefix="brl_types", ttl=CACHE_TIMEOUT)
20
+ CACHE = RedisCache(prefix="clue_types", ttl=CACHE_TIMEOUT)
21
21
 
22
22
 
23
23
  def get_types_regular_expressions(user: dict[str, Any]):
@@ -156,7 +156,7 @@ suppress-none-returning = true
156
156
  ###################
157
157
  [tool.pytest.ini_options]
158
158
  log_cli = true
159
- log_cli_level = "INFO"
159
+ log_cli_level = "WARN"
160
160
 
161
161
  ###################
162
162
  # Poetry settings #
@@ -164,7 +164,7 @@ log_cli_level = "INFO"
164
164
  [tool.poetry]
165
165
  package-mode = true
166
166
  name = "clue-api"
167
- version = "1.0.0.dev34"
167
+ version = "1.0.0.dev45"
168
168
  description = "Clue distributed enrichment service"
169
169
  authors = ["Canadian Centre for Cyber Security <contact@cyber.gc.ca>"]
170
170
  license = "MIT"
@@ -1,52 +0,0 @@
1
- import random
2
- from datetime import datetime
3
- from typing import Union, get_args, get_origin
4
-
5
- from pydantic import BaseModel
6
- from pydantic_core import Url
7
-
8
-
9
- def generate_example_model(model: type[BaseModel], as_list: bool = False): # noqa: C901
10
- """Populate fields with example values"""
11
- result = {}
12
-
13
- if as_list:
14
- full_list = []
15
- for _ in range(random.randint(1, 3)): # noqa: S311
16
- full_list.append(generate_example_model(model))
17
-
18
- return full_list
19
-
20
- for name, field_info in model.model_fields.items():
21
- field_type = field_info.annotation
22
- if field_type:
23
- if get_origin(field_type) is Union and type(None) in get_args(field_type):
24
- field_type = (
25
- next((_type for _type in get_args(field_type) if _type is not type(None)), None) or field_type
26
- )
27
-
28
- _as_list = False
29
- if get_origin(field_type) is list:
30
- _as_list = True
31
- field_type = get_args(field_type)[0]
32
-
33
- _issubclass = False
34
- try:
35
- if field_type:
36
- _issubclass = issubclass(field_type, BaseModel)
37
- except TypeError:
38
- pass
39
-
40
- if field_type and _issubclass:
41
- result[name] = generate_example_model(field_type, as_list=_as_list)
42
- elif field_info.examples:
43
- result[name] = random.choice(field_info.examples) # noqa: S311
44
- elif field_info.default:
45
- result[name] = field_info.default
46
-
47
- if isinstance(result[name], datetime):
48
- result[name] = result[name].isoformat()
49
- elif isinstance(result[name], Url):
50
- result[name] = str(result[name])
51
-
52
- return dict(result)
File without changes
File without changes