fred-core 3.4.2__tar.gz → 3.4.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fred_core-3.4.2 → fred_core-3.4.3}/PKG-INFO +1 -1
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/__init__.py +10 -9
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/common/__init__.py +2 -0
- fred_core-3.4.3/fred_core/common/resilient_sink.py +188 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/kpi/kpi_factory.py +14 -7
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/kpi/kpi_writer.py +11 -2
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/kpi/log_kpi_store.py +1 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/kpi/prometheus_kpi_store.py +44 -11
- fred_core-3.4.3/fred_core/logs/audit_log.py +66 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/logs/base_log_store.py +6 -16
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/logs/log_setup.py +69 -3
- fred_core-3.4.3/fred_core/logs/log_store_factory.py +84 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/logs/log_structures.py +14 -32
- fred_core-3.4.3/fred_core/logs/memory_log_store.py +57 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/logs/null_log_store.py +2 -5
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/logs/opensearch_log_store.py +26 -88
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/security/models.py +3 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/security/oidc.py +21 -21
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/security/rebac/noop_engine.py +1 -1
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/security/rebac/openfga_engine.py +1 -41
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/security/rebac/rebac_engine.py +98 -6
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/security/rebac/rebac_factory.py +1 -1
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/security/rebac/schema.fga +69 -0
- fred_core-3.4.3/fred_core/security/rebac/schema.fga.json +1 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/security/structure.py +12 -16
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tasks/models.py +46 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tasks/store.py +43 -1
- fred_core-3.4.3/fred_core/tests/common/test_log_setup.py +240 -0
- fred_core-3.4.3/fred_core/tests/common/test_resilient_sink.py +148 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/integration/test_rebac.py +302 -53
- fred_core-3.4.3/fred_core/tests/logs/test_audit_log.py +70 -0
- fred_core-3.4.3/fred_core/tests/logs/test_log_store_factory.py +93 -0
- fred_core-3.4.3/fred_core/tests/logs/test_memory_log_store.py +99 -0
- fred_core-3.4.3/fred_core/tests/logs/test_opensearch_log_store_ensure_ready.py +122 -0
- fred_core-3.4.3/fred_core/tests/security/test_capability_scoping_1980.py +152 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/security/test_rebac_engine_team_helpers.py +2 -2
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/tasks/test_models.py +98 -0
- fred_core-3.4.3/fred_core/tests/test_kpi_writer_fail_open.py +57 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/test_prometheus_kpi_store.py +29 -7
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core.egg-info/PKG-INFO +1 -1
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core.egg-info/SOURCES.txt +9 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/pyproject.toml +1 -1
- fred_core-3.4.2/fred_core/logs/memory_log_store.py +0 -126
- fred_core-3.4.2/fred_core/security/rebac/schema.fga.json +0 -1
- fred_core-3.4.2/fred_core/tests/common/test_log_setup.py +0 -71
- fred_core-3.4.2/fred_core/tests/logs/test_memory_log_store.py +0 -301
- {fred_core-3.4.2 → fred_core-3.4.3}/README.md +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/cli/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/cli/auth.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/cli/ui.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/common/config_files.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/common/config_loader.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/common/env.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/common/fastapi_handlers.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/common/lru_cache.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/common/structures.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/common/team_id.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/common/utils.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/documents/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/documents/document_models.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/documents/document_store.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/documents/document_structures.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/documents/postgres_document_store.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/documents/tag_models.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/filesystem/gcs_filesystem.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/filesystem/local_filesystem.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/filesystem/minio_filesystem.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/filesystem/structures.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/history/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/history/base_history_store.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/history/history_models.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/history/history_schema.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/history/postgres_history_store.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/kpi/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/kpi/base_kpi_store.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/kpi/base_kpi_writer.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/kpi/http_middleware.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/kpi/kpi_phase_metric.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/kpi/kpi_process.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/kpi/kpi_reader_structures.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/kpi/kpi_writer_structures.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/kpi/noop_kpi_writer.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/kpi/opensearch_kpi_store.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/logs/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/model/factory.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/model/http_clients.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/model/models.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/models/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/models/base.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/portable/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/portable/observability.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/py.typed +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/scheduler/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/scheduler/backend.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/scheduler/scheduler_structures.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/scheduler/temporal_client_provider.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/security/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/security/authorization.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/security/backend_to_backend_auth.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/security/keycloak/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/security/keycloak/keycloack_admin_client.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/security/outbound.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/security/rebac/openfga_schema.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/security/whitelist_access_control/access_control.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/session/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/session/session_schema.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/session/stores/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/session/stores/base_session_store.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/session/stores/postgres_session_store.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/session/stores/session_models.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/sql/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/sql/alembic_env.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/sql/async_session.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/sql/base_sql.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/sql/mixin.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/store/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/store/base_content_store.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/store/local_content_store.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/store/minio_content_store.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/store/opensearch_mapping_validator.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/store/vector_search.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tasks/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tasks/authz.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tasks/bus.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tasks/orm_models.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tasks/service.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tasks/sse.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tasks/workflow_control.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/teams/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/teams/metadata_store.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/teams/team_metatada_models.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/cli/test_auth.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/cli/test_ui.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/common/test_config_loader.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/common/test_env.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/common/test_fastapi_handlers.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/common/test_lru_cache.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/common/test_structures.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/filesystem/test_gcs_filesystem.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/filesystem/test_local_filesystem.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/integration/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/kpi/test_http_middleware.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/kpi/test_noop_kpi_writer.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/kpi/test_opensearch_kpi_store_anonymise.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/kpi/test_opensearch_kpi_store_ensure_ready.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/model/test_anthropic_factory.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/model/test_embedding_factory.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/model/test_http_clients.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/model/test_vertex_model_garden_auth.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/portable/test_observability.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/scheduler/test_backend.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/scheduler/test_temporal_client_provider.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/security/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/security/test_oidc_strict.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/security/test_rebac_config_defaults.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/security/test_rebac_schema_authz05.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/security/test_security_profile.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/security/test_service_agent.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/security/test_whitelist_access_control.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/session/test_postgres_json_session_store_sqlite.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/store/test_local_content_store.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/tasks/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/tasks/test_authz.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/tasks/test_bus.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/tasks/test_reconcile.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/tasks/test_scheduling.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/tasks/test_sse.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/tests/test_log_kpi_store.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/users/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/users/store/__init__.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/users/store/base_user_store.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/users/store/postgres_user_store.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core/users/user_models.py +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core.egg-info/dependency_links.txt +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core.egg-info/requires.txt +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/fred_core.egg-info/top_level.txt +0 -0
- {fred_core-3.4.2 → fred_core-3.4.3}/setup.cfg +0 -0
|
@@ -43,16 +43,15 @@ from fred_core.filesystem.structures import (
|
|
|
43
43
|
FilesystemResourceInfo,
|
|
44
44
|
FilesystemResourceInfoResult,
|
|
45
45
|
)
|
|
46
|
+
from fred_core.logs.audit_log import emit_audit_log
|
|
46
47
|
from fred_core.logs.base_log_store import BaseLogStore
|
|
47
|
-
from fred_core.logs.log_setup import StoreEmitHandler, log_setup
|
|
48
|
+
from fred_core.logs.log_setup import AUDIT_LOGGER_NAME, StoreEmitHandler, log_setup
|
|
49
|
+
from fred_core.logs.log_store_factory import build_log_store
|
|
48
50
|
from fred_core.logs.log_structures import (
|
|
49
51
|
InMemoryLogStorageConfig,
|
|
52
|
+
LogCategory,
|
|
50
53
|
LogEventDTO,
|
|
51
|
-
LogFilter,
|
|
52
|
-
LogQuery,
|
|
53
|
-
LogQueryResult,
|
|
54
54
|
LogStorageConfig,
|
|
55
|
-
TailFileResponse,
|
|
56
55
|
)
|
|
57
56
|
from fred_core.logs.memory_log_store import RamLogStore
|
|
58
57
|
from fred_core.logs.opensearch_log_store import OpenSearchLogStore
|
|
@@ -93,6 +92,7 @@ from fred_core.security.rebac.rebac_engine import (
|
|
|
93
92
|
ORGANIZATION_ID,
|
|
94
93
|
SERVICE_AGENT_ALLOWED_TEAM_PERMISSIONS,
|
|
95
94
|
AgentPermission,
|
|
95
|
+
CapabilityPermission,
|
|
96
96
|
DocumentPermission,
|
|
97
97
|
OrganizationPermission,
|
|
98
98
|
RebacDisabledResult,
|
|
@@ -134,14 +134,14 @@ from .users import (
|
|
|
134
134
|
|
|
135
135
|
__all__ = [
|
|
136
136
|
"BaseLogStore",
|
|
137
|
+
"LogCategory",
|
|
137
138
|
"LogEventDTO",
|
|
138
|
-
"LogFilter",
|
|
139
|
-
"LogQuery",
|
|
140
|
-
"LogQueryResult",
|
|
141
139
|
"OpenSearchLogStore",
|
|
140
|
+
"AUDIT_LOGGER_NAME",
|
|
142
141
|
"RamLogStore",
|
|
143
142
|
"StoreEmitHandler",
|
|
144
|
-
"
|
|
143
|
+
"build_log_store",
|
|
144
|
+
"emit_audit_log",
|
|
145
145
|
"log_setup",
|
|
146
146
|
"LogStorageConfig",
|
|
147
147
|
"InMemoryLogStorageConfig",
|
|
@@ -191,6 +191,7 @@ __all__ = [
|
|
|
191
191
|
"ORGANIZATION_ID",
|
|
192
192
|
"AgentPermission",
|
|
193
193
|
"OrganizationPermission",
|
|
194
|
+
"CapabilityPermission",
|
|
194
195
|
"RebacPermission",
|
|
195
196
|
"RebacDisabledResult",
|
|
196
197
|
"RebacEngine",
|
|
@@ -21,6 +21,7 @@ from .config_loader import (
|
|
|
21
21
|
from .env import coerce_bool, read_env_bool
|
|
22
22
|
from .fastapi_handlers import register_exception_handlers
|
|
23
23
|
from .lru_cache import ThreadSafeLRUCache
|
|
24
|
+
from .resilient_sink import ResilientSinkStore
|
|
24
25
|
from .structures import (
|
|
25
26
|
BaseModelWithId,
|
|
26
27
|
DuckdbStoreConfig,
|
|
@@ -60,6 +61,7 @@ __all__ = [
|
|
|
60
61
|
"TeamId",
|
|
61
62
|
"is_personal_team_id",
|
|
62
63
|
"personal_team_id",
|
|
64
|
+
"ResilientSinkStore",
|
|
63
65
|
"TemporalSchedulerConfig",
|
|
64
66
|
"ThreadSafeLRUCache",
|
|
65
67
|
"coerce_bool",
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# Copyright Thales 2026
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""
|
|
16
|
+
Resilient sink decorator — bounded queue + circuit breaker (issue #2009).
|
|
17
|
+
|
|
18
|
+
Why this exists:
|
|
19
|
+
- `BaseLogStore` and `BaseKPIStore` share the same write shape (`index_event`,
|
|
20
|
+
`bulk_index`) and the same contract note: "must not block or crash the
|
|
21
|
+
caller on outage." Nothing previously enforced that for the OpenSearch-
|
|
22
|
+
backed implementations — a down/slow cluster could block the calling
|
|
23
|
+
thread for a full HTTP timeout on every write, or (for `KPIWriter`, which
|
|
24
|
+
calls `index_event` synchronously and inline) let the exception propagate
|
|
25
|
+
into the business request that triggered the metric.
|
|
26
|
+
- This wraps any such store: writes are handed to a single background
|
|
27
|
+
thread via a bounded queue (never blocks the caller — full queue means
|
|
28
|
+
drop, not wait) and short-circuited once the wrapped store has failed
|
|
29
|
+
repeatedly, so a known-down cluster stops being hammered from the hot path.
|
|
30
|
+
|
|
31
|
+
How to use:
|
|
32
|
+
- `ResilientSinkStore(wrapped_store)` — same write shape as the store it
|
|
33
|
+
wraps (`ensure_ready`/`index_event`/`bulk_index`), so it's a drop-in
|
|
34
|
+
decorator. `ensure_ready()` passes straight through: startup provisioning
|
|
35
|
+
is not this decorator's concern, only the steady-state write path is.
|
|
36
|
+
`query()` also passes through, best-effort, for wrapped stores that
|
|
37
|
+
support it (KPI stores do; the log store no longer does since Fred stopped
|
|
38
|
+
exposing its own log-query surface — OpenSearch Dashboards reads that
|
|
39
|
+
index directly instead).
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
from __future__ import annotations
|
|
43
|
+
|
|
44
|
+
import logging
|
|
45
|
+
import queue
|
|
46
|
+
import threading
|
|
47
|
+
import time
|
|
48
|
+
from typing import Any, Protocol
|
|
49
|
+
|
|
50
|
+
logger = logging.getLogger(__name__)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# fmt: off
|
|
54
|
+
class _WritableStore(Protocol):
|
|
55
|
+
def ensure_ready(self) -> None:
|
|
56
|
+
...
|
|
57
|
+
|
|
58
|
+
def index_event(self, event: Any) -> None:
|
|
59
|
+
...
|
|
60
|
+
|
|
61
|
+
def bulk_index(self, events: list[Any]) -> None:
|
|
62
|
+
...
|
|
63
|
+
# fmt: on
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class _CircuitBreaker:
|
|
67
|
+
"""Opens after `failure_threshold` consecutive failures; half-opens
|
|
68
|
+
(allows one trial) after `cooldown_s`; closes again on a trial success."""
|
|
69
|
+
|
|
70
|
+
def __init__(self, *, failure_threshold: int, cooldown_s: float) -> None:
|
|
71
|
+
self._threshold = failure_threshold
|
|
72
|
+
self._cooldown_s = cooldown_s
|
|
73
|
+
self._consecutive_failures = 0
|
|
74
|
+
self._opened_at: float | None = None
|
|
75
|
+
self._lock = threading.Lock()
|
|
76
|
+
|
|
77
|
+
def allow(self) -> bool:
|
|
78
|
+
with self._lock:
|
|
79
|
+
if self._opened_at is None:
|
|
80
|
+
return True
|
|
81
|
+
return (time.monotonic() - self._opened_at) >= self._cooldown_s
|
|
82
|
+
|
|
83
|
+
def record_success(self) -> None:
|
|
84
|
+
with self._lock:
|
|
85
|
+
self._consecutive_failures = 0
|
|
86
|
+
self._opened_at = None
|
|
87
|
+
|
|
88
|
+
def record_failure(self) -> None:
|
|
89
|
+
with self._lock:
|
|
90
|
+
self._consecutive_failures += 1
|
|
91
|
+
if (
|
|
92
|
+
self._consecutive_failures >= self._threshold
|
|
93
|
+
and self._opened_at is None
|
|
94
|
+
):
|
|
95
|
+
self._opened_at = time.monotonic()
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def is_open(self) -> bool:
|
|
99
|
+
with self._lock:
|
|
100
|
+
if self._opened_at is None:
|
|
101
|
+
return False
|
|
102
|
+
return (time.monotonic() - self._opened_at) < self._cooldown_s
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class ResilientSinkStore:
|
|
106
|
+
"""Fail-open, non-blocking decorator around a `BaseLogStore`/`BaseKPIStore`.
|
|
107
|
+
|
|
108
|
+
Contract with the caller:
|
|
109
|
+
- `index_event`/`bulk_index` never raise and never block beyond a
|
|
110
|
+
non-blocking queue put.
|
|
111
|
+
- A full queue (sink can't keep up) or an open circuit (sink is known
|
|
112
|
+
down) both mean "drop the event", counted separately for diagnostics.
|
|
113
|
+
"""
|
|
114
|
+
|
|
115
|
+
def __init__(
|
|
116
|
+
self,
|
|
117
|
+
wrapped: _WritableStore,
|
|
118
|
+
*,
|
|
119
|
+
queue_size: int = 1000,
|
|
120
|
+
failure_threshold: int = 5,
|
|
121
|
+
cooldown_s: float = 30.0,
|
|
122
|
+
) -> None:
|
|
123
|
+
self._wrapped = wrapped
|
|
124
|
+
self._queue: queue.Queue[Any] = queue.Queue(maxsize=queue_size)
|
|
125
|
+
self._breaker = _CircuitBreaker(
|
|
126
|
+
failure_threshold=failure_threshold, cooldown_s=cooldown_s
|
|
127
|
+
)
|
|
128
|
+
self.dropped_queue_full = 0
|
|
129
|
+
self.dropped_circuit_open = 0
|
|
130
|
+
self._thread = threading.Thread(
|
|
131
|
+
target=self._worker, name="resilient-sink-writer", daemon=True
|
|
132
|
+
)
|
|
133
|
+
self._thread.start()
|
|
134
|
+
|
|
135
|
+
@property
|
|
136
|
+
def wrapped(self) -> _WritableStore:
|
|
137
|
+
"""The underlying store — exposed read-only for introspection/tests."""
|
|
138
|
+
return self._wrapped
|
|
139
|
+
|
|
140
|
+
# -- passthrough: not this decorator's concern -----------------------------
|
|
141
|
+
def ensure_ready(self) -> None:
|
|
142
|
+
self._wrapped.ensure_ready()
|
|
143
|
+
|
|
144
|
+
def query(self, q: Any) -> Any:
|
|
145
|
+
"""Best-effort passthrough for wrapped stores that support querying
|
|
146
|
+
(KPI stores do; the log store no longer does)."""
|
|
147
|
+
query_fn = getattr(self._wrapped, "query", None)
|
|
148
|
+
if query_fn is None:
|
|
149
|
+
raise AttributeError(
|
|
150
|
+
f"{type(self._wrapped).__name__} does not support query()"
|
|
151
|
+
)
|
|
152
|
+
return query_fn(q)
|
|
153
|
+
|
|
154
|
+
# -- writes: bounded, non-blocking, fail-open ------------------------------
|
|
155
|
+
def index_event(self, event: Any) -> None:
|
|
156
|
+
if not self._breaker.allow():
|
|
157
|
+
self.dropped_circuit_open += 1
|
|
158
|
+
return
|
|
159
|
+
try:
|
|
160
|
+
self._queue.put_nowait(event)
|
|
161
|
+
except queue.Full:
|
|
162
|
+
self.dropped_queue_full += 1
|
|
163
|
+
|
|
164
|
+
def bulk_index(self, events: list[Any]) -> None:
|
|
165
|
+
for event in events:
|
|
166
|
+
self.index_event(event)
|
|
167
|
+
|
|
168
|
+
# -- background worker ------------------------------------------------------
|
|
169
|
+
def _worker(self) -> None:
|
|
170
|
+
while True:
|
|
171
|
+
event = self._queue.get()
|
|
172
|
+
try:
|
|
173
|
+
if not self._breaker.allow():
|
|
174
|
+
self.dropped_circuit_open += 1
|
|
175
|
+
continue
|
|
176
|
+
try:
|
|
177
|
+
self._wrapped.index_event(event)
|
|
178
|
+
self._breaker.record_success()
|
|
179
|
+
except Exception:
|
|
180
|
+
self._breaker.record_failure()
|
|
181
|
+
logger.warning(
|
|
182
|
+
"Resilient sink write failed; dropping event", exc_info=True
|
|
183
|
+
)
|
|
184
|
+
finally:
|
|
185
|
+
self._queue.task_done()
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
__all__ = ["ResilientSinkStore"]
|
|
@@ -19,6 +19,7 @@ from __future__ import annotations
|
|
|
19
19
|
import logging
|
|
20
20
|
from typing import Optional
|
|
21
21
|
|
|
22
|
+
from fred_core.common.resilient_sink import ResilientSinkStore
|
|
22
23
|
from fred_core.common.structures import KpiObservabilityConfig, OpenSearchStoreConfig
|
|
23
24
|
from fred_core.kpi.base_kpi_store import BaseKPIStore
|
|
24
25
|
from fred_core.kpi.base_kpi_writer import BaseKPIWriter
|
|
@@ -65,13 +66,19 @@ def build_kpi_writer(
|
|
|
65
66
|
"— falling back to log sink"
|
|
66
67
|
)
|
|
67
68
|
else:
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
69
|
+
# ResilientSinkStore: writes are handed to a background thread via
|
|
70
|
+
# a bounded queue and short-circuited on repeated failure, so a
|
|
71
|
+
# slow/down OpenSearch cluster can never block or fail the
|
|
72
|
+
# business request that triggered a metric (issue #2009).
|
|
73
|
+
base_store = ResilientSinkStore( # type: ignore[assignment]
|
|
74
|
+
OpenSearchKPIStore(
|
|
75
|
+
host=opensearch_config.host,
|
|
76
|
+
username=opensearch_config.username,
|
|
77
|
+
password=opensearch_config.password,
|
|
78
|
+
secure=opensearch_config.secure,
|
|
79
|
+
verify_certs=opensearch_config.verify_certs,
|
|
80
|
+
index=os_cfg.index,
|
|
81
|
+
)
|
|
75
82
|
)
|
|
76
83
|
|
|
77
84
|
if base_store is None and log_cfg.enabled:
|
|
@@ -36,10 +36,11 @@ from fred_core.kpi.kpi_writer_structures import (
|
|
|
36
36
|
Quantities,
|
|
37
37
|
Trace,
|
|
38
38
|
)
|
|
39
|
+
from fred_core.logs.log_setup import KPI_LOGGER_NAME
|
|
39
40
|
from fred_core.security.structure import KeycloakUser
|
|
40
41
|
|
|
41
42
|
logger = logging.getLogger(__name__)
|
|
42
|
-
summary_logger = logging.getLogger(
|
|
43
|
+
summary_logger = logging.getLogger(KPI_LOGGER_NAME)
|
|
43
44
|
|
|
44
45
|
|
|
45
46
|
@dataclass
|
|
@@ -209,7 +210,15 @@ class KPIWriter(BaseKPIWriter):
|
|
|
209
210
|
trace=(Trace(**trace) if trace else None),
|
|
210
211
|
)
|
|
211
212
|
self._record_summary(event)
|
|
212
|
-
|
|
213
|
+
# Fail-open: a KPI sink outage (e.g. OpenSearch down) must never break
|
|
214
|
+
# or propagate into the business request that triggered this metric —
|
|
215
|
+
# see docs/swift/platform/OBSERVABILITY-AND-AUDIT.md §6.
|
|
216
|
+
try:
|
|
217
|
+
self.store.index_event(event)
|
|
218
|
+
except Exception:
|
|
219
|
+
logger.warning(
|
|
220
|
+
"KPI store write failed; dropping event %r", name, exc_info=True
|
|
221
|
+
)
|
|
213
222
|
|
|
214
223
|
# ---- simple helpers ------------------------------------------------------
|
|
215
224
|
def count(
|
|
@@ -26,11 +26,46 @@ from fred_core.kpi.kpi_writer_structures import KPIEvent
|
|
|
26
26
|
|
|
27
27
|
logger = logging.getLogger(__name__)
|
|
28
28
|
|
|
29
|
-
|
|
29
|
+
# Explicit allow-list, not a deny-list: Prometheus/Grafana serves platform
|
|
30
|
+
# operators, not team- or user-scoped product analytics (that's OBSERV-02's
|
|
31
|
+
# job, ReBAC-scoped, backed by OpenSearch — see
|
|
32
|
+
# docs/swift/platform/OBSERVABILITY-AND-AUDIT.md §3.1). A deny-list only
|
|
33
|
+
# protects against the specific fields someone remembered to list; any new
|
|
34
|
+
# per-call or per-user dim added to a KPI event later would silently leak
|
|
35
|
+
# into a scraped, platform-wide-visible label unless it's caught here too.
|
|
36
|
+
# An allow-list fails safe: a new dim needs a deliberate decision to appear
|
|
37
|
+
# as a label at all.
|
|
38
|
+
#
|
|
39
|
+
# Deliberately excluded, and why:
|
|
40
|
+
# - user_id, session_id, exchange_id: direct/pseudonymous identity.
|
|
41
|
+
# - team_id, agent_instance_id, agent_instance_name, agent_id: "usage by
|
|
42
|
+
# team/agent instance" is a product-analytics question already answered,
|
|
43
|
+
# correctly scoped, by OBSERV-02's presets — duplicating it here would
|
|
44
|
+
# mean a second, unsynchronized access-control model for the same fact.
|
|
45
|
+
# - trace_id, correlation_id, checkpoint_id, doc_uid, scope_id: unique per
|
|
46
|
+
# call/resource, no aggregate value as a label, and a pivot vector from a
|
|
47
|
+
# wide-audience dashboard back into raw logs.
|
|
48
|
+
PROMETHEUS_ALLOWED_LABELS = frozenset(
|
|
30
49
|
{
|
|
31
|
-
"
|
|
32
|
-
"
|
|
33
|
-
"
|
|
50
|
+
"tool_name",
|
|
51
|
+
"status",
|
|
52
|
+
"error_code",
|
|
53
|
+
"exception_type",
|
|
54
|
+
"http_status",
|
|
55
|
+
"template_agent_id",
|
|
56
|
+
"model",
|
|
57
|
+
"model_name",
|
|
58
|
+
"finish_reason",
|
|
59
|
+
"runtime_id",
|
|
60
|
+
"source_runtime_id",
|
|
61
|
+
"service",
|
|
62
|
+
"env",
|
|
63
|
+
"cluster",
|
|
64
|
+
"route",
|
|
65
|
+
"method",
|
|
66
|
+
"actor_type",
|
|
67
|
+
"file_type",
|
|
68
|
+
"agent_step",
|
|
34
69
|
}
|
|
35
70
|
)
|
|
36
71
|
|
|
@@ -54,21 +89,19 @@ def _prometheus_label_dims(event: KPIEvent) -> Dict[str, str]:
|
|
|
54
89
|
Return the bounded subset of KPI dims that may become Prometheus labels.
|
|
55
90
|
|
|
56
91
|
Why this exists:
|
|
57
|
-
- KPI events can carry per-user/per-turn identifiers for
|
|
58
|
-
but Prometheus labels must stay
|
|
92
|
+
- KPI events can carry per-user/per-team/per-turn identifiers for
|
|
93
|
+
structured stores (OpenSearch), but Prometheus labels must stay both
|
|
94
|
+
low-cardinality and free of anything RGPD-relevant — see
|
|
95
|
+
PROMETHEUS_ALLOWED_LABELS above for the reasoning per field.
|
|
59
96
|
|
|
60
97
|
How to use it:
|
|
61
98
|
- call before resolving or emitting Prometheus labels; do not mutate the
|
|
62
99
|
original event because delegates such as OpenSearch still need full dims.
|
|
63
|
-
|
|
64
|
-
Example:
|
|
65
|
-
- `_prometheus_label_dims(event)` drops `session_id`, `user_id`, and
|
|
66
|
-
`exchange_id` from the label candidate set.
|
|
67
100
|
"""
|
|
68
101
|
return {
|
|
69
102
|
k: str(v)
|
|
70
103
|
for k, v in (event.dims or {}).items()
|
|
71
|
-
if v is not None and k
|
|
104
|
+
if v is not None and k in PROMETHEUS_ALLOWED_LABELS
|
|
72
105
|
}
|
|
73
106
|
|
|
74
107
|
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# Copyright Thales 2026
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""
|
|
16
|
+
Shared emission primitive for the security/audit logger.
|
|
17
|
+
|
|
18
|
+
Why this exists:
|
|
19
|
+
- Every audit-worthy event (an authorization decision, a tool invocation) needs
|
|
20
|
+
to land as one structured line on `AUDIT_LOGGER_NAME`, formatted the same way,
|
|
21
|
+
so a downstream log pipeline can select the whole family on one field.
|
|
22
|
+
- Application layers may additionally keep their own short-lived view (e.g. an
|
|
23
|
+
in-memory ring buffer backing an admin UI) — that is layered on top of this,
|
|
24
|
+
not a replacement for it. This function only ever writes the log line.
|
|
25
|
+
|
|
26
|
+
How to use it:
|
|
27
|
+
- call with a stable `event_name` (e.g. "agent.tool.invocation.completed") and
|
|
28
|
+
whatever identifying/correlation fields are available; omit fields that are
|
|
29
|
+
not, never fabricate them.
|
|
30
|
+
- never pass tool arguments, tool results, prompts, documents, tokens, or any
|
|
31
|
+
other secret — this channel is audit-grade and is expected to be routed to
|
|
32
|
+
long-retention storage.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
import logging
|
|
38
|
+
|
|
39
|
+
from fred_core.logs.log_setup import AUDIT_LOGGER_NAME
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def emit_audit_log(
|
|
43
|
+
event_name: str,
|
|
44
|
+
level: str = "info",
|
|
45
|
+
/,
|
|
46
|
+
**fields: object,
|
|
47
|
+
) -> None:
|
|
48
|
+
"""Write one structured line to the shared security/audit logger.
|
|
49
|
+
|
|
50
|
+
`event_name` and `level` are positional-only so that spreading a caller's
|
|
51
|
+
dims dict via `**fields` (see ContextAwareTool._emit_tool_call_audit) can
|
|
52
|
+
never collide with — or be mistaken by a type checker for — these two
|
|
53
|
+
fixed parameters.
|
|
54
|
+
|
|
55
|
+
`fields` becomes the JSON payload's `extra` object once CompactJsonFormatter
|
|
56
|
+
formats the record (see log_setup.py). `None` values are dropped rather than
|
|
57
|
+
serialized, so a caller can pass every optional field unconditionally.
|
|
58
|
+
"""
|
|
59
|
+
payload: dict[str, object] = {"audit_event": event_name}
|
|
60
|
+
payload.update({k: v for k, v in fields.items() if v is not None})
|
|
61
|
+
logger = logging.getLogger(AUDIT_LOGGER_NAME)
|
|
62
|
+
log_fn = getattr(logger, level)
|
|
63
|
+
log_fn("[SECURITY] %s", event_name, extra=payload)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
__all__ = ["emit_audit_log", "AUDIT_LOGGER_NAME"]
|
|
@@ -22,8 +22,9 @@ Why this exists in Fred:
|
|
|
22
22
|
- We want **one emission API** (logging handler / writer) that can plug into
|
|
23
23
|
**many backends** (OpenSearch, in-memory ring for dev, CSV for tests, etc.).
|
|
24
24
|
- Backends differ in capabilities and SLOs; we keep the interface minimal but
|
|
25
|
-
sufficient: readiness, single-event ingest, bulk ingest
|
|
26
|
-
|
|
25
|
+
sufficient: readiness, single-event ingest, and bulk ingest. Reading these
|
|
26
|
+
logs back is done directly against the backing store (OpenSearch Dashboards),
|
|
27
|
+
not through Fred — this module is write-path only.
|
|
27
28
|
|
|
28
29
|
Key ideas:
|
|
29
30
|
- `BaseLogStore` is a `Protocol` (static duck-typing) → implementations don’t
|
|
@@ -35,14 +36,14 @@ from __future__ import annotations
|
|
|
35
36
|
|
|
36
37
|
from typing import Protocol
|
|
37
38
|
|
|
38
|
-
from fred_core.logs.log_structures import LogEventDTO
|
|
39
|
+
from fred_core.logs.log_structures import LogEventDTO
|
|
39
40
|
|
|
40
41
|
|
|
41
42
|
class BaseLogStore(Protocol):
|
|
42
|
-
"""Abstract Log store API used by the logging pipeline
|
|
43
|
+
"""Abstract Log store API used by the logging pipeline.
|
|
43
44
|
|
|
44
45
|
Architectural role:
|
|
45
|
-
- Decouple **log emission** from **physical storage
|
|
46
|
+
- Decouple **log emission** from **physical storage**.
|
|
46
47
|
- Keep the contract intentionally small so backends remain easy to implement.
|
|
47
48
|
- Allow services to boot even if the store is slow to initialize
|
|
48
49
|
(callers may handle failures without crashing business logic).
|
|
@@ -50,8 +51,6 @@ class BaseLogStore(Protocol):
|
|
|
50
51
|
Implementation notes:
|
|
51
52
|
- `index_event` must be non-blocking or bounded in latency (hot paths).
|
|
52
53
|
- `bulk_index` is for throughput (batching) when the caller can coalesce.
|
|
53
|
-
- `query` is intentionally lightweight: time range + simple filters
|
|
54
|
-
for the built-in console; not meant to replace a full SIEM.
|
|
55
54
|
"""
|
|
56
55
|
|
|
57
56
|
def ensure_ready(self) -> None:
|
|
@@ -80,12 +79,3 @@ class BaseLogStore(Protocol):
|
|
|
80
79
|
- Caller decides batching cadence; store optimizes the write path.
|
|
81
80
|
"""
|
|
82
81
|
...
|
|
83
|
-
|
|
84
|
-
def query(self, q: LogQuery) -> LogQueryResult:
|
|
85
|
-
"""Run a best-effort query over stored logs.
|
|
86
|
-
|
|
87
|
-
Design:
|
|
88
|
-
- Used by the Fred monitoring UI (time window, filters, limit/order).
|
|
89
|
-
- Implementations may support a subset of filters; document trade-offs.
|
|
90
|
-
"""
|
|
91
|
-
...
|
|
@@ -23,6 +23,7 @@ import threading
|
|
|
23
23
|
from typing import Any, Optional
|
|
24
24
|
|
|
25
25
|
from fred_core.logs.base_log_store import BaseLogStore, LogEventDTO
|
|
26
|
+
from fred_core.logs.log_structures import LogCategory
|
|
26
27
|
|
|
27
28
|
try:
|
|
28
29
|
from rich.logging import RichHandler
|
|
@@ -31,12 +32,33 @@ except Exception: # optional in prod images
|
|
|
31
32
|
|
|
32
33
|
logger = logging.getLogger(__name__)
|
|
33
34
|
|
|
35
|
+
# Single source of truth for the security/audit logger name — shared by
|
|
36
|
+
# log_setup() (which gives it its dedicated JSON stdout path, see below) and
|
|
37
|
+
# every call site that emits an audit event (authz decisions, tool-call
|
|
38
|
+
# invocations), so both sides can never drift out of sync on the string.
|
|
39
|
+
AUDIT_LOGGER_NAME = "fred.security.audit"
|
|
40
|
+
|
|
41
|
+
# Single source of truth for the reserved KPI-summary logger name — shared by
|
|
42
|
+
# StoreEmitHandler (which derives LogEventDTO.category from it) and
|
|
43
|
+
# kpi_writer.py's periodic rollup lines, so both sides can never drift out of
|
|
44
|
+
# sync on the string (mirrors AUDIT_LOGGER_NAME above).
|
|
45
|
+
KPI_LOGGER_NAME = "KPI"
|
|
46
|
+
|
|
34
47
|
LEVEL_MAP = {
|
|
35
48
|
"DETAIL": "DEBUG",
|
|
36
49
|
"TRACE": "DEBUG",
|
|
37
50
|
}
|
|
38
51
|
|
|
39
52
|
|
|
53
|
+
# Attributes a fresh LogRecord carries by default — anything else on a record's
|
|
54
|
+
# __dict__ came from a caller's `extra={...}` and should be surfaced, not dropped.
|
|
55
|
+
# Introspected from a throwaway record instead of hand-maintained so it tracks
|
|
56
|
+
# whatever the running Python version's logging module actually sets.
|
|
57
|
+
_STANDARD_LOG_RECORD_ATTRS = frozenset(
|
|
58
|
+
vars(logging.LogRecord("", 0, "", 0, "", (), None)).keys()
|
|
59
|
+
) | {"message", "asctime"}
|
|
60
|
+
|
|
61
|
+
|
|
40
62
|
# --- JSON formatter kept tiny and portable ---
|
|
41
63
|
class CompactJsonFormatter(logging.Formatter):
|
|
42
64
|
def __init__(self, service_name: str):
|
|
@@ -55,7 +77,14 @@ class CompactJsonFormatter(logging.Formatter):
|
|
|
55
77
|
"service": self.service,
|
|
56
78
|
"msg": record.getMessage(),
|
|
57
79
|
}
|
|
58
|
-
|
|
80
|
+
extra = {
|
|
81
|
+
k: v
|
|
82
|
+
for k, v in record.__dict__.items()
|
|
83
|
+
if k not in _STANDARD_LOG_RECORD_ATTRS
|
|
84
|
+
}
|
|
85
|
+
if extra:
|
|
86
|
+
base["extra"] = extra
|
|
87
|
+
return json.dumps(base, ensure_ascii=False, default=str)
|
|
59
88
|
|
|
60
89
|
|
|
61
90
|
# --- Minimal handler that pushes to a BaseLogStore (or a lazy getter) ---
|
|
@@ -71,6 +100,13 @@ class StoreEmitHandler(logging.Handler):
|
|
|
71
100
|
self._tls = threading.local()
|
|
72
101
|
|
|
73
102
|
def emit(self, record: logging.LogRecord) -> None:
|
|
103
|
+
# Hard drop, independent of the audit logger's own propagate=False:
|
|
104
|
+
# real security/audit events (fred.security.audit) must never land in
|
|
105
|
+
# the generic app-log store, even if a future refactor accidentally
|
|
106
|
+
# reattaches a handler or flips propagate to True (see
|
|
107
|
+
# docs/swift/platform/OBSERVABILITY-AND-AUDIT.md §6).
|
|
108
|
+
if record.name == AUDIT_LOGGER_NAME:
|
|
109
|
+
return
|
|
74
110
|
if getattr(self._tls, "in_emit", False):
|
|
75
111
|
return
|
|
76
112
|
self._tls.in_emit = True
|
|
@@ -88,6 +124,15 @@ class StoreEmitHandler(logging.Handler):
|
|
|
88
124
|
else:
|
|
89
125
|
mapped_level = record.levelname
|
|
90
126
|
|
|
127
|
+
# Closed, structural classification — derived from the LogRecord's
|
|
128
|
+
# own logger identity, never from message text (a message that
|
|
129
|
+
# happens to contain "[KPI]" does not become that category; only
|
|
130
|
+
# a record actually emitted on the reserved KPI logger does). See
|
|
131
|
+
# docs/swift/platform/OBSERVABILITY-AND-AUDIT.md §6.
|
|
132
|
+
category: LogCategory = (
|
|
133
|
+
"kpi" if record.name == KPI_LOGGER_NAME else "application"
|
|
134
|
+
)
|
|
135
|
+
|
|
91
136
|
e = LogEventDTO(
|
|
92
137
|
ts=payload.get("ts", record.created) if payload else record.created,
|
|
93
138
|
level=mapped_level, # type: ignore
|
|
@@ -101,6 +146,7 @@ class StoreEmitHandler(logging.Handler):
|
|
|
101
146
|
if payload
|
|
102
147
|
else self.service,
|
|
103
148
|
extra=payload.get("extra") if payload else None,
|
|
149
|
+
category=category,
|
|
104
150
|
)
|
|
105
151
|
|
|
106
152
|
# Never block the app on logging:
|
|
@@ -230,8 +276,13 @@ def log_setup(
|
|
|
230
276
|
|
|
231
277
|
# 1) Human console (Rich or plain)
|
|
232
278
|
formatter = logging.Formatter(
|
|
233
|
-
# Include process ID, task name,
|
|
234
|
-
|
|
279
|
+
# Include process ID, task name, thread name, and the logger's own
|
|
280
|
+
# dotted module name for concurrency diagnostics *and* provenance —
|
|
281
|
+
# %(name)s is why a message needs no hand-invented [TAG] to say where
|
|
282
|
+
# it came from. Reserve bracket prefixes in message text for the two
|
|
283
|
+
# real routed channels ([SECURITY] via emit_audit_log, [KPI] via
|
|
284
|
+
# logging.getLogger("KPI")); everything else should rely on this.
|
|
285
|
+
fmt="%(asctime)s | %(levelname)s | [pid=%(process)d %(threadName)s/%(task_name)s] | %(name)s | %(message)s",
|
|
235
286
|
datefmt="%Y-%m-%d %H:%M:%S",
|
|
236
287
|
)
|
|
237
288
|
if use_rich and RichHandler is not None:
|
|
@@ -306,4 +357,19 @@ def log_setup(
|
|
|
306
357
|
lg.propagate = True # forward to our root handlers
|
|
307
358
|
logging.getLogger("uvicorn.error").addFilter(UvicornWebsocketNoiseFilter())
|
|
308
359
|
|
|
360
|
+
# 5) Dedicated single-line JSON path for the security/audit logger.
|
|
361
|
+
# Structured audit events (authz decisions, tool-call invocations) must
|
|
362
|
+
# reach stdout as valid, self-contained JSON so a platform-native log
|
|
363
|
+
# pipeline (Cloud Logging or equivalent) can parse and route them without
|
|
364
|
+
# scraping free text — the human-readable console formatter above is not
|
|
365
|
+
# JSON. propagate=False keeps an audit line from also printing through
|
|
366
|
+
# that console handler.
|
|
367
|
+
audit_logger = logging.getLogger(AUDIT_LOGGER_NAME)
|
|
368
|
+
audit_logger.handlers.clear()
|
|
369
|
+
audit_handler = logging.StreamHandler()
|
|
370
|
+
audit_handler.setFormatter(CompactJsonFormatter(service_name))
|
|
371
|
+
audit_logger.addHandler(audit_handler)
|
|
372
|
+
audit_logger.setLevel(log_level.upper())
|
|
373
|
+
audit_logger.propagate = False
|
|
374
|
+
|
|
309
375
|
setattr(root, marker, True)
|