codebuff-agent 2.1.74__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codebuff_agent-2.1.74.data/data/etc/jupyter/jupyter_server_config.d/codebuff_agent.json +7 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/package.json +157 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/104.2b8f20f9d1ac2c9db01b.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/150.39fa98117cc8ca590465.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/166.386ff9b2c37ac865fdee.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/216.25cf81131bf7274dabf4.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/353.581bd1cb00746399b62e.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/367.575f9621b63c9a641aa9.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/421.11dac223a2c2da1b09f5.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/432.8513b4c6cc1d1232d165.js +2 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/432.8513b4c6cc1d1232d165.js.LICENSE.txt +19 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/439.ba5b5ef70ec6de214b8c.js +2 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/439.ba5b5ef70ec6de214b8c.js.LICENSE.txt +8 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/481.cc2dec9fa869d66f1407.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/487.0796e931d6e651482c14.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/49.d326f554b6f9ccde6580.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/541.26897ea32845659e36be.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/579.e569d3edd8391fd8fbb3.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/58.eb7cc679f7535cc8a2f1.js +2 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/58.eb7cc679f7535cc8a2f1.js.LICENSE.txt +8 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/666.d9bfdb23f98489d0a188.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/677.d266c39dbed33bc388b3.js +2 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/677.d266c39dbed33bc388b3.js.LICENSE.txt +8 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/68.26b6f59c8454adb17f20.js +204 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/694.4ac443814260057e68f7.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/791.61affb8d3480d3d60e0d.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/848.4d2db09cfbb09374b299.js +2 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/848.4d2db09cfbb09374b299.js.LICENSE.txt +9 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/868.5cfe7e1a348ec5af1b8c.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/97.ea559a5b8a9a09f17c6b.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/remoteEntry.5292fc9c6df7e7920cc8.js +1 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/style.js +4 -0
- codebuff_agent-2.1.74.data/data/share/jupyter/labextensions/codebuff/static/third-party-licenses.json +724 -0
- codebuff_agent-2.1.74.dist-info/METADATA +142 -0
- codebuff_agent-2.1.74.dist-info/RECORD +217 -0
- codebuff_agent-2.1.74.dist-info/WHEEL +4 -0
- codebuff_agent-2.1.74.dist-info/licenses/LICENSE +21 -0
- codebuff_core/__init__.py +153 -0
- codebuff_core/codebuff_config_defaults.json +64 -0
- codebuff_core/core/__init__.py +35 -0
- codebuff_core/core/code_validator.py +1238 -0
- codebuff_core/core/error_classifier.py +579 -0
- codebuff_core/core/exceptions.py +49 -0
- codebuff_core/data/tiktoken_cache/9b5ad71b2ce5302211f9c61530b329a4922fc6a4 +100256 -0
- codebuff_core/directory_policy.py +32 -0
- codebuff_core/factory.py +155 -0
- codebuff_core/interfaces.py +197 -0
- codebuff_core/knowledge/__init__.py +31 -0
- codebuff_core/knowledge/chunking.py +375 -0
- codebuff_core/knowledge/loader.py +339 -0
- codebuff_core/knowledge/watchdog_service.py +354 -0
- codebuff_core/legacy_paths.py +69 -0
- codebuff_core/llm/__init__.py +13 -0
- codebuff_core/llm/service.py +407 -0
- codebuff_core/managers/__init__.py +22 -0
- codebuff_core/managers/config_manager.py +690 -0
- codebuff_core/managers/remote_config.py +673 -0
- codebuff_core/managers/remote_skills.py +539 -0
- codebuff_core/managers/session_manager.py +255 -0
- codebuff_core/middleware/__init__.py +3 -0
- codebuff_core/middleware/error_recovery.py +394 -0
- codebuff_core/middleware/logging_utils.py +33 -0
- codebuff_core/models/__init__.py +111 -0
- codebuff_core/models/agent.py +306 -0
- codebuff_core/models/chat.py +57 -0
- codebuff_core/models/common.py +82 -0
- codebuff_core/models/rag.py +300 -0
- codebuff_core/platform_rules.py +127 -0
- codebuff_core/prompts/__init__.py +44 -0
- codebuff_core/prompts/auto_agent_prompts.py +1278 -0
- codebuff_core/services/__init__.py +30 -0
- codebuff_core/services/agent_service.py +345 -0
- codebuff_core/services/analytics_aggregate.py +837 -0
- codebuff_core/services/chat_service.py +898 -0
- codebuff_core/services/context_condenser.py +306 -0
- codebuff_core/services/pii_guardrail.py +393 -0
- codebuff_core/services/rag_chat_agent.py +2993 -0
- codebuff_core/services/rag_collections.py +2740 -0
- codebuff_core/services/rag_manager.py +890 -0
- codebuff_core/services/rag_service.py +145 -0
- codebuff_core/services/retriever.py +260 -0
- codebuff_core/services/vllm_embedding_service.py +290 -0
- codebuff_core/tests/__init__.py +1 -0
- codebuff_core/tests/conftest.py +81 -0
- codebuff_core/tests/test_error_recovery.py +647 -0
- codebuff_core/tests/test_factory.py +135 -0
- codebuff_core/tests/test_services.py +150 -0
- jupyter_ext/__init__.py +266 -0
- jupyter_ext/_version.py +4 -0
- jupyter_ext/agent/__init__.py +49 -0
- jupyter_ext/agent/agent_factory.py +2270 -0
- jupyter_ext/agent/ast_tools.py +451 -0
- jupyter_ext/agent/autofix_middleware.py +481 -0
- jupyter_ext/agent/background_procs.py +372 -0
- jupyter_ext/agent/code_history_middleware.py +211 -0
- jupyter_ext/agent/codex_autofix.py +160 -0
- jupyter_ext/agent/codex_bridge.py +3339 -0
- jupyter_ext/agent/codex_hooks.py +320 -0
- jupyter_ext/agent/codex_slash.py +685 -0
- jupyter_ext/agent/codex_threads.py +114 -0
- jupyter_ext/agent/content_injection.py +377 -0
- jupyter_ext/agent/continuation_middleware.py +548 -0
- jupyter_ext/agent/custom_skills.py +335 -0
- jupyter_ext/agent/file_diffs.py +99 -0
- jupyter_ext/agent/git_history.py +188 -0
- jupyter_ext/agent/guardrail_middleware.py +421 -0
- jupyter_ext/agent/json_repair_utils.py +54 -0
- jupyter_ext/agent/langchain_tools.py +2673 -0
- jupyter_ext/agent/mermaid_fixer.py +289 -0
- jupyter_ext/agent/mermaid_middleware.py +135 -0
- jupyter_ext/agent/middleware.py +359 -0
- jupyter_ext/agent/middleware_prompts.py +53 -0
- jupyter_ext/agent/observability.py +334 -0
- jupyter_ext/agent/patch_tool_calls.py +57 -0
- jupyter_ext/agent/prompts.py +112 -0
- jupyter_ext/agent/skill_middleware.py +257 -0
- jupyter_ext/agent/skills/airflow_dag_development.md +36 -0
- jupyter_ext/agent/skills/anomaly_detection.md +12 -0
- jupyter_ext/agent/skills/athena_sql.md +156 -0
- jupyter_ext/agent/skills/bdp_table_search.md +33 -0
- jupyter_ext/agent/skills/bigquery.md +65 -0
- jupyter_ext/agent/skills/bigquery_oracle.md +231 -0
- jupyter_ext/agent/skills/bigquery_tuning.md +307 -0
- jupyter_ext/agent/skills/classification_evaluation.md +11 -0
- jupyter_ext/agent/skills/clustering_analysis.md +12 -0
- jupyter_ext/agent/skills/comprehensive_eda.md +11 -0
- jupyter_ext/agent/skills/data_analysis.md +236 -0
- jupyter_ext/agent/skills/data_cleaning_pipeline.md +12 -0
- jupyter_ext/agent/skills/data_loading.md +158 -0
- jupyter_ext/agent/skills/data_quality_report.md +13 -0
- jupyter_ext/agent/skills/dimensionality_reduction.md +12 -0
- jupyter_ext/agent/skills/emr_script_development.md +17 -0
- jupyter_ext/agent/skills/experiment_tracking.md +13 -0
- jupyter_ext/agent/skills/feature_importance_analysis.md +10 -0
- jupyter_ext/agent/skills/hyperparameter_tuning.md +12 -0
- jupyter_ext/agent/skills/hypothesis_testing.md +12 -0
- jupyter_ext/agent/skills/imbalanced_data_handling.md +12 -0
- jupyter_ext/agent/skills/inference.md +392 -0
- jupyter_ext/agent/skills/lambda_function_development.md +19 -0
- jupyter_ext/agent/skills/model_comparison.md +11 -0
- jupyter_ext/agent/skills/model_explainability.md +12 -0
- jupyter_ext/agent/skills/model_training.md +318 -0
- jupyter_ext/agent/skills/pyspark.md +352 -0
- jupyter_ext/agent/skills/python_code_quality.md +35 -0
- jupyter_ext/agent/skills/regression_evaluation.md +12 -0
- jupyter_ext/agent/skills/spring_java_development.md +35 -0
- jupyter_ext/agent/skills/text_preprocessing.md +12 -0
- jupyter_ext/agent/skills/time_series_analysis.md +13 -0
- jupyter_ext/agent/skills/unit_testing.md +203 -0
- jupyter_ext/agent/todo_prompts.py +57 -0
- jupyter_ext/agent/tool_report.py +45 -0
- jupyter_ext/analytics_ingest.py +599 -0
- jupyter_ext/etc/jupyter/jupyter_server_config.d/codebuff_agent.json +7 -0
- jupyter_ext/handlers/__init__.py +8 -0
- jupyter_ext/handlers/analytics_handler.py +170 -0
- jupyter_ext/handlers/benchmark_handler.py +794 -0
- jupyter_ext/handlers/eda_benchmark_handler.py +1328 -0
- jupyter_ext/handlers/rag_admin_handler.py +689 -0
- jupyter_ext/handlers/websocket.py +497 -0
- jupyter_ext/http_handlers.py +8744 -0
- jupyter_ext/idle_monitor.py +346 -0
- jupyter_ext/labextension/package.json +157 -0
- jupyter_ext/labextension/static/104.2b8f20f9d1ac2c9db01b.js +1 -0
- jupyter_ext/labextension/static/150.39fa98117cc8ca590465.js +1 -0
- jupyter_ext/labextension/static/166.386ff9b2c37ac865fdee.js +1 -0
- jupyter_ext/labextension/static/216.25cf81131bf7274dabf4.js +1 -0
- jupyter_ext/labextension/static/353.581bd1cb00746399b62e.js +1 -0
- jupyter_ext/labextension/static/367.575f9621b63c9a641aa9.js +1 -0
- jupyter_ext/labextension/static/421.11dac223a2c2da1b09f5.js +1 -0
- jupyter_ext/labextension/static/432.8513b4c6cc1d1232d165.js +2 -0
- jupyter_ext/labextension/static/432.8513b4c6cc1d1232d165.js.LICENSE.txt +19 -0
- jupyter_ext/labextension/static/439.ba5b5ef70ec6de214b8c.js +2 -0
- jupyter_ext/labextension/static/439.ba5b5ef70ec6de214b8c.js.LICENSE.txt +8 -0
- jupyter_ext/labextension/static/481.cc2dec9fa869d66f1407.js +1 -0
- jupyter_ext/labextension/static/487.0796e931d6e651482c14.js +1 -0
- jupyter_ext/labextension/static/49.d326f554b6f9ccde6580.js +1 -0
- jupyter_ext/labextension/static/541.26897ea32845659e36be.js +1 -0
- jupyter_ext/labextension/static/579.e569d3edd8391fd8fbb3.js +1 -0
- jupyter_ext/labextension/static/58.eb7cc679f7535cc8a2f1.js +2 -0
- jupyter_ext/labextension/static/58.eb7cc679f7535cc8a2f1.js.LICENSE.txt +8 -0
- jupyter_ext/labextension/static/666.d9bfdb23f98489d0a188.js +1 -0
- jupyter_ext/labextension/static/677.d266c39dbed33bc388b3.js +2 -0
- jupyter_ext/labextension/static/677.d266c39dbed33bc388b3.js.LICENSE.txt +8 -0
- jupyter_ext/labextension/static/68.26b6f59c8454adb17f20.js +204 -0
- jupyter_ext/labextension/static/694.4ac443814260057e68f7.js +1 -0
- jupyter_ext/labextension/static/791.61affb8d3480d3d60e0d.js +1 -0
- jupyter_ext/labextension/static/848.4d2db09cfbb09374b299.js +2 -0
- jupyter_ext/labextension/static/848.4d2db09cfbb09374b299.js.LICENSE.txt +9 -0
- jupyter_ext/labextension/static/868.5cfe7e1a348ec5af1b8c.js +1 -0
- jupyter_ext/labextension/static/97.ea559a5b8a9a09f17c6b.js +1 -0
- jupyter_ext/labextension/static/remoteEntry.5292fc9c6df7e7920cc8.js +1 -0
- jupyter_ext/labextension/static/style.js +4 -0
- jupyter_ext/labextension/static/third-party-licenses.json +724 -0
- jupyter_ext/log_paths.py +113 -0
- jupyter_ext/resource_usage.py +519 -0
- jupyter_ext/secrets_bootstrap.py +116 -0
- jupyter_ext/team_access.py +105 -0
- jupyter_ext/tests/__init__.py +1 -0
- jupyter_ext/tests/test_context.py +234 -0
- jupyter_ext/tests/test_factory.py +563 -0
- jupyter_ext/tests/test_file_finder.py +139 -0
- jupyter_ext/tests/test_handlers.py +58 -0
- jupyter_ext/tests/test_mcp_selftest.py +204 -0
- jupyter_ext/tests/test_mermaid_fixer.py +272 -0
- jupyter_ext/tests/test_resource_tiers.py +298 -0
- jupyter_ext/tools/__init__.py +15 -0
- jupyter_ext/tools/base.py +90 -0
- jupyter_ext/tools/context.py +268 -0
- jupyter_ext/tools/dir_complete.py +86 -0
- jupyter_ext/tools/file_finder.py +303 -0
- jupyter_ext/tools/file_tools.py +244 -0
- jupyter_ext/tools/jupyter_tools.py +296 -0
- jupyter_ext/tools/registry.py +159 -0
- jupyter_ext/tools/search_tools.py +230 -0
- jupyter_ext/tools/security_tools.py +657 -0
- jupyter_ext/tools/shell_tools.py +109 -0
- jupyter_ext/tools/user_tools.py +132 -0
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Context Condenser for intelligent context compression.
|
|
3
|
+
|
|
4
|
+
Provides token-aware context management with multiple compression strategies
|
|
5
|
+
to optimize LLM input while preserving important information.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import logging
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from enum import Enum
|
|
11
|
+
from typing import Dict, List, Optional, Tuple
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class CompressionStrategy(Enum):
|
|
17
|
+
"""Available compression strategies."""
|
|
18
|
+
|
|
19
|
+
TRUNCATE = "truncate" # Keep only recent messages
|
|
20
|
+
SUMMARIZE = "summarize" # Summarize old messages, keep recent
|
|
21
|
+
ADAPTIVE = "adaptive" # Auto-select based on context size
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class CompressionStats:
|
|
26
|
+
"""Statistics from a compression operation."""
|
|
27
|
+
|
|
28
|
+
original_tokens: int
|
|
29
|
+
compressed_tokens: int
|
|
30
|
+
compression_ratio: float
|
|
31
|
+
strategy_used: str
|
|
32
|
+
messages_kept: int
|
|
33
|
+
messages_removed: int
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class ContextCondenser:
|
|
37
|
+
"""
|
|
38
|
+
Context compressor - optimizes conversation context within token budget.
|
|
39
|
+
|
|
40
|
+
Supports multiple LLM providers with different token limits.
|
|
41
|
+
Uses rule-based compression without additional LLM calls.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
# Provider-specific token limits for context
|
|
45
|
+
TOKEN_LIMITS = {
|
|
46
|
+
"vllm": 8000,
|
|
47
|
+
"default": 4000,
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
# Token estimation: average tokens per word (conservative)
|
|
51
|
+
TOKENS_PER_WORD = 1.3
|
|
52
|
+
|
|
53
|
+
def __init__(self, provider: str = "default"):
|
|
54
|
+
"""Initialize condenser with provider-specific settings.
|
|
55
|
+
|
|
56
|
+
Args:
|
|
57
|
+
provider: LLM provider name for token limit selection
|
|
58
|
+
"""
|
|
59
|
+
self._provider = provider
|
|
60
|
+
self._stats_history: List[CompressionStats] = []
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def provider(self) -> str:
|
|
64
|
+
"""Current LLM provider."""
|
|
65
|
+
return self._provider
|
|
66
|
+
|
|
67
|
+
@provider.setter
|
|
68
|
+
def provider(self, value: str) -> None:
|
|
69
|
+
"""Update LLM provider."""
|
|
70
|
+
self._provider = value
|
|
71
|
+
|
|
72
|
+
def estimate_tokens(self, text: str) -> int:
|
|
73
|
+
"""Estimate token count for text.
|
|
74
|
+
|
|
75
|
+
Uses word count with a conservative multiplier.
|
|
76
|
+
More accurate than character count for most text.
|
|
77
|
+
|
|
78
|
+
Args:
|
|
79
|
+
text: Input text to estimate
|
|
80
|
+
|
|
81
|
+
Returns:
|
|
82
|
+
Estimated token count
|
|
83
|
+
"""
|
|
84
|
+
if not text:
|
|
85
|
+
return 0
|
|
86
|
+
words = len(text.split())
|
|
87
|
+
return int(words * self.TOKENS_PER_WORD)
|
|
88
|
+
|
|
89
|
+
def get_token_limit(self) -> int:
|
|
90
|
+
"""Get token limit for current provider.
|
|
91
|
+
|
|
92
|
+
Returns:
|
|
93
|
+
Maximum tokens for context
|
|
94
|
+
"""
|
|
95
|
+
return self.TOKEN_LIMITS.get(self._provider, self.TOKEN_LIMITS["default"])
|
|
96
|
+
|
|
97
|
+
def condense(
|
|
98
|
+
self,
|
|
99
|
+
messages: List[Dict[str, str]],
|
|
100
|
+
target_tokens: Optional[int] = None,
|
|
101
|
+
strategy: CompressionStrategy = CompressionStrategy.ADAPTIVE,
|
|
102
|
+
) -> Tuple[List[Dict[str, str]], CompressionStats]:
|
|
103
|
+
"""Compress message list to fit within token budget.
|
|
104
|
+
|
|
105
|
+
Args:
|
|
106
|
+
messages: List of message dicts with 'role' and 'content' keys
|
|
107
|
+
target_tokens: Target token count (default: 50% of provider limit)
|
|
108
|
+
strategy: Compression strategy to use
|
|
109
|
+
|
|
110
|
+
Returns:
|
|
111
|
+
Tuple of (compressed_messages, compression_stats)
|
|
112
|
+
"""
|
|
113
|
+
if not messages:
|
|
114
|
+
return [], CompressionStats(0, 0, 1.0, "none", 0, 0)
|
|
115
|
+
|
|
116
|
+
target = target_tokens or (self.get_token_limit() // 2)
|
|
117
|
+
original_tokens = sum(
|
|
118
|
+
self.estimate_tokens(m.get("content", "")) for m in messages
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
# Already within budget - no compression needed
|
|
122
|
+
if original_tokens <= target:
|
|
123
|
+
stats = CompressionStats(
|
|
124
|
+
original_tokens=original_tokens,
|
|
125
|
+
compressed_tokens=original_tokens,
|
|
126
|
+
compression_ratio=1.0,
|
|
127
|
+
strategy_used="none",
|
|
128
|
+
messages_kept=len(messages),
|
|
129
|
+
messages_removed=0,
|
|
130
|
+
)
|
|
131
|
+
return messages, stats
|
|
132
|
+
|
|
133
|
+
# Select strategy if adaptive
|
|
134
|
+
if strategy == CompressionStrategy.ADAPTIVE:
|
|
135
|
+
strategy = self._select_strategy(original_tokens, target)
|
|
136
|
+
|
|
137
|
+
# Apply selected strategy
|
|
138
|
+
if strategy == CompressionStrategy.TRUNCATE:
|
|
139
|
+
compressed, stats = self._truncate(messages, target)
|
|
140
|
+
elif strategy == CompressionStrategy.SUMMARIZE:
|
|
141
|
+
compressed, stats = self._summarize(messages, target)
|
|
142
|
+
else:
|
|
143
|
+
compressed, stats = self._truncate(messages, target)
|
|
144
|
+
|
|
145
|
+
self._stats_history.append(stats)
|
|
146
|
+
logger.info(
|
|
147
|
+
f"Context compressed: {stats.original_tokens} → {stats.compressed_tokens} "
|
|
148
|
+
f"tokens ({stats.compression_ratio:.1%}), strategy={stats.strategy_used}"
|
|
149
|
+
)
|
|
150
|
+
return compressed, stats
|
|
151
|
+
|
|
152
|
+
def _select_strategy(self, original: int, target: int) -> CompressionStrategy:
|
|
153
|
+
"""Select best compression strategy based on reduction needed.
|
|
154
|
+
|
|
155
|
+
Args:
|
|
156
|
+
original: Original token count
|
|
157
|
+
target: Target token count
|
|
158
|
+
|
|
159
|
+
Returns:
|
|
160
|
+
Selected compression strategy
|
|
161
|
+
"""
|
|
162
|
+
ratio = target / original
|
|
163
|
+
# If we need to keep more than 50%, simple truncation works
|
|
164
|
+
if ratio >= 0.5:
|
|
165
|
+
return CompressionStrategy.TRUNCATE
|
|
166
|
+
# For more aggressive compression, use summarization
|
|
167
|
+
return CompressionStrategy.SUMMARIZE
|
|
168
|
+
|
|
169
|
+
def _truncate(
|
|
170
|
+
self, messages: List[Dict[str, str]], target: int
|
|
171
|
+
) -> Tuple[List[Dict[str, str]], CompressionStats]:
|
|
172
|
+
"""Keep only recent messages within token budget.
|
|
173
|
+
|
|
174
|
+
Preserves most recent messages, dropping oldest first.
|
|
175
|
+
|
|
176
|
+
Args:
|
|
177
|
+
messages: Original messages
|
|
178
|
+
target: Target token count
|
|
179
|
+
|
|
180
|
+
Returns:
|
|
181
|
+
Tuple of (truncated_messages, stats)
|
|
182
|
+
"""
|
|
183
|
+
original_tokens = sum(
|
|
184
|
+
self.estimate_tokens(m.get("content", "")) for m in messages
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
# Keep messages from the end (most recent)
|
|
188
|
+
kept: List[Dict[str, str]] = []
|
|
189
|
+
current_tokens = 0
|
|
190
|
+
|
|
191
|
+
for msg in reversed(messages):
|
|
192
|
+
msg_tokens = self.estimate_tokens(msg.get("content", ""))
|
|
193
|
+
if current_tokens + msg_tokens <= target:
|
|
194
|
+
kept.insert(0, msg)
|
|
195
|
+
current_tokens += msg_tokens
|
|
196
|
+
else:
|
|
197
|
+
break
|
|
198
|
+
|
|
199
|
+
return kept, CompressionStats(
|
|
200
|
+
original_tokens=original_tokens,
|
|
201
|
+
compressed_tokens=current_tokens,
|
|
202
|
+
compression_ratio=(
|
|
203
|
+
current_tokens / original_tokens if original_tokens else 1.0
|
|
204
|
+
),
|
|
205
|
+
strategy_used="truncate",
|
|
206
|
+
messages_kept=len(kept),
|
|
207
|
+
messages_removed=len(messages) - len(kept),
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
def _summarize(
|
|
211
|
+
self, messages: List[Dict[str, str]], target: int
|
|
212
|
+
) -> Tuple[List[Dict[str, str]], CompressionStats]:
|
|
213
|
+
"""Summarize old messages, keep recent ones intact.
|
|
214
|
+
|
|
215
|
+
Rule-based summarization without LLM calls.
|
|
216
|
+
Extracts first sentence from each old message.
|
|
217
|
+
|
|
218
|
+
Args:
|
|
219
|
+
messages: Original messages
|
|
220
|
+
target: Target token count
|
|
221
|
+
|
|
222
|
+
Returns:
|
|
223
|
+
Tuple of (summary + recent_messages, stats)
|
|
224
|
+
"""
|
|
225
|
+
original_tokens = sum(
|
|
226
|
+
self.estimate_tokens(m.get("content", "")) for m in messages
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
# Keep last 3 messages intact
|
|
230
|
+
recent_count = min(3, len(messages))
|
|
231
|
+
recent = messages[-recent_count:]
|
|
232
|
+
old = messages[:-recent_count] if len(messages) > recent_count else []
|
|
233
|
+
|
|
234
|
+
recent_tokens = sum(self.estimate_tokens(m.get("content", "")) for m in recent)
|
|
235
|
+
remaining = target - recent_tokens
|
|
236
|
+
|
|
237
|
+
# If recent messages already exceed budget, fallback to truncate
|
|
238
|
+
if remaining <= 0 or not old:
|
|
239
|
+
return self._truncate(messages, target)
|
|
240
|
+
|
|
241
|
+
# Summarize old messages (extract first sentence, max 100 chars)
|
|
242
|
+
summary_parts = []
|
|
243
|
+
for msg in old:
|
|
244
|
+
content = msg.get("content", "")
|
|
245
|
+
# Get first sentence or first 100 chars
|
|
246
|
+
first_sentence = content.split(".")[0][:100]
|
|
247
|
+
if first_sentence:
|
|
248
|
+
role = "User" if msg.get("role") == "user" else "Assistant"
|
|
249
|
+
summary_parts.append(f"[{role}]: {first_sentence}...")
|
|
250
|
+
|
|
251
|
+
summary_text = "\n".join(summary_parts)
|
|
252
|
+
summary_tokens = self.estimate_tokens(summary_text)
|
|
253
|
+
|
|
254
|
+
# If summary exceeds remaining budget, fallback to truncate
|
|
255
|
+
if summary_tokens > remaining:
|
|
256
|
+
return self._truncate(messages, target)
|
|
257
|
+
|
|
258
|
+
# Combine summary with recent messages
|
|
259
|
+
summary_msg = {
|
|
260
|
+
"role": "system",
|
|
261
|
+
"content": f"[Previous conversation summary]\n{summary_text}",
|
|
262
|
+
}
|
|
263
|
+
result = [summary_msg] + recent
|
|
264
|
+
|
|
265
|
+
total_tokens = summary_tokens + recent_tokens
|
|
266
|
+
return result, CompressionStats(
|
|
267
|
+
original_tokens=original_tokens,
|
|
268
|
+
compressed_tokens=total_tokens,
|
|
269
|
+
compression_ratio=total_tokens / original_tokens
|
|
270
|
+
if original_tokens
|
|
271
|
+
else 1.0,
|
|
272
|
+
strategy_used="summarize",
|
|
273
|
+
messages_kept=len(recent),
|
|
274
|
+
messages_removed=len(old),
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
def get_stats_history(self) -> List[CompressionStats]:
|
|
278
|
+
"""Get history of compression operations.
|
|
279
|
+
|
|
280
|
+
Returns:
|
|
281
|
+
List of CompressionStats from previous operations
|
|
282
|
+
"""
|
|
283
|
+
return self._stats_history.copy()
|
|
284
|
+
|
|
285
|
+
def clear_stats_history(self) -> None:
|
|
286
|
+
"""Clear compression statistics history."""
|
|
287
|
+
self._stats_history.clear()
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
# Singleton accessor
|
|
291
|
+
_context_condenser: Optional[ContextCondenser] = None
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def get_context_condenser(provider: str = "default") -> ContextCondenser:
|
|
295
|
+
"""Get or create singleton ContextCondenser instance.
|
|
296
|
+
|
|
297
|
+
Args:
|
|
298
|
+
provider: LLM provider name (only used on first call)
|
|
299
|
+
|
|
300
|
+
Returns:
|
|
301
|
+
Singleton ContextCondenser instance
|
|
302
|
+
"""
|
|
303
|
+
global _context_condenser
|
|
304
|
+
if _context_condenser is None:
|
|
305
|
+
_context_condenser = ContextCondenser(provider)
|
|
306
|
+
return _context_condenser
|
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
"""PII Guardrail — 사용자 입력에서 PII 14종을 정규식+체크섬으로 검출 후 마스킹.
|
|
2
|
+
|
|
3
|
+
설계 원칙:
|
|
4
|
+
- specific → generic 우선순위로 순차 in-place 마스킹 (충돌 자동 회피)
|
|
5
|
+
- 매칭된 substring을 동일 길이 '*'로 교체 → 다음 패턴은 '*'를 매칭 못 함
|
|
6
|
+
- 체크섬 검증으로 false positive 최소화 (Luhn / 가중치 합 체크섬)
|
|
7
|
+
- 원본 PII는 로깅/저장 금지. label만 노출
|
|
8
|
+
|
|
9
|
+
Reference: docs/superpowers/refs/2026-06-10-pii-regex-reference.md
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import re
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
from typing import Callable, Optional
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class PIIMatch:
|
|
21
|
+
"""단일 PII 매치 정보."""
|
|
22
|
+
|
|
23
|
+
type_label: str
|
|
24
|
+
start: int
|
|
25
|
+
end: int
|
|
26
|
+
original: str # debug-only, 외부 로깅 금지
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class MaskResult:
|
|
31
|
+
"""mask_pii 결과."""
|
|
32
|
+
|
|
33
|
+
masked_text: str
|
|
34
|
+
detected_types: list[str] = field(default_factory=list)
|
|
35
|
+
matches: list[PIIMatch] = field(default_factory=list)
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def has_pii(self) -> bool:
|
|
39
|
+
return bool(self.matches)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# ---------------------------------------------------------------------------
|
|
43
|
+
# 정규식 패턴
|
|
44
|
+
# ---------------------------------------------------------------------------
|
|
45
|
+
|
|
46
|
+
# 이메일 (HTML5 input validation, RFC 5322 lite)
|
|
47
|
+
_EMAIL_RE = re.compile(
|
|
48
|
+
r"\b[A-Za-z0-9._%+\-]+"
|
|
49
|
+
r"@[A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])?"
|
|
50
|
+
r"(?:\.[A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])?)+\b"
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
# IPv6 — RFC 4291. dual notation 포함
|
|
54
|
+
_IPV6_RE = re.compile(
|
|
55
|
+
r"(?:"
|
|
56
|
+
# full + zero-compressed
|
|
57
|
+
r"(?:[0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}"
|
|
58
|
+
r"|(?:[0-9A-Fa-f]{1,4}:){1,7}:"
|
|
59
|
+
r"|(?:[0-9A-Fa-f]{1,4}:){1,6}:[0-9A-Fa-f]{1,4}"
|
|
60
|
+
r"|(?:[0-9A-Fa-f]{1,4}:){1,5}(?::[0-9A-Fa-f]{1,4}){1,2}"
|
|
61
|
+
r"|(?:[0-9A-Fa-f]{1,4}:){1,4}(?::[0-9A-Fa-f]{1,4}){1,3}"
|
|
62
|
+
r"|(?:[0-9A-Fa-f]{1,4}:){1,3}(?::[0-9A-Fa-f]{1,4}){1,4}"
|
|
63
|
+
r"|(?:[0-9A-Fa-f]{1,4}:){1,2}(?::[0-9A-Fa-f]{1,4}){1,5}"
|
|
64
|
+
r"|[0-9A-Fa-f]{1,4}:(?::[0-9A-Fa-f]{1,4}){1,6}"
|
|
65
|
+
r"|:(?:(?::[0-9A-Fa-f]{1,4}){1,7}|:)"
|
|
66
|
+
# dual notation (IPv4-mapped)
|
|
67
|
+
r"|::(?:[Ff]{4}(?::0{1,4})?:)?"
|
|
68
|
+
r"(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}"
|
|
69
|
+
r"(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)"
|
|
70
|
+
r"|(?:[0-9A-Fa-f]{1,4}:){1,4}:"
|
|
71
|
+
r"(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}"
|
|
72
|
+
r"(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)"
|
|
73
|
+
r")"
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
# IPv4 strict (no leading zeros) — RFC 5321 / CVE-2021-29921
|
|
77
|
+
_IPV4_RE = re.compile(
|
|
78
|
+
r"\b(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}"
|
|
79
|
+
r"(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\b"
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
# MAC 주소 — IEEE 802 (콜론/하이픈) 또는 Cisco dotted
|
|
83
|
+
_MAC_RE = re.compile(
|
|
84
|
+
r"\b(?:[0-9A-Fa-f]{2}[:\-]){5}[0-9A-Fa-f]{2}\b"
|
|
85
|
+
r"|\b(?:[0-9A-Fa-f]{4}\.){2}[0-9A-Fa-f]{4}\b"
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
# 한국 여권 — [MSROD] + 8 digits OR + 3 digits + letter + 4 digits
|
|
89
|
+
_PASSPORT_RE = re.compile(r"\b[MSROD](?:\d{8}|\d{3}[A-Z]\d{4})\b")
|
|
90
|
+
|
|
91
|
+
# 운전면허 12자리 (지역코드)
|
|
92
|
+
_DL_12_RE = re.compile(r"\b(?:1[1-9]|2[0-6]|28)[\- ]\d{2}[\- ]\d{6}[\- ]\d{2}\b")
|
|
93
|
+
|
|
94
|
+
# 운전면허 10자리 (2-6-2)
|
|
95
|
+
_DL_10_RE = re.compile(r"\b\d{2}[\- ]\d{6}[\- ]\d{2}\b")
|
|
96
|
+
|
|
97
|
+
# 신용카드 — 13~19 digits (with optional space/dash). Luhn 필수.
|
|
98
|
+
_CC_RE = re.compile(r"\b\d(?:[ \-]?\d){12,18}\b")
|
|
99
|
+
|
|
100
|
+
# 주민/외국인/법인등록 — 13자리 dash 형식 (validator로 disambiguate)
|
|
101
|
+
_RRN_FRN_CRN_RE = re.compile(r"\b\d{6}[\- ]?\d{7}\b")
|
|
102
|
+
|
|
103
|
+
# 사업자등록 — 3-2-5 dash 형식
|
|
104
|
+
_BIZ_RE = re.compile(r"\b\d{3}[\- ]\d{2}[\- ]\d{5}\b")
|
|
105
|
+
|
|
106
|
+
# 한국 전화번호
|
|
107
|
+
# 일반: 휴대폰(010-XXXX-XXXX) / 지역(02-XXXX-XXXX, 031-XXX-XXXX) / 인터넷전화(070-XXXX-XXXX)
|
|
108
|
+
# 대표번호: 1588-1234 형식 (prefix + 4-4)
|
|
109
|
+
_PHONE_RE = re.compile(
|
|
110
|
+
r"\b(?:"
|
|
111
|
+
# 대표번호 (1577/1588/1599/1644 등) — prefix 4 + 4
|
|
112
|
+
r"1[5-9]\d{2}[\- ]?\d{4}"
|
|
113
|
+
# 일반 (국제 +82 옵션) — prefix + 3-4 + 4
|
|
114
|
+
r"|(?:\+?82[\- ]?)?"
|
|
115
|
+
r"(?:0?1[016789]|02|0[3-6]\d|070|050\d)"
|
|
116
|
+
r"[\- ]?\d{3,4}[\- ]?\d{4}"
|
|
117
|
+
r")\b"
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
# 건강보험 — 11자리 + 1로 시작 (low confidence)
|
|
121
|
+
_HEALTH_RE = re.compile(r"\b1\d{10}\b")
|
|
122
|
+
|
|
123
|
+
# 계좌번호 — dash 형식만 (last resort)
|
|
124
|
+
_ACCOUNT_RE = re.compile(r"\b\d{2,6}[\- ]\d{2,6}[\- ]\d{2,6}(?:[\- ]\d{2,6})?\b")
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
# ---------------------------------------------------------------------------
|
|
128
|
+
# 체크섬 / 검증 함수
|
|
129
|
+
# ---------------------------------------------------------------------------
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def luhn_valid(s: str) -> bool:
|
|
133
|
+
"""Luhn 알고리즘 검증 (신용카드)."""
|
|
134
|
+
digits = [int(c) for c in s if c.isdigit()]
|
|
135
|
+
if not 13 <= len(digits) <= 19:
|
|
136
|
+
return False
|
|
137
|
+
checksum = 0
|
|
138
|
+
parity = len(digits) % 2
|
|
139
|
+
for i, d in enumerate(digits):
|
|
140
|
+
if i % 2 == parity:
|
|
141
|
+
d *= 2
|
|
142
|
+
if d > 9:
|
|
143
|
+
d -= 9
|
|
144
|
+
checksum += d
|
|
145
|
+
return checksum % 10 == 0
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _valid_date(yy: int, mm: int, dd: int) -> bool:
|
|
149
|
+
"""YYMMDD 기초 유효성 (월 1~12, 일 1~31). 윤년 등은 엄격 X."""
|
|
150
|
+
return 1 <= mm <= 12 and 1 <= dd <= 31
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _rrn_frn_checksum(digits: list[int]) -> bool:
|
|
154
|
+
"""주민/외국인등록 동일 체크섬 (가중치 [2,3,4,5,6,7,8,9,2,3,4,5] mod 11)."""
|
|
155
|
+
weights = [2, 3, 4, 5, 6, 7, 8, 9, 2, 3, 4, 5]
|
|
156
|
+
s_sum = sum(d * w for d, w in zip(digits[:12], weights))
|
|
157
|
+
check = (11 - s_sum % 11) % 10
|
|
158
|
+
return check == digits[12]
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def rrn_valid(s: str) -> bool:
|
|
162
|
+
"""주민등록번호: 13자리 + 날짜 유효 + 7번째 자리 0/1/2/3/4/9 + 체크섬."""
|
|
163
|
+
digits = [int(c) for c in s if c.isdigit()]
|
|
164
|
+
if len(digits) != 13:
|
|
165
|
+
return False
|
|
166
|
+
yy = digits[0] * 10 + digits[1]
|
|
167
|
+
mm = digits[2] * 10 + digits[3]
|
|
168
|
+
dd = digits[4] * 10 + digits[5]
|
|
169
|
+
if not _valid_date(yy, mm, dd):
|
|
170
|
+
return False
|
|
171
|
+
# 한국인: 1/2 (1900s), 3/4 (2000s), 9/0 (1800s)
|
|
172
|
+
if digits[6] not in (0, 1, 2, 3, 4, 9):
|
|
173
|
+
return False
|
|
174
|
+
return _rrn_frn_checksum(digits)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def frn_valid(s: str) -> bool:
|
|
178
|
+
"""외국인등록번호: 7번째 자리 5/6/7/8 + 체크섬 (RRN과 동일)."""
|
|
179
|
+
digits = [int(c) for c in s if c.isdigit()]
|
|
180
|
+
if len(digits) != 13:
|
|
181
|
+
return False
|
|
182
|
+
yy = digits[0] * 10 + digits[1]
|
|
183
|
+
mm = digits[2] * 10 + digits[3]
|
|
184
|
+
dd = digits[4] * 10 + digits[5]
|
|
185
|
+
if not _valid_date(yy, mm, dd):
|
|
186
|
+
return False
|
|
187
|
+
if digits[6] not in (5, 6, 7, 8):
|
|
188
|
+
return False
|
|
189
|
+
return _rrn_frn_checksum(digits)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def crn_valid(s: str) -> bool:
|
|
193
|
+
"""법인등록번호: 13자리 + RRN/FRN 모두 fail (둘 다 형식 동일)."""
|
|
194
|
+
digits = [int(c) for c in s if c.isdigit()]
|
|
195
|
+
if len(digits) != 13:
|
|
196
|
+
return False
|
|
197
|
+
# RRN/FRN로 분류 가능하면 CRN 아님
|
|
198
|
+
if rrn_valid(s) or frn_valid(s):
|
|
199
|
+
return False
|
|
200
|
+
# 폴백: 13자리 dash 형식이면 CRN으로 분류
|
|
201
|
+
return True
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def biz_no_valid(s: str) -> bool:
|
|
205
|
+
"""사업자등록번호 체크섬 (egovframe 알고리즘, 가중치 [1,3,7,1,3,7,1,3,5])."""
|
|
206
|
+
digits = [int(c) for c in s if c.isdigit()]
|
|
207
|
+
if len(digits) != 10:
|
|
208
|
+
return False
|
|
209
|
+
weights = [1, 3, 7, 1, 3, 7, 1, 3, 5]
|
|
210
|
+
s_sum = sum(d * w for d, w in zip(digits[:9], weights))
|
|
211
|
+
s_sum += (digits[8] * 5) // 10 # 9번째 × 5의 십의 자리
|
|
212
|
+
check = (10 - s_sum % 10) % 10
|
|
213
|
+
return check == digits[9]
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def account_valid(s: str) -> bool:
|
|
217
|
+
"""계좌번호 후보 검증 — 날짜(YYYY-MM-DD)로 보이면 계좌번호 아님(오탐 방지).
|
|
218
|
+
|
|
219
|
+
계좌 정규식은 dash 구분 숫자(2~6자리 그룹)를 잡는 last-resort 패턴이라
|
|
220
|
+
'2026-07-23' 같은 날짜를 계좌번호로 오탐한다. 앞 3그룹이 유효한 연/월/일이면
|
|
221
|
+
계좌번호에서 제외한다. 뒤에 시각(예: '2026-07-20 03')이 붙어 4그룹이 돼도
|
|
222
|
+
날짜로 시작하면 제외한다(타임스탬프 오탐 방지).
|
|
223
|
+
"""
|
|
224
|
+
parts = re.split(r"[\- ]", s.strip())
|
|
225
|
+
if len(parts) >= 3 and len(parts[0]) == 4 and parts[0].isdigit():
|
|
226
|
+
try:
|
|
227
|
+
y, mo, d = int(parts[0]), int(parts[1]), int(parts[2])
|
|
228
|
+
except ValueError:
|
|
229
|
+
return True
|
|
230
|
+
if 1900 <= y <= 2999 and 1 <= mo <= 12 and 1 <= d <= 31:
|
|
231
|
+
return False # 날짜/타임스탬프 → 계좌 아님
|
|
232
|
+
return True
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
# ---------------------------------------------------------------------------
|
|
236
|
+
# 우선순위 (specific → generic)
|
|
237
|
+
# ---------------------------------------------------------------------------
|
|
238
|
+
|
|
239
|
+
_PATTERNS: list[tuple[str, re.Pattern, Optional[Callable[[str], bool]]]] = [
|
|
240
|
+
("이메일", _EMAIL_RE, None),
|
|
241
|
+
("IPv6", _IPV6_RE, None),
|
|
242
|
+
("IPv4", _IPV4_RE, None),
|
|
243
|
+
("MAC", _MAC_RE, None),
|
|
244
|
+
("여권", _PASSPORT_RE, None),
|
|
245
|
+
("운전면허", _DL_12_RE, None), # 12자리 먼저 (longest match)
|
|
246
|
+
("운전면허", _DL_10_RE, None), # 10자리 폴백
|
|
247
|
+
# 13자리 dash 형식 → 신용카드(13~19 digit Luhn pass)와 우연히 겹칠 수 있어
|
|
248
|
+
# 주민/외국인/법인등록을 신용카드보다 먼저 적용 (체크섬 + 7번째 자리로 정확히 disambiguate)
|
|
249
|
+
("주민등록번호", _RRN_FRN_CRN_RE, rrn_valid),
|
|
250
|
+
("외국인등록번호", _RRN_FRN_CRN_RE, frn_valid),
|
|
251
|
+
("법인등록번호", _RRN_FRN_CRN_RE, crn_valid),
|
|
252
|
+
("신용카드", _CC_RE, luhn_valid),
|
|
253
|
+
("사업자등록번호", _BIZ_RE, biz_no_valid),
|
|
254
|
+
("전화번호", _PHONE_RE, None),
|
|
255
|
+
("건강보험", _HEALTH_RE, None),
|
|
256
|
+
("계좌번호", _ACCOUNT_RE, account_valid),
|
|
257
|
+
]
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
# ---------------------------------------------------------------------------
|
|
261
|
+
# 도구 결과 모드 — 문맥 검증 (이슈 #155)
|
|
262
|
+
# ---------------------------------------------------------------------------
|
|
263
|
+
#
|
|
264
|
+
# 도구 결과(파일 내용·명령 출력·쿼리 결과)에는 식별자 숫자가 흔하다. 체크섬이 없는
|
|
265
|
+
# 건강보험(11자리)은 그런 숫자를 전부 잡고, 19자리 snapshot_id 는 10% 확률로 Luhn 을
|
|
266
|
+
# 통과한다. 그래서 도구 결과에서만 두 유형에 문맥 조건을 더한다. 사용자 입력은
|
|
267
|
+
# 기본 모드 그대로다 — 사람이 직접 친 번호는 문맥 없이도 가린다.
|
|
268
|
+
|
|
269
|
+
# 문맥을 보는 범위 — 같은 줄의 앞뒤 이만큼
|
|
270
|
+
_CONTEXT_CHARS = 40
|
|
271
|
+
_CARD_CONTEXT_RE = re.compile(
|
|
272
|
+
# `pan_no`·`PAN_ID` 도 잡도록 `_` 를 낱말 경계로 본다 (`\b` 는 `_` 를 낱말로 본다)
|
|
273
|
+
r"카드|신용|결제|card|credit|(?<![a-z])pan(?![a-z])|(?<![a-z])cc_?(?:no|num)",
|
|
274
|
+
re.I,
|
|
275
|
+
)
|
|
276
|
+
_HEALTH_CONTEXT_RE = re.compile(r"건강보험|건보|보험증|증번호|nhis|health|insur", re.I)
|
|
277
|
+
# 이 낱말로 끝나는 키 바로 뒤의 숫자는 식별자다 (`snapshot_id": `, `seq=`)
|
|
278
|
+
_ID_KEY_WORDS = frozenset(
|
|
279
|
+
{
|
|
280
|
+
"id",
|
|
281
|
+
"ids",
|
|
282
|
+
"snapshot",
|
|
283
|
+
"seq",
|
|
284
|
+
"sequence",
|
|
285
|
+
"offset",
|
|
286
|
+
"timestamp",
|
|
287
|
+
"ts",
|
|
288
|
+
"epoch",
|
|
289
|
+
"version",
|
|
290
|
+
"hash",
|
|
291
|
+
"nonce",
|
|
292
|
+
"trace",
|
|
293
|
+
"span",
|
|
294
|
+
"commit",
|
|
295
|
+
}
|
|
296
|
+
)
|
|
297
|
+
_KEY_BEFORE_VALUE_RE = re.compile(r"([A-Za-z][A-Za-z0-9_]*)[\"']?\s*[:=]\s*[\"']?$")
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _context(text: str, start: int, end: int) -> tuple[str, str]:
|
|
301
|
+
"""매치 앞뒤로 같은 줄 안의 `_CONTEXT_CHARS` 글자."""
|
|
302
|
+
line_start = text.rfind("\n", 0, start) + 1
|
|
303
|
+
line_end = text.find("\n", end)
|
|
304
|
+
if line_end < 0:
|
|
305
|
+
line_end = len(text)
|
|
306
|
+
before = text[max(line_start, start - _CONTEXT_CHARS) : start]
|
|
307
|
+
after = text[end : min(line_end, end + _CONTEXT_CHARS)]
|
|
308
|
+
return before, after
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _after_id_key(before: str) -> bool:
|
|
312
|
+
m = _KEY_BEFORE_VALUE_RE.search(before)
|
|
313
|
+
if not m:
|
|
314
|
+
return False
|
|
315
|
+
key = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", m.group(1)).lower()
|
|
316
|
+
return any(part in _ID_KEY_WORDS for part in key.split("_"))
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _tool_output_skip(label: str, text: str, start: int, end: int) -> bool:
|
|
320
|
+
"""도구 결과에서 이 매치를 PII 로 보지 않을 것인가."""
|
|
321
|
+
if label not in ("신용카드", "건강보험"):
|
|
322
|
+
return False
|
|
323
|
+
before, after = _context(text, start, end)
|
|
324
|
+
if label == "건강보험":
|
|
325
|
+
# 체크섬이 없다 — 건강보험 문맥이 있을 때만 가린다
|
|
326
|
+
return not (
|
|
327
|
+
_HEALTH_CONTEXT_RE.search(before) or _HEALTH_CONTEXT_RE.search(after)
|
|
328
|
+
)
|
|
329
|
+
if _CARD_CONTEXT_RE.search(before) or _CARD_CONTEXT_RE.search(after):
|
|
330
|
+
return False
|
|
331
|
+
digits = text[start:end]
|
|
332
|
+
if not digits.isdigit():
|
|
333
|
+
return False # 구분자(공백·대시)로 끊어 쓴 번호는 카드 표기다
|
|
334
|
+
# 구분자 없는 17~19자리는 카드보다 식별자(int64 등)가 흔하다
|
|
335
|
+
return len(digits) >= 17 or _after_id_key(before)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
# ---------------------------------------------------------------------------
|
|
339
|
+
# Main API
|
|
340
|
+
# ---------------------------------------------------------------------------
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def mask_pii(text: str, tool_output: bool = False) -> MaskResult:
|
|
344
|
+
"""텍스트에서 PII를 검출하여 동일 길이 '*'로 마스킹한 결과 반환.
|
|
345
|
+
|
|
346
|
+
우선순위 순서대로 정규식 + 체크섬 적용. 한 패턴이 매칭하면 그 substring을
|
|
347
|
+
즉시 '*'로 치환 → 다음 패턴은 그 자리를 재매칭 못 함.
|
|
348
|
+
|
|
349
|
+
Args:
|
|
350
|
+
text: 입력 텍스트
|
|
351
|
+
tool_output: 도구 결과(파일·명령 출력)면 True. 카드·건강보험에 문맥 조건을
|
|
352
|
+
더해 식별자 숫자 오탐을 줄인다 (`_tool_output_skip`, 이슈 #155)
|
|
353
|
+
|
|
354
|
+
Returns:
|
|
355
|
+
MaskResult(masked_text, detected_types, matches)
|
|
356
|
+
"""
|
|
357
|
+
if not text or not text.strip():
|
|
358
|
+
return MaskResult(masked_text=text)
|
|
359
|
+
|
|
360
|
+
buf = list(text)
|
|
361
|
+
matches: list[PIIMatch] = []
|
|
362
|
+
seen_labels: list[str] = []
|
|
363
|
+
|
|
364
|
+
for label, regex, validator in _PATTERNS:
|
|
365
|
+
current = "".join(buf)
|
|
366
|
+
for m in regex.finditer(current):
|
|
367
|
+
substr = m.group(0)
|
|
368
|
+
if validator is not None and not validator(substr):
|
|
369
|
+
continue
|
|
370
|
+
start, end = m.start(), m.end()
|
|
371
|
+
if tool_output and _tool_output_skip(label, current, start, end):
|
|
372
|
+
continue
|
|
373
|
+
# 이미 마스킹된 영역은 skip (방어)
|
|
374
|
+
if all(buf[i] == "*" for i in range(start, end)):
|
|
375
|
+
continue
|
|
376
|
+
for i in range(start, end):
|
|
377
|
+
buf[i] = "*"
|
|
378
|
+
matches.append(
|
|
379
|
+
PIIMatch(
|
|
380
|
+
type_label=label,
|
|
381
|
+
start=start,
|
|
382
|
+
end=end,
|
|
383
|
+
original=substr,
|
|
384
|
+
)
|
|
385
|
+
)
|
|
386
|
+
if label not in seen_labels:
|
|
387
|
+
seen_labels.append(label)
|
|
388
|
+
|
|
389
|
+
return MaskResult(
|
|
390
|
+
masked_text="".join(buf),
|
|
391
|
+
detected_types=seen_labels,
|
|
392
|
+
matches=matches,
|
|
393
|
+
)
|