whatap-python 2.1.0__py3-none-any.whl

whatap/llm/providers/openai/responses/responses_extractor.py

@@ -0,0 +1,126 @@
+"""OpenAI Responses API 응답에서 토큰 사용량 및 출력 텍스트를 추출하는 모듈."""
+import json
+
+from whatap import logging
+from whatap.llm.features import LlmFeature
+from whatap.llm.providers.stream_accumulator import StreamAccumulator
+
+
+def extract_usage(usage):
+    """usage 객체에서 토큰 사용량 정보를 딕셔너리로 추출한다."""
+    if not usage:
+        return {}
+    inp = getattr(usage, "input_tokens", 0) or 0
+    out = getattr(usage, "output_tokens", 0) or 0
+    data = {"input_tokens": inp, "output_tokens": out, "total_tokens_count": inp + out}
+    inp_d = getattr(usage, 'input_tokens_details', None)
+    if inp_d:
+        data["cached_tokens"] = getattr(inp_d, 'cached_tokens', 0) or 0
+    out_d = getattr(usage, 'output_tokens_details', None)
+    if out_d:
+        data["reasoning_tokens"] = getattr(out_d, 'reasoning_tokens', 0) or 0
+    return data
+
+
+def _extract_output_text(response):
+    """응답 객체에서 출력 텍스트를 추출한다."""
+    try:
+        if hasattr(response, 'output_text'):
+            return response.output_text
+        parts = []
+        for item in response.output:
+            if hasattr(item, 'type') and item.type == 'message':
+                for block in item.content:
+                    if hasattr(block, 'type') and block.type == 'output_text':
+                        parts.append(block.text)
+        return "".join(parts)
+    except Exception as e:
+        logging.warning('[LLM] _extract_responses_output_text failed: %s' % e, extra={'id': 'LLM015'})
+        return ""
+
+
+def _extract_tool_calls(response):
+    """응답에서 function_call 타입의 도구 호출 목록을 추출한다."""
+    return [
+        {
+            "id": getattr(item, 'call_id', '') or getattr(item, 'id', ''),
+            "function": getattr(item, 'name', ''),
+            "arguments": getattr(item, 'arguments', ''),
+        }
+        for item in getattr(response, 'output', [])
+        if getattr(item, 'type', '') == 'function_call'
+    ]
+
+
+def finalize(response, pack, features):
+    """비스트리밍 Responses 응답에서 출력 텍스트, 토큰, 도구 호출 정보를 팩에 기록한다."""
+    for item in getattr(response, 'output', []):
+        t = getattr(item, 'type', '')
+        if t == 'function_call' and LlmFeature.TOOL_USE not in features:
+            features.append(LlmFeature.TOOL_USE)
+        elif 'web_search' in t and LlmFeature.WEBSEARCH not in features:
+            features.append(LlmFeature.WEBSEARCH)
+    pack.features = ",".join(features)
+
+    tool_calls = _extract_tool_calls(response)
+    if tool_calls:
+        pack.tool_calls_text = json.dumps(tool_calls, ensure_ascii=False)
+
+    pack.set_tokens(extract_usage(getattr(response, "usage", None)))
+    pack.completion_text = _extract_output_text(response)
+    pack.finish_reason = getattr(response, 'status', None)
+
+
+class ResponsesStream(StreamAccumulator):
+    """OpenAI Responses 스트리밍 이벤트를 누적하는 어큐뮬레이터."""
+
+    def __init__(self, pack, active_key):
+        super().__init__(pack, active_key)
+        self.text = ""
+        self.usage = {}
+        self.output_features = set()
+        self.tool_calls = {}
+
+    def on_chunk(self, event):
+        t = getattr(event, 'type', '')
+        if t == 'response.output_text.delta':
+            delta = getattr(event, 'delta', '') or ''
+            if delta:
+                self.on_first_token()
+                self.text += delta
+        elif t == 'response.output_item.added':
+            item = getattr(event, 'item', None)
+            it = getattr(item, 'type', '') if item else ''
+            if it == 'function_call':
+                self.output_features.add(LlmFeature.TOOL_USE)
+                call_id = getattr(item, 'call_id', '') or getattr(item, 'id', '')
+                if call_id and call_id not in self.tool_calls:
+                    self.tool_calls[call_id] = {"id": call_id, "function": getattr(item, 'name', ''), "arguments": ""}
+            elif 'web_search' in it:
+                self.output_features.add(LlmFeature.WEBSEARCH)
+        elif t == 'response.function_call_arguments.delta':
+            call_id = getattr(event, 'call_id', '')
+            if call_id in self.tool_calls:
+                self.tool_calls[call_id]["arguments"] += getattr(event, 'delta', '') or ''
+        elif t == 'response.function_call_arguments.done':
+            call_id = getattr(event, 'call_id', '')
+            if call_id in self.tool_calls:
+                self.tool_calls[call_id]["arguments"] = getattr(event, 'arguments', '') or self.tool_calls[call_id]["arguments"]
+        elif t == 'response.completed':
+            resp = getattr(event, 'response', None)
+            if resp:
+                self.usage = extract_usage(getattr(resp, 'usage', None))
+                self.pack.finish_reason = getattr(resp, 'status', None)
+
+    def _apply(self):
+        pack = self.pack
+        if self.output_features:
+            parts = [f for f in pack.features.split(",") if f]
+            for f in self.output_features:
+                if f not in parts:
+                    parts.append(f)
+            pack.features = ",".join(parts)
+        if self.tool_calls:
+            pack.tool_calls_text = json.dumps(list(self.tool_calls.values()), ensure_ascii=False)
+        pack.set_tokens(self.usage)
+        pack.completion_text = self.text

whatap/llm/providers/stream_accumulator.py

@@ -0,0 +1,73 @@
+"""스트리밍 응답 누적 처리.
+
+각 프로바이더의 Stream 클래스가 StreamAccumulator를 상속하여
+on_chunk()로 청크 데이터를 누적하고, _apply()로 pack에 반영한다.
+"""
+import time
+
+from whatap.llm.providers.interceptor import _dispatch, _ensure_end
+
+
+class StreamAccumulator(object):
+    """스트리밍 응답 누적 기반 클래스.
+
+    서브클래스가 on_chunk()와 _apply()를 구현한다.
+    finalize()에서 latency/ttft 계산 → _apply() → 로그싱크 전송.
+    """
+
+    def __init__(self, pack, active_key=None):
+        self.pack = pack
+        self.active_key = active_key
+        self.first_token_time = None
+
+    def __del__(self):
+        """GC 안전망: finalize 없이 제너레이터가 소멸될 때 active 카운터 정리."""
+        if self.active_key:
+            _ensure_end(self.pack, self.active_key)
+
+    def on_chunk(self, chunk):
+        """스트림 청크 수신 시 호출. 서브클래스에서 구현."""
+        raise NotImplementedError
+
+    def on_first_token(self):
+        """첫 번째 토큰 수신 시각 기록 (TTFT 계산용)."""
+        if self.first_token_time is None:
+            self.first_token_time = time.monotonic()
+
+    def finalize(self):
+        """스트림 종료 시: latency/ttft 계산 → _apply() → 전송 → active 카운터 감소."""
+        try:
+            end_time = time.monotonic()
+            self.pack.latency = round((end_time - self.pack._start_time) * 1000)
+            if self.first_token_time is not None:
+                self.pack.ttft = round((self.first_token_time - self.pack._start_time) * 1000)
+            self._apply()
+            self.pack.success = True
+            _dispatch(self.pack)
+        finally:
+            if self.active_key:
+                _ensure_end(self.pack, self.active_key)
+
+    def _apply(self):
+        """누적된 데이터를 pack에 반영. 서브클래스에서 구현."""
+        pass
+
+
+def sync_stream(response, acc):
+    """동기 스트림 래퍼. 원본 응답을 그대로 yield하면서 청크를 누적한다."""
+    try:
+        for chunk in response:
+            acc.on_chunk(chunk)
+            yield chunk
+    finally:
+        acc.finalize()
+
+
+async def async_stream(response, acc):
+    """비동기 스트림 래퍼. 원본 응답을 그대로 yield하면서 청크를 누적한다."""
+    try:
+        async for chunk in response:
+            acc.on_chunk(chunk)
+            yield chunk
+    finally:
+        acc.finalize()

whatap/llm/stats/__init__.py

@@ -0,0 +1,35 @@
+from whatap.llm.stats.token_usage_stat import TokenUsageStat
+from whatap.llm.stats.active_stat import ActiveStat
+from whatap.llm.stats.perf_stat import PerfStat
+from whatap.llm.stats.error_stat import ErrorStat
+from whatap.llm.stats.feature_stat import FeatureStat
+from whatap.llm.stats.finish_stat import FinishStat
+from whatap.llm.stats.api_status_stat import ApiStatusStat
+from whatap.llm.stats.eval_stat import EvalStat
+from whatap.llm.stats.hallucination_eval_stat import HallucinationEvalStat
+from whatap.llm.stats.answer_relevance_eval_stat import AnswerRelevanceEvalStat
+from whatap.llm.stats.toxicity_eval_stat import ToxicityEvalStat
+from whatap.llm.stats.combined_judge_eval_stat import CombinedJudgeEvalStat
+from whatap.llm.stats.prompt_injection_eval_stat import PromptInjectionEvalStat
+from whatap.llm.stats.factuality_eval_stat import FactualityEvalStat
+from whatap.llm.stats.pii_leak_eval_stat import PIILeakEvalStat
+from whatap.llm.stats.url_scan_eval_stat import URLScanEvalStat
+
+LLM_STAT_CLASSES = [
+    TokenUsageStat,
+    ActiveStat,
+    PerfStat,
+    ErrorStat,
+    FeatureStat,
+    FinishStat,
+    ApiStatusStat,
+    EvalStat,
+    HallucinationEvalStat,
+    AnswerRelevanceEvalStat,
+    ToxicityEvalStat,
+    CombinedJudgeEvalStat,
+    PromptInjectionEvalStat,
+    FactualityEvalStat,
+    PIILeakEvalStat,
+    URLScanEvalStat,
+]

whatap/llm/stats/active_stat.py

@@ -0,0 +1,86 @@
+from collections import defaultdict
+
+from whatap.llm.stats.base_stat import BaseStat
+
+
+class ActiveStat(BaseStat):
+    _category = "llm_active_stat"
+    _conf_enabled_key = 'llm_enabled'
+
+    def __init__(self):
+        super().__init__()
+        self._model_hosts = {}
+        self._model_urls = {}
+
+    def _empty_stats(self):
+        return defaultdict(int)
+
+    def interval(self):
+        return 5
+
+    def on_start(self, model, operation_type, prompt_version='v1'):
+        key = (model or 'unknown', operation_type or 'default', prompt_version or 'v1')
+        with self._lock:
+            self._stats[key] += 1
+
+    def set_host(self, model, host, url=''):
+        with self._lock:
+            if host:
+                self._model_hosts[model or 'unknown'] = host
+            if url:
+                self._model_urls[model or 'unknown'] = url
+
+    def on_end(self, model, operation_type, prompt_version='v1', host='', url=''):
+        key = (model or 'unknown', operation_type or 'default', prompt_version or 'v1')
+        with self._lock:
+            self._stats[key] -= 1
+            if self._stats[key] <= 0:
+                del self._stats[key]
+            if host:
+                self._model_hosts[model or 'unknown'] = host
+            if url:
+                self._model_urls[model or 'unknown'] = url
+
+    def process(self):
+        from whatap.conf.configure import Configure as conf
+        if not getattr(conf, self._conf_enabled_key, False):
+            return
+
+        with self._lock:
+            snapshot = dict(self._stats)
+            hosts = dict(self._model_hosts)
+            urls = dict(self._model_urls)
+
+        if not snapshot:
+            return
+
+        for key, count in snapshot.items():
+            try:
+                # 3-tuple: (model, operation_type=prompt_name, prompt_version)
+                # backward-compat: 2-tuple key 도 처리
+                if len(key) == 3:
+                    model, op_type, prompt_version = key
+                else:
+                    model, op_type = key
+                    prompt_version = 'v1'
+
+                from whatap.pack.tagCountPack import TagCountPack
+                from whatap import DateUtil
+                from whatap.llm.stats.base_stat import currentpid
+
+                p = TagCountPack()
+                p.time = DateUtil.now() // 5000 * 5000
+                p.Category = self._category
+                p.tags.putAuto("pid", currentpid)
+                p.tags.putAuto("model", model)
+                p.tags.putAuto("provider", hosts.get(model, ''))
+                p.tags.putAuto("operation_type", op_type)
+                p.tags.putAuto("url", urls.get(model, ''))
+                p.tags.putAuto("prompt_version", prompt_version)
+
+                p.fields.putAuto("count", count)
+
+                self.send_pack(p)
+            except Exception:
+                import traceback
+                traceback.print_exc()

whatap/llm/stats/answer_relevance_eval_stat.py

@@ -0,0 +1,10 @@
+"""llm_eval_answer_relevance 카테고리 — Answer Relevance 점수 히스토그램.
+
+11-bucket 분포 (value0~value10). 자세한 동작은 ``eval_stat.ScoreHistogramStat``
+docstring 참고.
+"""
+from whatap.llm.stats.eval_stat import ScoreHistogramStat
+
+
+class AnswerRelevanceEvalStat(ScoreHistogramStat):
+    _category = 'llm_eval_answer_relevance'

whatap/llm/stats/api_status_stat.py

@@ -0,0 +1,35 @@
+from collections import defaultdict
+
+from whatap.llm.stats.base_stat import BaseStat
+
+
+class ApiStatusStat(BaseStat):
+    _category = "llm_api_status"
+
+    def _empty_stats(self):
+        return {
+            '4xx': defaultdict(int),
+            '5xx': defaultdict(int),
+        }
+
+    def _get_keys(self, stats):
+        return set(stats['4xx'].keys()) | set(stats['5xx'].keys())
+
+    def update_stats(self, model_name, provider, operation_type, status_code, url='',
+                     prompt_version='v1'):
+        key = (model_name or 'unknown', provider or '', operation_type or 'unknown',
+               url or '', prompt_version or 'v1')
+        with self._lock:
+            if 400 <= status_code < 500:
+                self._stats['4xx'][key] += 1
+            elif 500 <= status_code < 600:
+                self._stats['5xx'][key] += 1
+
+    def update_from_pack(self, pack):
+        # 외부 호출자가 update_stats 를 직접 부르므로 update_from_pack 은 그쪽으로 prompt_version 전파
+        # 가 안 되는 케이스 대비 — 현재는 별도 쓰임 없음
+        pass
+
+    def _build_fields(self, pack, stats, key):
+        pack.fields.putAuto("4xx_total_count", stats['4xx'][key])
+        pack.fields.putAuto("5xx_total_count", stats['5xx'][key])

whatap/llm/stats/base_stat.py

@@ -0,0 +1,107 @@
+import os
+import threading
+from collections import defaultdict
+
+from whatap.counter.tasks.base_task import BaseTask
+from whatap.pack.tagCountPack import TagCountPack
+from whatap import DateUtil
+
+currentpid = os.getpid()
+
+
+class BaseStat(BaseTask):
+    _llm_task = True
+    _category = None
+    _conf_enabled_key = 'llm_enabled'
+
+    # 리스트형 메트릭 (!rectype=2 + newList 사용) 은 True 로 override.
+    # True 면 5초 윈도우당 단일 TagCountPack 만 만들고, 키 차원 (model/provider/
+    # operation_type/url/prompt_version) 을 태그가 아닌 row 단위 평행 배열 필드로
+    # 평탄화한다. 서브클래스가 ``_build_fields_listed(pack, stats, keys)`` 구현.
+    _is_listed = False
+
+    def __init__(self):
+        self._lock = threading.Lock()
+        self._stats = self._empty_stats()
+
+    def _empty_stats(self):
+        raise NotImplementedError
+
+    def interval(self):
+        from whatap.conf.configure import Configure as conf
+        return 5
+
+    def process(self):
+        from whatap.conf.configure import Configure as conf
+        if not getattr(conf, self._conf_enabled_key, False):
+            return
+
+        with self._lock:
+            stats = self._stats
+            self._stats = self._empty_stats()
+
+        keys = list(self._get_keys(stats))
+        if not keys:
+            return
+
+        # 리스트형: 단일 pack + 모든 키 차원을 row 단위 평행 배열로 평탄화.
+        if self._is_listed:
+            try:
+                p = TagCountPack()
+                p.time = DateUtil.now() // 5000 * 5000
+                p.Category = self._category
+                p.tags.putAuto("!rectype", 2)
+                self._build_fields_listed(p, stats, keys, currentpid)
+                self.send_pack(p)
+            except Exception:
+                import traceback
+                traceback.print_exc()
+            return
+
+        # 일반(스칼라) 메트릭: 키별 pack + 키 차원은 태그.
+        for key in keys:
+            # 5-tuple: (model, provider, op_type=prompt_name, url, prompt_version)
+            try:
+                model, provider, op_type, url, prompt_version = key
+            except ValueError:
+                # backward-compat: 4-tuple 키도 처리
+                model, provider, op_type, url = key
+                prompt_version = 'v1'
+            try:
+                p = TagCountPack()
+                p.time = DateUtil.now() // 5000 * 5000
+                p.Category = self._category
+                p.tags.putAuto("pid", currentpid)
+                p.tags.putAuto("model", model)
+                p.tags.putAuto("provider", provider)
+                p.tags.putAuto("operation_type", op_type)
+                p.tags.putAuto("url", url)
+                p.tags.putAuto("prompt_version", prompt_version)
+
+                self._build_fields(p, stats, key)
+                self.send_pack(p)
+
+            except Exception:
+                import traceback
+                traceback.print_exc()
+
+    def _get_keys(self, stats):
+        raise NotImplementedError
+
+    def _build_fields(self, pack, stats, key):
+        """일반(스칼라) 메트릭의 필드 빌드. ``_is_listed=False`` 인 경우만 호출."""
+        raise NotImplementedError
+
+    def _build_fields_listed(self, pack, stats, keys, pid):
+        """리스트형 메트릭의 필드 빌드. ``_is_listed=True`` 인 경우만 호출.
+
+        ``pack`` 한 개에 ``keys`` 전체의 데이터를 평행 배열 column 으로 채워야
+        한다. 키 차원 6종 (pid/model/provider/operation_type/url/prompt_version)
+        은 모두 row 단위 평행 배열로 펼치고, key-level scalar metric (call_count
+        등) 은 같은 key 의 첫 row 에만 값을 적고 나머지는 0 으로 채워 합산
+        정확성을 보장한다.
+        """
+        raise NotImplementedError
+
+    def update_from_pack(self, pack):
+        pass

whatap/llm/stats/combined_judge_eval_stat.py

@@ -0,0 +1,11 @@
+"""llm_eval_combined_judge 카테고리 — Combined Judge 종합 위험도 히스토그램.
+
+CombinedJudgeEvaluator 가 hallucination/answer_relevance/toxicity 의 worst-aspect
+risk 를 primary 로 송출. 11-bucket 분포 (value0~value10). 자세한 동작은
+``eval_stat.ScoreHistogramStat`` docstring 참고.
+"""
+from whatap.llm.stats.eval_stat import ScoreHistogramStat
+
+
+class CombinedJudgeEvalStat(ScoreHistogramStat):
+    _category = 'llm_eval_combined_judge'

whatap/llm/stats/error_stat.py

@@ -0,0 +1,59 @@
+from collections import defaultdict
+
+from whatap.llm.stats.base_stat import BaseStat
+
+
+class ErrorStat(BaseStat):
+    _category = "llm_error_stat"
+
+    def _empty_stats(self):
+        return {
+            'api_error_count': defaultdict(int),
+            'program_error_count': defaultdict(int),
+            'last_api_error_count': defaultdict(int),
+        }
+
+    def _get_keys(self, stats):
+        return (set(stats['api_error_count'].keys())
+                | set(stats['program_error_count'].keys())
+                | set(stats['last_api_error_count'].keys()))
+
+    def update_stats(self, model_name, provider, operation_type, url='',
+                     error_type='', prompt_version='v1'):
+        key = (model_name or 'unknown', provider or '', operation_type or 'unknown',
+               url or '', prompt_version or 'v1')
+        with self._lock:
+            if error_type == 'api_error':
+                self._stats['api_error_count'][key] += 1
+            elif error_type == 'program_error':
+                self._stats['program_error_count'][key] += 1
+
+    def update_last_error(self, model_name, provider, operation_type, url='',
+                          error_type='', prompt_version='v1'):
+        if error_type != 'api_error':
+            return
+        key = (model_name or 'unknown', provider or '', operation_type or 'unknown',
+               url or '', prompt_version or 'v1')
+        with self._lock:
+            self._stats['last_api_error_count'][key] += 1
+
+    def update_from_pack(self, pack):
+        if pack.success:
+            return
+        self.update_stats(
+            pack.model or 'unknown',
+            pack.provider or '',
+            pack.operation_type or 'unknown',
+            url=pack.url or '',
+            error_type=pack.error_type or '',
+            prompt_version=getattr(pack, 'prompt_version', 'v1') or 'v1',
+        )
+
+    def _build_fields(self, pack, stats, key):
+        api_err = stats['api_error_count'][key]
+        prog_err = stats['program_error_count'][key]
+
+        pack.fields.putAuto("error_count", api_err + prog_err)
+        pack.fields.putAuto("api_error_count", api_err)
+        pack.fields.putAuto("program_error_count", prog_err)
+        pack.fields.putAuto("last_api_error_count", stats['last_api_error_count'][key])