arthexis 0.1.26__py3-none-any.whl → 0.1.28__py3-none-any.whl

ocpp/store.py

@@ -3,11 +3,18 @@
 from __future__ import annotations
 
 import asyncio
+import concurrent.futures
+from collections import deque
+from dataclasses import dataclass
 from datetime import datetime, timezone
 import json
+import os
 from pathlib import Path
 import re
 import threading
+import heapq
+import itertools
+from typing import Iterable, Iterator
 
 from core.log_paths import select_log_dir
 
@@ -19,7 +26,10 @@ MAX_CONNECTIONS_PER_IP = 2
 
 connections: dict[str, object] = {}
 transactions: dict[str, object] = {}
-logs: dict[str, dict[str, list[str]]] = {"charger": {}, "simulator": {}}
+# Maximum number of recent log entries to keep in memory per identity.
+MAX_IN_MEMORY_LOG_ENTRIES = 1000
+
+logs: dict[str, dict[str, deque[str]]] = {"charger": {}, "simulator": {}}
 # store per charger session logs before they are flushed to disk
 history: dict[str, dict[str, object]] = {}
 simulators = {}
@@ -28,6 +38,7 @@ pending_calls: dict[str, dict[str, object]] = {}
 _pending_call_events: dict[str, threading.Event] = {}
 _pending_call_results: dict[str, dict[str, object]] = {}
 _pending_call_lock = threading.Lock()
+_pending_call_handles: dict[str, asyncio.TimerHandle] = {}
 triggered_followups: dict[str, list[dict[str, object]]] = {}
 
 # mapping of charger id / cp_path to friendly names used for log files
@@ -42,6 +53,20 @@ LOCK_DIR.mkdir(exist_ok=True)
 SESSION_LOCK = LOCK_DIR / "charging.lck"
 _lock_task: asyncio.Task | None = None
 
+_scheduler_loop: asyncio.AbstractEventLoop | None = None
+_scheduler_thread: threading.Thread | None = None
+_scheduler_lock = threading.Lock()
+
+SESSION_LOG_BUFFER_LIMIT = 16
+
+
+@dataclass(frozen=True)
+class LogEntry:
+    """Structured log entry returned by :func:`iter_log_entries`."""
+
+    timestamp: datetime
+    text: str
+
 
 def connector_slug(value: int | str | None) -> str:
     """Return the canonical slug for a connector value."""
@@ -202,13 +227,20 @@ def register_pending_call(message_id: str, metadata: dict[str, object]) -> None:
         event = threading.Event()
         _pending_call_events[message_id] = event
         _pending_call_results.pop(message_id, None)
+        handle = _pending_call_handles.pop(message_id, None)
+    if handle:
+        _cancel_timer_handle(handle)
 
 
 def pop_pending_call(message_id: str) -> dict[str, object] | None:
     """Return and remove metadata for a previously registered call."""
 
     with _pending_call_lock:
-        return pending_calls.pop(message_id, None)
+        metadata = pending_calls.pop(message_id, None)
+        handle = _pending_call_handles.pop(message_id, None)
+    if handle:
+        _cancel_timer_handle(handle)
+    return metadata
 
 
 def record_pending_call_result(
@@ -234,6 +266,9 @@ def record_pending_call_result(
     with _pending_call_lock:
         _pending_call_results[message_id] = result
         event = _pending_call_events.pop(message_id, None)
+        handle = _pending_call_handles.pop(message_id, None)
+    if handle:
+        _cancel_timer_handle(handle)
     if event:
         event.set()
 
@@ -269,29 +304,50 @@ def schedule_call_timeout(
 ) -> None:
     """Schedule a timeout notice if a pending call is not answered."""
 
+    loop = _ensure_scheduler_loop()
+
     def _notify() -> None:
+        target_log: str | None = None
+        entry_label: str | None = None
         with _pending_call_lock:
+            _pending_call_handles.pop(message_id, None)
             metadata = pending_calls.get(message_id)
-        if not metadata:
-            return
-        if action and metadata.get("action") != action:
-            return
-        if metadata.get("timeout_notice_sent"):
-            return
-        target_log = log_key or metadata.get("log_key")
-        if not target_log:
+            if not metadata:
+                return
+            if action and metadata.get("action") != action:
+                return
+            if metadata.get("timeout_notice_sent"):
+                return
+            target_log = log_key or metadata.get("log_key")
+            if not target_log:
+                metadata["timeout_notice_sent"] = True
+                return
+            entry_label = message
+            if not entry_label:
+                action_label = action or str(metadata.get("action") or "Call")
+                entry_label = f"{action_label} request timed out"
             metadata["timeout_notice_sent"] = True
+        if target_log and entry_label:
+            add_log(target_log, entry_label, log_type=log_type)
+
+    future: concurrent.futures.Future[asyncio.TimerHandle] = concurrent.futures.Future()
+
+    def _schedule_timer() -> None:
+        try:
+            handle = loop.call_later(timeout, _notify)
+        except Exception as exc:  # pragma: no cover - defensive
+            future.set_exception(exc)
             return
-        label = message
-        if not label:
-            action_label = action or str(metadata.get("action") or "Call")
-            label = f"{action_label} request timed out"
-        add_log(target_log, label, log_type=log_type)
-        metadata["timeout_notice_sent"] = True
+        future.set_result(handle)
 
-    timer = threading.Timer(timeout, _notify)
-    timer.daemon = True
-    timer.start()
+    loop.call_soon_threadsafe(_schedule_timer)
+    handle = future.result()
+
+    with _pending_call_lock:
+        previous = _pending_call_handles.pop(message_id, None)
+        _pending_call_handles[message_id] = handle
+    if previous:
+        _cancel_timer_handle(previous)
 
 
 def register_triggered_followup(
@@ -342,6 +398,7 @@ def consume_triggered_followup(
 def clear_pending_calls(serial: str) -> None:
     """Remove any pending calls associated with the provided charger id."""
 
+    to_cancel: list[asyncio.TimerHandle] = []
     with _pending_call_lock:
         to_remove = [
             key
@@ -352,7 +409,52 @@ def clear_pending_calls(serial: str) -> None:
             pending_calls.pop(key, None)
             _pending_call_events.pop(key, None)
             _pending_call_results.pop(key, None)
-    triggered_followups.pop(serial, None)
+            handle = _pending_call_handles.pop(key, None)
+            if handle:
+                to_cancel.append(handle)
+    for handle in to_cancel:
+        _cancel_timer_handle(handle)
+
+
+def _run_scheduler_loop(
+    loop: asyncio.AbstractEventLoop, ready: threading.Event
+) -> None:
+    asyncio.set_event_loop(loop)
+    ready.set()
+    loop.run_forever()
+
+
+def _ensure_scheduler_loop() -> asyncio.AbstractEventLoop:
+    global _scheduler_loop, _scheduler_thread
+
+    loop = _scheduler_loop
+    if loop and loop.is_running():
+        return loop
+    with _scheduler_lock:
+        loop = _scheduler_loop
+        if loop and loop.is_running():
+            return loop
+        loop = asyncio.new_event_loop()
+        ready = threading.Event()
+        thread = threading.Thread(
+            target=_run_scheduler_loop,
+            args=(loop, ready),
+            name="ocpp-store-scheduler",
+            daemon=True,
+        )
+        thread.start()
+        ready.wait()
+        _scheduler_loop = loop
+        _scheduler_thread = thread
+        return loop
+
+
+def _cancel_timer_handle(handle: asyncio.TimerHandle) -> None:
+    loop = _scheduler_loop
+    if loop and loop.is_running():
+        loop.call_soon_threadsafe(handle.cancel)
+    else:  # pragma: no cover - loop stopped during shutdown
+        handle.cancel()
 
 
 def reassign_identity(old_key: str, new_key: str) -> str:
@@ -433,8 +535,21 @@ def add_log(cid: str, entry: str, log_type: str = "charger") -> None:
     store = logs[log_type]
     # Store log entries under the cid as provided but allow retrieval using
     # any casing by recording entries in a case-insensitive manner.
-    key = next((k for k in store.keys() if k.lower() == cid.lower()), cid)
-    store.setdefault(key, []).append(entry)
+    buffer = None
+    lower = cid.lower()
+    key = cid
+    for existing_key, entries in store.items():
+        if existing_key.lower() == lower:
+            key = existing_key
+            buffer = entries
+            break
+    if buffer is None:
+        buffer = deque(maxlen=MAX_IN_MEMORY_LOG_ENTRIES)
+        store[key] = buffer
+    elif buffer.maxlen != MAX_IN_MEMORY_LOG_ENTRIES:
+        buffer = deque(buffer, maxlen=MAX_IN_MEMORY_LOG_ENTRIES)
+        store[key] = buffer
+    buffer.append(entry)
     path = _file_path(key, log_type)
     with path.open("a", encoding="utf-8") as handle:
         handle.write(entry + "\n")
@@ -452,10 +567,29 @@ def _session_folder(cid: str) -> Path:
 def start_session_log(cid: str, tx_id: int) -> None:
     """Begin logging a session for the given charger and transaction id."""
 
+    existing = history.pop(cid, None)
+    if existing:
+        try:
+            _finalize_session(existing)
+        except Exception:
+            # If finalizing the previous session fails we still want to reset
+            # the session metadata so the new session can proceed.
+            pass
+
+    start = datetime.now(timezone.utc)
+    folder = _session_folder(cid)
+    date = start.strftime("%Y%m%d")
+    filename = f"{date}_{tx_id}.json"
+    path = folder / filename
+    handle = path.open("w", encoding="utf-8")
+    handle.write("[")
     history[cid] = {
         "transaction": tx_id,
-        "start": datetime.now(timezone.utc),
-        "messages": [],
+        "start": start,
+        "path": path,
+        "handle": handle,
+        "buffer": [],
+        "first": True,
     }
 
 
@@ -465,14 +599,19 @@ def add_session_message(cid: str, message: str) -> None:
     sess = history.get(cid)
     if not sess:
         return
-    sess["messages"].append(
+    buffer: list[str] = sess.setdefault("buffer", [])
+    payload = json.dumps(
         {
             "timestamp": datetime.now(timezone.utc)
             .isoformat()
             .replace("+00:00", "Z"),
             "message": message,
-        }
+        },
+        ensure_ascii=False,
     )
+    buffer.append(payload)
+    if len(buffer) >= SESSION_LOG_BUFFER_LIMIT:
+        _flush_session_buffer(sess)
 
 
 def end_session_log(cid: str) -> None:
@@ -481,13 +620,42 @@ def end_session_log(cid: str) -> None:
     sess = history.pop(cid, None)
     if not sess:
         return
-    folder = _session_folder(cid)
-    date = sess["start"].strftime("%Y%m%d")
-    tx_id = sess.get("transaction")
-    filename = f"{date}_{tx_id}.json"
-    path = folder / filename
-    with path.open("w", encoding="utf-8") as handle:
-        json.dump(sess["messages"], handle, ensure_ascii=False, indent=2)
+    _finalize_session(sess)
+
+
+def _flush_session_buffer(sess: dict[str, object]) -> None:
+    handle = sess.get("handle")
+    buffer = sess.get("buffer")
+    if not handle or not buffer:
+        return
+    first = bool(sess.get("first", True))
+    for raw in list(buffer):
+        if not first:
+            handle.write(",")
+        handle.write("\n  ")
+        handle.write(raw)
+        first = False
+    handle.flush()
+    buffer.clear()
+    sess["first"] = first
+
+
+def _finalize_session(sess: dict[str, object]) -> None:
+    handle = sess.get("handle")
+    try:
+        _flush_session_buffer(sess)
+        if handle:
+            if sess.get("first", True):
+                handle.write("]\n")
+            else:
+                handle.write("\n]\n")
+            handle.flush()
+    finally:
+        if handle:
+            try:
+                handle.close()
+            except Exception:
+                pass
 
 
 def _log_key_candidates(cid: str, log_type: str) -> list[str]:
@@ -546,10 +714,17 @@ def _resolve_log_identifier(cid: str, log_type: str) -> tuple[str, str | None]:
 def _log_file_for_identifier(cid: str, name: str | None, log_type: str) -> Path:
     path = _file_path(cid, log_type)
     if not path.exists():
-        target = f"{log_type}.{_safe_name(name or cid).lower()}"
-        for file in LOG_DIR.glob(f"{log_type}.*.log"):
-            if file.stem.lower() == target:
-                path = file
+        candidates = [_safe_name(name or cid).lower()]
+        cid_candidate = _safe_name(cid).lower()
+        if cid_candidate not in candidates:
+            candidates.append(cid_candidate)
+        for candidate in candidates:
+            target = f"{log_type}.{candidate}"
+            for file in LOG_DIR.glob(f"{log_type}.*.log"):
+                if file.stem.lower() == target:
+                    path = file
+                    break
+            if path.exists():
                 break
     return path
 
@@ -559,10 +734,194 @@ def _memory_logs_for_identifier(cid: str, log_type: str) -> list[str]:
     lower = cid.lower()
     for key, entries in store.items():
         if key.lower() == lower:
-            return entries
+            return list(entries)
     return []
 
 
+def _parse_log_timestamp(entry: str) -> datetime | None:
+    """Return the parsed timestamp for a log entry, if available."""
+
+    if len(entry) < 24:
+        return None
+    try:
+        timestamp = datetime.strptime(entry[:23], "%Y-%m-%d %H:%M:%S.%f")
+    except ValueError:
+        return None
+    return timestamp.replace(tzinfo=timezone.utc)
+
+
+def _iter_file_lines_reverse(path: Path, *, limit: int | None = None) -> Iterator[str]:
+    """Yield lines from ``path`` starting with the newest entries."""
+
+    if not path.exists():
+        return
+
+    chunk_size = 4096
+    remaining = limit
+    with path.open("rb") as handle:
+        handle.seek(0, os.SEEK_END)
+        position = handle.tell()
+        buffer = b""
+        while position > 0:
+            read_size = min(chunk_size, position)
+            position -= read_size
+            handle.seek(position)
+            chunk = handle.read(read_size)
+            buffer = chunk + buffer
+            lines = buffer.split(b"\n")
+            buffer = lines.pop(0)
+            for line in reversed(lines):
+                if not line:
+                    continue
+                try:
+                    text = line.decode("utf-8")
+                except UnicodeDecodeError:
+                    text = line.decode("utf-8", errors="ignore")
+                yield text
+                if remaining is not None:
+                    remaining -= 1
+                    if remaining <= 0:
+                        return
+        if buffer:
+            try:
+                text = buffer.decode("utf-8")
+            except UnicodeDecodeError:
+                text = buffer.decode("utf-8", errors="ignore")
+            if text:
+                yield text
+
+
+def _iter_log_entries_for_key(
+    cid: str,
+    name: str | None,
+    log_type: str,
+    *,
+    since: datetime | None = None,
+    limit: int | None = None,
+) -> Iterator[LogEntry]:
+    """Yield structured log entries for a specific identifier."""
+
+    yielded = 0
+    seen_for_key: set[str] = set()
+    memory_entries = _memory_logs_for_identifier(cid, log_type)
+    for entry in reversed(memory_entries):
+        if entry in seen_for_key:
+            continue
+        timestamp = _parse_log_timestamp(entry)
+        if timestamp is None:
+            continue
+        seen_for_key.add(entry)
+        yield LogEntry(timestamp=timestamp, text=entry)
+        yielded += 1
+        if since is not None and timestamp < since:
+            return
+        if limit is not None and yielded >= limit:
+            return
+
+    path = _log_file_for_identifier(cid, name, log_type)
+    file_limit = None
+    if limit is not None:
+        file_limit = max(limit - yielded, 0)
+        if file_limit == 0:
+            return
+    for entry in _iter_file_lines_reverse(path, limit=file_limit):
+        if entry in seen_for_key:
+            continue
+        timestamp = _parse_log_timestamp(entry)
+        if timestamp is None:
+            continue
+        seen_for_key.add(entry)
+        yield LogEntry(timestamp=timestamp, text=entry)
+        yielded += 1
+        if since is not None and timestamp < since:
+            return
+        if limit is not None and yielded >= limit:
+            return
+
+
+def iter_log_entries(
+    identifiers: str | Iterable[str],
+    log_type: str = "charger",
+    *,
+    since: datetime | None = None,
+    limit: int | None = None,
+) -> Iterator[LogEntry]:
+    """Yield log entries ordered from newest to oldest.
+
+    ``identifiers`` may be a single charger identifier or an iterable of
+    identifiers. Results are de-duplicated across matching memory and file
+    sources and iteration stops once entries fall before ``since`` or ``limit``
+    is reached.
+    """
+
+    if isinstance(identifiers, str):
+        requested: list[str] = [identifiers]
+    else:
+        requested = list(identifiers)
+
+    seen_keys: set[str] = set()
+    sources: list[tuple[str, str | None]] = []
+    for identifier in requested:
+        for key in _log_key_candidates(identifier, log_type):
+            lower_key = key.lower()
+            if lower_key in seen_keys:
+                continue
+            seen_keys.add(lower_key)
+            resolved, name = _resolve_log_identifier(key, log_type)
+            sources.append((resolved, name))
+
+    heap: list[tuple[float, int, LogEntry, Iterator[LogEntry]]] = []
+    counter = itertools.count()
+    seen_entries: set[str] = set()
+    total_yielded = 0
+
+    for resolved, name in sources:
+        iterator = _iter_log_entries_for_key(
+            resolved,
+            name,
+            log_type,
+            since=since,
+            limit=limit,
+        )
+        try:
+            entry = next(iterator)
+        except StopIteration:
+            continue
+        heapq.heappush(
+            heap,
+            (
+                -entry.timestamp.timestamp(),
+                next(counter),
+                entry,
+                iterator,
+            ),
+        )
+
+    while heap:
+        _, _, entry, iterator = heapq.heappop(heap)
+        if entry.text not in seen_entries:
+            seen_entries.add(entry.text)
+            yield entry
+            total_yielded += 1
+            if limit is not None and total_yielded >= limit:
+                return
+            if since is not None and entry.timestamp < since:
+                return
+        try:
+            next_entry = next(iterator)
+        except StopIteration:
+            continue
+        heapq.heappush(
+            heap,
+            (
+                -next_entry.timestamp.timestamp(),
+                next(counter),
+                next_entry,
+                iterator,
+            ),
+        )
+
+
 def get_logs(cid: str, log_type: str = "charger") -> list[str]:
     """Return all log entries for the given id and type."""
 
@@ -573,14 +932,21 @@ def get_logs(cid: str, log_type: str = "charger") -> list[str]:
         resolved, name = _resolve_log_identifier(key, log_type)
         path = _log_file_for_identifier(resolved, name, log_type)
         if path.exists() and path not in seen_paths:
-            entries.extend(path.read_text(encoding="utf-8").splitlines())
+            if max_entries is None:
+                entries.extend(path.read_text(encoding="utf-8").splitlines())
+            else:
+                with path.open("r", encoding="utf-8") as handle:
+                    for line in handle:
+                        entries.append(line.rstrip("\r\n"))
             seen_paths.add(path)
         memory_entries = _memory_logs_for_identifier(resolved, log_type)
         lower_key = resolved.lower()
         if memory_entries and lower_key not in seen_keys:
             entries.extend(memory_entries)
             seen_keys.add(lower_key)
-    return entries
+    if max_entries is None:
+        return entries_list
+    return list(entries_deque)
 
 
 def clear_log(cid: str, log_type: str = "charger") -> None: