cr_proc 0.2.0__py3-none-any.whl

cr_proc/__init__.py

@@ -0,0 +1,7 @@
+"""Code Recorder Processor - A tool for processing BYU CS code recording files."""
+from importlib.metadata import version, PackageNotFoundError
+
+try:
+    __version__ = version("cr_proc")
+except PackageNotFoundError:
+    __version__ = "unknown"

cr_proc/api/build.py

@@ -0,0 +1,184 @@
+"""Replay edit events to reconstruct document state."""
+
+from __future__ import annotations
+
+import sys
+from typing import Any
+
+from ..timeutil import parse_timestamp
+from .document import filter_events_by_document_with_rename_handling
+from .load import filter_edit_events
+
+
+def _normalize_newlines(text: str) -> str:
+    """Normalize CRLF to LF for stable replay and diff behavior."""
+    return text.replace("\r\n", "\n")
+
+
+def _ordered_edit_events(events: tuple[dict[str, Any], ...]) -> list[dict[str, Any]]:
+    decorated: list[tuple[int, object, dict[str, Any]]] = []
+    for index, event in enumerate(events):
+        timestamp = event.get("timestamp")
+        if timestamp:
+            try:
+                decorated.append((0, parse_timestamp(str(timestamp)), event))
+                continue
+            except ValueError:
+                pass
+        decorated.append((1, index, event))
+
+    decorated.sort(key=lambda item: (item[0], item[1]))
+    return [event for _, _, event in decorated]
+
+
+def _utf16_units_to_index(text: str, units: int) -> int:
+    if units <= 0:
+        return 0
+
+    consumed = 0
+    index = 0
+    for char in text:
+        if consumed >= units:
+            break
+        consumed += 2 if ord(char) > 0xFFFF else 1
+        index += 1
+    return index
+
+
+def _resolve_offset(document: str, old_fragment: str, offset: int, window: int) -> int:
+    if old_fragment == "":
+        return max(0, min(offset, len(document)))
+
+    if 0 <= offset <= len(document) and document[offset : offset + len(old_fragment)] == old_fragment:
+        return offset
+
+    start = max(0, offset - window)
+    end = min(len(document), offset + window + len(old_fragment))
+
+    best_match: tuple[int, int] | None = None
+    search_at = start
+    while True:
+        found = document.find(old_fragment, search_at, end)
+        if found == -1:
+            break
+        distance = abs(found - offset)
+        candidate = (distance, found)
+        if best_match is None or candidate < best_match:
+            best_match = candidate
+        search_at = found + 1
+
+    if best_match is None:
+        raise ValueError(
+            f"Old fragment not found near offset {offset}.\n"
+            f"old={old_fragment!r}\nnew fragment length={len(old_fragment)}"
+        )
+
+    return best_match[1]
+
+
+def _apply_edit(
+    document: str,
+    *,
+    old_fragment: str,
+    new_fragment: str,
+    offset: int,
+    window: int,
+    utf16_mode: bool,
+) -> str:
+    text_offset = _utf16_units_to_index(document, offset) if utf16_mode else offset
+    resolved_offset = _resolve_offset(document, old_fragment, text_offset, window)
+    return (
+        document[:resolved_offset]
+        + new_fragment
+        + document[resolved_offset + len(old_fragment) :]
+    )
+
+
+def reconstruct_file_from_events(
+    events: tuple[dict[str, Any], ...],
+    template: str,
+    document_path: str | None = None,
+    *,
+    utf16_mode: bool = False,
+    window: int = 200,
+    normalize_newlines: bool = True,
+    skip_unreplayable: bool = True,
+) -> str:
+    """Replay edit events to reconstruct the final document state."""
+    edit_events = filter_edit_events(events)
+    if not edit_events:
+        return _normalize_newlines(template) if normalize_newlines else template
+
+    target_document = document_path
+    if target_document is None:
+        recorded_docs = {
+            str(event["document"])
+            for event in edit_events
+            if event.get("document") is not None
+        }
+        if len(recorded_docs) == 1:
+            target_document = next(iter(recorded_docs))
+        else:
+            raise ValueError(
+                "Ambiguous target document: provide document_path explicitly."
+            )
+
+    doc_events = tuple(
+        filter_events_by_document_with_rename_handling(edit_events, target_document)
+    )
+    ordered_events = _ordered_edit_events(doc_events)
+    if not ordered_events:
+        return _normalize_newlines(template) if normalize_newlines else template
+
+    document = _normalize_newlines(template) if normalize_newlines else template
+    skipped = 0
+
+    for event_index, event in enumerate(ordered_events):
+        old_fragment = str(event.get("oldFragment", ""))
+        new_fragment = str(event.get("newFragment", ""))
+        if normalize_newlines:
+            old_fragment = _normalize_newlines(old_fragment)
+            new_fragment = _normalize_newlines(new_fragment)
+
+        try:
+            offset = int(event.get("offset", 0) or 0)
+        except (TypeError, ValueError):
+            offset = 0
+
+        if old_fragment == new_fragment and offset == 0:
+            if old_fragment:
+                document = old_fragment
+            continue
+
+        if old_fragment == new_fragment:
+            continue
+
+        try:
+            document = _apply_edit(
+                document,
+                old_fragment=old_fragment,
+                new_fragment=new_fragment,
+                offset=offset,
+                window=window,
+                utf16_mode=utf16_mode,
+            )
+        except ValueError as exc:
+            if not skip_unreplayable:
+                raise
+
+            skipped += 1
+            print(
+                "Warning: "
+                f"Skipping event #{event_index} "
+                f"(timestamp: {event.get('timestamp', 'unknown')}): "
+                f"{exc} - document offset may have drifted",
+                file=sys.stderr,
+            )
+
+    if skipped:
+        print(
+            f"Warning: Skipped {skipped} event(s) due to offset drift",
+            file=sys.stderr,
+        )
+
+    return document

cr_proc/api/document.py

@@ -0,0 +1,249 @@
+"""Document selection and template matching utilities."""
+
+from __future__ import annotations
+
+import difflib
+from pathlib import Path, PurePosixPath, PureWindowsPath
+from typing import Any
+
+
+def normalize_path_string(path_str: str) -> str:
+    """Normalize a Windows or POSIX path string into POSIX form."""
+    if "\\" in path_str:
+        return PureWindowsPath(path_str).as_posix()
+    return PurePosixPath(path_str).as_posix()
+
+
+def _path_parts(path_str: str) -> PurePosixPath:
+    return PurePosixPath(normalize_path_string(path_str))
+
+
+def _stem_candidates(stem: str) -> list[str]:
+    candidates = [stem]
+    current = stem
+    while True:
+        last_dash = current.rfind("-")
+        last_underscore = current.rfind("_")
+        split_index = max(last_dash, last_underscore)
+        if split_index <= 0:
+            break
+        current = current[:split_index]
+        if current not in candidates:
+            candidates.append(current)
+    return candidates
+
+
+def _is_recording_artifact(path: Path) -> bool:
+    name = path.name.casefold()
+    return name.endswith(".recording.jsonl.gz") or name.endswith(".jsonl.gz") or name.endswith(".jsonl")
+
+
+def get_normalized_document_key(doc_path: str) -> tuple[str, str]:
+    """Return a normalized filename and suffix pair for a document path."""
+    path = _path_parts(doc_path)
+    return path.name, path.suffix
+
+
+def group_documents_by_name(docs: list[str]) -> dict[tuple[str, str], list[str]]:
+    """Group recorder document paths by normalized filename and suffix."""
+    groups: dict[tuple[str, str], list[str]] = {}
+    for doc in docs:
+        groups.setdefault(get_normalized_document_key(doc), []).append(doc)
+    return groups
+
+
+def get_recorded_documents(events: tuple[dict[str, Any], ...]) -> list[str]:
+    """Return sorted unique document paths found in the recording."""
+    return sorted(
+        {
+            str(event["document"])
+            for event in events
+            if event.get("document") is not None
+        }
+    )
+
+
+def _matching_documents(recorded_docs: list[str], requested: str) -> list[str]:
+    target = _path_parts(requested)
+    exact_matches = [
+        doc for doc in recorded_docs if normalize_path_string(doc) == target.as_posix()
+    ]
+    if exact_matches:
+        return exact_matches
+
+    by_name = [
+        doc for doc in recorded_docs if _path_parts(doc).name.casefold() == target.name.casefold()
+    ]
+    if by_name:
+        return by_name
+
+    by_stem = [
+        doc
+        for doc in recorded_docs
+        if _path_parts(doc).stem.casefold() == target.stem.casefold()
+        and (
+            not target.suffix
+            or _path_parts(doc).suffix.casefold() == target.suffix.casefold()
+        )
+    ]
+    if by_stem:
+        return by_stem
+
+    if target.suffix:
+        same_suffix = [
+            doc for doc in recorded_docs if _path_parts(doc).suffix.casefold() == target.suffix.casefold()
+        ]
+        if len(same_suffix) == 1:
+            return same_suffix
+        if same_suffix:
+            names = [_path_parts(doc).name for doc in same_suffix]
+            match = difflib.get_close_matches(target.name, names, n=1, cutoff=0.4)
+            if match:
+                return [doc for doc in same_suffix if _path_parts(doc).name == match[0]]
+
+    return []
+
+
+def filter_events_by_document(
+    events: tuple[dict[str, Any], ...],
+    document: str | None,
+) -> tuple[dict[str, Any], ...]:
+    """Filter events to an exact document path match."""
+    if document is None:
+        return events
+
+    target = normalize_path_string(document)
+    return tuple(
+        event
+        for event in events
+        if normalize_path_string(str(event.get("document", ""))) == target
+    )
+
+
+def filter_events_by_document_with_rename_handling(
+    events: tuple[dict[str, Any], ...],
+    document: str | None,
+) -> tuple[dict[str, Any], ...]:
+    """Filter events to the best matching document, allowing renamed paths."""
+    if document is None:
+        return events
+
+    recorded_docs = get_recorded_documents(events)
+    matching_docs = _matching_documents(recorded_docs, document)
+    if not matching_docs:
+        return ()
+
+    normalized_matches = {normalize_path_string(doc) for doc in matching_docs}
+    return tuple(
+        event
+        for event in events
+        if normalize_path_string(str(event.get("document", ""))) in normalized_matches
+    )
+
+
+def find_matching_template(template_dir: Path, document_path: str) -> Path | None:
+    """Find the template file that best matches a recorded document."""
+    if not template_dir.is_dir():
+        return None
+
+    candidates = sorted(
+        path
+        for path in template_dir.iterdir()
+        if path.is_file() and not _is_recording_artifact(path)
+    )
+    if not candidates:
+        return None
+
+    document = _path_parts(document_path)
+    exact = template_dir / document.name
+    if exact.is_file() and not _is_recording_artifact(exact):
+        return exact
+
+    suffix_candidates = candidates
+    if document.suffix:
+        suffix_candidates = [
+            path for path in candidates if path.suffix.casefold() == document.suffix.casefold()
+        ]
+
+    for stem in _stem_candidates(document.stem):
+        same_stem = [
+            path
+            for path in suffix_candidates
+            if path.stem == stem
+        ]
+        if same_stem:
+            return same_stem[0]
+
+    fuzzy_candidates = suffix_candidates if suffix_candidates else candidates
+    if document.suffix and not suffix_candidates:
+        return None
+
+    names = [path.name for path in fuzzy_candidates]
+    name_match = difflib.get_close_matches(document.name, names, n=1, cutoff=0.4)
+    if name_match:
+        return template_dir / name_match[0]
+
+    stems = [path.stem for path in fuzzy_candidates]
+    stem_match = difflib.get_close_matches(document.stem, stems, n=1, cutoff=0.4)
+    if stem_match:
+        for path in fuzzy_candidates:
+            if path.stem == stem_match[0]:
+                return path
+
+    return None
+
+
+def resolve_document(
+    recorded_docs: list[str],
+    template_base: Path | None,
+    document_override: str | None,
+) -> str | None:
+    """Choose the document to process from a recording."""
+    if not recorded_docs:
+        return None
+
+    if document_override:
+        matches = _matching_documents(recorded_docs, document_override)
+        return matches[0] if matches else document_override
+
+    if len(recorded_docs) == 1:
+        return recorded_docs[0]
+
+    if template_base and template_base.is_file():
+        matches = _matching_documents(recorded_docs, str(template_base))
+        if matches:
+            return matches[0]
+
+    if template_base and template_base.suffix:
+        same_suffix = [
+            doc for doc in recorded_docs if _path_parts(doc).suffix == template_base.suffix
+        ]
+        if len(same_suffix) == 1:
+            return same_suffix[0]
+
+    python_docs = [doc for doc in recorded_docs if _path_parts(doc).suffix == ".py"]
+    if len(python_docs) == 1:
+        return python_docs[0]
+
+    return recorded_docs[0]
+
+
+def resolve_template_file(
+    template_path: Path | None,
+    template_dir: Path | None,
+    document_path: str | None,
+) -> tuple[str, Path | None]:
+    """Load the template text for a document from a file or template directory."""
+    if template_dir is not None:
+        if document_path is None:
+            return "", None
+        match = find_matching_template(template_dir, document_path)
+        if match is None:
+            return "", None
+        return match.read_text(), match
+
+    if template_path is None:
+        return "", None
+    if not template_path.exists():
+        raise FileNotFoundError(f"Template file not found: {template_path}")
+    return template_path.read_text(), template_path

cr_proc/api/load.py

@@ -0,0 +1,165 @@
+"""Utilities for loading and classifying recorder event streams."""
+
+from __future__ import annotations
+
+import gzip
+import json
+import zlib
+from gzip import BadGzipFile
+from io import StringIO
+from pathlib import Path
+from typing import Any, TextIO
+
+
+Event = dict[str, Any]
+_GZIP_MAGIC = b"\x1f\x8b"
+
+
+def _parse_jsonl(stream: TextIO) -> tuple[Event, ...]:
+    events: list[Event] = []
+    for line_number, raw_line in enumerate(stream, start=1):
+        line = raw_line.strip()
+        if not line:
+            continue
+        try:
+            payload = json.loads(line)
+        except json.JSONDecodeError as exc:
+            raise ValueError(f"Invalid JSON on line {line_number}: {exc.msg}") from exc
+        if not isinstance(payload, dict):
+            raise ValueError(f"Line {line_number} must decode to a JSON object")
+        events.append(payload)
+
+    if not events:
+        raise ValueError("JSONL file is empty")
+
+    return tuple(events)
+
+
+def _salvage_gzip_text(raw: bytes) -> str | None:
+    """Best-effort recovery for truncated or partially corrupted gzip data."""
+    chunks: list[bytes] = []
+    remaining = raw
+
+    while remaining:
+        decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
+        try:
+            chunk = decompressor.decompress(remaining) + decompressor.flush()
+        except zlib.error:
+            break
+
+        if chunk:
+            chunks.append(chunk)
+
+        if not decompressor.unused_data:
+            break
+        if decompressor.unused_data == remaining:
+            break
+
+        remaining = decompressor.unused_data
+
+    if not chunks:
+        return None
+
+    return b"".join(chunks).decode("utf-8", errors="replace")
+
+
+def _decode_jsonl_bytes(raw: bytes, *, prefer_gzip: bool) -> tuple[Event, ...]:
+    gzip_error: Exception | None = None
+    text_error: Exception | None = None
+
+    if prefer_gzip:
+        try:
+            text = gzip.decompress(raw).decode("utf-8")
+            return _parse_jsonl(StringIO(text))
+        except (
+            BadGzipFile,
+            EOFError,
+            OSError,
+            UnicodeDecodeError,
+            ValueError,
+        ) as exc:
+            gzip_error = exc
+
+        salvaged_text = _salvage_gzip_text(raw)
+        if salvaged_text is not None:
+            try:
+                return _parse_jsonl(StringIO(salvaged_text))
+            except ValueError as exc:
+                gzip_error = exc
+
+    try:
+        text = raw.decode("utf-8")
+        return _parse_jsonl(StringIO(text))
+    except (UnicodeDecodeError, ValueError) as exc:
+        text_error = exc
+
+    if gzip_error is not None and text_error is not None:
+        raise IOError(
+            f"Unable to decode recording as gzip ({gzip_error}) "
+            f"or plain text ({text_error})"
+        ) from gzip_error
+    if gzip_error is not None:
+        raise IOError(f"Unable to decode recording as gzip: {gzip_error}") from gzip_error
+
+    assert text_error is not None
+    raise IOError(f"Unable to decode recording as plain text: {text_error}") from text_error
+
+
+def load_jsonl(file: Path) -> tuple[Event, ...]:
+    """Load a plain or gzipped JSONL recorder file."""
+    if not file.exists():
+        raise FileNotFoundError(f"File not found: {file}")
+    if not file.is_file():
+        raise ValueError(f"Path is not a file: {file}")
+
+    try:
+        raw = file.read_bytes()
+    except OSError as exc:
+        raise IOError(f"Error reading file: {exc}") from exc
+
+    if not raw:
+        raise ValueError("JSONL file is empty")
+
+    prefer_gzip = raw.startswith(_GZIP_MAGIC) or file.suffix == ".gz"
+    return _decode_jsonl_bytes(raw, prefer_gzip=prefer_gzip)
+
+
+def is_edit_event(event: Event) -> bool:
+    """Return True for recorder edit events, including legacy untyped entries."""
+    return event.get("type") in (None, "edit")
+
+
+def filter_edit_events(events: tuple[Event, ...]) -> tuple[Event, ...]:
+    """Keep edit events and prefer the typed stream when mixed streams appear."""
+    edit_events = tuple(event for event in events if is_edit_event(event))
+    if not edit_events:
+        return ()
+
+    first_typed_idx = next(
+        (index for index, event in enumerate(edit_events) if event.get("type") == "edit"),
+        None,
+    )
+    first_legacy_idx = next(
+        (index for index, event in enumerate(edit_events) if event.get("type") is None),
+        None,
+    )
+
+    if (
+        first_typed_idx is not None
+        and first_legacy_idx is not None
+        and first_typed_idx < first_legacy_idx
+    ):
+        return tuple(event for event in edit_events if event.get("type") == "edit")
+
+    return edit_events
+
+
+def get_focus_events(events: tuple[Event, ...]) -> tuple[Event, ...]:
+    """Return focus status events from a recording."""
+    return tuple(event for event in events if event.get("type") == "focusStatus")
+
+
+def get_editors(events: tuple[Event, ...]) -> set[str]:
+    """Return the set of editors observed in the recording."""
+    editors = {str(event["editor"]) for event in events if event.get("editor")}
+    return editors or {"jetbrains"}

cr_proc/api/output.py

@@ -0,0 +1,75 @@
+"""JSON output helpers for processed recordings."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+from .document import normalize_path_string
+
+
+def _sanitize_time_info(time_info: dict[str, Any] | None) -> dict[str, Any] | None:
+    if time_info is None:
+        return None
+    return {key: value for key, value in time_info.items() if key != "active_intervals"}
+
+
+def write_batch_json_output(
+    output_path: Path,
+    results: list[dict[str, Any]],
+    combined_time_info: dict[str, Any] | None,
+    all_verified: bool,
+    batch_mode: bool = False,
+    submitted_all_passed: bool | None = None,
+    version: str | None = None,
+) -> None:
+    """Write processing results to a JSON file."""
+    files_data: list[dict[str, Any]] = []
+    all_editors: set[str] = set()
+
+    for result in results:
+        editors = list(result.get("editors", []))
+        all_editors.update(editors)
+
+        file_data = {
+            "jsonl_file": normalize_path_string(str(result["jsonl_file"])),
+            "document": result.get("target_document"),
+            "verified": result.get("verified", False),
+            "time_info": _sanitize_time_info(result.get("time_info")),
+            "suspicious_events": result.get("suspicious_events", []),
+            "template_diff": result.get("template_diff", ""),
+            "reconstructed_code": result.get("reconstructed", ""),
+        }
+        if editors:
+            file_data["editors"] = editors
+        if result.get("warnings"):
+            file_data["warnings"] = result["warnings"]
+        if result.get("submitted_comparison") is not None:
+            file_data["submitted_comparison"] = result["submitted_comparison"]
+
+        files_data.append(file_data)
+
+    payload: dict[str, Any] = {
+        "version": version,
+        "batch_mode": batch_mode,
+        "total_files": len(results),
+        "verified_count": sum(1 for result in results if result.get("verified")),
+        "all_verified": all_verified,
+        "files": files_data,
+    }
+    if combined_time_info is not None:
+        payload["combined_time_info"] = _sanitize_time_info(combined_time_info)
+    if all_editors:
+        payload["editors"] = sorted(all_editors)
+    if submitted_all_passed is not None:
+        payload["submitted_all_passed"] = submitted_all_passed
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(json.dumps(payload, indent=2))
+
+    if batch_mode:
+        print(f"Batch results written to {output_path}", file=sys.stderr)
+    else:
+        print(f"Results written to {output_path}", file=sys.stderr)