remarkable-tools 0.2.1__py3-none-any.whl

remarkable/helpers.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import importlib.metadata as importlib_metadata
+import logging
+import os
+import tomllib
+import typing
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+
+def deep_merge(
+  parent_source: dict[str, typing.Any], source: dict[str, typing.Any]
+) -> dict[str, typing.Any]:
+  result = parent_source.copy()
+  for k, v in source.items():
+    if k in result and isinstance(result[k], dict) and isinstance(v, dict):
+      result[k] = deep_merge(result[k], v)
+    else:
+      result[k] = v
+  return result
+
+
+def get_config_paths(platform: str | None = None) -> list[Path]:
+  home = Path.home()
+  if (platform or os.name) == "nt":
+    config_paths: list[Path] = []
+    for env_var in ("PROGRAMDATA", "APPDATA", "LOCALAPPDATA"):
+      config_dir = os.getenv(env_var)
+      if config_dir:
+        config_paths.append(Path(config_dir) / "remarkable" / "config")
+    config_paths.append(home / ".remarkableconfig")
+    return config_paths
+
+  config_paths = [Path("/") / "usr" / "local" / "etc" / "remarkableconfig"]
+  xdg_config_home = os.getenv("XDG_CONFIG_HOME")
+  if xdg_config_home:
+    config_paths.append(Path(xdg_config_home) / "remarkable" / "config")
+  config_paths.append(home / ".config" / "remarkable" / "config")
+  config_paths.append(home / ".remarkableconfig")
+  return config_paths
+
+
+def parse_config(default: dict[str, typing.Any]) -> dict[str, typing.Any]:
+  config_paths = get_config_paths()
+  config = default
+  for path in config_paths:
+    if not path.is_file():
+      continue
+    log.info("Loading configs from: %s", path)
+    with path.open("rb") as f:
+      data = tomllib.load(f)
+    log.info("Loaded config from %s: %s", path, data)
+    config = deep_merge(config, data)
+  return config
+
+
+def get_runtime_version(dist_name: str = "remarkable") -> str:
+  """Return installed package version, or fall back to pyproject.toml."""
+  try:
+    return importlib_metadata.version(dist_name)
+  except importlib_metadata.PackageNotFoundError:
+    pass
+
+  pyproject_path = Path(__file__).resolve().parents[2] / "pyproject.toml"
+  try:
+    with pyproject_path.open("rb") as f:
+      pyproject_data = tomllib.load(f)
+    return str(pyproject_data["project"]["version"])
+  except OSError, tomllib.TOMLDecodeError, KeyError, TypeError:
+    return "unknown"

remarkable/hl_fix.py

@@ -0,0 +1,166 @@
+"""Lower reMarkable highlight opacity and optionally calibrate highlight colors."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from remarkable import helpers
+from remarkable.hl_palette import (
+  HL_RGB_COLORS,
+  detect_highlight_palette,
+  format_palette_block,
+)
+from remarkable.hl_processing import (
+  ALPHA_THRESHOLD,
+  MIN_STROKE_WIDTH_FOR_TRANSPARENCY,
+  TARGET_ALPHA,
+  set_hl_transparency,
+)
+
+
+def build_parser() -> argparse.ArgumentParser:
+  parser = argparse.ArgumentParser(
+    description=(
+      "Reduce reMarkable-style highlight fill and stroke opacity from "
+      "1.0 to 0.4 in a PDF file."
+    )
+  )
+  parser.add_argument(
+    "input_pdf",
+    nargs="?",
+    default=Path("input.pdf"),
+    type=Path,
+    help="Path to the input PDF (default: %(default)s)",
+  )
+  parser.add_argument(
+    "output_pdf",
+    nargs="?",
+    default=Path("output.pdf"),
+    type=Path,
+    help="Path to the output PDF (default: %(default)s)",
+  )
+  parser.add_argument(
+    "--version",
+    action="version",
+    version=f"%(prog)s {helpers.get_runtime_version()}",
+  )
+  parser.add_argument(
+    "--all-colors",
+    action="store_true",
+    help=(
+      "Apply opacity reduction to all fully opaque filled shapes, not only "
+      "known reMarkable highlight colors"
+    ),
+  )
+  parser.add_argument(
+    "--min-stroke-width",
+    type=float,
+    default=MIN_STROKE_WIDTH_FOR_TRANSPARENCY,
+    help=(
+      "Minimum stroke width required before stroke opacity is reduced "
+      "(default: %(default)s)."
+    ),
+  )
+  parser.add_argument(
+    "--calibrate-palette",
+    action="store_true",
+    help=(
+      "Detect highlight colors from a single page and print a paste-ready "
+      "HIGHLIGHT_RGB_COLORS block, then exit."
+    ),
+  )
+  parser.add_argument(
+    "--calibration-page",
+    type=int,
+    default=1,
+    help="1-based page number for --calibrate-palette (default: %(default)s).",
+  )
+  return parser
+
+
+# ---------------------------------------------------------------------------
+# High-Level Commands
+# ---------------------------------------------------------------------------
+
+
+def run_calibration(input_pdf: Path, calibration_page: int) -> int:
+  detected_palette = detect_highlight_palette(
+    input_pdf,
+    page_number=calibration_page,
+  )
+  if not detected_palette:
+    print(
+      f"No known highlight colors were detected on calibration page {calibration_page}."
+    )
+    return 0
+
+  print(
+    f"Detected {len(detected_palette)} highlight colors on page {calibration_page}:"
+  )
+  print(format_palette_block(detected_palette))
+  return 0
+
+
+def run_transparency_pass(
+  input_pdf: Path,
+  output_pdf: Path,
+  *,
+  all_colors: bool,
+  min_stroke_width: float,
+  target_alpha: float,
+  alpha_threshold: float,
+) -> int:
+  with input_pdf.open("rb") as f:
+    output_stream, changed, total = set_hl_transparency(
+      f,
+      all_colors=all_colors,
+      hl_palette=HL_RGB_COLORS,
+      min_stroke_width=min_stroke_width,
+      target_alpha=target_alpha,
+      alpha_threshold=alpha_threshold,
+    )
+  with output_pdf.open("wb") as f:
+    f.write(output_stream)
+  print(
+    f"Processed {total} drawing instructions. Updated {changed} highlight paints in "
+    f"{output_pdf}."
+  )
+  return 0
+
+
+def main() -> None:
+  config = helpers.parse_config(
+    default={
+      "hl_fix": {
+        "alpha": TARGET_ALPHA,
+        "min_stroke_width": MIN_STROKE_WIDTH_FOR_TRANSPARENCY,
+        "alpha_threshold": ALPHA_THRESHOLD,
+      }
+    }
+  )
+
+  parser = build_parser()
+  parser.set_defaults(**config.get("hl_fix", {}))
+  args = parser.parse_args()
+
+  if not args.input_pdf.exists():
+    parser.error(f"Input file not found: {args.input_pdf}")
+
+  if args.calibrate_palette:
+    raise SystemExit(run_calibration(args.input_pdf, args.calibration_page))
+
+  raise SystemExit(
+    run_transparency_pass(
+      args.input_pdf,
+      args.output_pdf,
+      all_colors=args.all_colors,
+      min_stroke_width=args.min_stroke_width,
+      target_alpha=config["hl_fix"]["alpha"],
+      alpha_threshold=config["hl_fix"]["alpha_threshold"],
+    )
+  )
+
+
+if __name__ == "__main__":
+  main()

remarkable/hl_palette.py

@@ -0,0 +1,142 @@
+"""Utilities for highlight color matching and one-shot palette calibration."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Iterator, Sequence, cast
+
+import pikepdf
+
+# Tuned highlight colors detected from input.pdf
+RGB = tuple[float, float, float]
+Palette = Sequence[RGB]
+HL_RGB_COLORS: Palette = (
+  (1.0, 0.929412, 0.458824),  # yellow
+  (1.0, 0.764706, 0.54902),  # orange
+  (0.67451, 1.0, 0.521569),  # green
+  (0.94902, 0.619608, 1.0),  # pink
+  (0.745098, 0.917647, 0.996078),  # light blue
+  (0.780392, 0.780392, 0.776471),  # gray
+)
+COLOR_TOLERANCE = 0.1
+
+
+def color_from_instruction(op: str, operands: Sequence[Any]) -> RGB | None:
+  """Return RGB color for grayscale/rgb paint operators, else None."""
+  if op in {"rg", "RG"}:
+    if len(operands) < 3:
+      return None
+    return (float(operands[0]), float(operands[1]), float(operands[2]))
+  if op in {"g", "G"}:
+    if len(operands) < 1:
+      return None
+    gray = float(operands[0])
+    return (gray, gray, gray)
+  return None
+
+
+def iter_content_streams(contents: Any) -> Iterator[pikepdf.Stream]:
+  """Normalize page contents to a list of content streams."""
+  if isinstance(contents, pikepdf.Stream):
+    yield contents
+    return
+  yield from (
+    stream
+    for stream in cast(Sequence[Any], contents)
+    if isinstance(stream, pikepdf.Stream)
+  )
+
+
+def palette_index_for_color(color: RGB | None, palette: Palette) -> int | None:
+  if color is None:
+    return None
+
+  best_cost = 100.0  # arbitrary large number
+  index = None
+  for i, expected in enumerate(palette):
+    if all(abs(c - e) <= COLOR_TOLERANCE for c, e in zip(color, expected)):
+      cost = sum(abs(c - e) for c, e in zip(color, expected))
+      if cost < best_cost:
+        best_cost = cost
+        index = i
+  return index
+
+
+def _collect_palette_indices_from_stream(stream: pikepdf.Stream) -> set[int]:
+  instructions = pikepdf.parse_content_stream(stream)
+  indices: set[int] = set()
+
+  for instruction in instructions:
+    if not hasattr(instruction, "operator"):
+      continue
+
+    op = str(instruction.operator)
+    operands = list(instruction.operands)
+    color = color_from_instruction(op, operands)
+
+    idx = palette_index_for_color(color, HL_RGB_COLORS)
+    if idx is not None:
+      indices.add(idx)
+
+  return indices
+
+
+def _collect_palette_indices_from_resources(resources: pikepdf.Dictionary) -> set[int]:
+  indices: set[int] = set()
+  xobjects_obj = resources.get("/XObject")
+  if not isinstance(xobjects_obj, pikepdf.Dictionary):
+    return indices
+
+  for _, xobj in xobjects_obj.items():
+    if not isinstance(xobj, pikepdf.Stream):
+      continue
+    if xobj.get("/Subtype") != pikepdf.Name("/Form"):
+      continue
+
+    indices |= _collect_palette_indices_from_stream(xobj)
+
+    form_resources_obj = xobj.get("/Resources")
+    if isinstance(form_resources_obj, pikepdf.Dictionary):
+      indices |= _collect_palette_indices_from_resources(form_resources_obj)
+
+  return indices
+
+
+def detect_highlight_palette(
+  input_path: Path,
+  *,
+  page_number: int = 3,
+) -> tuple[RGB, ...]:
+  """Detect which tuned highlight colors are present on the given page."""
+  if page_number < 1:
+    msg = "page_number must be >= 1"
+    raise ValueError(msg)
+
+  with pikepdf.open(input_path) as pdf:
+    if page_number > len(pdf.pages):
+      msg = (
+        f"page_number {page_number} exceeds number of pages in PDF ({len(pdf.pages)})"
+      )
+      raise ValueError(msg)
+
+    page = pdf.pages[page_number - 1]
+    indices: set[int] = set()
+
+    resources_obj = page.obj.get("/Resources")
+    resources = resources_obj if isinstance(resources_obj, pikepdf.Dictionary) else None
+    if resources is not None:
+      indices |= _collect_palette_indices_from_resources(resources)
+
+    for stream in iter_content_streams(page.Contents):
+      indices |= _collect_palette_indices_from_stream(stream)
+
+  return tuple(HL_RGB_COLORS[i] for i in sorted(indices))
+
+
+def format_palette_block(palette: Palette) -> str:
+  """Format a palette as a paste-ready HIGHLIGHT_RGB_COLORS block."""
+  lines = ["HIGHLIGHT_RGB_COLORS = ("]
+  for r, g, b in palette:
+    lines.append(f"  ({r:.6f}, {g:.6f}, {b:.6f}),")
+  lines.append(")")
+  return "\n".join(lines)

remarkable/hl_processing.py

@@ -0,0 +1,287 @@
+"""PDF content-stream processing for lowering highlight fill/stroke opacity."""
+
+from __future__ import annotations
+
+import io
+from dataclasses import dataclass, replace
+from typing import Any, BinaryIO
+
+import pikepdf
+
+from remarkable.hl_palette import (
+  HL_RGB_COLORS,
+  RGB,
+  Palette,
+  color_from_instruction,
+  iter_content_streams,
+  palette_index_for_color,
+)
+
+
+@dataclass(slots=True)
+class State:
+  fill_alpha: float
+  stroke_alpha: float
+  line_width: float
+  fill_rgb: RGB | None
+  stroke_rgb: RGB | None
+
+
+TARGET_ALPHA = 0.4
+ALPHA_THRESHOLD = 0.999
+MIN_STROKE_WIDTH_FOR_TRANSPARENCY = 0.1
+GS_NAME_FILL = pikepdf.Name("/GS_HIGHLIGHT_FILL_20")
+GS_NAME_STROKE = pikepdf.Name("/GS_HIGHLIGHT_STROKE_20")
+GS_NAME_BOTH = pikepdf.Name("/GS_HIGHLIGHT_20")
+
+FILL_OPERATORS = {"f", "f*", "B", "B*", "b", "b*"}
+STROKE_OPERATORS = {"S", "s", "B", "B*", "b", "b*"}
+
+
+def _ensure_extgstate(resources: pikepdf.Dictionary, target_alpha: float) -> None:
+  if "/ExtGState" not in resources:
+    resources["/ExtGState"] = pikepdf.Dictionary()
+
+  extgstate = resources["/ExtGState"]
+  extgstate[GS_NAME_FILL] = pikepdf.Dictionary(
+    {
+      "/Type": pikepdf.Name("/ExtGState"),
+      "/ca": target_alpha,
+    }
+  )
+  extgstate[GS_NAME_STROKE] = pikepdf.Dictionary(
+    {
+      "/Type": pikepdf.Name("/ExtGState"),
+      "/CA": target_alpha,
+    }
+  )
+  extgstate[GS_NAME_BOTH] = pikepdf.Dictionary(
+    {
+      "/Type": pikepdf.Name("/ExtGState"),
+      "/ca": target_alpha,
+      "/CA": target_alpha,
+    }
+  )
+
+
+def _extgstate_alpha_map(
+  resources: pikepdf.Dictionary,
+) -> dict[str, tuple[float | None, float | None]]:
+  mapping: dict[str, tuple[float | None, float | None]] = {}
+  extgstate = resources.get("/ExtGState")
+  if extgstate is None:
+    return mapping
+
+  for name, state in extgstate.items():
+    fill_alpha = float(state["/ca"]) if "/ca" in state else None
+    stroke_alpha = float(state["/CA"]) if "/CA" in state else None
+    mapping[str(name)] = (fill_alpha, stroke_alpha)
+
+  return mapping
+
+
+def _process_stream(
+  stream: pikepdf.Stream,
+  resources: pikepdf.Dictionary,
+  *,
+  all_colors: bool,
+  hl_palette: Palette,
+  min_stroke_width: float,
+  target_alpha: float,
+  alpha_threshold: float,
+) -> int:
+  instructions = pikepdf.parse_content_stream(stream)
+  if not instructions:
+    return 0
+
+  alpha_by_gs = _extgstate_alpha_map(resources)
+
+  state = State(
+    fill_alpha=1.0,
+    stroke_alpha=1.0,
+    line_width=1.0,
+    fill_rgb=None,
+    stroke_rgb=None,
+  )
+  state_stack: list[State] = []
+  out: list[Any] = []
+  changed = 0
+
+  for instruction in instructions:
+    if not hasattr(instruction, "operator"):
+      out.append(instruction)
+      continue
+
+    op = str(instruction.operator)
+    operands = list(instruction.operands)
+
+    if op == "q":
+      # q/Q saves and restores graphics state. Snapshot values to prevent
+      # alpha and color state from leaking across sibling blocks.
+      state_stack.append(replace(state))
+    elif op == "Q" and state_stack:
+      state = state_stack.pop()
+    elif op in ("rg", "g"):
+      state.fill_rgb = color_from_instruction(op, operands)
+    elif op in ("RG", "G"):
+      state.stroke_rgb = color_from_instruction(op, operands)
+    elif op == "k":
+      state.fill_rgb = None
+    elif op == "K":
+      state.stroke_rgb = None
+    elif op == "gs" and operands:
+      fill_alpha, stroke_alpha = alpha_by_gs.get(str(operands[0]), (None, None))
+      if fill_alpha is not None:
+        state.fill_alpha = fill_alpha
+      if stroke_alpha is not None:
+        state.stroke_alpha = stroke_alpha
+    elif op == "w" and operands:
+      try:
+        state.line_width = float(operands[0])
+      except TypeError, ValueError:
+        pass
+
+    should_update_fill = all_colors or (
+      palette_index_for_color(state.fill_rgb, hl_palette) is not None
+    )
+    should_update_stroke = all_colors or (
+      palette_index_for_color(state.stroke_rgb, hl_palette) is not None
+    )
+    apply_fill_opacity = (
+      op in FILL_OPERATORS
+      and should_update_fill
+      and state.fill_alpha >= alpha_threshold
+    )
+    apply_stroke_opacity = (
+      op in STROKE_OPERATORS
+      and should_update_stroke
+      and state.stroke_alpha >= alpha_threshold
+      and state.line_width >= min_stroke_width
+    )
+    if apply_fill_opacity or apply_stroke_opacity:
+      if apply_fill_opacity and apply_stroke_opacity:
+        gs_name = GS_NAME_BOTH
+        state.fill_alpha = target_alpha
+        state.stroke_alpha = target_alpha
+      elif apply_fill_opacity:
+        gs_name = GS_NAME_FILL
+        state.fill_alpha = target_alpha
+      else:
+        gs_name = GS_NAME_STROKE
+        state.stroke_alpha = target_alpha
+
+      out.append(pikepdf.ContentStreamInstruction([gs_name], pikepdf.Operator("gs")))
+      changed += 1
+
+    out.append(instruction)
+
+  if changed:
+    stream.write(pikepdf.unparse_content_stream(out))
+
+  return changed
+
+
+def _get_or_create_resources(page: pikepdf.Page) -> pikepdf.Dictionary:
+  page_dict = page.obj
+  resources_obj = page_dict.get("/Resources")
+  if isinstance(resources_obj, pikepdf.Dictionary):
+    return resources_obj
+
+  resources = pikepdf.Dictionary()
+  page_dict["/Resources"] = resources
+  return resources
+
+
+def _process_resources(
+  resources: pikepdf.Dictionary,
+  *,
+  all_colors: bool,
+  hl_palette: Palette,
+  min_stroke_width: float,
+  target_alpha: float,
+  alpha_threshold: float,
+) -> int:
+  _ensure_extgstate(resources, target_alpha)
+  changed = 0
+
+  xobjects_obj = resources.get("/XObject")
+  if not isinstance(xobjects_obj, pikepdf.Dictionary):
+    return changed
+
+  for _, xobj in xobjects_obj.items():
+    if not isinstance(xobj, pikepdf.Stream):
+      continue
+
+    if xobj.get("/Subtype") != pikepdf.Name("/Form"):
+      continue
+
+    form_resources_obj = xobj.get("/Resources")
+    form_resources = (
+      form_resources_obj if isinstance(form_resources_obj, pikepdf.Dictionary) else None
+    )
+    active_resources = form_resources if form_resources is not None else resources
+    _ensure_extgstate(active_resources, target_alpha)
+    changed += _process_stream(
+      xobj,
+      active_resources,
+      all_colors=all_colors,
+      hl_palette=hl_palette,
+      min_stroke_width=min_stroke_width,
+      target_alpha=target_alpha,
+      alpha_threshold=alpha_threshold,
+    )
+
+    if form_resources is not None:
+      changed += _process_resources(
+        form_resources,
+        all_colors=all_colors,
+        hl_palette=hl_palette,
+        min_stroke_width=min_stroke_width,
+        target_alpha=target_alpha,
+        alpha_threshold=alpha_threshold,
+      )
+
+  return changed
+
+
+def set_hl_transparency(
+  input_stream: BinaryIO,
+  *,
+  all_colors: bool,
+  hl_palette: Palette = HL_RGB_COLORS,
+  min_stroke_width: float,
+  target_alpha: float,
+  alpha_threshold: float,
+) -> tuple[bytes, int, int]:
+  """Apply transparency and return (updated_paints, instructions_seen)."""
+  changed = 0
+  instructions_seen = 0
+
+  with pikepdf.open(input_stream) as pdf:
+    for page in pdf.pages:
+      resources = _get_or_create_resources(page)
+      _ensure_extgstate(resources, target_alpha)
+      changed += _process_resources(
+        resources,
+        all_colors=all_colors,
+        hl_palette=hl_palette,
+        min_stroke_width=min_stroke_width,
+        target_alpha=target_alpha,
+        alpha_threshold=alpha_threshold,
+      )
+
+      for stream in iter_content_streams(page.Contents):
+        instructions_seen += len(pikepdf.parse_content_stream(stream))
+        changed += _process_stream(
+          stream,
+          resources,
+          all_colors=all_colors,
+          hl_palette=hl_palette,
+          min_stroke_width=min_stroke_width,
+          target_alpha=target_alpha,
+          alpha_threshold=alpha_threshold,
+        )
+    buffer = io.BytesIO()
+    pdf.save(buffer)
+
+  return buffer.getvalue(), changed, instructions_seen