remarkablesync 2.0.0__tar.gz → 2.0.1__tar.gz

{remarkablesync-2.0.0/remarkablesync.egg-info → remarkablesync-2.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: remarkablesync
-Version: 2.0.0
+Version: 2.0.1
 Summary: Backup and convert reMarkable tablet notebooks to PDF
 Home-page: https://github.com/JeffSteinbok/RemarkableSync
 Author: Jeff Steinbok
@@ -54,7 +54,7 @@ Dynamic: summary
 
 # RemarkableSync
 
-![RemarkableSync](githubSocial.png)
+![RemarkableSync](https://raw.githubusercontent.com/JeffSteinbok/RemarkableSync/main/githubSocial.png)
 
 [![GitHub](https://img.shields.io/badge/GitHub-RemarkableSync-blue?logo=github)](https://github.com/JeffSteinbok/RemarkableSync)
 [![GitHub release](https://img.shields.io/github/v/release/JeffSteinbok/RemarkableSync)](https://github.com/JeffSteinbok/RemarkableSync/releases)

{remarkablesync-2.0.0 → remarkablesync-2.0.1}/README.md

@@ -1,6 +1,6 @@
 # RemarkableSync
 
-![RemarkableSync](githubSocial.png)
+![RemarkableSync](https://raw.githubusercontent.com/JeffSteinbok/RemarkableSync/main/githubSocial.png)
 
 [![GitHub](https://img.shields.io/badge/GitHub-RemarkableSync-blue?logo=github)](https://github.com/JeffSteinbok/RemarkableSync)
 [![GitHub release](https://img.shields.io/github/v/release/JeffSteinbok/RemarkableSync)](https://github.com/JeffSteinbok/RemarkableSync/releases)

{remarkablesync-2.0.0 → remarkablesync-2.0.1/remarkablesync.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: remarkablesync
-Version: 2.0.0
+Version: 2.0.1
 Summary: Backup and convert reMarkable tablet notebooks to PDF
 Home-page: https://github.com/JeffSteinbok/RemarkableSync
 Author: Jeff Steinbok
@@ -54,7 +54,7 @@ Dynamic: summary
 
 # RemarkableSync
 
-![RemarkableSync](githubSocial.png)
+![RemarkableSync](https://raw.githubusercontent.com/JeffSteinbok/RemarkableSync/main/githubSocial.png)
 
 [![GitHub](https://img.shields.io/badge/GitHub-RemarkableSync-blue?logo=github)](https://github.com/JeffSteinbok/RemarkableSync)
 [![GitHub release](https://img.shields.io/github/v/release/JeffSteinbok/RemarkableSync)](https://github.com/JeffSteinbok/RemarkableSync/releases)

{remarkablesync-2.0.0 → remarkablesync-2.0.1}/src/__version__.py

@@ -1,4 +1,4 @@
 """Version information for RemarkableSync."""
 
-__version__ = "2.0.0"
+__version__ = "2.0.1"
 __repository__ = "https://github.com/JeffSteinbok/RemarkableSync"

{remarkablesync-2.0.0 → remarkablesync-2.0.1}/src/commands/pipeline.py

@@ -249,6 +249,7 @@ def run_pipeline(
             force=force_export,
             converted_pages=converted_pages,
             page_filter=page_filter,
+            updated_pages=updated_pages,
         )
 
     # ------------------------------------------------------------------

{remarkablesync-2.0.0 → remarkablesync-2.0.1}/src/commands/watch_command.py

@@ -738,7 +738,7 @@ class _TrayLogHandler(logging.Handler):
                 return
             self._tray.set_detail(msg)
 
-            # Parse progress from page callbacks: "PDF: Work (page 3/21)"
+            # Parse progress from page callbacks: "PDF: Work (page 3/21)" or "PDF: Work (page 3/21) [cached]"
             import re
 
             m = re.search(r"(PDF|MD): (.+?) \(page (\d+)/(\d+)\)", msg)

{remarkablesync-2.0.0 → remarkablesync-2.0.1}/src/hybrid_converter.py

@@ -446,6 +446,7 @@ def convert_notebook(
     template_renderer: Optional[TemplateRenderer] = None,
     changed_page_ids: Optional[set] = None,
     on_page_done: Optional[callable] = None,
+    on_page_start: Optional[callable] = None,
 ) -> Dict:
     """Convert a notebook using appropriate tools for each file type.
 
@@ -462,6 +463,8 @@ def convert_notebook(
         template_renderer: Optional template renderer for backgrounds.
         changed_page_ids: Set of page IDs whose ``.rm`` files changed.
             When *None* all pages are (re-)converted.
+        on_page_done: Callback ``(cached: bool)`` called after each page.
+            *cached* is True when the page was served from cache.
     """
     # Create safe filename
     safe_name = "".join(c for c in notebook["name"] if c.isalnum() or c in (" ", "-", "_")).rstrip()
@@ -505,7 +508,7 @@ def convert_notebook(
         template_temp_dir = Path(tempfile.mkdtemp(prefix="remarkable_templates_"))
 
     try:
-        # Resolve ordered pages using .content file if present (v5 ordering)
+        # Resolve ordered pages using .content file if present
         metadata_file = notebook.get("metadata_file")
         content_path = metadata_file.with_suffix(".content") if metadata_file else None
 
@@ -514,28 +517,52 @@ def convert_notebook(
         if template_renderer and content_path:
             page_templates = get_page_templates(content_path)
 
-        ordered_v5_pages: List[Path] = []
+        # Build a set of all known .rm files by page ID for fast lookup
+        all_rm_by_id: Dict[str, Path] = {}
+        for rm_file in (
+            notebook.get("v5_files", [])
+            + notebook.get("v6_files", [])
+            + notebook.get("v4_files", [])
+        ):
+            all_rm_by_id[rm_file.stem] = rm_file
+
+        # Determine version for each page
+        v6_ids = {f.stem for f in notebook.get("v6_files", [])}
+        v4_ids = {f.stem for f in notebook.get("v4_files", [])}
+
+        # Order pages using .content file (applies to all versions)
+        ordered_pages: List[Path] = []
         if content_path and content_path.exists():
             try:
                 with open(content_path, "r", encoding="utf-8") as cf:
                     content_json = json.load(cf)
                 page_ids = content_json.get("pages", [])
+                # v6 notebooks use cPages.pages with {id, idx, ...} dicts
+                if not page_ids:
+                    cpages = content_json.get("cPages", {}).get("pages", [])
+                    page_ids = [p["id"] for p in cpages if "id" in p]
                 base_dir = content_path.parent / content_path.stem
                 for pid in page_ids:
-                    candidate = base_dir / f"{pid}.rm"
-                    if candidate.exists():
-                        ordered_v5_pages.append(candidate)
+                    if pid in all_rm_by_id:
+                        ordered_pages.append(all_rm_by_id[pid])
                     else:
-                        # fallback: find rm page anywhere under files matching page id
-                        alt = list((content_path.parent).glob(f"{pid}.rm"))
-                        if alt:
-                            ordered_v5_pages.append(alt[0])
+                        candidate = base_dir / f"{pid}.rm"
+                        if candidate.exists():
+                            ordered_pages.append(candidate)
+                        else:
+                            alt = list((content_path.parent).glob(f"{pid}.rm"))
+                            if alt:
+                                ordered_pages.append(alt[0])
             except Exception as e:
                 logging.debug("Failed reading content ordering for %s: %s", notebook["name"], e)
 
         # Fallback to unsorted list if ordering extraction failed
-        if not ordered_v5_pages:
-            ordered_v5_pages = notebook["v5_files"]
+        if not ordered_pages:
+            ordered_pages = (
+                notebook.get("v5_files", [])
+                + notebook.get("v6_files", [])
+                + notebook.get("v4_files", [])
+            )
 
         def _needs_conversion(page_id: str) -> bool:
             """Check if a page needs (re-)conversion."""
@@ -543,24 +570,24 @@ def convert_notebook(
                 return True  # No change info → convert all
             return page_id in changed_page_ids
 
-        def _convert_page(
-            rm_file: Path, version_tag: str, convert_fn, result_key: str
-        ) -> Optional[Path]:
+        def _convert_page(rm_file: Path, version_tag: str, convert_fn, result_key: str) -> tuple:
             """Convert a single page, using cache when possible.
 
-            Returns the path to the cached page PDF, or None on failure.
+            Returns ``(path, cached)`` where *cached* is True when the
+            page was served from the persistent cache without conversion.
+            Returns ``(None, False)`` on failure.
             """
             page_id = rm_file.stem
             cached_pdf = page_cache_dir / f"{page_id}.pdf"
 
             # Use cached PDF if page hasn't changed
             if not _needs_conversion(page_id) and cached_pdf.exists():
-                return cached_pdf
+                return cached_pdf, True
 
             # Convert the .rm file to a content PDF
             content_pdf = page_cache_dir / f"{page_id}_content.pdf"
             if not convert_fn(rm_file, content_pdf):
-                return None
+                return None, False
 
             # Apply template if available
             if template_renderer and template_temp_dir:
@@ -575,7 +602,7 @@ def convert_notebook(
                             except OSError:
                                 pass
                             results[result_key] += 1
-                            return cached_pdf
+                            return cached_pdf, False
 
             # No template or template merge failed — content PDF is the final
             if content_pdf != cached_pdf:
@@ -585,35 +612,34 @@ def convert_notebook(
                 except OSError:
                     cached_pdf = content_pdf
             results[result_key] += 1
-            return cached_pdf
+            return cached_pdf, False
 
-        # Convert v5 files in determined order
-        for rm_file in ordered_v5_pages:
-            pdf = _convert_page(rm_file, "v5", convert_v5_file_with_rmrl, "v5_converted")
-            if pdf:
-                page_pdfs.append(pdf)
-            if on_page_done:
-                on_page_done()
-
-        # Convert v6 files
-        for rm_file in notebook["v6_files"]:
-            pdf = _convert_page(rm_file, "v6", convert_v6_file_with_rmc, "v6_converted")
-            if pdf:
-                page_pdfs.append(pdf)
-            if on_page_done:
-                on_page_done()
-
-        # Convert v4 files (best-effort; may not succeed)
-        for rm_file in notebook.get("v4_files", []):
-            pdf = _convert_page(rm_file, "v4", convert_v4_file_with_rmrl, "v4_converted")
+        # Convert all pages in content-file order
+        for rm_file in ordered_pages:
+            page_id = rm_file.stem
+            if on_page_start:
+                on_page_start()
+            if page_id in v6_ids:
+                pdf, cached = _convert_page(rm_file, "v6", convert_v6_file_with_rmc, "v6_converted")
+            elif page_id in v4_ids:
+                pdf, cached = _convert_page(
+                    rm_file, "v4", convert_v4_file_with_rmrl, "v4_converted"
+                )
+            else:
+                pdf, cached = _convert_page(
+                    rm_file, "v5", convert_v5_file_with_rmrl, "v5_converted"
+                )
             if pdf:
                 page_pdfs.append(pdf)
             if on_page_done:
-                on_page_done()
+                on_page_done(cached=cached)
 
         # Copy existing PDFs
         for i, pdf_file in enumerate(notebook["pdf_files"]):
+            if on_page_start:
+                on_page_start()
             cached_pdf = page_cache_dir / f"existing_{i+1:03d}.pdf"
+            was_cached = False
             if not cached_pdf.exists() or changed_page_ids is None:
                 if copy_existing_pdf(pdf_file, cached_pdf):
                     page_pdfs.append(cached_pdf)
@@ -621,8 +647,12 @@ def convert_notebook(
             else:
                 page_pdfs.append(cached_pdf)
                 results["pdfs_copied"] += 1
+                was_cached = True
             if on_page_done:
-                on_page_done()
+                on_page_done(cached=was_cached)
+
+        # Store ordered page PDFs in results for downstream consumers
+        results["page_pdfs"] = list(page_pdfs)
 
         # Create merged PDF if we have any pages
         if page_pdfs:

{remarkablesync-2.0.0 → remarkablesync-2.0.1}/src/pdf_md_converter.py

@@ -115,6 +115,26 @@ class MarkdownExporter:
         except OSError as exc:
             logging.error("Failed to save Markdown export state: %s", exc)
 
+    @staticmethod
+    def _get_content_page_order(notebook: Dict) -> Optional[List[str]]:
+        """Return ordered page IDs from the notebook's .content file."""
+        metadata_file = notebook.get("metadata_file")
+        if not metadata_file:
+            return None
+        content_path = metadata_file.with_suffix(".content")
+        if not content_path.exists():
+            return None
+        try:
+            with open(content_path, "r", encoding="utf-8") as fh:
+                data = json.load(fh)
+            page_ids = data.get("pages", [])
+            if not page_ids:
+                cpages = data.get("cPages", {}).get("pages", [])
+                page_ids = [p["id"] for p in cpages if "id" in p]
+            return page_ids if page_ids else None
+        except Exception:
+            return None
+
     def _needs_export(self, notebook_uuid: str, pdf_path: Path) -> bool:
         """Return True if the notebook has changed since the last export."""
         if notebook_uuid not in self._state:
@@ -334,6 +354,7 @@ class MarkdownExporter:
         force: bool = False,
         page_pdfs: Optional[List[Path]] = None,
         on_page_done: Optional[callable] = None,
+        changed_page_ids: Optional[set] = None,
     ) -> Optional[Path]:
         """Export a notebook as a folder with one Markdown file per page.
 
@@ -342,8 +363,12 @@ class MarkdownExporter:
             pdf_path: Path to the converted PDF for this notebook.
             force: Re-export even if the notebook hasn't changed.
             page_pdfs: Optional list of cached per-page PDF paths.
-            on_page_done: Callback ``(page_num, total_pages)`` called after
-                each page is processed.
+            on_page_done: Callback ``(page_num, total_pages, cached=False)``
+                called after each page is processed.  *cached* is True when
+                the page was skipped because its PDF hash was unchanged.
+            changed_page_ids: Set of page IDs (UUID stems) known to have
+                changed in the backup.  When provided, pages in this set
+                are always re-exported regardless of hash state.
 
         Returns:
             Path to the notebook folder, or *None* on failure.
@@ -352,8 +377,9 @@ class MarkdownExporter:
         name = notebook["name"]
         folder_path = notebook.get("folder_path", "")
 
-        # Skip if nothing changed
-        if not force and not self._needs_export(uuid, pdf_path):
+        # Skip if nothing changed (notebook-level check only when we don't
+        # have per-page PDFs — per-page hashing handles the granular case).
+        if not force and not page_pdfs and not self._needs_export(uuid, pdf_path):
             logging.debug("Skipping unchanged notebook: %s", name)
             return self._state.get(uuid, {}).get("md_path")
 
@@ -385,6 +411,17 @@ class MarkdownExporter:
             rate_limited = False
 
             for pg_idx, pg_pdf in enumerate(pages_to_process, start=1):
+                # Skip pages that haven't changed
+                if (
+                    not force
+                    and changed_page_ids is not None
+                    and pg_pdf.stem not in changed_page_ids
+                ):
+                    logging.debug("Skipping unchanged page %d of '%s'", pg_idx, name)
+                    if on_page_done:
+                        on_page_done(pg_idx, total_pages, cached=True)
+                    continue
+
                 # Rasterise page to image
                 page_image: Optional[Path] = None
                 page_images: List[Path] = []  # noqa: F841
@@ -485,6 +522,7 @@ class MarkdownExporter:
         force: bool = False,
         converted_pages: Optional[Dict[str, List[Path]]] = None,
         page_filter: Optional[int] = None,
+        updated_pages: Optional[Dict[str, set]] = None,
     ) -> Tuple[int, int]:
         """Export all notebooks to Markdown.
 
@@ -496,6 +534,8 @@ class MarkdownExporter:
             converted_pages: Dict mapping notebook UUID to list of per-page
                 PDF paths produced by the PDF conversion step.  When provided,
                 these are used directly instead of scanning the cache dir.
+            updated_pages: Dict mapping notebook UUID to set of changed page
+                IDs from the backup stage.
 
         Returns:
             ``(exported_count, skipped_count)`` tuple.
@@ -509,6 +549,8 @@ class MarkdownExporter:
         # Count total pages for progress bar
         total_pages = 0
         nb_page_counts = []
+        # Count total pages that actually need OCR processing
+        total_ocr_pages = 0
         for nb in doc_notebooks:
             count = 0
             if converted_pages and nb["uuid"] in converted_pages:
@@ -517,10 +559,23 @@ class MarkdownExporter:
                 cache = self.backup_dir / "PagePDFs" / nb["uuid"]
                 if cache.exists():
                     count = len([p for p in cache.glob("*.pdf") if not p.stem.endswith("_content")])
-            count = max(count, 1)  # at least 1 so progress always advances
+            count = max(count, 1)
             nb_page_counts.append(count)
             total_pages += count
 
+            # Count only pages that will actually be processed
+            nb_changed = None
+            if updated_pages is not None:
+                nb_changed = updated_pages.get(nb["uuid"], set())
+            if nb_changed is not None and converted_pages and nb["uuid"] in converted_pages:
+                total_ocr_pages += sum(
+                    1 for p in converted_pages[nb["uuid"]] if p.stem in nb_changed
+                )
+            else:
+                total_ocr_pages += count
+
+        ocr_counter = [0]
+
         with create_progress("Exporting") as progress:
             task = progress.add_task("Exporting", total=total_pages)
 
@@ -549,10 +604,20 @@ class MarkdownExporter:
                 else:
                     page_cache_dir = self.backup_dir / "PagePDFs" / notebook["uuid"]
                     if page_cache_dir.exists():
-                        pdfs = sorted(page_cache_dir.glob("*.pdf"))
-                        pdfs = [p for p in pdfs if not p.stem.endswith("_content")]
-                        if pdfs:
-                            page_pdfs_list = pdfs
+                        pdfs_on_disk = {
+                            p.stem: p
+                            for p in page_cache_dir.glob("*.pdf")
+                            if not p.stem.endswith("_content")
+                        }
+                        if pdfs_on_disk:
+                            # Order by .content file if available
+                            ordered = self._get_content_page_order(notebook)
+                            if ordered:
+                                page_pdfs_list = [
+                                    pdfs_on_disk[pid] for pid in ordered if pid in pdfs_on_disk
+                                ]
+                            else:
+                                page_pdfs_list = sorted(pdfs_on_disk.values())
 
                 # Filter to specific page if requested
                 if page_filter and page_pdfs_list:
@@ -565,13 +630,31 @@ class MarkdownExporter:
                             len(page_pdfs_list),
                         )
 
-                def _on_page(pg_num, pg_total, _nb_name=nb_name):
-                    progress.update(
-                        task,
-                        advance=1,
-                        description=f"{_nb_name} (page {pg_num} of {pg_total})",
-                    )
-                    logging.info("MD: %s (page %d/%d)", _nb_name, pg_num, pg_total)
+                def _on_page(
+                    pg_num,
+                    pg_total,
+                    _nb_name=nb_name,
+                    _oc=ocr_counter,
+                    _total_ocr=total_ocr_pages,
+                    cached=False,
+                ):
+                    if cached:
+                        logging.info("MD: %s (page %d/%d) [cached]", _nb_name, pg_num, pg_total)
+                    else:
+                        _oc[0] += 1
+                        desc = f"OCR page {_oc[0]} of {_total_ocr} ({_nb_name} page {pg_num})"
+                        progress.update(task, advance=1, description=desc)
+                        logging.info(
+                            "MD: OCR page %d of %d (%s page %d)",
+                            _oc[0],
+                            _total_ocr,
+                            _nb_name,
+                            pg_num,
+                        )
+
+                nb_changed_pages = None
+                if updated_pages and notebook["uuid"] in updated_pages:
+                    nb_changed_pages = updated_pages[notebook["uuid"]]
 
                 result = self.export_notebook(
                     notebook,
@@ -579,6 +662,7 @@ class MarkdownExporter:
                     force=force,
                     page_pdfs=page_pdfs_list,
                     on_page_done=_on_page,
+                    changed_page_ids=nb_changed_pages,
                 )
                 # Ensure we advance the full count even if pages were fewer
                 remaining = nb_pages - (nb_pages if result else 0)

{remarkablesync-2.0.0 → remarkablesync-2.0.1}/src/rm_pdf_converter.py

@@ -167,21 +167,24 @@ def run_conversion(
             )
             page_counter = [0]  # mutable so the lambda can update it
 
-            def _on_page_done(_pc=page_counter, _nb=nb_name, _nbt=nb_total):
+            def _on_page_done(_pc=page_counter, _nb=nb_name, _nbt=nb_total, cached=False):
                 _pc[0] += 1
+                progress.update(task, advance=1)
+                suffix = " [cached]" if cached else ""
+                logging.info("PDF: %s (page %d/%d)%s", _nb, _pc[0], _nbt, suffix)
+
+            def _on_page_start(_pc=page_counter, _nb=nb_name, _nbt=nb_total):
                 progress.update(
                     task,
-                    advance=1,
-                    description=f"{_nb} (page {_pc[0]} of {_nbt})",
+                    description=f"{_nb} (page {_pc[0] + 1} of {_nbt})",
                 )
-                logging.info("PDF: %s (page %d/%d)", _nb, _pc[0], _nbt)
 
             progress.update(task, description=f"{nb_name} (page 0 of {nb_total})")
 
             try:
                 notebook_changed_pages = None
-                if updated_pages and notebook["uuid"] in updated_pages:
-                    notebook_changed_pages = updated_pages[notebook["uuid"]]
+                if updated_pages is not None:
+                    notebook_changed_pages = updated_pages.get(notebook["uuid"], set())
 
                 results = convert_notebook(
                     notebook,
@@ -190,15 +193,13 @@ def run_conversion(
                     template_renderer,
                     changed_page_ids=notebook_changed_pages,
                     on_page_done=_on_page_done,
+                    on_page_start=_on_page_start,
                 )
                 if results["output_files"]:
                     successful += 1
-                    # Collect per-page PDFs for downstream (MD export)
-                    cache_dir = results.get("page_cache_dir")
-                    if cache_dir and cache_dir.exists():
-                        page_pdfs = sorted(cache_dir.glob("*.pdf"))
-                        # Exclude intermediate *_content.pdf files
-                        page_pdfs = [p for p in page_pdfs if not p.stem.endswith("_content")]
+                    # Use ordered page PDFs from converter (preserves .content order)
+                    page_pdfs = results.get("page_pdfs", [])
+                    if page_pdfs:
                         converted[notebook["uuid"]] = page_pdfs
             except Exception as e:
                 print_error(f"  [ERR] Failed to convert {notebook['name']}: {e}")