rosetta-cli 3.0.6__tar.gz → 3.0.7__tar.gz

{rosetta_cli-3.0.6/rosetta_cli.egg-info → rosetta_cli-3.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rosetta-cli
-Version: 3.0.6
+Version: 3.0.7
 Summary: Rosetta CLI for publishing knowledge base content to RAGFlow
 Author-email: Rosetta Team <rosetta-support@griddynamics.com>
 License-Expression: Apache-2.0

{rosetta_cli-3.0.6 → rosetta_cli-3.0.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rosetta-cli"
-version = "3.0.6"
+version = "3.0.7"
 description = "Rosetta CLI for publishing knowledge base content to RAGFlow"
 readme = "README.md"
 requires-python = ">=3.12"

{rosetta_cli-3.0.6 → rosetta_cli-3.0.7}/rosetta_cli/rosetta_publisher.py

@@ -23,7 +23,7 @@ from pathlib import Path
 from typing import cast
 
 from .services.document_service import DocumentService
-from .services.document_data import DocumentData
+from .services.document_data import DocumentData, InvalidTextEncodingError
 from .ragflow_client import DocumentMetadata, RAGFlowClient, RAGFlowClientError
 from .rosetta_config import DEFAULT_PARSE_TIMEOUT
 from .typing_utils import DocumentLike, JsonDict
@@ -171,6 +171,7 @@ class ContentPublisher:
         print("Reading files...")
         all_caches = []   # ALL successfully read caches (for cleanup/orphan detection)
         results = []
+        read_failures: list[Path] = []
 
         for file in files:
             try:
@@ -178,17 +179,32 @@ class ContentPublisher:
                 cache = DocumentData.from_file(
                     file,
                     self.workspace_root,
-                    self.file_extensions,
                     publish_root=folder,
                 )
                 all_caches.append(cache)
 
             except Exception as e:
+                read_failures.append(file)
                 print(f"✗ Error reading {file.name}: {e}")
-                continue
-
-        # Duplicate cleanup: remove stale server copies of the same file (before publish)
-        self._cleanup_duplicates(all_caches, dry_run)
+                results.append(PublishResult(
+                    success=False,
+                    document_id="",
+                    file_path=str(file),
+                    tags=[],
+                    error=str(e),
+                ))
+
+        # Destructive cleanup requires a complete local inventory. If even one
+        # file could not be read, preserve remote documents until a full run can
+        # determine which paths are truly stale.
+        if read_failures:
+            print(
+                "\nDuplicate cleanup skipped "
+                f"({len(read_failures)} local file(s) could not be read)"
+            )
+        else:
+            # Remove stale server copies of the same file before publishing.
+            self._cleanup_duplicates(all_caches, dry_run)
 
         # Check for changes after cleanup so skip decisions reflect current server state.
         doc_caches = []   # Only changed caches (for publishing)
@@ -271,15 +287,23 @@ class ContentPublisher:
         # docs — running orphan detection would delete everything else.
         is_full_publish = folder.name == "instructions" or folder.parent == self.workspace_root
         if is_full_publish:
-            managed_domains_by_dataset: dict[str, set[str]] = {}
-            for cache in all_caches:
-                dataset_name = self._resolve_dataset_name({"release": cache.release})
-                managed_domains_by_dataset.setdefault(dataset_name, set()).add(cache.domain)
-            self._cleanup_orphans(
-                all_caches,
-                dry_run,
-                managed_domains_by_dataset=managed_domains_by_dataset,
-            )
+            if read_failures:
+                # The local inventory is incomplete, so a failed read must not
+                # make an existing server document look like an orphan.
+                print(
+                    "\nOrphan detection skipped "
+                    f"({len(read_failures)} local file(s) could not be read)"
+                )
+            else:
+                managed_domains_by_dataset: dict[str, set[str]] = {}
+                for cache in all_caches:
+                    dataset_name = self._resolve_dataset_name({"release": cache.release})
+                    managed_domains_by_dataset.setdefault(dataset_name, set()).add(cache.domain)
+                self._cleanup_orphans(
+                    all_caches,
+                    dry_run,
+                    managed_domains_by_dataset=managed_domains_by_dataset,
+                )
         else:
             print("\nOrphan detection skipped (subfolder publish)")
 
@@ -347,12 +371,20 @@ class ContentPublisher:
                 )
             
             # Create cache on-the-fly (still better than reading multiple times)
-            cache = DocumentData.from_file(
-                file,
-                self.workspace_root,
-                self.file_extensions,
-                publish_root=file.parent,
-            )
+            try:
+                cache = DocumentData.from_file(
+                    file,
+                    self.workspace_root,
+                    publish_root=file.parent,
+                )
+            except InvalidTextEncodingError as e:
+                return PublishResult(
+                    success=False,
+                    document_id="",
+                    file_path=str(file),
+                    tags=[],
+                    error=str(e),
+                )
             
             ims_doc_id = cache.ims_doc_id
             content = cache.content

{rosetta_cli-3.0.6 → rosetta_cli-3.0.7}/rosetta_cli/services/document_data.py

@@ -18,6 +18,13 @@ except Exception:  # pragma: no cover - guarded at runtime
     frontmatter = None
 
 
+class InvalidTextEncodingError(ValueError):
+    """Raised when a text-suffix file cannot be decoded as UTF-8."""
+
+    def __init__(self, file_path: Path):
+        super().__init__(f"{file_path} is not valid UTF-8")
+
+
 def _is_r2_or_later(release: str) -> bool:
     """Check if release is r2 or later (r2, r3, r2.5, etc.)."""
     if not release or not release.startswith("r"):
@@ -85,8 +92,8 @@ class DocumentData:
         if is_text:
             try:
                 content_str = content.decode("utf-8")
-            except UnicodeDecodeError:
-                content_str = content.decode("utf-8", errors="ignore")
+            except UnicodeDecodeError as error:
+                raise InvalidTextEncodingError(file_path) from error
 
         # Count lines platform-independently: \r\n, \n\r, \r, \n all count as separators
         line_count = None
@@ -111,7 +118,7 @@ class DocumentData:
         doc_title = cls._compute_doc_title(parsed_path, file_path.name)
 
         content_hash = cls._calculate_hash(
-            content_str if content_str is not None else str(len(content)),
+            content_str if content_str is not None else hashlib.sha256(content).hexdigest(),
             tags,
             domain,
             release,

{rosetta_cli-3.0.6 → rosetta_cli-3.0.7/rosetta_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rosetta-cli
-Version: 3.0.6
+Version: 3.0.7
 Summary: Rosetta CLI for publishing knowledge base content to RAGFlow
 Author-email: Rosetta Team <rosetta-support@griddynamics.com>
 License-Expression: Apache-2.0