rosetta-cli 3.0.5__tar.gz → 3.0.7__tar.gz

{rosetta_cli-3.0.5/rosetta_cli.egg-info → rosetta_cli-3.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rosetta-cli
-Version: 3.0.5
+Version: 3.0.7
 Summary: Rosetta CLI for publishing knowledge base content to RAGFlow
 Author-email: Rosetta Team <rosetta-support@griddynamics.com>
 License-Expression: Apache-2.0

{rosetta_cli-3.0.5 → rosetta_cli-3.0.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rosetta-cli"
-version = "3.0.5"
+version = "3.0.7"
 description = "Rosetta CLI for publishing knowledge base content to RAGFlow"
 readme = "README.md"
 requires-python = ">=3.12"

{rosetta_cli-3.0.5 → rosetta_cli-3.0.7}/rosetta_cli/commands/cleanup_command.py

@@ -87,7 +87,16 @@ class CleanupCommand(BaseCommand):
         dataset_service: DatasetService,
         args: CommandArgs,
     ) -> list[DocumentLike]:
-        """Get documents to delete based on filters."""
+        """Get documents to delete based on filters.
+
+        Accepted limitation: every branch below reads one page, bounded by
+        RAGFLOW_PAGE_SIZE (default 1000). On a dataset larger than that, this
+        command deletes the documents on the first page and reports success -
+        the remainder is left in place with no warning. The instruction set is
+        not expected to approach the cap, and this command serves the optional
+        MCP publishing pipeline rather than plugin delivery. Anyone pointing
+        `--dataset` at a larger dataset should know the delete is partial.
+        """
         document_service = DocumentService(self.client)
         
         if args.tags:

{rosetta_cli-3.0.5 → rosetta_cli-3.0.7}/rosetta_cli/commands/parse_command.py

@@ -11,7 +11,7 @@ from ..services.auth_service import AuthService
 
 
 from .base_command import BaseCommand
-from ..typing_utils import CommandArgs, DatasetLike, JsonDict
+from ..typing_utils import CommandArgs, DatasetLike, DocumentLike, JsonDict
 
 
 class ParseCommand(BaseCommand):
@@ -102,58 +102,57 @@ class ParseCommand(BaseCommand):
             traceback.print_exc()
             return 1
     
+    # RAGFlow `run` values that mean a document still needs parsing.
+    PARSE_REQUIRED_STATUSES = ("FAIL", "UNSTART", "CANCEL")
+
     def _get_documents_to_parse(self, dataset: DatasetLike, args: CommandArgs) -> tuple[list[JsonDict], dict[str, int]]:
         """Get documents that need parsing."""
-        document_service = DocumentService(self.client)
         docs_to_parse: list[JsonDict] = []
         status_counts = {'done': 0, 'running': 0}
         
         print(f"Checking parsing status for documents...")
         
+        # One unfiltered list call serves both modes: force selects everything,
+        # default partitions the same list client-side on `run`. Default mode used
+        # to make two calls - a server-side run= filter, plus a full list purely to
+        # tally done/running for the summary.
+        #
+        # Deliberate tradeoff, not a no-op: the server-filtered call surfaced up to
+        # page_size documents *needing parsing*, whereas page 1 of an unfiltered
+        # list surfaces only the needing-parse documents that happen to fall inside
+        # it. Once a dataset exceeds RAGFLOW_PAGE_SIZE this can therefore select
+        # fewer documents than before.
+        #
+        # Accepted limitation, not a deferred fix: every list path in this command
+        # reads page 1 only, bounded by RAGFLOW_PAGE_SIZE (default 1000). The
+        # instruction set is not expected to approach that cap - it is roughly 470
+        # files - and this command serves the optional MCP publishing pipeline,
+        # which is secondary to plugin delivery. The page-1-only read is also
+        # pre-existing: it already applied to the force branch and to the
+        # done/running tally.
+        documents = dataset.list_documents(page_size=self.config.page_size)
+        
+        selected: list[tuple[DocumentLike, str]] = []
+        for document in documents:
+            status = self._count_parse_status(status_counts, document)
+            if args.force or status in self.PARSE_REQUIRED_STATUSES:
+                selected.append((document, status))
+        
         if args.force:
-            # Force mode: all documents
-            documents = dataset.list_documents(page_size=self.config.page_size)
-            
-            print(f"Found {len(documents)} document(s) (force mode - parsing all)\n")
-            
-            for document in documents:
-                status = self._count_parse_status(status_counts, document)
-                doc_id = getattr(document, 'id', None)
-                if doc_id:
-                    docs_to_parse.append({
-                        "id": doc_id,
-                        "name": getattr(document, 'name', 'Untitled'),
-                        "dataset_id": dataset.id,
-                        "folder": ".",
-                        "status": status
-                    })
+            print(f"Found {len(selected)} document(s) (force mode - parsing all)\n")
         else:
-            # Default mode: filter by status (need parsing)
-            documents_needing_parse = document_service.list_documents_by_status(
-                dataset,
-                statuses=["FAIL", "UNSTART", "CANCEL"],
-                limit=self.config.page_size
-            )
-
-            # Tally skipped documents from the full list so the summary reflects
-            # what this run will not re-trigger.
-            all_documents = dataset.list_documents(page_size=self.config.page_size)
-            for document in all_documents:
-                self._count_parse_status(status_counts, document)
-            
-            print(f"Found {len(documents_needing_parse)} document(s) needing parsing\n")
-            
-            for document in documents_needing_parse:
-                status = self._count_parse_status(status_counts, document)
-                doc_id = getattr(document, 'id', None)
-                if doc_id:
-                    docs_to_parse.append({
-                        "id": doc_id,
-                        "name": getattr(document, 'name', 'Untitled'),
-                        "dataset_id": dataset.id,
-                        "folder": ".",
-                        "status": status
-                    })
+            print(f"Found {len(selected)} document(s) needing parsing\n")
+        
+        for document, status in selected:
+            doc_id = getattr(document, 'id', None)
+            if doc_id:
+                docs_to_parse.append({
+                    "id": doc_id,
+                    "name": getattr(document, 'name', 'Untitled'),
+                    "dataset_id": dataset.id,
+                    "folder": ".",
+                    "status": status
+                })
         
         return docs_to_parse, status_counts
 

{rosetta_cli-3.0.5 → rosetta_cli-3.0.7}/rosetta_cli/rosetta_config.py

@@ -9,7 +9,6 @@ import logging
 import os
 from dataclasses import dataclass
 from pathlib import Path
-from urllib.parse import urlparse
 
 from dotenv import load_dotenv
 
@@ -69,14 +68,40 @@ def find_env_file(env_name: str | None = None) -> Path | None:
     return None
 
 
-def _first_subdomain(url: str) -> str | None:
-    """Extract first subdomain from URL. 'https://ims-dev.example.com/' -> 'ims-dev'."""
-    try:
-        hostname = urlparse(url).hostname or ""
-        parts = hostname.split(".")
-        return parts[0] if len(parts) > 1 else None
-    except Exception:
-        return None
+# Environment keywords searched for in the RAGFlow URL, in scan order.
+#
+# ORDER IS LOAD-BEARING - do not sort or alphabetize this tuple. The scan takes the
+# first keyword that appears anywhere in the URL, so a keyword that contains another
+# keyword must be listed before the one it contains. "prod" is a substring of
+# "preprod" - the only such pair in this set - so putting "prod" first would make
+# preprod.example.com resolve to "prod". test_environment_keyword_order_is_safe in
+# tests/test_rosetta_config_environment.py enforces this invariant.
+ENVIRONMENT_KEYWORDS: tuple[str, ...] = (
+    "preprod",  # must precede "prod", which it contains
+    "prod",
+    "stag",
+    "qa",
+    "dev",
+    "test",
+    "uat",
+    "sandbox",
+    "perf",
+)
+
+
+def _environment_from_url(url: str) -> str | None:
+    """Detect an environment name from keywords in a RAGFlow URL.
+
+    Case-insensitive substring scan over the URL exactly as given (scheme, host,
+    port and path all count). The first keyword in ENVIRONMENT_KEYWORDS to appear
+    wins, and the resolved value is the keyword itself, not the surrounding label:
+    'https://ims-dev.example.com/' -> 'dev'. Returns None when no keyword appears.
+    """
+    lowered = (url or "").lower()
+    for keyword in ENVIRONMENT_KEYWORDS:
+        if keyword in lowered:
+            return keyword
+    return None
 
 
 @dataclass
@@ -98,7 +123,12 @@ class RosettaConfig:
         RAGFLOW_PAGE_SIZE: Page size for listing operations (default: 1000)
         RAGFLOW_PARSE_TIMEOUT: Timeout for parsing operations in seconds (default: 1200)
         RAGFLOW_TIMEOUT: HTTP request timeout in seconds (default: 30)
-        ENVIRONMENT: Environment name (default: "local")
+        ENVIRONMENT: Environment display label. Takes precedence over URL keyword
+            detection, but not over an explicit --env argument. When unset, the
+            label is detected from RAGFLOW_BASE_URL by a case-insensitive scan for
+            the first of preprod, prod, stag, qa, dev, test, uat, sandbox, perf to
+            appear in it (the resolved value is that keyword itself, so
+            "https://ims-dev.example.com" yields "dev"); "local" if none appear.
     
     Examples:
         >>> config = RosettaConfig.from_env(".env")
@@ -194,9 +224,11 @@ class RosettaConfig:
         Args:
             environment: Optional explicit environment name (e.g., "local",
                 "dev", "remote"). If provided, this value is used and takes
-                precedence over the ENVIRONMENT environment variable. If not
-                provided, the ENVIRONMENT variable is used, defaulting to
-                "local" when unset.
+                precedence over everything else. If not provided, the
+                ENVIRONMENT variable is used; failing that, the label is
+                detected from a keyword in RAGFLOW_BASE_URL (see
+                ENVIRONMENT_KEYWORDS), defaulting to "local" when no keyword
+                appears.
         
         Returns:
             RosettaConfig instance
@@ -213,9 +245,11 @@ class RosettaConfig:
         api_key = os.getenv("RAGFLOW_API_KEY", "")
         dataset_default = os.getenv("RAGFLOW_DATASET_DEFAULT", "aia")
         dataset_template = os.getenv("RAGFLOW_DATASET_TEMPLATE", "aia-{release}")
-        # fallback to ENVIRONMENT env var, or default to "local"
+        # Environment is a display label only - nothing branches on its value.
+        # Precedence: explicit argument (--env) > ENVIRONMENT > keyword detected in
+        # the RAGFlow URL > "local".
         if not environment:
-            environment = os.getenv("ENVIRONMENT") or _first_subdomain(base_url) or "local"
+            environment = os.getenv("ENVIRONMENT") or _environment_from_url(base_url) or "local"
         
         # Dataset creation settings
         embedding_model = os.getenv("RAGFLOW_EMBEDDING_MODEL") or None

{rosetta_cli-3.0.5 → rosetta_cli-3.0.7}/rosetta_cli/rosetta_publisher.py

@@ -23,7 +23,7 @@ from pathlib import Path
 from typing import cast
 
 from .services.document_service import DocumentService
-from .services.document_data import DocumentData
+from .services.document_data import DocumentData, InvalidTextEncodingError
 from .ragflow_client import DocumentMetadata, RAGFlowClient, RAGFlowClientError
 from .rosetta_config import DEFAULT_PARSE_TIMEOUT
 from .typing_utils import DocumentLike, JsonDict
@@ -171,6 +171,7 @@ class ContentPublisher:
         print("Reading files...")
         all_caches = []   # ALL successfully read caches (for cleanup/orphan detection)
         results = []
+        read_failures: list[Path] = []
 
         for file in files:
             try:
@@ -178,17 +179,32 @@ class ContentPublisher:
                 cache = DocumentData.from_file(
                     file,
                     self.workspace_root,
-                    self.file_extensions,
                     publish_root=folder,
                 )
                 all_caches.append(cache)
 
             except Exception as e:
+                read_failures.append(file)
                 print(f"✗ Error reading {file.name}: {e}")
-                continue
-
-        # Duplicate cleanup: remove stale server copies of the same file (before publish)
-        self._cleanup_duplicates(all_caches, dry_run)
+                results.append(PublishResult(
+                    success=False,
+                    document_id="",
+                    file_path=str(file),
+                    tags=[],
+                    error=str(e),
+                ))
+
+        # Destructive cleanup requires a complete local inventory. If even one
+        # file could not be read, preserve remote documents until a full run can
+        # determine which paths are truly stale.
+        if read_failures:
+            print(
+                "\nDuplicate cleanup skipped "
+                f"({len(read_failures)} local file(s) could not be read)"
+            )
+        else:
+            # Remove stale server copies of the same file before publishing.
+            self._cleanup_duplicates(all_caches, dry_run)
 
         # Check for changes after cleanup so skip decisions reflect current server state.
         doc_caches = []   # Only changed caches (for publishing)
@@ -271,15 +287,23 @@ class ContentPublisher:
         # docs — running orphan detection would delete everything else.
         is_full_publish = folder.name == "instructions" or folder.parent == self.workspace_root
         if is_full_publish:
-            managed_domains_by_dataset: dict[str, set[str]] = {}
-            for cache in all_caches:
-                dataset_name = self._resolve_dataset_name({"release": cache.release})
-                managed_domains_by_dataset.setdefault(dataset_name, set()).add(cache.domain)
-            self._cleanup_orphans(
-                all_caches,
-                dry_run,
-                managed_domains_by_dataset=managed_domains_by_dataset,
-            )
+            if read_failures:
+                # The local inventory is incomplete, so a failed read must not
+                # make an existing server document look like an orphan.
+                print(
+                    "\nOrphan detection skipped "
+                    f"({len(read_failures)} local file(s) could not be read)"
+                )
+            else:
+                managed_domains_by_dataset: dict[str, set[str]] = {}
+                for cache in all_caches:
+                    dataset_name = self._resolve_dataset_name({"release": cache.release})
+                    managed_domains_by_dataset.setdefault(dataset_name, set()).add(cache.domain)
+                self._cleanup_orphans(
+                    all_caches,
+                    dry_run,
+                    managed_domains_by_dataset=managed_domains_by_dataset,
+                )
         else:
             print("\nOrphan detection skipped (subfolder publish)")
 
@@ -347,12 +371,20 @@ class ContentPublisher:
                 )
             
             # Create cache on-the-fly (still better than reading multiple times)
-            cache = DocumentData.from_file(
-                file,
-                self.workspace_root,
-                self.file_extensions,
-                publish_root=file.parent,
-            )
+            try:
+                cache = DocumentData.from_file(
+                    file,
+                    self.workspace_root,
+                    publish_root=file.parent,
+                )
+            except InvalidTextEncodingError as e:
+                return PublishResult(
+                    success=False,
+                    document_id="",
+                    file_path=str(file),
+                    tags=[],
+                    error=str(e),
+                )
             
             ims_doc_id = cache.ims_doc_id
             content = cache.content
@@ -641,13 +673,16 @@ class ContentPublisher:
                 return name[: -len(path.name)] + new_name
 
             # Collect all docs to delete into one list: (doc, label)
-            # Start with unmanaged: incomplete metadata (ims_doc_id or original_path absent)
+            # Start with managed docs carrying incomplete metadata: original_path is
+            # set (so the doc is ours) but ims_doc_id is absent. A doc with no
+            # original_path is custom/unmanaged and is never touched - see the
+            # docstring guarantee, which _cleanup_orphans enforces the same way.
             duplicates: list[tuple[DocumentLike, str]] = []
             for doc in all_docs:
                 meta = getattr(doc, "meta_fields", {}) or {}
                 ims_doc_id = meta.get("ims_doc_id") if isinstance(meta, dict) else getattr(meta, "ims_doc_id", None)
                 doc_original_path = meta.get("original_path", "") if isinstance(meta, dict) else getattr(meta, "original_path", "")
-                if not ims_doc_id or not doc_original_path:
+                if doc_original_path and not ims_doc_id:
                     doc_name = getattr(doc, "name", "") or doc.id
                     duplicates.append((doc, doc_name))
 
@@ -674,8 +709,10 @@ class ContentPublisher:
                     duplicates.append((doc, original_path))
 
             # Name duplicates: foo.md + foo(1).md + foo(2).md ...
+            # Grouped over managed_docs, not all_docs: a custom doc whose stripped
+            # name happens to collide with a managed one must not be deleted.
             name_groups: dict[str, list[DocumentLike]] = defaultdict(list)
-            for doc in all_docs:
+            for doc in managed_docs:
                 doc_name = getattr(doc, "name", "") or ""
                 if not doc_name:
                     continue

{rosetta_cli-3.0.5 → rosetta_cli-3.0.7}/rosetta_cli/services/document_data.py

@@ -18,6 +18,13 @@ except Exception:  # pragma: no cover - guarded at runtime
     frontmatter = None
 
 
+class InvalidTextEncodingError(ValueError):
+    """Raised when a text-suffix file cannot be decoded as UTF-8."""
+
+    def __init__(self, file_path: Path):
+        super().__init__(f"{file_path} is not valid UTF-8")
+
+
 def _is_r2_or_later(release: str) -> bool:
     """Check if release is r2 or later (r2, r3, r2.5, etc.)."""
     if not release or not release.startswith("r"):
@@ -85,8 +92,8 @@ class DocumentData:
         if is_text:
             try:
                 content_str = content.decode("utf-8")
-            except UnicodeDecodeError:
-                content_str = content.decode("utf-8", errors="ignore")
+            except UnicodeDecodeError as error:
+                raise InvalidTextEncodingError(file_path) from error
 
         # Count lines platform-independently: \r\n, \n\r, \r, \n all count as separators
         line_count = None
@@ -111,7 +118,7 @@ class DocumentData:
         doc_title = cls._compute_doc_title(parsed_path, file_path.name)
 
         content_hash = cls._calculate_hash(
-            content_str if content_str is not None else str(len(content)),
+            content_str if content_str is not None else hashlib.sha256(content).hexdigest(),
             tags,
             domain,
             release,

{rosetta_cli-3.0.5 → rosetta_cli-3.0.7/rosetta_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rosetta-cli
-Version: 3.0.5
+Version: 3.0.7
 Summary: Rosetta CLI for publishing knowledge base content to RAGFlow
 Author-email: Rosetta Team <rosetta-support@griddynamics.com>
 License-Expression: Apache-2.0