rosetta-cli 3.0.5__tar.gz → 3.0.6__tar.gz

{rosetta_cli-3.0.5/rosetta_cli.egg-info → rosetta_cli-3.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rosetta-cli
-Version: 3.0.5
+Version: 3.0.6
 Summary: Rosetta CLI for publishing knowledge base content to RAGFlow
 Author-email: Rosetta Team <rosetta-support@griddynamics.com>
 License-Expression: Apache-2.0

{rosetta_cli-3.0.5 → rosetta_cli-3.0.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rosetta-cli"
-version = "3.0.5"
+version = "3.0.6"
 description = "Rosetta CLI for publishing knowledge base content to RAGFlow"
 readme = "README.md"
 requires-python = ">=3.12"

{rosetta_cli-3.0.5 → rosetta_cli-3.0.6}/rosetta_cli/commands/cleanup_command.py

@@ -87,7 +87,16 @@ class CleanupCommand(BaseCommand):
         dataset_service: DatasetService,
         args: CommandArgs,
     ) -> list[DocumentLike]:
-        """Get documents to delete based on filters."""
+        """Get documents to delete based on filters.
+
+        Accepted limitation: every branch below reads one page, bounded by
+        RAGFLOW_PAGE_SIZE (default 1000). On a dataset larger than that, this
+        command deletes the documents on the first page and reports success -
+        the remainder is left in place with no warning. The instruction set is
+        not expected to approach the cap, and this command serves the optional
+        MCP publishing pipeline rather than plugin delivery. Anyone pointing
+        `--dataset` at a larger dataset should know the delete is partial.
+        """
         document_service = DocumentService(self.client)
         
         if args.tags:

{rosetta_cli-3.0.5 → rosetta_cli-3.0.6}/rosetta_cli/commands/parse_command.py

@@ -11,7 +11,7 @@ from ..services.auth_service import AuthService
 
 
 from .base_command import BaseCommand
-from ..typing_utils import CommandArgs, DatasetLike, JsonDict
+from ..typing_utils import CommandArgs, DatasetLike, DocumentLike, JsonDict
 
 
 class ParseCommand(BaseCommand):
@@ -102,58 +102,57 @@ class ParseCommand(BaseCommand):
             traceback.print_exc()
             return 1
     
+    # RAGFlow `run` values that mean a document still needs parsing.
+    PARSE_REQUIRED_STATUSES = ("FAIL", "UNSTART", "CANCEL")
+
     def _get_documents_to_parse(self, dataset: DatasetLike, args: CommandArgs) -> tuple[list[JsonDict], dict[str, int]]:
         """Get documents that need parsing."""
-        document_service = DocumentService(self.client)
         docs_to_parse: list[JsonDict] = []
         status_counts = {'done': 0, 'running': 0}
         
         print(f"Checking parsing status for documents...")
         
+        # One unfiltered list call serves both modes: force selects everything,
+        # default partitions the same list client-side on `run`. Default mode used
+        # to make two calls - a server-side run= filter, plus a full list purely to
+        # tally done/running for the summary.
+        #
+        # Deliberate tradeoff, not a no-op: the server-filtered call surfaced up to
+        # page_size documents *needing parsing*, whereas page 1 of an unfiltered
+        # list surfaces only the needing-parse documents that happen to fall inside
+        # it. Once a dataset exceeds RAGFLOW_PAGE_SIZE this can therefore select
+        # fewer documents than before.
+        #
+        # Accepted limitation, not a deferred fix: every list path in this command
+        # reads page 1 only, bounded by RAGFLOW_PAGE_SIZE (default 1000). The
+        # instruction set is not expected to approach that cap - it is roughly 470
+        # files - and this command serves the optional MCP publishing pipeline,
+        # which is secondary to plugin delivery. The page-1-only read is also
+        # pre-existing: it already applied to the force branch and to the
+        # done/running tally.
+        documents = dataset.list_documents(page_size=self.config.page_size)
+        
+        selected: list[tuple[DocumentLike, str]] = []
+        for document in documents:
+            status = self._count_parse_status(status_counts, document)
+            if args.force or status in self.PARSE_REQUIRED_STATUSES:
+                selected.append((document, status))
+        
         if args.force:
-            # Force mode: all documents
-            documents = dataset.list_documents(page_size=self.config.page_size)
-            
-            print(f"Found {len(documents)} document(s) (force mode - parsing all)\n")
-            
-            for document in documents:
-                status = self._count_parse_status(status_counts, document)
-                doc_id = getattr(document, 'id', None)
-                if doc_id:
-                    docs_to_parse.append({
-                        "id": doc_id,
-                        "name": getattr(document, 'name', 'Untitled'),
-                        "dataset_id": dataset.id,
-                        "folder": ".",
-                        "status": status
-                    })
+            print(f"Found {len(selected)} document(s) (force mode - parsing all)\n")
         else:
-            # Default mode: filter by status (need parsing)
-            documents_needing_parse = document_service.list_documents_by_status(
-                dataset,
-                statuses=["FAIL", "UNSTART", "CANCEL"],
-                limit=self.config.page_size
-            )
-
-            # Tally skipped documents from the full list so the summary reflects
-            # what this run will not re-trigger.
-            all_documents = dataset.list_documents(page_size=self.config.page_size)
-            for document in all_documents:
-                self._count_parse_status(status_counts, document)
-            
-            print(f"Found {len(documents_needing_parse)} document(s) needing parsing\n")
-            
-            for document in documents_needing_parse:
-                status = self._count_parse_status(status_counts, document)
-                doc_id = getattr(document, 'id', None)
-                if doc_id:
-                    docs_to_parse.append({
-                        "id": doc_id,
-                        "name": getattr(document, 'name', 'Untitled'),
-                        "dataset_id": dataset.id,
-                        "folder": ".",
-                        "status": status
-                    })
+            print(f"Found {len(selected)} document(s) needing parsing\n")
+        
+        for document, status in selected:
+            doc_id = getattr(document, 'id', None)
+            if doc_id:
+                docs_to_parse.append({
+                    "id": doc_id,
+                    "name": getattr(document, 'name', 'Untitled'),
+                    "dataset_id": dataset.id,
+                    "folder": ".",
+                    "status": status
+                })
         
         return docs_to_parse, status_counts
 

{rosetta_cli-3.0.5 → rosetta_cli-3.0.6}/rosetta_cli/rosetta_config.py

@@ -9,7 +9,6 @@ import logging
 import os
 from dataclasses import dataclass
 from pathlib import Path
-from urllib.parse import urlparse
 
 from dotenv import load_dotenv
 
@@ -69,14 +68,40 @@ def find_env_file(env_name: str | None = None) -> Path | None:
     return None
 
 
-def _first_subdomain(url: str) -> str | None:
-    """Extract first subdomain from URL. 'https://ims-dev.example.com/' -> 'ims-dev'."""
-    try:
-        hostname = urlparse(url).hostname or ""
-        parts = hostname.split(".")
-        return parts[0] if len(parts) > 1 else None
-    except Exception:
-        return None
+# Environment keywords searched for in the RAGFlow URL, in scan order.
+#
+# ORDER IS LOAD-BEARING - do not sort or alphabetize this tuple. The scan takes the
+# first keyword that appears anywhere in the URL, so a keyword that contains another
+# keyword must be listed before the one it contains. "prod" is a substring of
+# "preprod" - the only such pair in this set - so putting "prod" first would make
+# preprod.example.com resolve to "prod". test_environment_keyword_order_is_safe in
+# tests/test_rosetta_config_environment.py enforces this invariant.
+ENVIRONMENT_KEYWORDS: tuple[str, ...] = (
+    "preprod",  # must precede "prod", which it contains
+    "prod",
+    "stag",
+    "qa",
+    "dev",
+    "test",
+    "uat",
+    "sandbox",
+    "perf",
+)
+
+
+def _environment_from_url(url: str) -> str | None:
+    """Detect an environment name from keywords in a RAGFlow URL.
+
+    Case-insensitive substring scan over the URL exactly as given (scheme, host,
+    port and path all count). The first keyword in ENVIRONMENT_KEYWORDS to appear
+    wins, and the resolved value is the keyword itself, not the surrounding label:
+    'https://ims-dev.example.com/' -> 'dev'. Returns None when no keyword appears.
+    """
+    lowered = (url or "").lower()
+    for keyword in ENVIRONMENT_KEYWORDS:
+        if keyword in lowered:
+            return keyword
+    return None
 
 
 @dataclass
@@ -98,7 +123,12 @@ class RosettaConfig:
         RAGFLOW_PAGE_SIZE: Page size for listing operations (default: 1000)
         RAGFLOW_PARSE_TIMEOUT: Timeout for parsing operations in seconds (default: 1200)
         RAGFLOW_TIMEOUT: HTTP request timeout in seconds (default: 30)
-        ENVIRONMENT: Environment name (default: "local")
+        ENVIRONMENT: Environment display label. Takes precedence over URL keyword
+            detection, but not over an explicit --env argument. When unset, the
+            label is detected from RAGFLOW_BASE_URL by a case-insensitive scan for
+            the first of preprod, prod, stag, qa, dev, test, uat, sandbox, perf to
+            appear in it (the resolved value is that keyword itself, so
+            "https://ims-dev.example.com" yields "dev"); "local" if none appear.
     
     Examples:
         >>> config = RosettaConfig.from_env(".env")
@@ -194,9 +224,11 @@ class RosettaConfig:
         Args:
             environment: Optional explicit environment name (e.g., "local",
                 "dev", "remote"). If provided, this value is used and takes
-                precedence over the ENVIRONMENT environment variable. If not
-                provided, the ENVIRONMENT variable is used, defaulting to
-                "local" when unset.
+                precedence over everything else. If not provided, the
+                ENVIRONMENT variable is used; failing that, the label is
+                detected from a keyword in RAGFLOW_BASE_URL (see
+                ENVIRONMENT_KEYWORDS), defaulting to "local" when no keyword
+                appears.
         
         Returns:
             RosettaConfig instance
@@ -213,9 +245,11 @@ class RosettaConfig:
         api_key = os.getenv("RAGFLOW_API_KEY", "")
         dataset_default = os.getenv("RAGFLOW_DATASET_DEFAULT", "aia")
         dataset_template = os.getenv("RAGFLOW_DATASET_TEMPLATE", "aia-{release}")
-        # fallback to ENVIRONMENT env var, or default to "local"
+        # Environment is a display label only - nothing branches on its value.
+        # Precedence: explicit argument (--env) > ENVIRONMENT > keyword detected in
+        # the RAGFlow URL > "local".
         if not environment:
-            environment = os.getenv("ENVIRONMENT") or _first_subdomain(base_url) or "local"
+            environment = os.getenv("ENVIRONMENT") or _environment_from_url(base_url) or "local"
         
         # Dataset creation settings
         embedding_model = os.getenv("RAGFLOW_EMBEDDING_MODEL") or None

{rosetta_cli-3.0.5 → rosetta_cli-3.0.6}/rosetta_cli/rosetta_publisher.py

@@ -641,13 +641,16 @@ class ContentPublisher:
                 return name[: -len(path.name)] + new_name
 
             # Collect all docs to delete into one list: (doc, label)
-            # Start with unmanaged: incomplete metadata (ims_doc_id or original_path absent)
+            # Start with managed docs carrying incomplete metadata: original_path is
+            # set (so the doc is ours) but ims_doc_id is absent. A doc with no
+            # original_path is custom/unmanaged and is never touched - see the
+            # docstring guarantee, which _cleanup_orphans enforces the same way.
             duplicates: list[tuple[DocumentLike, str]] = []
             for doc in all_docs:
                 meta = getattr(doc, "meta_fields", {}) or {}
                 ims_doc_id = meta.get("ims_doc_id") if isinstance(meta, dict) else getattr(meta, "ims_doc_id", None)
                 doc_original_path = meta.get("original_path", "") if isinstance(meta, dict) else getattr(meta, "original_path", "")
-                if not ims_doc_id or not doc_original_path:
+                if doc_original_path and not ims_doc_id:
                     doc_name = getattr(doc, "name", "") or doc.id
                     duplicates.append((doc, doc_name))
 
@@ -674,8 +677,10 @@ class ContentPublisher:
                     duplicates.append((doc, original_path))
 
             # Name duplicates: foo.md + foo(1).md + foo(2).md ...
+            # Grouped over managed_docs, not all_docs: a custom doc whose stripped
+            # name happens to collide with a managed one must not be deleted.
             name_groups: dict[str, list[DocumentLike]] = defaultdict(list)
-            for doc in all_docs:
+            for doc in managed_docs:
                 doc_name = getattr(doc, "name", "") or ""
                 if not doc_name:
                     continue

{rosetta_cli-3.0.5 → rosetta_cli-3.0.6/rosetta_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rosetta-cli
-Version: 3.0.5
+Version: 3.0.6
 Summary: Rosetta CLI for publishing knowledge base content to RAGFlow
 Author-email: Rosetta Team <rosetta-support@griddynamics.com>
 License-Expression: Apache-2.0