rosetta-cli 3.0.4__tar.gz → 3.0.6__tar.gz

{rosetta_cli-3.0.4/rosetta_cli.egg-info → rosetta_cli-3.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rosetta-cli
-Version: 3.0.4
+Version: 3.0.6
 Summary: Rosetta CLI for publishing knowledge base content to RAGFlow
 Author-email: Rosetta Team <rosetta-support@griddynamics.com>
 License-Expression: Apache-2.0

{rosetta_cli-3.0.4 → rosetta_cli-3.0.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rosetta-cli"
-version = "3.0.4"
+version = "3.0.6"
 description = "Rosetta CLI for publishing knowledge base content to RAGFlow"
 readme = "README.md"
 requires-python = ">=3.12"

{rosetta_cli-3.0.4 → rosetta_cli-3.0.6}/rosetta_cli/cli.py

@@ -378,7 +378,8 @@ Frontmatter Metadata (publish flow):
             chunk_method=config.chunk_method,
             parser_config=config.parser_config,
             page_size=config.page_size,
-            timeout=config.timeout
+            dataset_default=config.dataset_default,
+            timeout=config.timeout,
         )
         
         return execute_command(args.command, args, client, config)

{rosetta_cli-3.0.4 → rosetta_cli-3.0.6}/rosetta_cli/commands/cleanup_command.py

@@ -87,7 +87,16 @@ class CleanupCommand(BaseCommand):
         dataset_service: DatasetService,
         args: CommandArgs,
     ) -> list[DocumentLike]:
-        """Get documents to delete based on filters."""
+        """Get documents to delete based on filters.
+
+        Accepted limitation: every branch below reads one page, bounded by
+        RAGFLOW_PAGE_SIZE (default 1000). On a dataset larger than that, this
+        command deletes the documents on the first page and reports success -
+        the remainder is left in place with no warning. The instruction set is
+        not expected to approach the cap, and this command serves the optional
+        MCP publishing pipeline rather than plugin delivery. Anyone pointing
+        `--dataset` at a larger dataset should know the delete is partial.
+        """
         document_service = DocumentService(self.client)
         
         if args.tags:

{rosetta_cli-3.0.4 → rosetta_cli-3.0.6}/rosetta_cli/commands/parse_command.py

@@ -11,7 +11,7 @@ from ..services.auth_service import AuthService
 
 
 from .base_command import BaseCommand
-from ..typing_utils import CommandArgs, DatasetLike, JsonDict
+from ..typing_utils import CommandArgs, DatasetLike, DocumentLike, JsonDict
 
 
 class ParseCommand(BaseCommand):
@@ -102,58 +102,57 @@ class ParseCommand(BaseCommand):
             traceback.print_exc()
             return 1
     
+    # RAGFlow `run` values that mean a document still needs parsing.
+    PARSE_REQUIRED_STATUSES = ("FAIL", "UNSTART", "CANCEL")
+
     def _get_documents_to_parse(self, dataset: DatasetLike, args: CommandArgs) -> tuple[list[JsonDict], dict[str, int]]:
         """Get documents that need parsing."""
-        document_service = DocumentService(self.client)
         docs_to_parse: list[JsonDict] = []
         status_counts = {'done': 0, 'running': 0}
         
         print(f"Checking parsing status for documents...")
         
+        # One unfiltered list call serves both modes: force selects everything,
+        # default partitions the same list client-side on `run`. Default mode used
+        # to make two calls - a server-side run= filter, plus a full list purely to
+        # tally done/running for the summary.
+        #
+        # Deliberate tradeoff, not a no-op: the server-filtered call surfaced up to
+        # page_size documents *needing parsing*, whereas page 1 of an unfiltered
+        # list surfaces only the needing-parse documents that happen to fall inside
+        # it. Once a dataset exceeds RAGFLOW_PAGE_SIZE this can therefore select
+        # fewer documents than before.
+        #
+        # Accepted limitation, not a deferred fix: every list path in this command
+        # reads page 1 only, bounded by RAGFLOW_PAGE_SIZE (default 1000). The
+        # instruction set is not expected to approach that cap - it is roughly 470
+        # files - and this command serves the optional MCP publishing pipeline,
+        # which is secondary to plugin delivery. The page-1-only read is also
+        # pre-existing: it already applied to the force branch and to the
+        # done/running tally.
+        documents = dataset.list_documents(page_size=self.config.page_size)
+        
+        selected: list[tuple[DocumentLike, str]] = []
+        for document in documents:
+            status = self._count_parse_status(status_counts, document)
+            if args.force or status in self.PARSE_REQUIRED_STATUSES:
+                selected.append((document, status))
+        
         if args.force:
-            # Force mode: all documents
-            documents = dataset.list_documents(page_size=self.config.page_size)
-            
-            print(f"Found {len(documents)} document(s) (force mode - parsing all)\n")
-            
-            for document in documents:
-                status = self._count_parse_status(status_counts, document)
-                doc_id = getattr(document, 'id', None)
-                if doc_id:
-                    docs_to_parse.append({
-                        "id": doc_id,
-                        "name": getattr(document, 'name', 'Untitled'),
-                        "dataset_id": dataset.id,
-                        "folder": ".",
-                        "status": status
-                    })
+            print(f"Found {len(selected)} document(s) (force mode - parsing all)\n")
         else:
-            # Default mode: filter by status (need parsing)
-            documents_needing_parse = document_service.list_documents_by_status(
-                dataset,
-                statuses=["FAIL", "UNSTART", "CANCEL"],
-                limit=self.config.page_size
-            )
-
-            # Tally skipped documents from the full list so the summary reflects
-            # what this run will not re-trigger.
-            all_documents = dataset.list_documents(page_size=self.config.page_size)
-            for document in all_documents:
-                self._count_parse_status(status_counts, document)
-            
-            print(f"Found {len(documents_needing_parse)} document(s) needing parsing\n")
-            
-            for document in documents_needing_parse:
-                status = self._count_parse_status(status_counts, document)
-                doc_id = getattr(document, 'id', None)
-                if doc_id:
-                    docs_to_parse.append({
-                        "id": doc_id,
-                        "name": getattr(document, 'name', 'Untitled'),
-                        "dataset_id": dataset.id,
-                        "folder": ".",
-                        "status": status
-                    })
+            print(f"Found {len(selected)} document(s) needing parsing\n")
+        
+        for document, status in selected:
+            doc_id = getattr(document, 'id', None)
+            if doc_id:
+                docs_to_parse.append({
+                    "id": doc_id,
+                    "name": getattr(document, 'name', 'Untitled'),
+                    "dataset_id": dataset.id,
+                    "folder": ".",
+                    "status": status
+                })
         
         return docs_to_parse, status_counts
 

{rosetta_cli-3.0.4 → rosetta_cli-3.0.6}/rosetta_cli/ragflow_client.py

@@ -116,7 +116,8 @@ class RAGFlowClient:
         embedding_model: str | None = None,
         chunk_method: str = "naive",
         parser_config: JsonDict | None = None,
-        page_size: int = 1000
+        page_size: int = 1000,
+        dataset_default: str = "aia",
     ):
         """
         Initialize RAGFlow client.
@@ -130,6 +131,7 @@ class RAGFlowClient:
             chunk_method: Chunking method (default: naive)
             parser_config: Parser configuration dict for chunk_method settings
             page_size: Default page size for list operations (default: 1000)
+            dataset_default: Default dataset name when no release-specific or explicit dataset name is available
             
         Raises:
             ValueError: If api_key or base_url is empty
@@ -147,6 +149,7 @@ class RAGFlowClient:
         self.chunk_method = chunk_method
         self.parser_config = parser_config or {}
         self.page_size = page_size
+        self.dataset_default = dataset_default
 
         # Initialize RAGFlow SDK client
         self._client = RAGFlow(api_key=api_key, base_url=base_url, version=version)
@@ -469,26 +472,38 @@ class RAGFlowClient:
         # Dataset doesn't exist, create it (gated by dry_run)
         return self.create_dataset(name, description, dry_run=dry_run)
     
-    def _resolve_dataset_name(self, template: str, release: str | None) -> str:
+    def _resolve_dataset_name(
+        self,
+        dataset_template: str,
+        release: str | None,
+        dataset_name: str | None,
+    ) -> str:
         """
-        Resolve dataset name from template.
+        Resolve dataset name using release + dataset_template, explicit dataset_name or default.
         
         Args:
-            template: Name template (e.g., "aia-{release}")
+            dataset_template: Name template (e.g., "aia-{release}")
             release: Release identifier (e.g., "r1")
+            dataset_name: Explicit dataset name, if provided (e.g., "aia-r1")
             
         Returns:
             Resolved dataset name
             
         Examples:
-            >>> _resolve_dataset_name("aia-{release}", "r1")
+            >>> _resolve_dataset_name("aia-{release}", "r1", "aia-r2")
             "aia-r1"
-            >>> _resolve_dataset_name("aia", None)
-            "aia"
+            >>> _resolve_dataset_name("aia-{release}", None, "aia-r2")
+            "aia-r2"
         """
-        if release and "{release}" in template:
-            return template.format(release=release)
-        return template
+        if release and "{release}" in dataset_template:
+            resolved_name = dataset_template.format(release=release)
+        else:
+            resolved_name = dataset_name or self.dataset_default
+
+        if "{release}" in resolved_name:
+            raise ValueError(f"Unresolved release placeholder in dataset name: {resolved_name}")
+
+        return resolved_name
     
     def _build_title_with_tags(self, tags: list[str], filename: str) -> str:
         """
@@ -521,7 +536,7 @@ class RAGFlowClient:
         OPTIMIZED: Now accepts pre-read content to avoid redundant file I/O.
         
         This method:
-        1. Resolves dataset name from template + release
+        1. Resolves the target dataset name
         2. Ensures dataset exists
         3. Builds title with tag prefixes
         4. Checks if document exists (by ims_doc_id)
@@ -529,6 +544,12 @@ class RAGFlowClient:
         6. Deletes existing document if changed
         7. Uploads new document with metadata
         
+        Dataset name resolution precedence:
+        1. If metadata.release is present, and dataset_template contains "{release}",
+           resolve dataset_template with the release.
+        2. Otherwise, use dataset_name when provided.
+        3. Otherwise, fall back to dataset_default.
+
         Args:
             file_path: Path to file (for filename, backward compatibility)
             metadata: Document metadata with pre-calculated hash
@@ -567,10 +588,7 @@ class RAGFlowClient:
         actual_hash = metadata.content_hash
         
         # Resolve dataset name
-        resolved_name = self._resolve_dataset_name(
-            dataset_template if "{release}" in dataset_template else (dataset_name or dataset_template),
-            metadata.release
-        )
+        resolved_name = self._resolve_dataset_name(dataset_template, metadata.release, dataset_name)
         
         # Ensure dataset exists (dry_run gates the underlying create_dataset call)
         dataset = self._ensure_dataset(

{rosetta_cli-3.0.4 → rosetta_cli-3.0.6}/rosetta_cli/rosetta_config.py

@@ -9,7 +9,6 @@ import logging
 import os
 from dataclasses import dataclass
 from pathlib import Path
-from urllib.parse import urlparse
 
 from dotenv import load_dotenv
 
@@ -69,14 +68,40 @@ def find_env_file(env_name: str | None = None) -> Path | None:
     return None
 
 
-def _first_subdomain(url: str) -> str | None:
-    """Extract first subdomain from URL. 'https://ims-dev.example.com/' -> 'ims-dev'."""
-    try:
-        hostname = urlparse(url).hostname or ""
-        parts = hostname.split(".")
-        return parts[0] if len(parts) > 1 else None
-    except Exception:
-        return None
+# Environment keywords searched for in the RAGFlow URL, in scan order.
+#
+# ORDER IS LOAD-BEARING - do not sort or alphabetize this tuple. The scan takes the
+# first keyword that appears anywhere in the URL, so a keyword that contains another
+# keyword must be listed before the one it contains. "prod" is a substring of
+# "preprod" - the only such pair in this set - so putting "prod" first would make
+# preprod.example.com resolve to "prod". test_environment_keyword_order_is_safe in
+# tests/test_rosetta_config_environment.py enforces this invariant.
+ENVIRONMENT_KEYWORDS: tuple[str, ...] = (
+    "preprod",  # must precede "prod", which it contains
+    "prod",
+    "stag",
+    "qa",
+    "dev",
+    "test",
+    "uat",
+    "sandbox",
+    "perf",
+)
+
+
+def _environment_from_url(url: str) -> str | None:
+    """Detect an environment name from keywords in a RAGFlow URL.
+
+    Case-insensitive substring scan over the URL exactly as given (scheme, host,
+    port and path all count). The first keyword in ENVIRONMENT_KEYWORDS to appear
+    wins, and the resolved value is the keyword itself, not the surrounding label:
+    'https://ims-dev.example.com/' -> 'dev'. Returns None when no keyword appears.
+    """
+    lowered = (url or "").lower()
+    for keyword in ENVIRONMENT_KEYWORDS:
+        if keyword in lowered:
+            return keyword
+    return None
 
 
 @dataclass
@@ -98,7 +123,12 @@ class RosettaConfig:
         RAGFLOW_PAGE_SIZE: Page size for listing operations (default: 1000)
         RAGFLOW_PARSE_TIMEOUT: Timeout for parsing operations in seconds (default: 1200)
         RAGFLOW_TIMEOUT: HTTP request timeout in seconds (default: 30)
-        ENVIRONMENT: Environment name (default: "local")
+        ENVIRONMENT: Environment display label. Takes precedence over URL keyword
+            detection, but not over an explicit --env argument. When unset, the
+            label is detected from RAGFLOW_BASE_URL by a case-insensitive scan for
+            the first of preprod, prod, stag, qa, dev, test, uat, sandbox, perf to
+            appear in it (the resolved value is that keyword itself, so
+            "https://ims-dev.example.com" yields "dev"); "local" if none appear.
     
     Examples:
         >>> config = RosettaConfig.from_env(".env")
@@ -194,9 +224,11 @@ class RosettaConfig:
         Args:
             environment: Optional explicit environment name (e.g., "local",
                 "dev", "remote"). If provided, this value is used and takes
-                precedence over the ENVIRONMENT environment variable. If not
-                provided, the ENVIRONMENT variable is used, defaulting to
-                "local" when unset.
+                precedence over everything else. If not provided, the
+                ENVIRONMENT variable is used; failing that, the label is
+                detected from a keyword in RAGFLOW_BASE_URL (see
+                ENVIRONMENT_KEYWORDS), defaulting to "local" when no keyword
+                appears.
         
         Returns:
             RosettaConfig instance
@@ -213,9 +245,11 @@ class RosettaConfig:
         api_key = os.getenv("RAGFLOW_API_KEY", "")
         dataset_default = os.getenv("RAGFLOW_DATASET_DEFAULT", "aia")
         dataset_template = os.getenv("RAGFLOW_DATASET_TEMPLATE", "aia-{release}")
-        # fallback to ENVIRONMENT env var, or default to "local"
+        # Environment is a display label only - nothing branches on its value.
+        # Precedence: explicit argument (--env) > ENVIRONMENT > keyword detected in
+        # the RAGFlow URL > "local".
         if not environment:
-            environment = os.getenv("ENVIRONMENT") or _first_subdomain(base_url) or "local"
+            environment = os.getenv("ENVIRONMENT") or _environment_from_url(base_url) or "local"
         
         # Dataset creation settings
         embedding_model = os.getenv("RAGFLOW_EMBEDDING_MODEL") or None

{rosetta_cli-3.0.4 → rosetta_cli-3.0.6}/rosetta_cli/rosetta_publisher.py

@@ -641,13 +641,16 @@ class ContentPublisher:
                 return name[: -len(path.name)] + new_name
 
             # Collect all docs to delete into one list: (doc, label)
-            # Start with unmanaged: incomplete metadata (ims_doc_id or original_path absent)
+            # Start with managed docs carrying incomplete metadata: original_path is
+            # set (so the doc is ours) but ims_doc_id is absent. A doc with no
+            # original_path is custom/unmanaged and is never touched - see the
+            # docstring guarantee, which _cleanup_orphans enforces the same way.
             duplicates: list[tuple[DocumentLike, str]] = []
             for doc in all_docs:
                 meta = getattr(doc, "meta_fields", {}) or {}
                 ims_doc_id = meta.get("ims_doc_id") if isinstance(meta, dict) else getattr(meta, "ims_doc_id", None)
                 doc_original_path = meta.get("original_path", "") if isinstance(meta, dict) else getattr(meta, "original_path", "")
-                if not ims_doc_id or not doc_original_path:
+                if doc_original_path and not ims_doc_id:
                     doc_name = getattr(doc, "name", "") or doc.id
                     duplicates.append((doc, doc_name))
 
@@ -674,8 +677,10 @@ class ContentPublisher:
                     duplicates.append((doc, original_path))
 
             # Name duplicates: foo.md + foo(1).md + foo(2).md ...
+            # Grouped over managed_docs, not all_docs: a custom doc whose stripped
+            # name happens to collide with a managed one must not be deleted.
             name_groups: dict[str, list[DocumentLike]] = defaultdict(list)
-            for doc in all_docs:
+            for doc in managed_docs:
                 doc_name = getattr(doc, "name", "") or ""
                 if not doc_name:
                     continue

{rosetta_cli-3.0.4 → rosetta_cli-3.0.6}/rosetta_cli/services/dataset_service.py

@@ -27,14 +27,27 @@ class DatasetService:
     
     def resolve_dataset_name(self, args_dataset: Optional[str]) -> tuple[Optional[str], bool]:
         """
-        Resolve dataset name with auto-detection.
-        
+        Resolve which dataset to act on.
+
+        How the name is picked:
+          1. --dataset given      -> use it               (False)
+          2. else list server datasets, keep those starting with the template
+             prefix ("aia-" from "aia-{release}")
+          3. exactly one match    -> use it               (True)
+          4. several matches      -> give up, ask user    (None, False)
+          5. no matches           -> use dataset_default  (True)
+
         Args:
             args_dataset: Dataset name from arguments (can be None)
-        
+
         Returns:
             Tuple of (dataset_name: str or None, auto_detected: bool)
             Returns (None, False) if resolution fails
+
+            auto_detected answers "did the caller supply this name?", not "did
+            we find it on the server". False only for step 1. True for steps 3
+            and 5 alike: the caller supplied nothing and this method chose.
+            True on step 5 is correct.
         """
         # Explicit dataset provided
         if args_dataset:

{rosetta_cli-3.0.4 → rosetta_cli-3.0.6/rosetta_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rosetta-cli
-Version: 3.0.4
+Version: 3.0.6
 Summary: Rosetta CLI for publishing knowledge base content to RAGFlow
 Author-email: Rosetta Team <rosetta-support@griddynamics.com>
 License-Expression: Apache-2.0