folder-classifier 0.2.3__py3-none-any.whl → 0.3.1__py3-none-any.whl

folder_classifier/app.py

@@ -3,14 +3,11 @@ import logging
 from fastapi import FastAPI
 from ray import serve
 
-from folder_classifier.dto import (ModelConfig, FolderClassificationResponse,
-                                   FolderClassificationRequest, Listing, Folder)
-from folder_classifier.util import flatten_folder
-from folder_classifier import classifier
+from folder_classifier.classifier import FolderClassifier
+from folder_classifier.dto import ModelConfig, FolderClassificationRequest, FolderClassificationResponse
 
 web_api = FastAPI(title=f"Folder Classifier API")
 
-
 @serve.deployment
 @serve.ingress(web_api)
 class FolderClassifierAPI:
@@ -19,13 +16,13 @@ class FolderClassifierAPI:
         assert model_config.app_name and model_config.deployment, "Invalid ModelConfig values"
         logging.basicConfig(level=logging.INFO)
         self.logger = logging.getLogger(__name__)
-        self.logger.info(f"Initializing model: {model_config}")
-        #self.model_handle = serve.get_deployment_handle(app_name=model_config.app_name, deployment_name=model_config.deployment)
+        self.logger.info(f"Initializing Folder Classifier model: {model_config}")
+        model_handle = serve.get_deployment_handle(app_name=model_config.app_name, deployment_name=model_config.deployment)
+        self.classifier = FolderClassifier(model_handle)
         self.logger.info(f"Successfully initialized Folder Classifier API")
 
     @web_api.post("/predict")
     async def predict(self, request: FolderClassificationRequest) -> FolderClassificationResponse:
-        listing = request if isinstance(request, Listing) else Listing(items=flatten_folder(request))
-        category, confidence = classifier.predict(listing)
         self.logger.info(f"Received request: {request}")
-        return FolderClassificationResponse(category=category, confidence=confidence)
+        category, reasoning = await self.classifier.predict(request)
+        return FolderClassificationResponse(category=category, reasoning=reasoning)

folder_classifier/classifier.py

@@ -1,25 +1,129 @@
-from typing import Tuple
-import torch
-from transformers import pipeline
-from folder_classifier.dto import Listing
-
-
-classifier = None
-
-
-def predict(listing: Listing) -> Tuple[str, float]:
-    global classifier
-    if classifier is None:
-        classifier = pipeline(
-            "text-classification",
-            model="/mnt/cluster_storage/models/corto-ai/ModernBERT-large-folder-classifier",
-            torch_dtype=torch.bfloat16,
-            device="cuda"
-        )
-    text = "\n".join(listing.items)
-    prediction = classifier(text)
-    predicted_label = prediction[0]["label"]
-    confidence = prediction[0]["score"]
-    return predicted_label, confidence
+import logging
+import os
+from typing import Tuple, Dict, Any
+
+from ray.serve.handle import DeploymentHandle
+
+from folder_classifier.dto import FolderClassificationRequest, FolderClassification
+from folder_classifier.util import build_folder, render_tree
+
+
+MODEL = os.getenv("MODEL", "Qwen3-4B-Instruct-2507-classifier-FP8")
+
+SYSTEM_PROMPT = """
+You are a strict text classifier.
+Output MUST be a single JSON object with exactly two keys: "category" and "reasoning".
+- "category" ∈ {"matter","other"} (lowercase).
+- "reasoning" ≤ 30 words.
+- Do not include any double-quote (") characters inside the value of "reasoning"; use single quotes ' instead.
+- No backticks, no code fences, no extra text. Return one JSON object only.
+If unsure, choose "other".
+""".strip()
+
+USER_PROMPT_TEMPLATE = """
+Classify the FOLDER TREE as "matter" or "other".
+
+Definitions
+- matter: The FOLDER TREE represents exactly one legal matter/container (one client/case/matter) AND includes at least one legal-work indicator.
+- other: Anything else, including:
+  (a) the FOLDER TREE appears to contain multiple distinct matters (a container of matters), or
+  (b) the ROOT NAME is a common subfolder/stage/type (e.g., "Correspondence", "Drafts", "Pleadings", "Court Documents", "Billing", "Evidence"), or
+  (c) legal-work indicators are absent, or
+  (d) there are zero files with extensions anywhere in the tree, or
+  (e) contents are exclusively non-legal domains (finance/accounting/IT/admin) with no legal-work indicators, or
+  (f) the ROOT NAME is generic/system-generated and not matter-specific (see R9).
+
+Decision Rules (apply in order; case-insensitive)
+RC_container: Any indication the FOLDER TREE holds more than one distinct matter (e.g., multiple top-level matter-like subfolders; repeated separate matter numbers/clients) → category=other.
+R0_subfolder: ROOT NAME equals a common matter subfolder/stage/type (e.g., Correspondence/Emails, File Notes/Attendance Notes, Searches/Certificates, Court Documents/Pleadings/Evidence/Disclosure/Discovery, Drafts/Final/Signed/Executed, Billing/Invoices/Time Records) → category=other.
+R7_files_present: Must contain ≥1 file with an extension (e.g., .pdf, .docx) anywhere in the tree; if none → category=other.
+R8_nonlegal_only: If the tree shows strong non-legal domain signals (finance/accounting/IT/admin) and NO legal-work indicators (R2–R4 or R6), classify as other.
+    • Finance/Accounting examples: BAS/Business Activity Statement, IAS, GST, Tax Return, PAYG, Payroll, Timesheets, Payslips, Superannuation, Xero/MYOB/QuickBooks exports, General Ledger, Trial Balance, Journals, Bank Statements.
+    • IT/Systems examples: Backups, Logs, Source Code, Git, DevOps, Server/Network/VPN, Mailboxes, Google Workspace/Microsoft 365 admin.
+    • Admin-only examples: generic receipts, vendor invoices, expense folders without legal context.
+    • Note: “Billing/Invoices/Time Records” inside a matter is a typical legal subfolder; R8 applies only when legal indicators are entirely absent.
+R9_generic_rootname: If the ROOT NAME is generic/system-generated and not matter-specific → category=other (even if legal documents appear underneath).
+    • Examples: "CORTO Generated", "Generated Files", "Exports", "Uploads", "Scans", "Shared", "Dropbox Shared", "Google Drive", "OneDrive", "SharePoint", "Archive", "Backup", "Temp", "Incoming", "Outbox", "Bulk Import".
+    • Heuristic: words like generated/export/import/sync/shared/archive/backup/temp/uploads/downloads indicate a system or generic container, not a single matter.
+R1_rootname: ROOT NAME resembles a single matter/container (matter/file/case number; client/surname/company; or a combination such as “12345 Smith” or “Smith – Contract Dispute”).
+R2_initial_docs: Early-stage matter docs (cost agreement/disclosure, retainer/engagement, intake/onboarding).
+R3_legal_docs: Legal document types (agreement, contract, deed, will, affidavit, statement, advice, brief, pleadings, court forms, subpoena, orders, judgment, undertaking, notice of appeal, docket/case forms).
+R4_legal_subfolders: Typical legal subfolders (correspondence/emails, file notes/attendance notes, searches/certificates, court documents/evidence/disclosure/discovery, drafts/final/signed/executed, billing/invoices/time records).
+R5_support_filename_patterns: Supportive only (not decisive): versioning (v1/v2/v3), “final”, “executed”, “signed”, eight-digit dates (YYYYMMDD/DDMMYYYY).
+R6_jurisdiction: Court/jurisdiction/case references (generic court acronyms, registry references, docket patterns).
+
+Decision
+- If RC_container → category=other (stop).
+- Else if R0_subfolder → category=other (stop).
+- Else if NOT R7_files_present → category=other (stop).
+- Else if R8_nonlegal_only → category=other (stop).
+- Else if R9_generic_rootname → category=other (stop).
+- Else if R1_rootname AND any of {R2_initial_docs, R3_legal_docs, R4_legal_subfolders, R6_jurisdiction} AND no multi-matter signal → category=matter.
+  (R5_support_filename_patterns cannot be used alone to justify "matter"; it is supportive only.)
+- Else → category=other.
+
+Normalization
+- “File with extension”: name containing a period followed by a 1–5 char alphanumeric extension (e.g., .pdf, .docx). Ignore leading/trailing periods.
+- Treat hyphens/underscores as separators. Ignore file extensions for semantic matching beyond the presence test. Tolerate minor typos.
+
+Output format (JSON only; no prose before/after):
+{"category": "<matter|other>", "reasoning": "≤30 words citing R# and 1–2 evidence tokens>"}
+
+FOLDER TREE:
+{folder_tree}
+""".strip()
+
+
+FOLDER_CLASSIFICATION_SCHEMA = FolderClassification.model_json_schema()
+
+
+class FolderClassifier:
+    def __init__(self, model_handle: DeploymentHandle):
+        self.logger = logging.getLogger(__name__)
+        self.model_handler = model_handle
+        self.logger.info(f"Successfully initialized FolderClassifier")
+
+    async def predict(self, request: FolderClassificationRequest) -> Tuple[str, str]:
+        try:
+            chat_completion_request = self._to_chat_completion_request(request)
+            response = await self.model_handler.create_chat_completion.remote(chat_completion_request)
+            content = response.choices[0].message.content
+            result = FolderClassification.model_validate_json(content)
+        except Exception as ex:
+            self.logger.error(f"Failed to parse response: {content}\n{ex}")
+            if '"category": "matter"' in content:
+                result = FolderClassification(category="matter", reasoning="NA")
+            else:
+                result = FolderClassification(category="other", reasoning="NA")
+
+        return result.category, result.reasoning
+
+    @staticmethod
+    def _to_chat_completion_request(request: FolderClassificationRequest) -> Dict[str, Any]:
+        input_paths = request.items
+        folder = build_folder(input_paths)
+        folder_tree = render_tree(folder)
+        chat_completion_request = {
+            "model": MODEL,
+            "messages": [
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {"role": "user", "content": USER_PROMPT_TEMPLATE.replace("{folder_tree}", folder_tree)}
+            ],
+            "max_tokens": 1024,
+            "temperature": 0.7,
+            "top_p": 0.8,
+            "response_format": {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": "FolderClassification",
+                    "schema": FOLDER_CLASSIFICATION_SCHEMA,
+                    "strict": True,
+                },
+            }
+        }
+        return chat_completion_request
+
+
+
 
 

folder_classifier/dto.py

@@ -1,9 +1,5 @@
-from __future__ import annotations
-
-from textwrap import dedent
 from typing import List, Union, Literal, Optional
-
-from pydantic import BaseModel, Field, confloat
+from pydantic import BaseModel, Field, ConfigDict
 
 
 class ModelConfig(BaseModel):
@@ -23,32 +19,19 @@ class File(BaseModel):
 class Folder(BaseModel):
     name: str
     type: Literal["folder"]
-    # Discriminated union: 'type' field is used to select between File and Folder
-    items: Optional[List[Union[File, Folder]]] = Field(default_factory=list)
-    model_config = {
-        "json_schema_extra": {
-            # Override the OpenAPI example to avoid the default 'string' entry
-            "example": dedent("""{
-                "name": "string",
-                "type": "folder",
-                "items": [
-                    {
-                      "name": "string",
-                      "type": "file"
-                    }
-                  ]
-                }""")
-        }
-    }
-
-
-class Listing(BaseModel):
-    items: List[str]
+    items: List[Union[File, 'Folder']] = Field(default_factory=list)
+
 
+class FolderClassificationRequest(BaseModel):
+    items: List[str]
 
-Folder.model_rebuild()
-FolderClassificationRequest = Listing
 
 class FolderClassificationResponse(BaseModel):
     category: Literal["matter", "other"]
-    confidence: confloat(ge=0.0, le=1.0)
+    reasoning: Optional[str] = None
+
+
+class FolderClassification(BaseModel):
+    category: Literal["matter", "other"]
+    reasoning: str
+    model_config = ConfigDict(extra="forbid")

folder_classifier/util.py

@@ -1,6 +1,78 @@
-from typing import List
+from typing import List, Union
 
-from folder_classifier.dto import Folder
+from folder_classifier.dto import Folder, File
+
+
+def build_folder(paths: List[str]) -> Folder:
+    """
+    Create a Folder tree from a list of file paths;
+    The file paths are delimited by "/" - leaf segments are assumed to be files
+    """
+    if not paths:
+        raise ValueError("No paths provided")
+
+    # Get all directory prefixes
+    prefix_set = set()
+    for p in paths:
+        parts = p.split('/')
+        for i in range(1, len(parts)):
+            prefix_set.add('/'.join(parts[:i]))
+
+    # Sort by depth so parents are created before children
+    sorted_paths = sorted(paths, key=lambda x: x.count('/'))
+
+    # Create root folder
+    root_name = sorted_paths[0].split('/')[0]
+    root = Folder(name=root_name, type="folder", items=[])
+
+    # Build the tree
+    for p in sorted_paths:
+        parts = p.split('/')
+        current = root
+        for idx, part in enumerate(parts[1:], start=1):
+            full_path = '/'.join(parts[:idx+1])
+            is_last = idx == len(parts) - 1
+
+            # existing item
+            existing = next((item for item in current.items if item.name == part), None)
+            if existing:
+                if isinstance(existing, Folder):
+                    current = existing
+                continue
+
+            # Determine type for new item
+            if is_last and full_path not in prefix_set:
+                new_item = File(name=part, type="file")
+            else:
+                new_item = Folder(name=part, type="folder", items=[])
+
+            current.items.append(new_item)
+            if isinstance(new_item, Folder):
+                current = new_item
+
+    return root
+
+
+def render_tree(folder: Folder) -> str:
+    """
+    Render Folder tree using ASCII tree characters (├──, └──, │).
+    """
+    lines: List[str] = []
+
+    def recurse(node: Union[Folder, File], prefix: str, is_last: bool):
+        connector = "└── " if is_last else "├── "
+        lines.append(f"{prefix}{connector}{node.name}")
+        if isinstance(node, Folder):
+            child_prefix = prefix + ("    " if is_last else "│   ")
+            for idx, child in enumerate(node.items):
+                recurse(child, child_prefix, idx == len(node.items) - 1)
+
+    # root
+    lines.append(folder.name)
+    for idx, child in enumerate(folder.items):
+        recurse(child, "", idx == len(folder.items) - 1)
+
+    return "\n".join(lines)
 
 
 def flatten_folder(folder: Folder, parent_path: str = "") -> List[str]:
@@ -18,4 +90,6 @@ def flatten_folder(folder: Folder, parent_path: str = "") -> List[str]:
         else:
             # Recursively flatten subfolders
             paths.extend(flatten_folder(item, current_path))
-    return paths
+    return paths
+
+

{folder_classifier-0.2.3.dist-info → folder_classifier-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: folder-classifier
-Version: 0.2.3
+Version: 0.3.1
 Summary: Deploy folder classifier API to a Ray cluster
 Author: Crispin Almodovar
 Author-email: 

folder_classifier-0.3.1.dist-info/RECORD

@@ -0,0 +1,10 @@
+folder_classifier/__init__.py,sha256=k0YWZyUNe7myJiKeX0OaXtJ30_3EGE-vsZiAUbqa-3E,46
+folder_classifier/app.py,sha256=doaXkKnpe-LNztkMkuROutjId7XPS_t9G7m9dSXt-ow,1330
+folder_classifier/classifier.py,sha256=3V4M-x5K7vASStsnPlrW2M9tA6uWsKA4L1rCMHqCQ_M,7705
+folder_classifier/deploy.py,sha256=UQTbQjR_JX92Xo8L5EbK4nPg_VJmwWDmzMQSjP3-7iQ,412
+folder_classifier/dto.py,sha256=pCRErTIflx8zMe7dD_GcnaBagPYpuZy2vIZCAbb0i0Q,767
+folder_classifier/util.py,sha256=4zk52bPVyoVU0FfDTmeK5JdM4y0IbnuaPS1PWc9txa4,3065
+folder_classifier-0.3.1.dist-info/METADATA,sha256=x4Dv4CCUXoo0GAL5DX3I3FPrHvTWYKowY1UrkMGD2Gs,392
+folder_classifier-0.3.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+folder_classifier-0.3.1.dist-info/top_level.txt,sha256=36ugc9pEbNQ-mnzz4Ot2WVjY3t_LzAN6XOCjDFP4p4k,18
+folder_classifier-0.3.1.dist-info/RECORD,,

folder_classifier-0.2.3.dist-info/RECORD

@@ -1,10 +0,0 @@
-folder_classifier/__init__.py,sha256=k0YWZyUNe7myJiKeX0OaXtJ30_3EGE-vsZiAUbqa-3E,46
-folder_classifier/app.py,sha256=FqoBp_KQ3yIfoHTagdDTVsLbIj8luCn7en533Q870x8,1443
-folder_classifier/classifier.py,sha256=YVKXvNAHny167H0Iv0GlRJEtRMAYwGea3cUVeGC4_sI,668
-folder_classifier/deploy.py,sha256=UQTbQjR_JX92Xo8L5EbK4nPg_VJmwWDmzMQSjP3-7iQ,412
-folder_classifier/dto.py,sha256=Xb1ozZQkfC45cbBoNOZ8xfkillJWFKXKWQc9CZxppXI,1260
-folder_classifier/util.py,sha256=t-ma2suHovfNutogJb9jailRbRpIg4qv-zph3dHb2og,692
-folder_classifier-0.2.3.dist-info/METADATA,sha256=MZ_BgIBlPrq86na7t9aMXA8EONA05MaHPofJV1P6hZM,392
-folder_classifier-0.2.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-folder_classifier-0.2.3.dist-info/top_level.txt,sha256=36ugc9pEbNQ-mnzz4Ot2WVjY3t_LzAN6XOCjDFP4p4k,18
-folder_classifier-0.2.3.dist-info/RECORD,,