projectdavid 1.31.0__py3-none-any.whl → 1.32.0__py3-none-any.whl

projectdavid/clients/file_processor.py

@@ -1,7 +1,6 @@
 import asyncio
 import csv
 import json
-import mimetypes
 import re
 import textwrap
 from concurrent.futures import ThreadPoolExecutor
@@ -10,10 +9,9 @@ from typing import Any, Dict, List, Tuple, Union
 
 try:  # Python 3.11+
     from typing import LiteralString
-except ImportError:  # 3.9 - 3.10
+except ImportError:  # 3.9–3.10
     from typing_extensions import LiteralString
 
-import magic
 import numpy as np
 import pdfplumber
 from docx import Document
@@ -54,54 +52,30 @@ class FileProcessor:
             raise ValueError(f"{file_path.name} > {mb} MB limit")
 
     # ------------------------------------------------------------------ #
-    #  File-type detection (extension + MIME)
+    #  File-type detection  (simple extension map – NO libmagic)
     # ------------------------------------------------------------------ #
     def _detect_file_type(self, file_path: Path) -> str:
         """
-        Return a handler tag:
+        Return one of:
 
-            • 'pdf'     • 'csv'
-            • 'json'    • 'office'
-            • 'text'
+            • 'pdf'   • 'csv'   • 'json'
+            • 'office' (.doc/.docx/.pptx)
+            • 'text'  (code / markup / plain text)
 
-        Raises *ValueError* on anything unknown.
+        Raises *ValueError* if the extension is not recognised.
         """
-        # 1️⃣  Best-effort MIME sniff
-        mime_type: str | None = None
-        if magic is not None:
-            try:
-                mime_type = magic.from_file(str(file_path), mime=True)
-            except Exception:
-                mime_type = None
-
-        # 2️⃣  Fallback → mimetypes
-        if not mime_type:
-            mime_type, _ = mimetypes.guess_type(file_path.name)
-
         suffix = file_path.suffix.lower()
 
-        PDF_MIMES = {"application/pdf"}
-        CSV_MIMES = {"text/csv", "application/csv"}
-        JSON_MIMES = {"application/json"}
-        OFFICE_MIMES = {
-            "application/msword",
-            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-            "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-        }
-        TEXT_MIMES = {
-            "text/plain",
-            "text/markdown",
-            "text/x-python",
-            "text/x-c",
-            "text/x-c++",
-            "text/x-java-source",
-            "text/x-script.python",
-            "text/html",
-            "text/css",
-            "application/typescript",
-            "text/javascript",
-        }
-        TEXT_EXTS = {
+        if suffix == ".pdf":
+            return "pdf"
+        if suffix == ".csv":
+            return "csv"
+        if suffix == ".json":
+            return "json"
+        if suffix in {".doc", ".docx", ".pptx"}:
+            return "office"
+
+        text_exts = {
             ".txt",
             ".md",
             ".rst",
@@ -120,32 +94,10 @@ class FileProcessor:
             ".html",
             ".css",
         }
-
-        # --- PDF ---
-        if mime_type in PDF_MIMES or suffix == ".pdf":
-            return "pdf"
-
-        # --- CSV ---
-        if mime_type in CSV_MIMES or suffix == ".csv":
-            return "csv"
-
-        # --- JSON ---
-        if mime_type in JSON_MIMES or suffix == ".json":
-            return "json"
-
-        # --- Office documents ---
-        if mime_type in OFFICE_MIMES or suffix in {".doc", ".docx", ".pptx"}:
-            return "office"
-
-        # --- Generic text / code / markup ---
-        if mime_type in TEXT_MIMES or suffix in TEXT_EXTS:
+        if suffix in text_exts:
             return "text"
 
-        # --- Unsupported ---
-        raise ValueError(
-            f"Unsupported file type for '{file_path.name}': "
-            f"MIME={mime_type or 'unknown'}  extension={suffix}"
-        )
+        raise ValueError(f"Unsupported file type: {file_path.name} (ext={suffix})")
 
     # ------------------------------------------------------------------ #
     #  Public entry-point
@@ -156,19 +108,17 @@ class FileProcessor:
         self.validate_file(file_path)
         ftype = self._detect_file_type(file_path)
 
-        if ftype == "pdf":
-            return await self._process_pdf(file_path)
-        if ftype == "text":
-            return await self._process_text(file_path)
-        if ftype == "csv":
-            return await self._process_csv(file_path)
-        if ftype == "office":
-            return await self._process_office(file_path)
-        if ftype == "json":
-            return await self._process_json(file_path)
-
-        # Safety net (should never hit)
-        raise ValueError(f"Unsupported file type: {file_path.suffix}")
+        dispatch_map = {
+            "pdf": self._process_pdf,
+            "text": self._process_text,
+            "csv": self._process_csv,
+            "office": self._process_office,
+            "json": self._process_json,
+        }
+        if ftype not in dispatch_map:
+            raise ValueError(f"Unsupported file type: {file_path.suffix}")
+
+        return await dispatch_map[ftype](file_path)
 
     # ------------------------------------------------------------------ #
     #  PDF

projectdavid/clients/synchronous_inference_wrapper.py

@@ -4,13 +4,19 @@ from typing import Generator, Optional
 
 from projectdavid_common import UtilsInterface
 
-logging_utility = UtilsInterface.LoggingUtility()
+from projectdavid.utils.function_call_suppressor import FunctionCallSuppressor
+from projectdavid.utils.peek_gate import PeekGate
+
+LOG = UtilsInterface.LoggingUtility()
 
 
 class SynchronousInferenceStream:
     _GLOBAL_LOOP = asyncio.new_event_loop()
     asyncio.set_event_loop(_GLOBAL_LOOP)
 
+    # --------------------------------------------------------------
+    # ctor / setup
+    # --------------------------------------------------------------
     def __init__(self, inference) -> None:
         self.inference_client = inference
         self.user_id: Optional[str] = None
@@ -36,31 +42,24 @@ class SynchronousInferenceStream:
         self.run_id = run_id
         self.api_key = api_key
 
+    # --------------------------------------------------------------
+    # main streaming entry-point
+    # --------------------------------------------------------------
     def stream_chunks(
         self,
         provider: str,
         model: str,
-        *,  # Following parameters are keyword-only.
+        *,  # keyword-only from here
         api_key: Optional[str] = None,
         timeout_per_chunk: float = 280.0,
+        suppress_fc: bool = True,  # ← NEW switch
     ) -> Generator[dict, None, None]:
-        """
-        Streams inference response chunks synchronously by wrapping an async generator.
-
-        Args:
-            provider (str): The provider name.
-            model (str): The model name.
-            api_key (Optional[str]): API key for authentication. If not provided, falls back to the value from setup().
-            timeout_per_chunk (float): Timeout per chunk in seconds.
-
-        Yields:
-            dict: A chunk of the inference response.
-        """
-        # Prefer direct input over self.api_key
+
         resolved_api_key = api_key or self.api_key
 
-        async def _stream_chunks_async() -> Generator[dict, None, None]:
-            async for chunk in self.inference_client.stream_inference_response(
+        # ------------- build async generator ----------------------
+        async def _stream_chunks_async():
+            async for chk in self.inference_client.stream_inference_response(
                 provider=provider,
                 model=model,
                 api_key=resolved_api_key,
@@ -69,30 +68,60 @@ class SynchronousInferenceStream:
                 run_id=self.run_id,
                 assistant_id=self.assistant_id,
             ):
-                yield chunk
+                yield chk
+
+        agen = _stream_chunks_async().__aiter__()
+
+        # ------------- optional suppression chain -----------------
+        if suppress_fc:
+            _suppressor = FunctionCallSuppressor()
+            _peek_gate = PeekGate(_suppressor)  # 2 KB default
+
+            def _filter_text(txt: str) -> str:
+                return _peek_gate.feed(txt)
+
+        else:
 
-        gen = _stream_chunks_async().__aiter__()
+            def _filter_text(txt: str) -> str:  # no-op
+                return txt
+
+        # ----------------------------------------------------------
 
         while True:
             try:
                 chunk = self._GLOBAL_LOOP.run_until_complete(
-                    asyncio.wait_for(gen.__anext__(), timeout=timeout_per_chunk)
+                    asyncio.wait_for(agen.__anext__(), timeout=timeout_per_chunk)
                 )
+
+                # --- provider-labelled function_call blocks -------
+                if suppress_fc and chunk.get("type") == "function_call":
+                    LOG.debug(
+                        "[SUPPRESSOR] blocked provider-labelled function_call chunk"
+                    )
+                    continue
+
+                # --- inline content filtering --------------------
+                if isinstance(chunk.get("content"), str):
+                    chunk["content"] = _filter_text(chunk["content"])
+                    if not chunk["content"]:
+                        # fully suppressed → skip forwarding
+                        continue
+
                 yield chunk
+
             except StopAsyncIteration:
-                logging_utility.info("Stream completed normally.")
+                LOG.info("Stream completed normally.")
                 break
             except asyncio.TimeoutError:
-                logging_utility.error(
-                    "[TimeoutError] Timeout occurred, stopping stream."
-                )
+                LOG.error("[TimeoutError] Timeout occurred, stopping stream.")
                 break
             except Exception as e:
-                logging_utility.error(
-                    "Unexpected error during streaming completions: %s", e
-                )
+                LOG.error("Unexpected error during streaming completions: %s", e)
                 break
 
+    # --------------------------------------------------------------
+    # housekeeping
+    # --------------------------------------------------------------
     @classmethod
     def shutdown_loop(cls) -> None:
         if cls._GLOBAL_LOOP and not cls._GLOBAL_LOOP.is_closed():

projectdavid/utils/function_call_suppressor.py

@@ -0,0 +1,41 @@
+import re
+
+from projectdavid_common.utilities.logging_service import LoggingUtility
+
+LOG = LoggingUtility()
+
+
+# ─────────────────────────────────────────────────────────────────────
+#  function-call filter helpers  (unchanged except for logger)
+# ─────────────────────────────────────────────────────────────────────
+class FunctionCallSuppressor:
+    OPEN_RE = re.compile(r"<\s*fc\s*>", re.I)
+    CLOSE_RE = re.compile(r"</\s*fc\s*>", re.I)
+
+    def __init__(self):
+        self.in_fc = False
+        self.buf = ""
+
+    def filter_chunk(self, chunk: str) -> str:
+        self.buf += chunk
+        out = ""
+
+        while self.buf:
+            if not self.in_fc:
+                m = self.OPEN_RE.search(self.buf)
+                if not m:
+                    out += self.buf
+                    self.buf = ""
+                    break
+                out += self.buf[: m.start()]
+                LOG.debug("[SUPPRESSOR] <fc> detected")
+                self.buf = self.buf[m.end() :]
+                self.in_fc = True
+            else:
+                m = self.CLOSE_RE.search(self.buf)
+                if not m:
+                    break  # wait for more tokens
+                LOG.debug("[SUPPRESSOR] </fc> detected — block suppressed")
+                self.buf = self.buf[m.end() :]
+                self.in_fc = False
+        return out

projectdavid/utils/peek_gate.py

@@ -0,0 +1,49 @@
+import re
+
+from projectdavid_common.utilities.logging_service import LoggingUtility
+
+from .function_call_suppressor import FunctionCallSuppressor
+
+LOG = LoggingUtility()
+
+
+class PeekGate:
+    """
+    Turns the suppressor on only if a <fc> block appears in the first
+    `peek_limit` characters; otherwise passes text through unchanged.
+    """
+
+    def __init__(self, downstream: FunctionCallSuppressor, peek_limit: int = 2048):
+        self.downstream = downstream
+        self.peek_limit = peek_limit
+        self.buf = ""
+        self.mode = "peeking"  # -> "normal" after decision
+        self.suppressing = False
+
+    def feed(self, txt: str) -> str:
+        # decision already taken
+        if self.mode == "normal":
+            return self.downstream.filter_chunk(txt) if self.suppressing else txt
+
+        # still peeking …
+        self.buf += txt
+        m = re.search(r"<\s*fc\s*>", self.buf, flags=re.I)
+        if m:  # found a tag
+            head = self.buf[: m.start()]
+            LOG.debug("[PEEK] <fc> located after leading text – engaging suppressor")
+            self.suppressing = True
+            self.mode = "normal"
+            tail = self.buf[m.start() :]
+            self.buf = ""
+            return head + self.downstream.filter_chunk(tail)
+
+        if len(self.buf) >= self.peek_limit:  # give up
+            LOG.debug(
+                "[PEEK] no <fc> tag within first %d chars – no suppression",
+                self.peek_limit,
+            )
+            self.mode = "normal"
+            self.suppressing = False
+            out, self.buf = self.buf, ""
+            return out
+        return ""

{projectdavid-1.31.0.dist-info → projectdavid-1.32.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: projectdavid
-Version: 1.31.0
+Version: 1.32.0
 Summary: Python SDK for interacting with the Entities Assistant API.
 Author-email: Francis Neequaye Armah <francis.neequaye@projectdavid.co.uk>
 License: PolyForm Noncommercial License 1.0.0
@@ -27,7 +27,6 @@ Requires-Dist: validators<0.35.0,>=0.29.0
 Requires-Dist: sentence-transformers<5.0,>=3.4.0
 Requires-Dist: sseclient-py
 Requires-Dist: requests
-Requires-Dist: python-magic
 Requires-Dist: python-docx
 Requires-Dist: python-pptx
 Provides-Extra: dev

{projectdavid-1.31.0.dist-info → projectdavid-1.32.0.dist-info}/RECORD

@@ -9,13 +9,13 @@ projectdavid/clients/assistants_client.py,sha256=SsIGa5wPr7ga9WX0ywam3djUF-uWFdk
 projectdavid/clients/base_client.py,sha256=UWl6nr6sxD1_xC6iyptQDR1tnNdFCOrEx5cEUPCRqJE,3417
 projectdavid/clients/base_vector_store.py,sha256=jXivmqAW1bgYcLgIeW-hPxOiWZbs2hCsLy4oWzSvpNI,2061
 projectdavid/clients/event_handler.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-projectdavid/clients/file_processor.py,sha256=-dRibUVVfGXjPRP3P2kzJaRZYvagIUKgDmfmi96V45w,15586
+projectdavid/clients/file_processor.py,sha256=t-Uw-kBP_VmlguMxO9PWY6ANuMAY0PstQDW37wLPF0Q,13980
 projectdavid/clients/file_search.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 projectdavid/clients/files_client.py,sha256=XkIDzbQFGDrd88taf0Kouc_4YJOPIYEHiIyWYLKDofI,15581
 projectdavid/clients/inference_client.py,sha256=xz4ACPv5Tkis604QxO5mJX1inH_TGDfQP-31geETYpE,6609
 projectdavid/clients/messages_client.py,sha256=467xeIt3VYs6cG8-bl-eDRi_auWOPmfd5tSJDmQSJUI,17232
 projectdavid/clients/runs.py,sha256=-fXOq5L9w2efDPmZkNxb0s2yjl6oN0XN4_aLXqaeceo,25270
-projectdavid/clients/synchronous_inference_wrapper.py,sha256=25dNSel5DVieu2aFg6G1e4ENo5WWB4EB8_jugIiVzlo,3433
+projectdavid/clients/synchronous_inference_wrapper.py,sha256=sikHDDmx3NFhK9eoFL0PVlZ0CaFG-CIWfDmgFkH48Do,4594
 projectdavid/clients/threads_client.py,sha256=ekzU5w14zftmtmFkiec3NC90Of-_KVSUY1qH9cmfSFg,6771
 projectdavid/clients/tools_client.py,sha256=GkCVOmwpAoPqVt6aYmH0G1HIFha3iEwR9IIf9teR0j8,11487
 projectdavid/clients/users_client.py,sha256=eCuUb9qvyH1GUFhZu6TRL9zdoK-qzHSs8-Vmrk_0mmg,13729
@@ -29,11 +29,13 @@ projectdavid/synthesis/prompt.py,sha256=Jlya6avnWV5ryaJaibt4W29XYxRlJRV6rwHfjgYy
 projectdavid/synthesis/reranker.py,sha256=aFjdLDoqnhgzxPIoRfs4y_Z_F6T88g5Sg8R5m0HMxdU,868
 projectdavid/synthesis/retriever.py,sha256=8R5HIgPXMKIQxV5KsN8u_gVVP4JY-wsDwJd92A-74-o,2300
 projectdavid/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+projectdavid/utils/function_call_suppressor.py,sha256=uTjB6e1JDs_nkSVtRuqretb-zYDbAAC3jN3gpzqVauo,1616
 projectdavid/utils/monitor_launcher.py,sha256=3YAgJdeuaUvq3JGvpA4ymqFsAnk29nH5q93cwStP4hc,2836
+projectdavid/utils/peek_gate.py,sha256=odQIZJYTygRPu5IfOBBgqqeSXiWfQ73-lRBYM2KbY4I,1630
 projectdavid/utils/run_monitor.py,sha256=F_WkqIP-qnWH-4llIbileWWLfRj2Q1Cg-ni23SR1rec,3786
 projectdavid/utils/vector_search_formatter.py,sha256=YTe3HPGec26qGY7uxY8_GS8lc4QaN6aNXMzkl29nZpI,1735
-projectdavid-1.31.0.dist-info/licenses/LICENSE,sha256=_8yjiEGttpS284BkfhXxfERqTRZW_tUaHiBB0GTJTMg,4563
-projectdavid-1.31.0.dist-info/METADATA,sha256=we616BQkChiuQYO_-UfQ1VL-7j-IfDfPI7OtheEMsUM,10809
-projectdavid-1.31.0.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-projectdavid-1.31.0.dist-info/top_level.txt,sha256=kil8GU4s7qYRfNnzGnFHhZnSNRSxgNG-J4HLgQMmMtw,13
-projectdavid-1.31.0.dist-info/RECORD,,
+projectdavid-1.32.0.dist-info/licenses/LICENSE,sha256=_8yjiEGttpS284BkfhXxfERqTRZW_tUaHiBB0GTJTMg,4563
+projectdavid-1.32.0.dist-info/METADATA,sha256=F3ZVLx0ylA3omK6BEsxsTrbaroIwcE42FO0GUcT0EkU,10781
+projectdavid-1.32.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+projectdavid-1.32.0.dist-info/top_level.txt,sha256=kil8GU4s7qYRfNnzGnFHhZnSNRSxgNG-J4HLgQMmMtw,13
+projectdavid-1.32.0.dist-info/RECORD,,

{projectdavid-1.31.0.dist-info → projectdavid-1.32.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.8.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any