projectdavid 1.31.0__tar.gz → 1.32.0__tar.gz

{projectdavid-1.31.0 → projectdavid-1.32.0}/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [1.32.0](https://github.com/frankie336/projectdavid/compare/v1.31.1...v1.32.0) (2025-06-08)
+
+
+### Features
+
+* Integrate function call suppression. The provides optional methods to clean <fc><\fc> wrapped function calls from stream. ([05b357e](https://github.com/frankie336/projectdavid/commit/05b357e17a5dfacc019bfea106d3be560878df4b))
+
+## [1.31.1](https://github.com/frankie336/projectdavid/compare/v1.31.0...v1.31.1) (2025-05-26)
+
+
+### Bug Fixes
+
+* Remove magic dependency when finding file type ([7063c14](https://github.com/frankie336/projectdavid/commit/7063c14c3d9f21bc9bd9579d4d7d2c55004a627f))
+
 # [1.31.0](https://github.com/frankie336/projectdavid/compare/v1.30.4...v1.31.0) (2025-05-26)
 
 

{projectdavid-1.31.0/src/projectdavid.egg-info → projectdavid-1.32.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: projectdavid
-Version: 1.31.0
+Version: 1.32.0
 Summary: Python SDK for interacting with the Entities Assistant API.
 Author-email: Francis Neequaye Armah <francis.neequaye@projectdavid.co.uk>
 License: PolyForm Noncommercial License 1.0.0
@@ -27,7 +27,6 @@ Requires-Dist: validators<0.35.0,>=0.29.0
 Requires-Dist: sentence-transformers<5.0,>=3.4.0
 Requires-Dist: sseclient-py
 Requires-Dist: requests
-Requires-Dist: python-magic
 Requires-Dist: python-docx
 Requires-Dist: python-pptx
 Provides-Extra: dev

{projectdavid-1.31.0 → projectdavid-1.32.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "projectdavid"
-version = "1.31.0"
+version = "1.32.0"
 description = "Python SDK for interacting with the Entities Assistant API."
 readme = "README.md"
 authors = [
@@ -27,7 +27,6 @@ dependencies = [
     "sentence-transformers>=3.4.0,<5.0",
     "sseclient-py",
      "requests",
-      "python-magic",
     "python-docx",
     "python-pptx",
 

{projectdavid-1.31.0 → projectdavid-1.32.0}/src/projectdavid/clients/file_processor.py

@@ -1,7 +1,6 @@
 import asyncio
 import csv
 import json
-import mimetypes
 import re
 import textwrap
 from concurrent.futures import ThreadPoolExecutor
@@ -10,10 +9,9 @@ from typing import Any, Dict, List, Tuple, Union
 
 try:  # Python 3.11+
     from typing import LiteralString
-except ImportError:  # 3.9 - 3.10
+except ImportError:  # 3.9–3.10
     from typing_extensions import LiteralString
 
-import magic
 import numpy as np
 import pdfplumber
 from docx import Document
@@ -54,54 +52,30 @@ class FileProcessor:
             raise ValueError(f"{file_path.name} > {mb} MB limit")
 
     # ------------------------------------------------------------------ #
-    #  File-type detection (extension + MIME)
+    #  File-type detection  (simple extension map – NO libmagic)
     # ------------------------------------------------------------------ #
     def _detect_file_type(self, file_path: Path) -> str:
         """
-        Return a handler tag:
+        Return one of:
 
-            • 'pdf'     • 'csv'
-            • 'json'    • 'office'
-            • 'text'
+            • 'pdf'   • 'csv'   • 'json'
+            • 'office' (.doc/.docx/.pptx)
+            • 'text'  (code / markup / plain text)
 
-        Raises *ValueError* on anything unknown.
+        Raises *ValueError* if the extension is not recognised.
         """
-        # 1️⃣  Best-effort MIME sniff
-        mime_type: str | None = None
-        if magic is not None:
-            try:
-                mime_type = magic.from_file(str(file_path), mime=True)
-            except Exception:
-                mime_type = None
-
-        # 2️⃣  Fallback → mimetypes
-        if not mime_type:
-            mime_type, _ = mimetypes.guess_type(file_path.name)
-
         suffix = file_path.suffix.lower()
 
-        PDF_MIMES = {"application/pdf"}
-        CSV_MIMES = {"text/csv", "application/csv"}
-        JSON_MIMES = {"application/json"}
-        OFFICE_MIMES = {
-            "application/msword",
-            "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-            "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-        }
-        TEXT_MIMES = {
-            "text/plain",
-            "text/markdown",
-            "text/x-python",
-            "text/x-c",
-            "text/x-c++",
-            "text/x-java-source",
-            "text/x-script.python",
-            "text/html",
-            "text/css",
-            "application/typescript",
-            "text/javascript",
-        }
-        TEXT_EXTS = {
+        if suffix == ".pdf":
+            return "pdf"
+        if suffix == ".csv":
+            return "csv"
+        if suffix == ".json":
+            return "json"
+        if suffix in {".doc", ".docx", ".pptx"}:
+            return "office"
+
+        text_exts = {
             ".txt",
             ".md",
             ".rst",
@@ -120,32 +94,10 @@ class FileProcessor:
             ".html",
             ".css",
         }
-
-        # --- PDF ---
-        if mime_type in PDF_MIMES or suffix == ".pdf":
-            return "pdf"
-
-        # --- CSV ---
-        if mime_type in CSV_MIMES or suffix == ".csv":
-            return "csv"
-
-        # --- JSON ---
-        if mime_type in JSON_MIMES or suffix == ".json":
-            return "json"
-
-        # --- Office documents ---
-        if mime_type in OFFICE_MIMES or suffix in {".doc", ".docx", ".pptx"}:
-            return "office"
-
-        # --- Generic text / code / markup ---
-        if mime_type in TEXT_MIMES or suffix in TEXT_EXTS:
+        if suffix in text_exts:
             return "text"
 
-        # --- Unsupported ---
-        raise ValueError(
-            f"Unsupported file type for '{file_path.name}': "
-            f"MIME={mime_type or 'unknown'}  extension={suffix}"
-        )
+        raise ValueError(f"Unsupported file type: {file_path.name} (ext={suffix})")
 
     # ------------------------------------------------------------------ #
     #  Public entry-point
@@ -156,19 +108,17 @@ class FileProcessor:
         self.validate_file(file_path)
         ftype = self._detect_file_type(file_path)
 
-        if ftype == "pdf":
-            return await self._process_pdf(file_path)
-        if ftype == "text":
-            return await self._process_text(file_path)
-        if ftype == "csv":
-            return await self._process_csv(file_path)
-        if ftype == "office":
-            return await self._process_office(file_path)
-        if ftype == "json":
-            return await self._process_json(file_path)
-
-        # Safety net (should never hit)
-        raise ValueError(f"Unsupported file type: {file_path.suffix}")
+        dispatch_map = {
+            "pdf": self._process_pdf,
+            "text": self._process_text,
+            "csv": self._process_csv,
+            "office": self._process_office,
+            "json": self._process_json,
+        }
+        if ftype not in dispatch_map:
+            raise ValueError(f"Unsupported file type: {file_path.suffix}")
+
+        return await dispatch_map[ftype](file_path)
 
     # ------------------------------------------------------------------ #
     #  PDF

projectdavid-1.32.0/src/projectdavid/clients/synchronous_inference_wrapper.py

@@ -0,0 +1,133 @@
+import asyncio
+from contextlib import suppress
+from typing import Generator, Optional
+
+from projectdavid_common import UtilsInterface
+
+from projectdavid.utils.function_call_suppressor import FunctionCallSuppressor
+from projectdavid.utils.peek_gate import PeekGate
+
+LOG = UtilsInterface.LoggingUtility()
+
+
+class SynchronousInferenceStream:
+    _GLOBAL_LOOP = asyncio.new_event_loop()
+    asyncio.set_event_loop(_GLOBAL_LOOP)
+
+    # --------------------------------------------------------------
+    # ctor / setup
+    # --------------------------------------------------------------
+    def __init__(self, inference) -> None:
+        self.inference_client = inference
+        self.user_id: Optional[str] = None
+        self.thread_id: Optional[str] = None
+        self.assistant_id: Optional[str] = None
+        self.message_id: Optional[str] = None
+        self.run_id: Optional[str] = None
+        self.api_key: Optional[str] = None
+
+    def setup(
+        self,
+        user_id: str,
+        thread_id: str,
+        assistant_id: str,
+        message_id: str,
+        run_id: str,
+        api_key: str,
+    ) -> None:
+        self.user_id = user_id
+        self.thread_id = thread_id
+        self.assistant_id = assistant_id
+        self.message_id = message_id
+        self.run_id = run_id
+        self.api_key = api_key
+
+    # --------------------------------------------------------------
+    # main streaming entry-point
+    # --------------------------------------------------------------
+    def stream_chunks(
+        self,
+        provider: str,
+        model: str,
+        *,  # keyword-only from here
+        api_key: Optional[str] = None,
+        timeout_per_chunk: float = 280.0,
+        suppress_fc: bool = True,  # ← NEW switch
+    ) -> Generator[dict, None, None]:
+
+        resolved_api_key = api_key or self.api_key
+
+        # ------------- build async generator ----------------------
+        async def _stream_chunks_async():
+            async for chk in self.inference_client.stream_inference_response(
+                provider=provider,
+                model=model,
+                api_key=resolved_api_key,
+                thread_id=self.thread_id,
+                message_id=self.message_id,
+                run_id=self.run_id,
+                assistant_id=self.assistant_id,
+            ):
+                yield chk
+
+        agen = _stream_chunks_async().__aiter__()
+
+        # ------------- optional suppression chain -----------------
+        if suppress_fc:
+            _suppressor = FunctionCallSuppressor()
+            _peek_gate = PeekGate(_suppressor)  # 2 KB default
+
+            def _filter_text(txt: str) -> str:
+                return _peek_gate.feed(txt)
+
+        else:
+
+            def _filter_text(txt: str) -> str:  # no-op
+                return txt
+
+        # ----------------------------------------------------------
+
+        while True:
+            try:
+                chunk = self._GLOBAL_LOOP.run_until_complete(
+                    asyncio.wait_for(agen.__anext__(), timeout=timeout_per_chunk)
+                )
+
+                # --- provider-labelled function_call blocks -------
+                if suppress_fc and chunk.get("type") == "function_call":
+                    LOG.debug(
+                        "[SUPPRESSOR] blocked provider-labelled function_call chunk"
+                    )
+                    continue
+
+                # --- inline content filtering --------------------
+                if isinstance(chunk.get("content"), str):
+                    chunk["content"] = _filter_text(chunk["content"])
+                    if not chunk["content"]:
+                        # fully suppressed → skip forwarding
+                        continue
+
+                yield chunk
+
+            except StopAsyncIteration:
+                LOG.info("Stream completed normally.")
+                break
+            except asyncio.TimeoutError:
+                LOG.error("[TimeoutError] Timeout occurred, stopping stream.")
+                break
+            except Exception as e:
+                LOG.error("Unexpected error during streaming completions: %s", e)
+                break
+
+    # --------------------------------------------------------------
+    # housekeeping
+    # --------------------------------------------------------------
+    @classmethod
+    def shutdown_loop(cls) -> None:
+        if cls._GLOBAL_LOOP and not cls._GLOBAL_LOOP.is_closed():
+            cls._GLOBAL_LOOP.stop()
+            cls._GLOBAL_LOOP.close()
+
+    def close(self) -> None:
+        with suppress(Exception):
+            self.inference_client.close()

projectdavid-1.32.0/src/projectdavid/utils/function_call_suppressor.py

@@ -0,0 +1,41 @@
+import re
+
+from projectdavid_common.utilities.logging_service import LoggingUtility
+
+LOG = LoggingUtility()
+
+
+# ─────────────────────────────────────────────────────────────────────
+#  function-call filter helpers  (unchanged except for logger)
+# ─────────────────────────────────────────────────────────────────────
+class FunctionCallSuppressor:
+    OPEN_RE = re.compile(r"<\s*fc\s*>", re.I)
+    CLOSE_RE = re.compile(r"</\s*fc\s*>", re.I)
+
+    def __init__(self):
+        self.in_fc = False
+        self.buf = ""
+
+    def filter_chunk(self, chunk: str) -> str:
+        self.buf += chunk
+        out = ""
+
+        while self.buf:
+            if not self.in_fc:
+                m = self.OPEN_RE.search(self.buf)
+                if not m:
+                    out += self.buf
+                    self.buf = ""
+                    break
+                out += self.buf[: m.start()]
+                LOG.debug("[SUPPRESSOR] <fc> detected")
+                self.buf = self.buf[m.end() :]
+                self.in_fc = True
+            else:
+                m = self.CLOSE_RE.search(self.buf)
+                if not m:
+                    break  # wait for more tokens
+                LOG.debug("[SUPPRESSOR] </fc> detected — block suppressed")
+                self.buf = self.buf[m.end() :]
+                self.in_fc = False
+        return out

projectdavid-1.32.0/src/projectdavid/utils/peek_gate.py

@@ -0,0 +1,49 @@
+import re
+
+from projectdavid_common.utilities.logging_service import LoggingUtility
+
+from .function_call_suppressor import FunctionCallSuppressor
+
+LOG = LoggingUtility()
+
+
+class PeekGate:
+    """
+    Turns the suppressor on only if a <fc> block appears in the first
+    `peek_limit` characters; otherwise passes text through unchanged.
+    """
+
+    def __init__(self, downstream: FunctionCallSuppressor, peek_limit: int = 2048):
+        self.downstream = downstream
+        self.peek_limit = peek_limit
+        self.buf = ""
+        self.mode = "peeking"  # -> "normal" after decision
+        self.suppressing = False
+
+    def feed(self, txt: str) -> str:
+        # decision already taken
+        if self.mode == "normal":
+            return self.downstream.filter_chunk(txt) if self.suppressing else txt
+
+        # still peeking …
+        self.buf += txt
+        m = re.search(r"<\s*fc\s*>", self.buf, flags=re.I)
+        if m:  # found a tag
+            head = self.buf[: m.start()]
+            LOG.debug("[PEEK] <fc> located after leading text – engaging suppressor")
+            self.suppressing = True
+            self.mode = "normal"
+            tail = self.buf[m.start() :]
+            self.buf = ""
+            return head + self.downstream.filter_chunk(tail)
+
+        if len(self.buf) >= self.peek_limit:  # give up
+            LOG.debug(
+                "[PEEK] no <fc> tag within first %d chars – no suppression",
+                self.peek_limit,
+            )
+            self.mode = "normal"
+            self.suppressing = False
+            out, self.buf = self.buf, ""
+            return out
+        return ""

{projectdavid-1.31.0 → projectdavid-1.32.0/src/projectdavid.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: projectdavid
-Version: 1.31.0
+Version: 1.32.0
 Summary: Python SDK for interacting with the Entities Assistant API.
 Author-email: Francis Neequaye Armah <francis.neequaye@projectdavid.co.uk>
 License: PolyForm Noncommercial License 1.0.0
@@ -27,7 +27,6 @@ Requires-Dist: validators<0.35.0,>=0.29.0
 Requires-Dist: sentence-transformers<5.0,>=3.4.0
 Requires-Dist: sseclient-py
 Requires-Dist: requests
-Requires-Dist: python-magic
 Requires-Dist: python-docx
 Requires-Dist: python-pptx
 Provides-Extra: dev

{projectdavid-1.31.0 → projectdavid-1.32.0}/src/projectdavid.egg-info/SOURCES.txt

@@ -58,7 +58,9 @@ src/projectdavid/synthesis/prompt.py
 src/projectdavid/synthesis/reranker.py
 src/projectdavid/synthesis/retriever.py
 src/projectdavid/utils/__init__.py
+src/projectdavid/utils/function_call_suppressor.py
 src/projectdavid/utils/monitor_launcher.py
+src/projectdavid/utils/peek_gate.py
 src/projectdavid/utils/run_monitor.py
 src/projectdavid/utils/vector_search_formatter.py
 tests/test_clients.py

{projectdavid-1.31.0 → projectdavid-1.32.0}/src/projectdavid.egg-info/requires.txt

@@ -10,7 +10,6 @@ validators<0.35.0,>=0.29.0
 sentence-transformers<5.0,>=3.4.0
 sseclient-py
 requests
-python-magic
 python-docx
 python-pptx
 

projectdavid-1.31.0/src/projectdavid/clients/synchronous_inference_wrapper.py

@@ -1,104 +0,0 @@
-import asyncio
-from contextlib import suppress
-from typing import Generator, Optional
-
-from projectdavid_common import UtilsInterface
-
-logging_utility = UtilsInterface.LoggingUtility()
-
-
-class SynchronousInferenceStream:
-    _GLOBAL_LOOP = asyncio.new_event_loop()
-    asyncio.set_event_loop(_GLOBAL_LOOP)
-
-    def __init__(self, inference) -> None:
-        self.inference_client = inference
-        self.user_id: Optional[str] = None
-        self.thread_id: Optional[str] = None
-        self.assistant_id: Optional[str] = None
-        self.message_id: Optional[str] = None
-        self.run_id: Optional[str] = None
-        self.api_key: Optional[str] = None
-
-    def setup(
-        self,
-        user_id: str,
-        thread_id: str,
-        assistant_id: str,
-        message_id: str,
-        run_id: str,
-        api_key: str,
-    ) -> None:
-        self.user_id = user_id
-        self.thread_id = thread_id
-        self.assistant_id = assistant_id
-        self.message_id = message_id
-        self.run_id = run_id
-        self.api_key = api_key
-
-    def stream_chunks(
-        self,
-        provider: str,
-        model: str,
-        *,  # Following parameters are keyword-only.
-        api_key: Optional[str] = None,
-        timeout_per_chunk: float = 280.0,
-    ) -> Generator[dict, None, None]:
-        """
-        Streams inference response chunks synchronously by wrapping an async generator.
-
-        Args:
-            provider (str): The provider name.
-            model (str): The model name.
-            api_key (Optional[str]): API key for authentication. If not provided, falls back to the value from setup().
-            timeout_per_chunk (float): Timeout per chunk in seconds.
-
-        Yields:
-            dict: A chunk of the inference response.
-        """
-        # Prefer direct input over self.api_key
-        resolved_api_key = api_key or self.api_key
-
-        async def _stream_chunks_async() -> Generator[dict, None, None]:
-            async for chunk in self.inference_client.stream_inference_response(
-                provider=provider,
-                model=model,
-                api_key=resolved_api_key,
-                thread_id=self.thread_id,
-                message_id=self.message_id,
-                run_id=self.run_id,
-                assistant_id=self.assistant_id,
-            ):
-                yield chunk
-
-        gen = _stream_chunks_async().__aiter__()
-
-        while True:
-            try:
-                chunk = self._GLOBAL_LOOP.run_until_complete(
-                    asyncio.wait_for(gen.__anext__(), timeout=timeout_per_chunk)
-                )
-                yield chunk
-            except StopAsyncIteration:
-                logging_utility.info("Stream completed normally.")
-                break
-            except asyncio.TimeoutError:
-                logging_utility.error(
-                    "[TimeoutError] Timeout occurred, stopping stream."
-                )
-                break
-            except Exception as e:
-                logging_utility.error(
-                    "Unexpected error during streaming completions: %s", e
-                )
-                break
-
-    @classmethod
-    def shutdown_loop(cls) -> None:
-        if cls._GLOBAL_LOOP and not cls._GLOBAL_LOOP.is_closed():
-            cls._GLOBAL_LOOP.stop()
-            cls._GLOBAL_LOOP.close()
-
-    def close(self) -> None:
-        with suppress(Exception):
-            self.inference_client.close()