OpenJarvis 1.0.4.dev872__tar.gz → 1.0.4.dev874__tar.gz

{openjarvis-1.0.4.dev872 → openjarvis-1.0.4.dev874}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: OpenJarvis
-Version: 1.0.4.dev872
+Version: 1.0.4.dev874
 Summary: OpenJarvis — modular AI assistant backend with composable intelligence primitives
 Project-URL: Homepage, https://github.com/open-jarvis/OpenJarvis
 Project-URL: Documentation, https://open-jarvis.github.io/OpenJarvis/

{openjarvis-1.0.4.dev872 → openjarvis-1.0.4.dev874}/src/openjarvis/agents/research_loop.py

@@ -143,10 +143,11 @@ Strategy:
   3. The `time_range` argument is a JSON object: `{{"start": "<ISO 8601>", "end": "<ISO 8601>"}}`. Either bound may be omitted, but pass at least one whenever the user gave you a temporal cue.
   4. When the user names a specific data source — "my Granola notes", "in Slack", "from my email" — you MUST pass `sources=[...]` with the matching connector ID. Only use IDs that appear in the connected-sources list above; do NOT invent or assume sources that are not connected. Common synonyms: "meeting notes"/"meetings"/"transcripts" → granola; "email"/"inbox" → gmail; "DMs"/"channels" → slack. Without this filter the search returns mail/messages ABOUT a tool instead of records FROM that tool.
   4a. Never apologize about sources that aren't in the connected-sources list — if the user asks about "Notion" but Notion isn't connected, just say "Notion isn't connected, but here's what I found in {available_sources}" and answer from what is available.
-  5. If the first structured search returns nothing useful, broaden with a semantic query and drop filters one at a time.
-  6. You have a clarify tool. Only use it AFTER at least one search attempt. Use it when: you found multiple ambiguous matches (e.g. 3 different people named John), search returned zero results and the query might need reframing, or the scope is too broad to synthesize meaningfully. Never use clarify before searching — always try first.
-  7. After receiving a clarify response, use the information to construct a precise search with the correct person, time_range, sources, and query parameters. Never send an empty query or a query with no parameters — extract every concrete signal from the user's reply (names, dates, topics, sources) and put it on the call.
-  8. Tool calls — search AND clarify — share a budget of 5 total. Spend wisely.
+  5. When the user asks for "next", "upcoming", "future", or "soon" calendar events/meetings/appointments, use `sources=["gcalendar"]` if gcalendar is connected, set `time_range={{"start": "{today}"}}`, and use `query=""` unless the user gave a specific topic such as "dentist" or "music lesson". This returns the nearest upcoming calendar items across calendars instead of keyword-matching only birthdays or event titles.
+  6. If the first structured search returns nothing useful, broaden with a semantic query and drop filters one at a time.
+  7. You have a clarify tool. Only use it AFTER at least one search attempt. Use it when: you found multiple ambiguous matches (e.g. 3 different people named John), search returned zero results and the query might need reframing, or the scope is too broad to synthesize meaningfully. Never use clarify before searching — always try first.
+  8. After receiving a clarify response, use the information to construct a precise search with the correct person, time_range, sources, and query parameters. Only use an empty query when structured filters carry the request; never send a search with no concrete parameters. Extract every concrete signal from the user's reply (names, dates, topics, sources) and put it on the call.
+  9. Tool calls — search AND clarify — share a budget of 5 total. Spend wisely.
 
 Synthesis rules:
   - Cite sources as individual numbers in square brackets. Always separate — write [4] [7] [20], never [4, 7, 20]. Never format citations as markdown links. Just the number in brackets: [1]. The `ref` field on each hit is the citation number.

{openjarvis-1.0.4.dev872 → openjarvis-1.0.4.dev874}/src/openjarvis/connectors/gcalendar.py

@@ -213,12 +213,13 @@ def _parse_event_timestamp(event: Dict[str, Any]) -> datetime:
     """
     start = event.get("start", {})
     date_time_str: str = start.get("dateTime", "")
-    if not date_time_str:
+    date_str: str = start.get("date", "")
+    if not date_time_str and not date_str:
         return datetime.now()
     try:
         # RFC3339 — Python 3.11+ fromisoformat handles the trailing 'Z'.
         # For older versions we replace 'Z' with '+00:00'.
-        normalized = date_time_str.replace("Z", "+00:00")
+        normalized = (date_time_str or date_str).replace("Z", "+00:00")
         return datetime.fromisoformat(normalized)
     except (ValueError, TypeError):
         return datetime.now()

{openjarvis-1.0.4.dev872 → openjarvis-1.0.4.dev874}/src/openjarvis/connectors/hybrid_search.py

@@ -20,8 +20,9 @@ from __future__ import annotations
 
 import json
 import logging
+import re
 from dataclasses import dataclass, field
-from datetime import datetime
+from datetime import date, datetime, timedelta, timezone
 from typing import Any, Dict, List, Optional, Sequence, Tuple
 
 # numpy imported lazily inside _vector_recall (see embeddings.py) so importing
@@ -32,6 +33,61 @@ from openjarvis.connectors.store import KnowledgeStore
 logger = logging.getLogger(__name__)
 
 
+_UPCOMING_TERMS = {
+    "next",
+    "upcoming",
+    "future",
+    "forthcoming",
+    "coming",
+    "soon",
+}
+_CALENDAR_TERMS = {
+    "calendar",
+    "calendars",
+    "event",
+    "events",
+}
+_CALENDAR_REQUEST_TERMS = _CALENDAR_TERMS | {
+    "appointment",
+    "appointments",
+    "meeting",
+    "meetings",
+    "schedule",
+}
+_GCALENDAR_GENERIC_TERMS = _UPCOMING_TERMS | _CALENDAR_TERMS | {
+    "appointment",
+    "appointments",
+    "meeting",
+    "meetings",
+    "schedule",
+}
+_QUERY_STOPWORDS = {
+    "a",
+    "all",
+    "am",
+    "are",
+    "do",
+    "for",
+    "have",
+    "i",
+    "in",
+    "is",
+    "list",
+    "me",
+    "my",
+    "on",
+    "s",
+    "show",
+    "tell",
+    "the",
+    "there",
+    "to",
+    "what",
+    "whats",
+    "when",
+}
+
+
 # ---------------------------------------------------------------------------
 # Result types
 # ---------------------------------------------------------------------------
@@ -120,6 +176,101 @@ def _snippet(content: str, max_chars: int = 500) -> str:
     return flat[:max_chars].rstrip() + "…"
 
 
+def _query_tokens(query: str) -> set[str]:
+    return set(re.findall(r"[a-z0-9_]+", query.lower()))
+
+
+def _sources_include_gcalendar(sources: Optional[Sequence[str]]) -> bool:
+    return any(str(source).lower() == "gcalendar" for source in sources or [])
+
+
+def _has_upcoming_calendar_intent(
+    query: str,
+    sources: Optional[Sequence[str]],
+) -> bool:
+    tokens = _query_tokens(query)
+    if not tokens or not (tokens & _UPCOMING_TERMS):
+        return False
+    if _sources_include_gcalendar(sources):
+        return True
+    if sources:
+        return False
+    return bool(tokens & _CALENDAR_REQUEST_TERMS)
+
+
+def _is_generic_calendar_timeline_query(query: str) -> bool:
+    tokens = _query_tokens(query)
+    if not tokens:
+        return True
+    topic_tokens = tokens - _GCALENDAR_GENERIC_TERMS - _QUERY_STOPWORDS
+    return not topic_tokens
+
+
+def _start_is_nowish_or_future(start: Optional[datetime]) -> bool:
+    if start is None:
+        return False
+    now = datetime.now(tz=start.tzinfo) if start.tzinfo else datetime.now()
+    return start >= now - timedelta(days=1)
+
+
+def _start_of_day(ts: datetime) -> datetime:
+    return ts.replace(hour=0, minute=0, second=0, microsecond=0)
+
+
+def _as_utc(ts: Optional[datetime]) -> Optional[datetime]:
+    if ts is None:
+        return None
+    if ts.tzinfo is None:
+        return ts.replace(tzinfo=timezone.utc)
+    return ts.astimezone(timezone.utc)
+
+
+def _parse_timestamp_for_timeline(
+    raw: Any,
+) -> Tuple[Optional[datetime], Optional[date]]:
+    if raw is None:
+        return None, None
+    text = str(raw).strip()
+    if not text:
+        return None, None
+    try:
+        parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
+    except ValueError:
+        return None, None
+    is_naive_midnight = (
+        parsed.tzinfo is None
+        and parsed.hour == 0
+        and parsed.minute == 0
+        and parsed.second == 0
+        and parsed.microsecond == 0
+    )
+    return _as_utc(parsed), parsed.date() if is_naive_midnight else None
+
+
+def _timestamp_in_range(
+    timestamp: Optional[datetime],
+    time_range: Optional[Tuple[Optional[datetime], Optional[datetime]]],
+    *,
+    all_day_date: Optional[date] = None,
+) -> bool:
+    if timestamp is None or time_range is None:
+        return timestamp is not None
+    start, end = time_range
+    if all_day_date is not None:
+        if start is not None and all_day_date < start.date():
+            return False
+        if end is not None and all_day_date > end.date():
+            return False
+        return True
+    start_utc = _as_utc(start)
+    end_utc = _as_utc(end)
+    if start_utc is not None and timestamp < start_utc:
+        return False
+    if end_utc is not None and timestamp > end_utc:
+        return False
+    return True
+
+
 # ---------------------------------------------------------------------------
 # HybridSearch
 # ---------------------------------------------------------------------------
@@ -377,6 +528,128 @@ class HybridSearch:
             for r in rows
         ]
 
+    def _normalise_calendar_timeline_scope(
+        self,
+        query: str,
+        time_range: Optional[Tuple[Optional[datetime], Optional[datetime]]],
+        sources: Optional[Sequence[str]],
+    ) -> Tuple[
+        Optional[Tuple[Optional[datetime], Optional[datetime]]],
+        Optional[Sequence[str]],
+        bool,
+        bool,
+    ]:
+        """Fill in structured filters for generic upcoming-calendar requests.
+
+        Queries like "what are my next calendar events?" often have no useful
+        lexical terms in the stored event text, so BM25/vector ranking can miss
+        nearby events. Treat that shape as a source-filtered timeline request.
+        """
+        scoped_sources = list(sources) if sources else None
+        has_upcoming_intent = _has_upcoming_calendar_intent(query, scoped_sources)
+
+        if has_upcoming_intent and (
+            scoped_sources is None or _sources_include_gcalendar(scoped_sources)
+        ):
+            scoped_sources = ["gcalendar"]
+
+        if not _sources_include_gcalendar(scoped_sources):
+            return time_range, scoped_sources, False, False
+
+        if has_upcoming_intent:
+            if time_range is None:
+                time_range = (_start_of_day(datetime.now(timezone.utc)), None)
+            else:
+                start, end = time_range
+                if start is None:
+                    time_range = (_start_of_day(datetime.now(timezone.utc)), end)
+                else:
+                    time_range = (_start_of_day(start), end)
+
+        chronological = has_upcoming_intent or (
+            time_range is not None
+            and time_range[1] is None
+            and _start_is_nowish_or_future(time_range[0])
+        )
+        metadata_only = chronological and _is_generic_calendar_timeline_query(query)
+        return time_range, scoped_sources, chronological, metadata_only
+
+    def _calendar_timeline_ids(
+        self,
+        *,
+        person: Optional[str],
+        time_range: Optional[Tuple[Optional[datetime], Optional[datetime]]],
+        sources: Optional[Sequence[str]],
+        limit: int,
+    ) -> List[str]:
+        """Return gcalendar rows sorted by normalized event start time."""
+        filter_sql, filter_params = self._build_filters(
+            person=person,
+            time_range=None,
+            sources=sources,
+        )
+        rows = self._store._conn.execute(
+            f"""
+            SELECT id, timestamp, created_at
+            FROM knowledge_chunks
+            WHERE {filter_sql}
+            """,
+            filter_params,
+        ).fetchall()
+
+        candidates: List[Tuple[str, datetime, float]] = []
+        for row in rows:
+            timestamp, all_day_date = _parse_timestamp_for_timeline(row["timestamp"])
+            if not _timestamp_in_range(
+                timestamp,
+                time_range,
+                all_day_date=all_day_date,
+            ):
+                continue
+            candidates.append(
+                (
+                    row["id"],
+                    timestamp or datetime.max.replace(tzinfo=timezone.utc),
+                    float(row["created_at"] or 0.0),
+                )
+            )
+
+        candidates.sort(key=lambda item: (item[1], item[2]))
+        return [chunk_id for chunk_id, *_ in candidates[:limit]]
+
+    def _filter_calendar_timeline_fused(
+        self,
+        fused: List[Tuple[str, float, float, float]],
+        time_range: Optional[Tuple[Optional[datetime], Optional[datetime]]],
+    ) -> List[Tuple[str, float, float, float]]:
+        """Apply normalized timestamp filtering to ranked calendar candidates."""
+        if not fused:
+            return fused
+        ids = [chunk_id for chunk_id, *_ in fused]
+        placeholders = ",".join("?" for _ in ids)
+        rows = self._store._conn.execute(
+            f"""
+            SELECT id, timestamp
+            FROM knowledge_chunks
+            WHERE id IN ({placeholders})
+            """,
+            ids,
+        ).fetchall()
+        timestamps = {
+            row["id"]: _parse_timestamp_for_timeline(row["timestamp"])
+            for row in rows
+        }
+
+        def _keeps_item(item: Tuple[str, float, float, float]) -> bool:
+            timestamp, all_day_date = timestamps.get(item[0], (None, None))
+            return _timestamp_in_range(
+                timestamp,
+                time_range,
+                all_day_date=all_day_date,
+            )
+
+        return [item for item in fused if _keeps_item(item)]
+
     # ------------------------------------------------------------------
     # Public entry point
     # ------------------------------------------------------------------
@@ -396,42 +669,65 @@ class HybridSearch:
         when callers want a pure metadata filter (e.g. "all mail from X in
         May") — in that case only the vector leg runs (and only if an
         embedder is configured); if neither leg yields anything the
-        structured filter is applied directly and the most recent rows are
-        returned.
+        structured filter is applied directly. Upcoming calendar timelines are
+        returned nearest-first; other fallbacks return the most recent rows.
         """
+        time_range, sources, chronological_order, metadata_only = (
+            self._normalise_calendar_timeline_scope(query, time_range, sources)
+        )
+        rank_query = "" if metadata_only else query
+        calendar_timeline = chronological_order and _sources_include_gcalendar(sources)
+        recall_time_range = None if calendar_timeline else time_range
+
         bm25_filter_sql, bm25_filter_params = self._build_filters(
-            person=person, time_range=time_range, sources=sources, alias="kc"
+            person=person, time_range=recall_time_range, sources=sources, alias="kc"
         )
         unaliased_filter_sql, unaliased_filter_params = self._build_filters(
-            person=person, time_range=time_range, sources=sources
+            person=person, time_range=recall_time_range, sources=sources
         )
 
         bm25 = (
-            self._bm25_recall(query, bm25_filter_sql, bm25_filter_params)
-            if query.strip()
+            self._bm25_recall(rank_query, bm25_filter_sql, bm25_filter_params)
+            if rank_query.strip()
             else []
         )
         vector = (
-            self._vector_recall(query, unaliased_filter_sql, unaliased_filter_params)
-            if query.strip()
+            self._vector_recall(
+                rank_query,
+                unaliased_filter_sql,
+                unaliased_filter_params,
+            )
+            if rank_query.strip()
             else []
         )
         fused = self._fuse(bm25, vector)
+        if calendar_timeline:
+            fused = self._filter_calendar_timeline_fused(fused, time_range)
 
         # Metadata-only fallback: empty query, or both legs produced nothing
-        # despite a non-empty query. Return the most recent rows matching the
-        # filter so the agent still gets a useful corpus snapshot.
+        # despite a non-empty query. Calendar timeline requests use start-time
+        # ascending; other searches use recency so the agent still gets a
+        # useful corpus snapshot.
         if not fused:
-            sql = f"""
-                SELECT id FROM knowledge_chunks
-                WHERE {unaliased_filter_sql}
-                ORDER BY timestamp DESC, created_at DESC
-                LIMIT ?
-            """
-            rows = self._store._conn.execute(
-                sql, [*unaliased_filter_params, limit]
-            ).fetchall()
-            fused = [(row["id"], 0.0, 0.0, 0.0) for row in rows]
+            if calendar_timeline:
+                chunk_ids = self._calendar_timeline_ids(
+                    person=person,
+                    time_range=time_range,
+                    sources=sources,
+                    limit=limit,
+                )
+                fused = [(chunk_id, 0.0, 0.0, 0.0) for chunk_id in chunk_ids]
+            else:
+                sql = f"""
+                    SELECT id FROM knowledge_chunks
+                    WHERE {unaliased_filter_sql}
+                    ORDER BY timestamp DESC, created_at DESC
+                    LIMIT ?
+                """
+                rows = self._store._conn.execute(
+                    sql, [*unaliased_filter_params, limit]
+                ).fetchall()
+                fused = [(row["id"], 0.0, 0.0, 0.0) for row in rows]
 
         # Materialise the top-N rows in one IN-clause round trip.
         top = fused[:limit]

{openjarvis-1.0.4.dev872 → openjarvis-1.0.4.dev874}/src/openjarvis/server/agent_manager_routes.py

@@ -2263,14 +2263,14 @@ def create_agent_manager_router(
         import httpx
 
         try:
-            resp = httpx.get(
-                "https://api.sendblue.co/api/lines",
-                headers={
-                    "sb-api-key-id": api_key_id,
-                    "sb-api-secret-key": api_secret_key,
-                },
-                timeout=15.0,
-            )
+            async with httpx.AsyncClient(timeout=15.0) as client:
+                resp = await client.get(
+                    "https://api.sendblue.co/api/lines",
+                    headers={
+                        "sb-api-key-id": api_key_id,
+                        "sb-api-secret-key": api_secret_key,
+                    },
+                )
             if resp.status_code == 401:
                 raise HTTPException(
                     status_code=401,
@@ -2290,12 +2290,16 @@ def create_agent_manager_router(
             )
             numbers = []
             for line in lines:
-                num = (
-                    line.get("number")
-                    or line.get("phone_number")
-                    or line.get("from_number")
-                    or (line if isinstance(line, str) else "")
-                )
+                if isinstance(line, str):
+                    num = line
+                elif isinstance(line, dict):
+                    num = (
+                        line.get("number")
+                        or line.get("phone_number")
+                        or line.get("from_number")
+                    )
+                else:
+                    num = None
                 if num:
                     numbers.append(num)
             return {
@@ -2327,18 +2331,18 @@ def create_agent_manager_router(
         import httpx
 
         try:
-            resp = httpx.post(
-                "https://api.sendblue.co/api/account/webhooks",
-                headers={
-                    "sb-api-key-id": api_key_id,
-                    "sb-api-secret-key": api_secret_key,
-                    "Content-Type": "application/json",
-                },
-                json={
-                    "receive": webhook_url,
-                },
-                timeout=15.0,
-            )
+            async with httpx.AsyncClient(timeout=15.0) as client:
+                resp = await client.post(
+                    "https://api.sendblue.co/api/account/webhooks",
+                    headers={
+                        "sb-api-key-id": api_key_id,
+                        "sb-api-secret-key": api_secret_key,
+                        "Content-Type": "application/json",
+                    },
+                    json={
+                        "receive": webhook_url,
+                    },
+                )
             return {
                 "registered": resp.status_code < 300,
                 "status": resp.status_code,
@@ -2378,16 +2382,16 @@ def create_agent_manager_router(
             if from_number:
                 payload["from_number"] = from_number
 
-            resp = httpx.post(
-                "https://api.sendblue.co/api/send-message",
-                headers={
-                    "sb-api-key-id": api_key_id,
-                    "sb-api-secret-key": api_secret_key,
-                    "Content-Type": "application/json",
-                },
-                json=payload,
-                timeout=15.0,
-            )
+            async with httpx.AsyncClient(timeout=15.0) as client:
+                resp = await client.post(
+                    "https://api.sendblue.co/api/send-message",
+                    headers={
+                        "sb-api-key-id": api_key_id,
+                        "sb-api-secret-key": api_secret_key,
+                        "Content-Type": "application/json",
+                    },
+                    json=payload,
+                )
             return {
                 "sent": resp.status_code < 300,
                 "status": resp.status_code,

{openjarvis-1.0.4.dev872 → openjarvis-1.0.4.dev874}/src/openjarvis/server/api_routes.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import inspect
 import json
 import logging
@@ -894,7 +895,12 @@ async def transcribe_speech(request: Request):
     ext = filename.rsplit(".", 1)[-1] if "." in filename else "wav"
 
     try:
-        result = backend.transcribe(audio_bytes, format=ext, language=language or None)
+        result = await asyncio.to_thread(
+            backend.transcribe,
+            audio_bytes,
+            format=ext,
+            language=language or None,
+        )
     except Exception as exc:
         logger.exception("Speech transcription failed")
         raise HTTPException(

{openjarvis-1.0.4.dev872 → openjarvis-1.0.4.dev874}/src/openjarvis/server/routes.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import logging
 import uuid
 from typing import Any
@@ -835,7 +836,7 @@ async def list_models(request: Request) -> ModelListResponse:
     # Filter out any cloud model IDs that may appear via MultiEngine.
     # Fall back to direct Ollama query only when the engine returns nothing.
     engine = request.app.state.engine
-    all_ids = engine.list_models()
+    all_ids = await asyncio.to_thread(engine.list_models)
     model_ids = [m for m in all_ids if not is_cloud_model(m)]
     if not model_ids:
         model_ids = await list_local_models()
@@ -865,12 +866,12 @@ async def pull_model(request: Request):
     import httpx as _httpx
 
     host = getattr(engine, "_host", "http://localhost:11434")
-    client = _httpx.Client(base_url=host, timeout=600.0)
     try:
-        resp = client.post(
-            "/api/pull",
-            json={"name": model_name, "stream": False},
-        )
+        async with _httpx.AsyncClient(base_url=host, timeout=600.0) as client:
+            resp = await client.post(
+                "/api/pull",
+                json={"name": model_name, "stream": False},
+            )
         resp.raise_for_status()
     except (_httpx.ConnectError, _httpx.TimeoutException) as exc:
         raise HTTPException(status_code=502, detail=f"Ollama unreachable: {exc}")
@@ -879,8 +880,6 @@ async def pull_model(request: Request):
             status_code=exc.response.status_code,
             detail=f"Ollama error: {exc.response.text[:300]}",
         )
-    finally:
-        client.close()
 
     return {"status": "ok", "model": model_name}
 
@@ -896,13 +895,13 @@ async def delete_model(model_name: str, request: Request):
     import httpx as _httpx
 
     host = getattr(engine, "_host", "http://localhost:11434")
-    client = _httpx.Client(base_url=host, timeout=30.0)
     try:
-        resp = client.request(
-            "DELETE",
-            "/api/delete",
-            json={"name": model_name},
-        )
+        async with _httpx.AsyncClient(base_url=host, timeout=30.0) as client:
+            resp = await client.request(
+                "DELETE",
+                "/api/delete",
+                json={"name": model_name},
+            )
         resp.raise_for_status()
     except (_httpx.ConnectError, _httpx.TimeoutException) as exc:
         raise HTTPException(status_code=502, detail=f"Ollama unreachable: {exc}")
@@ -911,8 +910,6 @@ async def delete_model(model_name: str, request: Request):
             status_code=exc.response.status_code,
             detail=f"Ollama error: {exc.response.text[:300]}",
         )
-    finally:
-        client.close()
 
     return {"status": "deleted", "model": model_name}
 

{openjarvis-1.0.4.dev872 → openjarvis-1.0.4.dev874}/src/openjarvis/telemetry/aggregator.py

@@ -90,6 +90,9 @@ class TelemetryAggregator:
     def __init__(self, db_path: str | Path) -> None:
         self._db_path = str(db_path)
         self._conn = sqlite3.connect(self._db_path, check_same_thread=False)
+        self._conn.execute("PRAGMA journal_mode=WAL")
+        self._conn.execute("PRAGMA synchronous=NORMAL")
+        self._conn.execute("PRAGMA busy_timeout=5000")
         self._conn.row_factory = sqlite3.Row
 
     def _time_filter(