chatify-app 0.2.0__py3-none-any.whl

chatify/configdoctor.py

@@ -0,0 +1,107 @@
+"""Offline configuration diagnostics and editor schema. Never runs connectors."""
+import copy, difflib, json, re
+from pathlib import Path
+from . import schema
+
+SOURCE_KEYS = {'read_users','document_users','id','label','type','weight','path','semantic_model','rerank_model','ocr','refresh_interval','url','results_path','fields','header_env','timeout','name','filter','category','entry','providers','provider','mailto','limit','language','endpoint','api_key_env'}
+
+def editor_schema():
+    def node(v):
+        if isinstance(v, dict): return {'type':'object','properties':{k:node(x) for k,x in v.items()},'additionalProperties':False}
+        if isinstance(v, bool): return {'type':'boolean'}
+        if isinstance(v, int): return {'type':'integer'}
+        if isinstance(v, float): return {'type':'number'}
+        if isinstance(v, list): return {'type':'array','items':{'type':'string'}}
+        if v is None: return {}
+        return {'type':'string'}
+    out=node(schema.DEFAULTS);out.update({'$schema':'https://json-schema.org/draft/2020-12/schema','title':'Chatify app configuration'})
+    props=out['properties'];props['schema_version']['const']=1
+    props['actions']['properties']['allow']={'type':['array','null'],'items':{'type':'string','enum':['list_notes','save_note']}}
+    p=props['provider']['properties'];p['type']['enum']=sorted(schema.PROVIDERS)
+    fallback=copy.deepcopy(p);fallback.pop('fallback',None)
+    p['fallback']={'anyOf':[{'type':'null'}, {'type':'object','properties':fallback,'additionalProperties':False}]}
+    props['auth']['properties']['mode']['enum']=sorted(schema.AUTH_MODES)
+    props['app']['properties']['theme']['enum']=['auto','light','dark']
+    props['prompt']['properties']['style']={'type':'object','additionalProperties':{'type':'string'}}
+    props['sources']={'type':'array','items':{'type':'object','required':['type'],'properties':{k:{} for k in sorted(SOURCE_KEYS)},'additionalProperties':True}}
+    sp=props['sources']['items']['properties'];sp['type']={'type':'string','enum':sorted(schema.CONNECTORS)}
+    for k in ('id','label','path','url','semantic_model','rerank_model','entry'):sp[k]={'type':'string'}
+    sp['ocr']={'type':'boolean'};sp['weight']={'type':'number','exclusiveMinimum':0}
+    props['retrieval']['properties']['top_k'].update(minimum=1,maximum=50)
+    props['server']['properties']['port'].update(minimum=1,maximum=65535)
+    return out
+
+def inspect(path):
+    """Return issues and validated config; never echoes scalar values or reads .env."""
+    import yaml
+    text=Path(path).read_text(encoding='utf-8');issues=[];positions={};duplicates=[]
+    def walk(n, key=''):
+        positions[key]=(n.start_mark.line+1,n.start_mark.column+1)
+        if isinstance(n,yaml.MappingNode):
+            seen=set()
+            for k,v in n.value:
+                child=(key+'.' if key else '')+str(k.value)
+                positions[child]=(k.start_mark.line+1,k.start_mark.column+1)
+                if k.value in seen:duplicates.append(child)
+                seen.add(k.value);walk(v,child)
+                positions[child]=(k.start_mark.line+1,k.start_mark.column+1)
+        elif isinstance(n,yaml.SequenceNode):
+            for i,v in enumerate(n.value):walk(v,key+'['+str(i)+']')
+    def issue(key,msg,fix=''):
+        line,col=positions.get(key,positions.get(key.split('.')[0],(1,1)))
+        issues.append({'key':key or '(root)','line':line,'column':col,'message':msg,'suggestion':fix})
+    try:
+        n=yaml.compose(text)
+        if n:walk(n)
+        raw=yaml.safe_load(text) if not str(path).endswith('.json') else json.loads(text)
+        if raw is None:raw={}
+    except (yaml.YAMLError,json.JSONDecodeError) as e:
+        mark=getattr(e,'problem_mark',None)
+        issues.append({'key':'(syntax)','line':mark.line+1 if mark else getattr(e,'lineno',1),'column':mark.column+1 if mark else getattr(e,'colno',1),'message':'Invalid YAML/JSON syntax.','suggestion':'Check indentation, colons and quoted strings near this line.'})
+        return issues,None
+    for key in duplicates:issue(key,'Duplicate key; only the last value would be used.','Keep one entry for this key.')
+    def check(v, spec, key=''):
+        if 'anyOf' in spec:
+            if v is None:return
+            spec=next(x for x in spec['anyOf'] if x.get('type')=='object')
+        typ=spec.get('type');typ=next((t for t in typ if t!='null'),'null') if isinstance(typ,list) and v is not None else ('null' if isinstance(typ,list) else typ);types={'object':dict,'array':list,'string':str,'integer':int,'number':(int,float),'boolean':bool}
+        if typ=='null' and v is None:return
+        if typ in types and (not isinstance(v,types[typ]) or typ in ('integer','number') and isinstance(v,bool)):
+            issue(key,'Expected '+typ+'.','Use a '+typ+' value.');return
+        if 'enum' in spec and v not in spec['enum']:issue(key,'Unsupported choice.','Choose: '+', '.join(spec['enum']))
+        for op in ('minimum','maximum','exclusiveMinimum'):
+            if op in spec and (v<spec[op] if op=='minimum' else v>spec[op] if op=='maximum' else v<=spec[op]):issue(key,'Number outside allowed range.','Use '+op+' '+str(spec[op])+'.')
+        if isinstance(v,dict):
+            known=spec.get('properties',{})
+            for k,x in v.items():
+                child=(key+'.' if key else '')+str(k)
+                if k in known:check(x,known[k],child)
+                elif spec.get('additionalProperties') is False:
+                    near=difflib.get_close_matches(str(k),known,n=1)
+                    issue(child,'Unknown key.','Use '+near[0]+'.' if near else 'Remove this key or check the schema.')
+                elif isinstance(spec.get('additionalProperties'),dict):check(x,spec['additionalProperties'],child)
+            for k in spec.get('required',[]):
+                if k not in v:issue(key+'.'+k,'Required key is missing.','Add '+k+'.')
+        if isinstance(v,list):
+            for i,x in enumerate(v):check(x,spec.get('items',{}),key+'['+str(i)+']')
+    check(raw,editor_schema())
+    cfg=None
+    if not issues:
+        try:cfg=schema.validate(raw)
+        except (schema.ConfigError,TypeError,KeyError,AttributeError,ValueError) as e:
+            msg=str(e) if isinstance(e,schema.ConfigError) else 'Configuration shape is invalid.'
+            if msg.startswith('Duplicate source id'):msg='sources: Duplicate source id. Each source needs a unique id.'
+            key=next((k for k in positions if k and k in msg),'')
+            match=re.search(r'(sources\[\d+\](?:\.\w+)?|(?:provider|retrieval|auth|budget|app)\.\w+)',msg)
+            if match:key=match.group(0)
+            issue(key,msg,'Check this key against the editor schema.')
+    return issues,cfg
+
+def report(path):
+    try:issues,cfg=inspect(path)
+    except OSError:
+        print('Cannot read config file. Check the path and permissions.');return False
+    if issues:
+        for i in issues:print('%s:%d:%d %s: %s %s' % (path,i['line'],i['column'],i['key'],i['message'],i['suggestion']))
+    else:print('Config OK: %d source(s). No connectors or models were called.' % len(cfg['sources']))
+    return not issues

chatify/connectors/__init__.py

@@ -0,0 +1,6 @@
+"""Connectors turn a source block from the app file into evidence.
+
+Every connector exposes search(query, k) -> list of Evidence dicts:
+  {id, title, text, url, authors, year, venue, source, score}
+"""
+from .base import Evidence, make_connector  # noqa: F401

chatify/connectors/arxiv.py

@@ -0,0 +1,37 @@
+"""arXiv Atom API (public). block.category narrows the search, e.g. eess.SP for signal processing."""
+import time, urllib.error, urllib.parse, urllib.request
+import xml.etree.ElementTree as ET
+from .base import Evidence
+
+NS = {"a": "http://www.w3.org/2005/Atom"}
+
+
+class Arxiv:
+    def __init__(self, block):
+        self.id, self.label, self.b = block["id"], block["label"], block
+
+    def search(self, query, k=6):
+        q = "all:" + urllib.parse.quote(query)
+        if self.b.get("category"):
+            q += "+AND+cat:" + urllib.parse.quote(self.b["category"])
+        url = "https://export.arxiv.org/api/query?search_query=%s&max_results=%d&sortBy=relevance" % (q, k)
+        for attempt in range(3):
+            try:
+                with urllib.request.urlopen(url, timeout=25) as r:
+                    root = ET.fromstring(r.read())
+                break
+            except (TimeoutError, urllib.error.URLError) as error:
+                if isinstance(error, urllib.error.HTTPError) and error.code not in (429, 500, 502, 503, 504):
+                    raise
+                if attempt == 2:
+                    raise
+                time.sleep(3 * (attempt + 1))
+        out = []
+        for i, e in enumerate(root.findall("a:entry", NS)):
+            link = e.findtext("a:id", "", NS)
+            out.append(Evidence(" ".join((e.findtext("a:title", "", NS) or "").split()),
+                                " ".join((e.findtext("a:summary", "", NS) or "").split()), link,
+                                [a.findtext("a:name", "", NS) for a in e.findall("a:author", NS)][:6],
+                                (e.findtext("a:published", "", NS) or "")[:4], "arXiv", self.id, 1.0 / (i + 1),
+                                "arXiv:" + link.rsplit("/", 1)[-1]))
+        return out

chatify/connectors/base.py

@@ -0,0 +1,45 @@
+import importlib, os, sys
+
+
+def Evidence(title="", text="", url="", authors=None, year="", venue="", source="", score=0.0, id=""):
+    return {"id": id or title[:60], "title": title, "text": text, "url": url, "authors": authors or [],
+            "year": str(year or ""), "venue": venue, "source": source, "score": float(score)}
+
+
+def make_connector(block, base_dir="."):
+    t = block["type"]
+    if t == "local_files":
+        from .local_files import LocalFiles
+        return LocalFiles(block, base_dir)
+    if t == "http_json":
+        from .http_json import HttpJson
+        return HttpJson(block)
+    if t == "pubmed":
+        from .pubmed import PubMed
+        return PubMed(block)
+    if t == "arxiv":
+        from .arxiv import Arxiv
+        return Arxiv(block)
+    if t in ("wikipedia", "crossref", "openalex"):
+        from . import open_sources
+        return {"wikipedia": open_sources.Wikipedia, "crossref": open_sources.Crossref, "openalex": open_sources.OpenAlex}[t](block)
+    if t == "web_search":
+        from .web_search import WebSearch
+        return WebSearch(block)
+    if t == "python":
+        if block.get('execution') == 'bounded':
+            from .bounded import BoundedPlugin
+            return BoundedPlugin(block, base_dir)
+        # plugin: module:function, function(query, k) -> list of Evidence dicts
+        mod, _, fn = block["entry"].partition(":")
+        if base_dir not in sys.path:
+            sys.path.insert(0, os.path.abspath(base_dir))
+        f = getattr(importlib.import_module(mod), fn)
+
+        class Plugin:
+            def __init__(self): self.id = block["id"]; self.label = block["label"]
+            def search(self, q, k):
+                from .recovery import recover_evidence
+                return recover_evidence(f(q,k),self.id,k)
+        return Plugin()
+    raise ValueError("Unknown connector " + t)

chatify/connectors/bounded.py

@@ -0,0 +1,44 @@
+"""Separate, bounded plugin process. NOT a security sandbox: host files/network remain accessible."""
+import json,os,re,subprocess,sys,tempfile
+from pathlib import Path
+SCRIPT = '''import importlib,json,sys,os
+if os.name!='nt':
+ import resource
+ resource.setrlimit(resource.RLIMIT_FSIZE,(2097152,2097152))
+ resource.setrlimit(resource.RLIMIT_CPU,(20,20))
+from pathlib import Path
+sys.path.insert(0,sys.argv[1])
+m,f=sys.argv[2].split(':',1)
+q,k=json.loads(sys.stdin.read())
+x=getattr(importlib.import_module(m),f)(q,k)
+if isinstance(x,list):x=x[:k]
+Path(sys.argv[3]).write_text(json.dumps(x))
+'''
+class BoundedPlugin:
+ def __init__(self,block,base):self.id=block['id'];self.label=block['label'];self.entry=block['entry'];self.base=os.path.abspath(base);self.key_env=block.get("api_key_env", "")
+ def search(self,q,k):
+  from .base import Evidence
+  k=max(1,min(int(k),50))
+  with tempfile.TemporaryDirectory(prefix='chatify-plugin-') as folder:
+   result=Path(folder)/'result.json';log=Path(folder)/'output.log'
+   allowed={'PATH','HOME','USERPROFILE','SYSTEMROOT','WINDIR','TEMP','TMP','TMPDIR','LANG','LC_ALL','LD_LIBRARY_PATH'}
+   env={a:b for a,b in os.environ.items() if a.upper() in allowed}
+   if re.fullmatch(r'[A-Z][A-Z0-9_]{2,100}',self.key_env) and self.key_env in os.environ:env[self.key_env]=os.environ[self.key_env]
+   with log.open('wb') as out:
+    p=subprocess.Popen([sys.executable,'-I','-c',SCRIPT,self.base,self.entry,str(result)],stdin=subprocess.PIPE,stdout=out,stderr=out,env=env,cwd=folder,start_new_session=os.name!='nt')
+    p.stdin.write(json.dumps([str(q)[:2000],k]).encode());p.stdin.close()
+    import time
+    deadline=time.monotonic()+20
+    while p.poll() is None:
+     if time.monotonic()>deadline or log.stat().st_size>1024*1024 or (result.exists() and result.stat().st_size>2*1024*1024):
+      if os.name!='nt':
+       import signal
+       try:os.killpg(p.pid,signal.SIGKILL)
+       except ProcessLookupError:pass
+      else:p.kill()
+      p.wait();raise ValueError('Connector exceeded its time/output limit')
+     time.sleep(.05)
+   if p.returncode or not result.exists():raise ValueError('Connector failed in bounded mode; inspect its code and dependencies')
+   if result.stat().st_size>2*1024*1024:raise ValueError('Connector result exceeds 2 MB')
+   from .recovery import recover_evidence
+   return recover_evidence(json.loads(result.read_text()),self.id,k)

chatify/connectors/http_json.py

@@ -0,0 +1,66 @@
+"""Generic REST connector: GET a URL template, map JSON fields to evidence. Covers most search APIs."""
+import json, urllib.parse, urllib.request
+from .base import Evidence
+from .. import source_credentials
+
+
+def dig(obj, path):
+    for part in [p for p in (path or "").split(".") if p]:
+        if isinstance(obj, list):
+            try: obj = obj[int(part)]
+            except (ValueError, IndexError): return None
+        elif isinstance(obj, dict):
+            obj = obj.get(part)
+        else:
+            return None
+    return obj
+
+
+class _NoCredentialRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        raise ValueError('Credential-bearing HTTP sources do not follow redirects; review the configured address')
+
+
+class HttpJson:
+    """block: url (with {query} and {k}), results_path, fields {title,text,url,authors,year,venue},
+    optional header_env: {Header-Name: ENV_VAR} so keys never live in the file."""
+    def __init__(self, block):
+        self.id, self.label, self.b = block["id"], block["label"], block
+
+    def search(self, query, k=6):
+        url = self.b["url"].replace("{query}", urllib.parse.quote(query)).replace("{k}", str(k))
+        headers = {"User-Agent": "chatify/0.1"}
+        if self.b.get("header_env"):source_credentials.declared({"sources":[self.b]})
+        for h, env in (self.b.get("header_env") or {}).items():
+            key = source_credentials.value(self.id, env, self.b["url"])
+            if not key: raise ValueError('Missing private HTTP source key: ' + env)
+            headers[h] = key
+        request = urllib.request.Request(url, headers=headers)
+        opener = urllib.request.build_opener(_NoCredentialRedirect()).open if self.b.get('header_env') else urllib.request.urlopen
+        with opener(request, timeout=self.b.get("timeout", 20)) as r:
+            raw = r.read(1500000).decode("utf-8", "replace")
+        try:
+            data = json.loads(raw)
+        except ValueError:
+            # Broken or non-JSON reply: keep it as one text passage so the model can still read it.
+            return [Evidence(self.b.get("name", "Source reply"), raw[:4000], url, [], "", "", self.id, 1.0)] if raw.strip() else []
+        if isinstance(data, dict) and (data.get('status') == 'error' or data.get('errors')):
+            raise ValueError('HTTP source rejected the query; check its key, quota and plan')
+        items = dig(data, self.b.get("results_path", "")) or []
+        if not isinstance(items, list) or not all(isinstance(item, dict) for item in items):
+            from .recovery import recover_evidence
+            recovered=recover_evidence(data,self.id,k)
+            if not recovered:raise ValueError('HTTP source result mapping has no usable evidence')
+            return recovered
+        f = self.b.get("fields", {})
+        out = []
+        for i, item in enumerate(items):
+            a = dig(item, f.get("authors", "authors")) or []
+            if isinstance(a, list):
+                a = [x.get("name", "") if isinstance(x, dict) else str(x) for x in a]
+            elif a: a = [str(a)]
+            else: a = []
+            out.append(Evidence(str(dig(item, f.get("title", "title")) or ""), str(dig(item, f.get("text", "abstract")) or ""),
+                                str(dig(item, f.get("url", "url")) or ""), a, dig(item, f.get("year", "year")) or "",
+                                str(dig(item, f.get("venue", "venue")) or ""), self.id, 1.0 / (i + 1)))
+        return out[:k]

chatify/connectors/local_files.py

@@ -0,0 +1,198 @@
+"""Search a folder of .md .txt .json .csv files with BM25. No network, no embeddings needed."""
+import glob, hashlib, json, math, os, re, threading, time
+from .base import Evidence
+
+TOKEN = re.compile(r"[a-z0-9]{2,}")
+STOP = set("the of and to in is are for on with that this as by an be or it at from was were which can".split())
+
+
+def tokens(s):
+    return [t for t in TOKEN.findall(s.lower()) if t not in STOP]
+
+
+def chunks(text, size=900):
+    paras, cur, out = [p.strip() for p in re.split(r"\n\s*\n", text) if p.strip()], "", []
+    for p in paras:
+        if cur and len(cur) + len(p) > size:
+            out.append(cur); cur = p
+        else:
+            cur = (cur + "\n\n" + p).strip()
+    if cur:
+        out.append(cur)
+    return out
+
+
+class LocalFiles:
+    def __init__(self, block, base_dir="."):
+        self.id, self.label = block["id"], block["label"]
+        root = block.get("path", "docs")
+        root = root if os.path.isabs(root) else os.path.join(base_dir, root)
+        self.root = os.path.abspath(root)
+        self.semantic_model = block.get("semantic_model", "")
+        self.rerank_model = block.get("rerank_model", "")
+        self._semantic = self._reranker = None
+        self._vectors = None; self._vector_revision = None
+        self.ocr = bool(block.get("ocr", False))
+        self.refresh_interval = float(block.get("refresh_interval", 30))
+        self._lock = threading.RLock()
+        self._snapshot = None
+        self.checked_at = self.indexed_at = 0
+        self.last_error = ""
+        self.last_check = {}
+        self.revision = ""
+        self.docs = []; self.idf = {}; self.avg = 0
+        try: self.refresh(force=True)
+        except OSError: pass  # Keep the app available; failed folders are excluded until recovery.
+
+    def _scan(self):
+        if os.path.realpath(self.root) != self.root:
+            raise OSError("Local document folder must not use symlinks")
+        if not os.path.isdir(self.root):
+            raise OSError("Local document folder is missing or unavailable")
+        rows = []
+        for p in sorted(glob.glob(os.path.join(self.root, "**", "*"), recursive=True)):
+            if not p.lower().endswith((".md", ".txt", ".json", ".csv", ".pdf")): continue
+            if os.path.islink(p) or os.path.realpath(p) != os.path.abspath(p):
+                raise OSError("Local documents must not use symlinks")
+            if os.path.isfile(p):
+                with open(p, "rb") as f: data = f.read()
+                rows.append((os.path.relpath(p, self.root), data, hashlib.sha256(data).hexdigest()))
+        return rows
+
+    def refresh(self, force=False):
+        """Swap a complete content-hashed index only after a successful scan."""
+        with self._lock:
+            if not force and not self.last_error and time.time() - self.checked_at < self.refresh_interval:
+                return False
+            self.checked_at = time.time()
+            try:
+                rows = self._scan()
+                old = dict(self._snapshot or ())
+                current = {name:digest for name,_,digest in rows}
+                old_chunks = len(self.docs)
+                previous_passages = {(d["id"],d["text"]) for d in self.docs}
+                added = len(current.keys()-old.keys()); removed = len(old.keys()-current.keys())
+                changed = sum(old[name] != current[name] for name in old.keys() & current.keys())
+                signature = tuple((name, digest) for name, _, digest in rows)
+                if signature == self._snapshot:
+                    self.last_check = {"state":"unchanged","documents_scanned":len(rows),"added":0,"removed":0,"changed":0,"passages_before":old_chunks,"passages_after":old_chunks,"passages_added":0,"passages_removed":0}
+                    self.last_error = ""
+                    return False
+                nxt = LocalFiles.__new__(LocalFiles)
+                nxt.docs = []
+                for name, data, digest in rows:
+                    if name.lower().endswith(".pdf"):
+                        from ..pdfread import pages
+                        for page in pages(data, ocr=self.ocr):
+                            nxt._add(os.path.join(self.root, name), self.root, page["text"].encode(), digest, page["page"], page["ocr"])
+                    else:
+                        nxt._add(os.path.join(self.root, name), self.root, data, digest)
+                n = len(nxt.docs) or 1
+                df = {}
+                for d in nxt.docs:
+                    for t in set(d["tok"]): df[t] = df.get(t, 0) + 1
+                idf = {t: math.log(1 + (n - c + 0.5) / (c + 0.5)) for t, c in df.items()}
+                avg = sum(len(d["tok"]) for d in nxt.docs) / n
+                self.docs, self.idf, self.avg = nxt.docs, idf, avg
+                self._snapshot = signature
+                self.revision = hashlib.sha256(json.dumps(signature).encode()).hexdigest()
+                self.indexed_at = time.time(); self.last_error = ""
+                self.last_check = {"state":"updated","documents_scanned":len(rows),"added":added,"removed":removed,"changed":changed,"passages_before":old_chunks,"passages_after":len(self.docs),"passages_added":len({(d["id"],d["text"]) for d in self.docs}-previous_passages),"passages_removed":len(previous_passages-{(d["id"],d["text"]) for d in self.docs})}
+                return True
+            except (OSError, ValueError) as exc:
+                self.last_check = {"state":"failed"}
+                self.last_error = "Local folder check failed: "+str(exc)+". Check that the folder exists, files are readable and PDF dependencies are installed. No old passages will be used until it recovers."
+                raise OSError("Local source could not be indexed; check PDF dependencies or readability") from None
+
+    def status(self):
+        with self._lock:
+            return {"id": self.id, "label": self.label, "documents": len(self._snapshot or ()),
+                    "chunks": len(self.docs), "revision": self.revision,
+                    "checked_at": self.checked_at, "indexed_at": self.indexed_at,
+                    "refresh_interval": self.refresh_interval, "error": self.last_error, "last_check":dict(self.last_check)}
+
+    def _add(self, path, root, data=None, digest="", page=None, ocr=False):
+        if data is None:
+            with open(path, "rb") as f: data = f.read()
+        raw = data.decode("utf-8", errors="replace")
+        digest = digest or hashlib.sha256(data).hexdigest()
+        title = os.path.splitext(os.path.relpath(path, root))[0]
+        m = re.match(r"\s*#\s+(.+)", raw)
+        if m:
+            title = m.group(1).strip()
+        if path.endswith(".json"):
+            try:
+                raw = json.dumps(json.loads(raw), indent=1)
+            except ValueError:
+                pass
+        sections = re.split(r"(?m)^(?=#{1,3}\s)", raw) if raw.lstrip().startswith("#") or "\n#" in raw else [raw]
+        n = 0
+        for sec in sections:
+            if not sec.strip():
+                continue
+            h = re.match(r"\s*#{1,3}\s+(.+)", sec)
+            sec_title = h.group(1).strip() if h else title
+            body = re.sub(r"^\s*#{1,3}\s+.*\n?", "", sec, count=1) if h else sec
+            for c in chunks(body):
+                self.docs.append({"title": sec_title, "text": c, "tok": tokens(sec_title + " " + c),
+                                  "id": "%s#%s%d" % (os.path.relpath(path, root), ("page-%d-" % page) if page else "", n), "document": os.path.relpath(path, root), "version": digest, "section": sec_title, "page": page, "ocr": ocr})
+                n += 1
+
+    def search(self, query, k=6, documents=None):
+        if hasattr(self, "_lock"):
+            with self._lock:
+                self.refresh()
+                return self._search(query, k, documents)
+        return self._search(query, k, documents)
+
+    def _search(self, query, k, documents=None):
+        q = tokens(query)
+        scored = []
+        for d in self.docs:
+            if documents is not None and d.get("document") not in documents: continue
+            tf = {}
+            for t in d["tok"]:
+                tf[t] = tf.get(t, 0) + 1
+            s = 0.0
+            for t in q:
+                if t in tf:
+                    s += self.idf.get(t, 0) * tf[t] * 2.2 / (tf[t] + 1.2 * (0.25 + 0.75 * len(d["tok"]) / (self.avg or 1)))
+            if s > 0:
+                scored.append((s, d))
+        if getattr(self, "semantic_model", "") and self.docs:
+            if not self._semantic:
+                self._semantic = self._model(self.semantic_model, False)
+            if self._vector_revision != self.revision:
+                self._vectors = self._semantic.encode([d["title"] + " " + d["text"] for d in self.docs], normalize_embeddings=True)
+                self._vector_revision = self.revision
+            vector = self._semantic.encode([query], normalize_embeddings=True)[0]
+            semantic = sorted([(sum(float(a) * float(b) for a, b in zip(v, vector)), d) for v, d in zip(self._vectors, self.docs) if documents is None or d.get("document") in documents], key=lambda x: -x[0])
+            keyword = sorted(scored, key=lambda x: -x[0])
+            # Reciprocal rank fusion: do not compare unlike lexical/vector scales.
+            merged = {}
+            for ranking in (keyword, semantic):
+                for rank, (_, doc) in enumerate(ranking):
+                    row = merged.setdefault(doc["id"], [0, doc]); row[0] += 1 / (60 + rank + 1)
+            scored = [tuple(v) for v in merged.values()]
+        scored.sort(key=lambda x: -x[0])
+        if documents is not None: scored = [(score, d) for score, d in scored if d.get("document") in documents]
+        if getattr(self, "rerank_model", "") and scored:
+            if not self._reranker: self._reranker = self._model(self.rerank_model, True)
+            candidate = scored[:min(30, max(k * 3, k))]
+            predictions = self._reranker.predict([(query, d["text"]) for _, d in candidate])
+            ranked = sorted([(float(value), d) for value, (_, d) in zip(predictions, candidate)], key=lambda x: -x[0])
+            scored = [(1.0 / (rank + 1), d) for rank, (_, d) in enumerate(ranked)]
+        return [{**Evidence(d["title"], d["text"], "", [], "", "", self.id, s, d["id"]),
+                 **({"document": d["document"], "version": d["version"], "section": d["section"], "page": d["page"], "ocr": d["ocr"]} if "version" in d else {})} for s, d in scored[:k]]
+
+    @staticmethod
+    def _model(path, rerank):
+        if not os.path.isabs(path) or not os.path.isdir(path):
+            raise ValueError("Optional retrieval models must be existing absolute local directories")
+        try:
+            from sentence_transformers import SentenceTransformer, CrossEncoder
+        except ImportError:
+            raise ValueError("Optional semantic retrieval needs chatify[semantic]") from None
+        # Explicit local paths and local_files_only prevent network model downloads.
+        cls = CrossEncoder if rerank else SentenceTransformer
+        return cls(path, local_files_only=True, trust_remote_code=False)

chatify/connectors/open_sources.py

@@ -0,0 +1,114 @@
+"""Reference search sources: key-free Wikipedia/Crossref and keyed OpenAlex.
+
+Shapes were checked against live responses on 5 Oct 2026. Each connector is tolerant of missing fields and
+raises on network errors, which the pipeline reports as a source error without blocking other sources.
+Crossref asks polite users to send a contact address; set `mailto` on the source block to do that.
+OpenAlex keys come from the private store or OPENALEX_API_KEY. No key pauses searches.
+"""
+import json, os, re, time, urllib.parse, urllib.request, urllib.error
+from .base import Evidence
+
+UA = "chatify/1.0 (+https://github.com/Harsh23Kashyap/chatify)"
+
+
+def _get(url, timeout=20):
+    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "application/json"})
+    for attempt in range(3):
+        try:
+            with urllib.request.urlopen(req, timeout=timeout) as r:
+                return json.loads(r.read(3000000).decode("utf-8", "replace"))
+        except urllib.error.HTTPError as error:
+            if attempt == 2 or error.code not in (429, 500, 502, 503, 504):
+                raise
+            time.sleep(attempt + 1)
+
+
+def _clean(t):
+    return " ".join(re.sub(r"<[^>]+>", " ", str(t or "")).split())
+
+
+class Wikipedia:
+    def __init__(self, block):
+        self.id, self.label, self.b = block["id"], block["label"], block
+        self.lang = re.sub(r"[^a-z\-]", "", str(block.get("lang", "en")).lower()) or "en"
+
+    def search(self, query, k=6):
+        q = urllib.parse.urlencode({"action": "query", "generator": "search", "gsrsearch": query, "gsrlimit": k, "prop": "extracts",
+                                    "exintro": 1, "explaintext": 1, "exlimit": "max", "format": "json", "formatversion": 2, "redirects": 1})
+        data = _get("https://%s.wikipedia.org/w/api.php?%s" % (self.lang, q))
+        pages = (data.get("query") or {}).get("pages") or []
+        pages = sorted((p for p in pages if isinstance(p, dict)), key=lambda p: p.get("index", 99))
+        out = []
+        for n, p in enumerate(pages):
+            text = _clean(p.get("extract"))
+            title = _clean(p.get("title"))
+            if not (text and title):
+                continue
+            url = "https://%s.wikipedia.org/wiki/%s" % (self.lang, urllib.parse.quote(title.replace(" ", "_")))
+            out.append(Evidence(title, text[:1500], url, [], "", "Wikipedia", self.id, 1.0 / (n + 1), "wiki:" + str(p.get("pageid", title))))
+        return out
+
+
+class Crossref:
+    def __init__(self, block):
+        self.id, self.label, self.b = block["id"], block["label"], block
+
+    def search(self, query, k=6):
+        args = {"query": query, "rows": k, "select": "DOI,title,abstract,URL,issued,container-title,author"}
+        if self.b.get("mailto"):
+            args["mailto"] = str(self.b["mailto"])[:200]
+        data = _get("https://api.crossref.org/works?" + urllib.parse.urlencode(args))
+        items = (data.get("message") or {}).get("items") or []
+        out = []
+        for n, it in enumerate(items):
+            if not isinstance(it, dict):
+                continue
+            title = _clean((it.get("title") or [""])[0] if isinstance(it.get("title"), list) else it.get("title"))
+            text = _clean(it.get("abstract"))
+            if not title:
+                continue
+            year = ""
+            try:
+                year = str(it["issued"]["date-parts"][0][0] or "")
+            except (KeyError, IndexError, TypeError):
+                pass
+            venue = _clean((it.get("container-title") or [""])[0] if isinstance(it.get("container-title"), list) else "")
+            authors = [" ".join(x for x in (a.get("given"), a.get("family")) if x) for a in (it.get("author") or []) if isinstance(a, dict)][:6]
+            out.append(Evidence(title, (text or title)[:1500], it.get("URL") or "", authors, year, venue or "Crossref", self.id, 1.0 / (n + 1), "doi:" + str(it.get("DOI", title[:40]))))
+        return out
+
+
+def _abstract(inv):
+    if not isinstance(inv, dict):
+        return ""
+    words = {}
+    for w, pos in inv.items():
+        for p in pos if isinstance(pos, list) else []:
+            if isinstance(p, int):
+                words[p] = w
+    return " ".join(words[i] for i in sorted(words))
+
+
+class OpenAlex:
+    def __init__(self, block):
+        self.id, self.label, self.b = block["id"], block["label"], block
+
+    def search(self, query, k=6):
+        args = {"search": re.sub(r"[?*]", " ", query).strip(), "per-page": k, "select": "id,title,publication_year,abstract_inverted_index,doi,authorships,primary_location"}
+        from .. import secrets
+        key = secrets.saved("catalog:openalex") or os.environ.get("OPENALEX_API_KEY", "")
+        if not key:
+            return []
+        if key:
+            args["api_key"] = key
+        data = _get("https://api.openalex.org/works?" + urllib.parse.urlencode(args))
+        out = []
+        for n, it in enumerate(data.get("results") or []):
+            if not isinstance(it, dict) or not it.get("title"):
+                continue
+            title = _clean(it["title"])
+            text = _abstract(it.get("abstract_inverted_index"))
+            authors = [((a.get("author") or {}).get("display_name") or "") for a in (it.get("authorships") or []) if isinstance(a, dict)][:6]
+            src = (((it.get("primary_location") or {}).get("source")) or {}).get("display_name") or "OpenAlex"
+            out.append(Evidence(title, (text or title)[:1500], it.get("doi") or it.get("id") or "", [a for a in authors if a], it.get("publication_year") or "", src, self.id, 1.0 / (n + 1), "openalex:" + str(it.get("id", title[:40]))))
+        return out