customchat-app 0.1.0__py3-none-any.whl

customchat/__init__.py

@@ -0,0 +1,2 @@
+"""CustomChat: build a chat application over any evidence source from one YAML file."""
+__version__ = "0.1.0"

customchat/__main__.py

@@ -0,0 +1,2 @@
+from .cli import main
+main()

customchat/accounts.py

@@ -0,0 +1,95 @@
+"""Accounts: sign up, sign in, sign out, change password. Standard library only.
+Passwords are stored as salted PBKDF2 hashes. Sessions are random tokens kept only as hashes."""
+import hashlib, hmac, os, re, secrets, time
+
+ITER = 200_000
+SESSION_DAYS = 30
+EMAIL = re.compile(r"^[^@\s]{1,64}@[^@\s]{1,190}\.[^@\s]{2,}$")
+
+TABLES = """
+CREATE TABLE IF NOT EXISTS users(id TEXT PRIMARY KEY, email TEXT UNIQUE, name TEXT, salt BLOB, hash BLOB, created REAL);
+CREATE TABLE IF NOT EXISTS sessions(token_hash TEXT PRIMARY KEY, user TEXT, expires REAL);
+CREATE TABLE IF NOT EXISTS attempts(key TEXT, ts REAL);
+"""
+
+
+def _hash(pw, salt):
+    return hashlib.pbkdf2_hmac("sha256", pw.encode(), salt, ITER)
+
+
+def _th(token):
+    return hashlib.sha256(token.encode()).hexdigest()
+
+
+class Accounts:
+    def __init__(self, store, allow_signup=True):
+        self.s = store
+        self.allow_signup = allow_signup
+        with store.c() as c:
+            c.executescript(TABLES)
+
+    def _check_pw(self, pw):
+        if not isinstance(pw, str) or not 8 <= len(pw) <= 200:
+            raise ValueError("Password must be 8 to 200 characters")
+
+    def signup(self, email, name, pw):
+        if not self.allow_signup and self.s.q("SELECT 1 FROM users LIMIT 1"):
+            raise PermissionError("Sign-ups are closed")
+        email = str(email or "").strip().lower()
+        if not EMAIL.match(email):
+            raise ValueError("Enter a valid email address")
+        self._check_pw(pw)
+        if self.s.q("SELECT 1 FROM users WHERE email=?", (email,), one=True):
+            raise ValueError("An account with that email already exists")
+        uid, salt = "u_" + secrets.token_hex(8), os.urandom(16)
+        self.s.q("INSERT INTO users VALUES(?,?,?,?,?,?)", (uid, email, str(name or "").strip()[:60] or email.split("@")[0], salt, _hash(pw, salt), time.time()), write=True)
+        return uid
+
+    def _throttle(self, key):
+        now = time.time()
+        self.s.q("DELETE FROM attempts WHERE ts<?", (now - 600,), write=True)
+        if len(self.s.q("SELECT 1 FROM attempts WHERE key=?", (key,))) >= 8:
+            raise PermissionError("Too many attempts, try again in a few minutes")
+
+    def login(self, email, pw, ip=""):
+        email = str(email or "").strip().lower()
+        self._throttle(email)
+        u = self.s.q("SELECT * FROM users WHERE email=?", (email,), one=True)
+        ok = bool(u) and hmac.compare_digest(_hash(str(pw or ""), u["salt"]), u["hash"])
+        if not ok:
+            self.s.q("INSERT INTO attempts VALUES(?,?)", (email, time.time()), write=True)
+            raise PermissionError("Wrong email or password")
+        return u["id"], self._session(u["id"])
+
+    def _session(self, uid):
+        t = secrets.token_urlsafe(32)
+        self.s.q("INSERT INTO sessions VALUES(?,?,?)", (_th(t), uid, time.time() + SESSION_DAYS * 86400), write=True)
+        return t
+
+    def user_for(self, token):
+        if not token:
+            return None
+        r = self.s.q("SELECT u.id,u.email,u.name FROM sessions s JOIN users u ON u.id=s.user WHERE s.token_hash=? AND s.expires>?", (_th(token), time.time()), one=True)
+        return dict(r) if r else None
+
+    def logout(self, token):
+        if token:
+            self.s.q("DELETE FROM sessions WHERE token_hash=?", (_th(token),), write=True)
+
+    def change_password(self, uid, old, new, keep_token=None):
+        u = self.s.q("SELECT * FROM users WHERE id=?", (uid,), one=True)
+        if not u or not hmac.compare_digest(_hash(str(old or ""), u["salt"]), u["hash"]):
+            raise PermissionError("Current password is wrong")
+        self._check_pw(new)
+        salt = os.urandom(16)
+        self.s.q("UPDATE users SET salt=?,hash=? WHERE id=?", (salt, _hash(new, salt), uid), write=True)
+        # sign out every other device
+        self.s.q("DELETE FROM sessions WHERE user=? AND token_hash!=?", (uid, _th(keep_token or "")), write=True)
+
+    def delete_account(self, uid, pw):
+        u = self.s.q("SELECT * FROM users WHERE id=?", (uid,), one=True)
+        if not u or not hmac.compare_digest(_hash(str(pw or ""), u["salt"]), u["hash"]):
+            raise PermissionError("Password is wrong")
+        for t, col in (("sessions", "user"), ("chats", "owner"), ("topics", "owner"), ("feedback", "owner"), ("states", "owner"), ("profile", "owner"), ("uploads", "owner")):
+            self.s.q(f"DELETE FROM {t} WHERE {col}=?", (uid,), write=True)
+        self.s.q("DELETE FROM users WHERE id=?", (uid,), write=True)

customchat/actions.py

@@ -0,0 +1,31 @@
+"""Finite, registered actions with owner-bound, expiring, one-use review tickets."""
+import copy,json,secrets,threading,time
+REGISTRY={
+ 'list_notes':{'label':'List my notes','mode':'read','fields':{}},
+ 'save_note':{'label':'Save a private note','mode':'write','fields':{'title':{'type':'string','maxLength':120},'text':{'type':'string','maxLength':4000}}},
+}
+class Actions:
+ def __init__(self,cfg,store):self.cfg,self.store=cfg,store;self.lock=threading.RLock()
+ def catalog(self):
+  policy=self.cfg.get('actions',{});allow=policy.get('allow')
+  return [dict(id=k,**copy.deepcopy(v)) for k,v in REGISTRY.items() if (allow is None or k in allow) and (v['mode']!='write' or policy.get('writes',True))]
+ def prepare(self,owner,tool,args):
+  spec=next((x for x in self.catalog() if x['id']==tool),None)
+  if not spec:raise PermissionError('Action unavailable')
+  if not isinstance(args,dict) or set(args)!=set(spec['fields']):raise ValueError('Use exactly the declared action fields')
+  for k,f in spec['fields'].items():
+   if not isinstance(args[k],str) or not args[k].strip() or len(args[k])>f['maxLength']:raise ValueError(k+' needs nonempty text within '+str(f['maxLength'])+' characters')
+  ticket=secrets.token_urlsafe(24);payload={'tool':tool,'args':copy.deepcopy(args),'expires':time.time()+300,'used':False}
+  self.store.save_state(owner,'action-'+ticket,payload)
+  return dict(ticket=ticket,tool=tool,label=spec['label'],mode=spec['mode'],args=args,expires_in=300)
+ def execute(self,owner,ticket):
+  with self.lock:
+   p=self.store.get_state(owner,'action-'+str(ticket))
+   if p['used'] or time.time()>p['expires']:raise PermissionError('Review expired or already used')
+   if p['tool'] not in {x['id'] for x in self.catalog()}:raise PermissionError('Action no longer allowed')
+   p['used']=True;self.store.save_state(owner,'action-'+str(ticket),p)
+   if p['tool']=='save_note':
+    name='note-'+secrets.token_hex(8);self.store.save_state(owner,name,p['args']);return {'saved':True,'id':name}
+   if p['tool']=='list_notes':
+    return {'notes':[self.store.get_state(owner,n) for n in self.store.states(owner) if n.startswith('note-')][:100]}
+   raise PermissionError('Action unavailable')

customchat/budget.py

@@ -0,0 +1,74 @@
+"""Durable, atomic request/call quotas. Dollar limits fail closed without a meter."""
+import copy, datetime, hashlib, json, math, sqlite3, threading
+from pathlib import Path
+from .providers import ProviderError
+
+DEFAULT = {'daily_questions': 0, 'user_daily_questions': 0, 'daily_model_calls': 0, 'spend_cap_usd': None}
+
+class BudgetError(ProviderError):
+    pass
+
+def clean(raw):
+    if not isinstance(raw, dict): raise ValueError('budget must be a mapping')
+    if set(raw) - set(DEFAULT): raise ValueError('Unknown budget key')
+    out = dict(DEFAULT, **raw)
+    for k in ('daily_questions','user_daily_questions','daily_model_calls'):
+        v=out[k]
+        if isinstance(v,bool) or not isinstance(v,int) or not math.isfinite(v) or not 0 <= v <= 10000000:
+            raise ValueError(k + ' must be a whole number from 0 to 10000000')
+    v=out['spend_cap_usd']
+    if v is not None and (isinstance(v,bool) or not isinstance(v,(int,float)) or not math.isfinite(v) or not 0 <= v <= 1000000):
+        raise ValueError('spend_cap_usd must be null or 0-1000000')
+    return out
+
+class Budget:
+    def __init__(self, cfg, folder=None):
+        self.value=clean(cfg.get('budget',{}));self.folder=Path(folder) if folder else None
+        self.lock=threading.RLock()
+        if self.folder:
+            self.folder.mkdir(parents=True,exist_ok=True)
+            path=self.folder/'budget.json'
+            if path.exists():self.value=clean(json.loads(path.read_text()))
+        self.db=sqlite3.connect(str(self.folder/'budget.db') if self.folder else ':memory:',check_same_thread=False)
+        self.db.execute('CREATE TABLE IF NOT EXISTS usage(day TEXT, scope TEXT, kind TEXT, amount INTEGER, PRIMARY KEY(day,scope,kind))');self.db.commit()
+    def day(self):return datetime.datetime.now(datetime.timezone.utc).date().isoformat()
+    def take(self, kind, owner=''):
+        with self.lock:
+            limits=[('*', self.value['daily_questions' if kind=='question' else 'daily_model_calls'])]
+            if kind=='question':limits.append(('user:' + hashlib.sha256(str(owner).encode()).hexdigest(),self.value['user_daily_questions']))
+            day=self.day()
+            try:
+                self.db.execute('BEGIN IMMEDIATE')
+                for scope,limit in limits:
+                    row=self.db.execute('SELECT amount FROM usage WHERE day=? AND scope=? AND kind=?',(day,scope,kind)).fetchone()
+                    if limit and (row[0] if row else 0)>=limit:raise BudgetError('Daily '+kind+' limit reached. Resets at midnight UTC.',429)
+                for scope,_ in limits:
+                    self.db.execute('INSERT INTO usage VALUES(?,?,?,1) ON CONFLICT(day,scope,kind) DO UPDATE SET amount=amount+1',(day,scope,kind))
+                self.db.commit()
+            except Exception:
+                self.db.rollback();raise
+    def save(self, raw):
+        value=clean(raw)
+        with self.lock:
+            if self.folder:
+                path=self.folder/'budget.json';tmp=self.folder/'budget.json.tmp'
+                tmp.write_text(json.dumps(value,indent=2));tmp.replace(path)
+            self.value=value
+        return self.view()
+    def view(self):
+        with self.lock:
+            rows=self.db.execute('SELECT kind,amount FROM usage WHERE day=? AND scope=?',(self.day(),'*')).fetchall()
+            return {'limits':copy.deepcopy(self.value),'used':dict(rows),'day':self.day(),'reset_timezone':'UTC',
+                    'spend_meter':'unavailable','spend_note':'Dollar caps block all non-Demo model calls. No verified provider billing meter is installed.'}
+    def wrap(self, provider):return Guarded(provider,self)
+
+class Guarded:
+    def __init__(self, provider, budget):self.provider,self.budget=provider,budget
+    def _check(self):
+        from .providers import Mock
+        if self.budget.value['spend_cap_usd'] is not None and not isinstance(self.provider,Mock):
+            raise BudgetError('Dollar cap is set, but verified billing is unavailable. Non-Demo model calls are blocked.',429)
+        self.budget.take('model')
+    def complete(self,messages):self._check();return self.provider.complete(messages)
+    def stream(self,messages):
+        self._check();yield from self.provider.stream(messages)

customchat/cli.py

@@ -0,0 +1,136 @@
+"""customchat init | validate | run | doctor | ask"""
+import argparse, json, os, shutil, sys, urllib.request
+from . import schema, __version__
+
+TEMPLATE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates", "minimal")
+
+
+def main(argv=None):
+    try:
+        _main(argv)
+    except schema.ConfigError as e:
+        sys.exit("There is a problem with the app file:\n  %s" % e)
+    except KeyboardInterrupt:
+        sys.exit(0)
+
+
+def _main(argv=None):
+    ap = argparse.ArgumentParser(prog="customchat", description="Build a chat app over any evidence source.")
+    ap.add_argument("--version", action="version", version=__version__)
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    i = sub.add_parser("init", help="scaffold a new app folder"); i.add_argument("name")
+    st = sub.add_parser("start", help="create a local Demo workspace if absent, then run it")
+    st.add_argument("--directory", default="customchat-app", help="persistent workspace, never the tool cache")
+    st.add_argument("--port", type=int, default=8080)
+    st.add_argument("--no-browser", action="store_true")
+    st.add_argument("--lock-config", action="store_true")
+    v = sub.add_parser("validate", help="check an app file"); v.add_argument("app")
+    r = sub.add_parser("run", help="serve an app"); r.add_argument("app"); r.add_argument("--host"); r.add_argument("--port", type=int)
+    df = sub.add_parser("config-diff", help="preview changed keys and effects without values"); df.add_argument("app"); df.add_argument("candidate")
+    ca = sub.add_parser("config-apply", help="apply exact previewed config with private rollback backup")
+    ca.add_argument("app"); ca.add_argument("candidate"); ca.add_argument("--current-sha", required=True); ca.add_argument("--candidate-sha", required=True)
+    sub.add_parser("config-schema", help="print JSON Schema for editor autocomplete")
+    cd = sub.add_parser("config-doctor", help="offline line/key configuration checks"); cd.add_argument("app")
+    d = sub.add_parser("doctor", help="check provider and sources"); d.add_argument("app")
+    ev = sub.add_parser("eval", help="run questions from a file and report citation coverage"); ev.add_argument("app"); ev.add_argument("questions", help="text file, one question per line")
+    a = sub.add_parser("ask", help="ask one question from the terminal"); a.add_argument("app"); a.add_argument("question"); a.add_argument("--json", action="store_true")
+    ex = sub.add_parser("export", help="export app YAML, look, prompts and local documents, without private state")
+    ex.add_argument("app"); ex.add_argument("output", help="new ZIP file; never overwritten")
+    if argv is None: argv = sys.argv[1:]
+    if not argv: argv = ["start"]
+    args = ap.parse_args(argv)
+    if args.cmd == "config-diff":
+        from .configchange import diff
+        print(json.dumps(diff(args.app,args.candidate),indent=2)); return
+    if args.cmd == "config-apply":
+        from .configchange import apply
+        backup=apply(args.app,args.candidate,args.current_sha,args.candidate_sha)
+        print("Applied file. Rollback backup: " + backup)
+        print("Restart the app to load it. Review auth/source/network changes before launch."); return
+    if args.cmd == "config-schema":
+        from .configdoctor import editor_schema
+        print(json.dumps(editor_schema(), indent=2)); return
+    if args.cmd == "config-doctor":
+        from .configdoctor import report
+        if not report(args.app): raise SystemExit(1)
+        return
+    if args.cmd == "init":
+        if os.path.exists(args.name):
+            sys.exit("%s already exists" % args.name)
+        shutil.copytree(TEMPLATE, args.name, ignore=shutil.ignore_patterns("__pycache__", "*.db"))
+        print("Created %s/. Next: customchat run %s/app.yaml" % (args.name, args.name))
+    elif args.cmd == "export":
+        from .portable import export_app, ExportError
+        try:
+            m = export_app(args.app, args.output)
+        except (ExportError, OSError) as e:
+            sys.exit("Export stopped: %s" % e)
+        print("Exported %d app files to %s. No keys or private session state copied." % (len(m["files"]), args.output))
+        print("Review included local documents before sharing.")
+    elif args.cmd == "start":
+        from .launcher import start
+        start(args.directory, args.port, args.no_browser, args.lock_config)
+    elif args.cmd == "validate":
+        try:
+            c = schema.load(args.app)
+        except (schema.ConfigError, OSError) as e:
+            sys.exit("Invalid: %s" % e)
+        print("OK: %s, %d source(s), provider %s" % (c["app"]["title"], len(c["sources"]), c["provider"]["type"]))
+    elif args.cmd == "run":
+        from .server import serve
+        serve(args.app, args.host, args.port)
+    elif args.cmd == "doctor":
+        doctor(args.app)
+    elif args.cmd == "eval":
+        from .pipeline import Engine
+        from .store import Store
+        c = schema.load(args.app)
+        qs = [x.strip() for x in open(args.questions, encoding="utf-8") if x.strip() and not x.startswith("#")]
+        e = Engine(c, Store(":memory:"))
+        tot = cited = 0
+        for q in qs:
+            res = e.ask("eval", e.store.new_chat("eval"), q)
+            led = res["ledger"]; ok = sum(1 for l in led if l["supported"])
+            tot += len(led); cited += ok
+            print("%-60s evidence=%d claims=%d supported=%d" % (q[:60], len(res["evidence"]), len(led), ok))
+        print("coverage: %d/%d claims cited (%.0f%%)" % (cited, tot, 100.0 * cited / tot if tot else 0))
+    elif args.cmd == "ask":
+        from .pipeline import Engine
+        from .store import Store
+        c = schema.load(args.app)
+        e = Engine(c, Store(":memory:"))
+        res = e.ask("local", e.store.new_chat("local"), args.question)
+        if args.json:
+            print(json.dumps({k: res[k] for k in ("question", "standalone", "answer", "evidence", "ledger")}, indent=2)); return
+        print(res["answer"])
+        for ev in res["evidence"]:
+            print("  [%d] %s %s" % (ev["n"], ev["title"], ev["url"]))
+
+
+def doctor(path):
+    c = schema.load(path)
+    p = c["provider"]
+    print("provider:", p["type"], p["model"])
+    if p["type"] == "ollama":
+        base = (p["base_url"] or os.environ.get("OLLAMA_BASE_URL") or "http://localhost:11434").rstrip("/")
+        try:
+            tags = json.loads(urllib.request.urlopen(base + "/api/tags", timeout=5).read())
+            names = [m["name"] for m in tags.get("models", [])]
+            ok = any(n == p["model"] or n.startswith(p["model"] + ":") for n in names)
+            print("  ollama reachable; model %s %s" % (p["model"], "installed" if ok else "NOT installed, run: ollama pull " + p["model"]))
+        except Exception as e:
+            print("  ollama NOT reachable at %s (%s). Install from https://ollama.com and run `ollama serve`." % (base, type(e).__name__))
+    elif p["api_key_env"]:
+        print("  key env %s: %s" % (p["api_key_env"], "set" if os.environ.get(p["api_key_env"]) else "NOT set"))
+    from .connectors import make_connector
+    for s in c["sources"]:
+        try:
+            conn = make_connector(s, c["_dir"])
+            n = len(getattr(conn, "docs", [])) if s["type"] == "local_files" else None
+            print("  source %s (%s): ready%s" % (s["id"], s["type"], "" if n is None else ", %d chunks" % n))
+        except Exception as e:
+            print("  source %s: FAILED %s" % (s["id"], e))
+
+
+if __name__ == "__main__":
+    main()

customchat/configchange.py

@@ -0,0 +1,46 @@
+"""Offline config comparison and revision-checked replacement with private backup."""
+import hashlib, os, tempfile, time
+from pathlib import Path
+from .configdoctor import inspect
+from .schema import ConfigError
+
+def digest(path):return hashlib.sha256(Path(path).read_bytes()).hexdigest()
+
+def checked(path):
+    issues,cfg=inspect(path)
+    if issues:raise ConfigError('Config has errors; run config-doctor first.')
+    return cfg
+
+def diff(current,candidate):
+    a,b=checked(current),checked(candidate);changes=[]
+    labels={'provider':'model behavior or remote requests','sources':'retrieval corpus/access','auth':'visitor access','prompt':'answer behavior','budget':'request/model quotas','storage':'saved data location','server':'network exposure','memory':'conversation context','citations':'evidence display','retrieval':'ranking/cache behavior','app':'brand/UI','schema_version':'config compatibility'}
+    def walk(a,b,key=''):
+        if isinstance(a,dict) and isinstance(b,dict):
+            for k in sorted(set(a)|set(b)):
+                walk(a.get(k),b.get(k),(key+'.' if key else '')+str(k))
+        elif a!=b:
+            changes.append({'key':key,'effect':labels.get(key.split('.')[0],'configuration')})
+    walk(a,b)
+    return {'current_sha256':digest(current),'candidate_sha256':digest(candidate),'changes':changes}
+
+def apply(current,candidate,current_sha,candidate_sha):
+    """Never copies .env or state, never follows a target symlink, no hot reload."""
+    dst,src=Path(current),Path(candidate)
+    if dst.is_symlink() or src.is_symlink():raise ConfigError('Use regular config files, not symlinks.')
+    checked(dst);checked(src)
+    old,new=dst.read_bytes(),src.read_bytes()
+    if hashlib.sha256(old).hexdigest()!=current_sha or hashlib.sha256(new).hexdigest()!=candidate_sha:
+        raise ConfigError('Config changed since preview. Run config-diff again.')
+    if old==new:raise ConfigError('No file change to apply.')
+    name=dst.with_name(dst.name+'.backup-'+str(time.time_ns()))
+    fd=os.open(str(name),os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
+    with os.fdopen(fd,'wb') as f:f.write(old);f.flush();os.fsync(f.fileno())
+    fd,tmp=tempfile.mkstemp(prefix='.'+dst.name+'.',dir=str(dst.parent))
+    try:
+        with os.fdopen(fd,'wb') as f:f.write(new);f.flush();os.fsync(f.fileno())
+        # Catch edits during backup creation. This is not a multi-writer lock.
+        if dst.is_symlink() or digest(dst)!=current_sha:raise ConfigError('Current file changed; replacement stopped. Backup retained.')
+        os.replace(tmp,dst)
+    finally:
+        if os.path.exists(tmp):os.unlink(tmp)
+    return str(name)

customchat/configdoctor.py

@@ -0,0 +1,106 @@
+"""Offline configuration diagnostics and editor schema. Never runs connectors."""
+import copy, difflib, json, re
+from pathlib import Path
+from . import schema
+
+SOURCE_KEYS = {'read_users','document_users','id','label','type','weight','path','semantic_model','rerank_model','ocr','refresh_interval','url','results_path','fields','header_env','timeout','name','filter','category','entry','providers','provider','mailto','limit','language','endpoint','api_key_env'}
+
+def editor_schema():
+    def node(v):
+        if isinstance(v, dict): return {'type':'object','properties':{k:node(x) for k,x in v.items()},'additionalProperties':False}
+        if isinstance(v, bool): return {'type':'boolean'}
+        if isinstance(v, int): return {'type':'integer'}
+        if isinstance(v, float): return {'type':'number'}
+        if isinstance(v, list): return {'type':'array','items':{'type':'string'}}
+        if v is None: return {}
+        return {'type':'string'}
+    out=node(schema.DEFAULTS);out.update({'$schema':'https://json-schema.org/draft/2020-12/schema','title':'CustomChat app configuration'})
+    props=out['properties'];props['schema_version']['const']=1
+    props['actions']['properties']['allow']={'type':['array','null'],'items':{'type':'string','enum':['list_notes','save_note']}}
+    p=props['provider']['properties'];p['type']['enum']=sorted(schema.PROVIDERS)
+    fallback=copy.deepcopy(p);fallback.pop('fallback',None)
+    p['fallback']={'anyOf':[{'type':'null'}, {'type':'object','properties':fallback,'additionalProperties':False}]}
+    props['auth']['properties']['mode']['enum']=sorted(schema.AUTH_MODES)
+    props['app']['properties']['theme']['enum']=['auto','light','dark']
+    props['prompt']['properties']['style']={'type':'object','additionalProperties':{'type':'string'}}
+    props['sources']={'type':'array','items':{'type':'object','required':['type'],'properties':{k:{} for k in sorted(SOURCE_KEYS)},'additionalProperties':True}}
+    sp=props['sources']['items']['properties'];sp['type']={'type':'string','enum':sorted(schema.CONNECTORS)}
+    for k in ('id','label','path','url','semantic_model','rerank_model','entry'):sp[k]={'type':'string'}
+    sp['ocr']={'type':'boolean'};sp['weight']={'type':'number','exclusiveMinimum':0}
+    props['retrieval']['properties']['top_k'].update(minimum=1,maximum=50)
+    props['server']['properties']['port'].update(minimum=1,maximum=65535)
+    return out
+
+def inspect(path):
+    """Return issues and validated config; never echoes scalar values or reads .env."""
+    import yaml
+    text=Path(path).read_text(encoding='utf-8');issues=[];positions={};duplicates=[]
+    def walk(n, key=''):
+        positions[key]=(n.start_mark.line+1,n.start_mark.column+1)
+        if isinstance(n,yaml.MappingNode):
+            seen=set()
+            for k,v in n.value:
+                child=(key+'.' if key else '')+str(k.value)
+                positions[child]=(k.start_mark.line+1,k.start_mark.column+1)
+                if k.value in seen:duplicates.append(child)
+                seen.add(k.value);walk(v,child)
+                positions[child]=(k.start_mark.line+1,k.start_mark.column+1)
+        elif isinstance(n,yaml.SequenceNode):
+            for i,v in enumerate(n.value):walk(v,key+'['+str(i)+']')
+    def issue(key,msg,fix=''):
+        line,col=positions.get(key,positions.get(key.split('.')[0],(1,1)))
+        issues.append({'key':key or '(root)','line':line,'column':col,'message':msg,'suggestion':fix})
+    try:
+        n=yaml.compose(text)
+        if n:walk(n)
+        raw=yaml.safe_load(text) if not str(path).endswith('.json') else json.loads(text)
+        if raw is None:raw={}
+    except (yaml.YAMLError,json.JSONDecodeError) as e:
+        mark=getattr(e,'problem_mark',None)
+        issues.append({'key':'(syntax)','line':mark.line+1 if mark else getattr(e,'lineno',1),'column':mark.column+1 if mark else getattr(e,'colno',1),'message':'Invalid YAML/JSON syntax.','suggestion':'Check indentation, colons and quoted strings near this line.'})
+        return issues,None
+    for key in duplicates:issue(key,'Duplicate key; only the last value would be used.','Keep one entry for this key.')
+    def check(v, spec, key=''):
+        if 'anyOf' in spec:
+            if v is None:return
+            spec=next(x for x in spec['anyOf'] if x.get('type')=='object')
+        typ=spec.get('type');types={'object':dict,'array':list,'string':str,'integer':int,'number':(int,float),'boolean':bool}
+        if typ in types and (not isinstance(v,types[typ]) or typ in ('integer','number') and isinstance(v,bool)):
+            issue(key,'Expected '+typ+'.','Use a '+typ+' value.');return
+        if 'enum' in spec and v not in spec['enum']:issue(key,'Unsupported choice.','Choose: '+', '.join(spec['enum']))
+        for op in ('minimum','maximum','exclusiveMinimum'):
+            if op in spec and (v<spec[op] if op=='minimum' else v>spec[op] if op=='maximum' else v<=spec[op]):issue(key,'Number outside allowed range.','Use '+op+' '+str(spec[op])+'.')
+        if isinstance(v,dict):
+            known=spec.get('properties',{})
+            for k,x in v.items():
+                child=(key+'.' if key else '')+str(k)
+                if k in known:check(x,known[k],child)
+                elif spec.get('additionalProperties') is False:
+                    near=difflib.get_close_matches(str(k),known,n=1)
+                    issue(child,'Unknown key.','Use '+near[0]+'.' if near else 'Remove this key or check the schema.')
+                elif isinstance(spec.get('additionalProperties'),dict):check(x,spec['additionalProperties'],child)
+            for k in spec.get('required',[]):
+                if k not in v:issue(key+'.'+k,'Required key is missing.','Add '+k+'.')
+        if isinstance(v,list):
+            for i,x in enumerate(v):check(x,spec.get('items',{}),key+'['+str(i)+']')
+    check(raw,editor_schema())
+    cfg=None
+    if not issues:
+        try:cfg=schema.validate(raw)
+        except (schema.ConfigError,TypeError,KeyError,AttributeError,ValueError) as e:
+            msg=str(e) if isinstance(e,schema.ConfigError) else 'Configuration shape is invalid.'
+            if msg.startswith('Duplicate source id'):msg='sources: Duplicate source id. Each source needs a unique id.'
+            key=next((k for k in positions if k and k in msg),'')
+            match=re.search(r'(sources\[\d+\](?:\.\w+)?|(?:provider|retrieval|auth|budget|app)\.\w+)',msg)
+            if match:key=match.group(0)
+            issue(key,msg,'Check this key against the editor schema.')
+    return issues,cfg
+
+def report(path):
+    try:issues,cfg=inspect(path)
+    except OSError:
+        print('Cannot read config file. Check the path and permissions.');return False
+    if issues:
+        for i in issues:print('%s:%d:%d %s: %s %s' % (path,i['line'],i['column'],i['key'],i['message'],i['suggestion']))
+    else:print('Config OK: %d source(s). No connectors or models were called.' % len(cfg['sources']))
+    return not issues

customchat/connectors/__init__.py

@@ -0,0 +1,6 @@
+"""Connectors turn a source block from the app file into evidence.
+
+Every connector exposes search(query, k) -> list of Evidence dicts:
+  {id, title, text, url, authors, year, venue, source, score}
+"""
+from .base import Evidence, make_connector  # noqa: F401

customchat/connectors/arxiv.py

@@ -0,0 +1,28 @@
+"""arXiv Atom API (public). block.category narrows the search, e.g. eess.SP for signal processing."""
+import urllib.parse, urllib.request
+import xml.etree.ElementTree as ET
+from .base import Evidence
+
+NS = {"a": "http://www.w3.org/2005/Atom"}
+
+
+class Arxiv:
+    def __init__(self, block):
+        self.id, self.label, self.b = block["id"], block["label"], block
+
+    def search(self, query, k=6):
+        q = "all:" + urllib.parse.quote(query)
+        if self.b.get("category"):
+            q += "+AND+cat:" + urllib.parse.quote(self.b["category"])
+        url = "http://export.arxiv.org/api/query?search_query=%s&max_results=%d&sortBy=relevance" % (q, k)
+        with urllib.request.urlopen(url, timeout=25) as r:
+            root = ET.fromstring(r.read())
+        out = []
+        for i, e in enumerate(root.findall("a:entry", NS)):
+            link = e.findtext("a:id", "", NS)
+            out.append(Evidence(" ".join((e.findtext("a:title", "", NS) or "").split()),
+                                " ".join((e.findtext("a:summary", "", NS) or "").split()), link,
+                                [a.findtext("a:name", "", NS) for a in e.findall("a:author", NS)][:6],
+                                (e.findtext("a:published", "", NS) or "")[:4], "arXiv", self.id, 1.0 / (i + 1),
+                                "arXiv:" + link.rsplit("/", 1)[-1]))
+        return out

customchat/connectors/base.py

@@ -0,0 +1,46 @@
+import importlib, os, sys
+
+
+def Evidence(title="", text="", url="", authors=None, year="", venue="", source="", score=0.0, id=""):
+    return {"id": id or title[:60], "title": title, "text": text, "url": url, "authors": authors or [],
+            "year": str(year or ""), "venue": venue, "source": source, "score": float(score)}
+
+
+def make_connector(block, base_dir="."):
+    t = block["type"]
+    if t == "local_files":
+        from .local_files import LocalFiles
+        return LocalFiles(block, base_dir)
+    if t == "http_json":
+        from .http_json import HttpJson
+        return HttpJson(block)
+    if t == "pubmed":
+        from .pubmed import PubMed
+        return PubMed(block)
+    if t == "arxiv":
+        from .arxiv import Arxiv
+        return Arxiv(block)
+    if t in ("wikipedia", "crossref", "openalex"):
+        from . import open_sources
+        return {"wikipedia": open_sources.Wikipedia, "crossref": open_sources.Crossref, "openalex": open_sources.OpenAlex}[t](block)
+    if t == "web_search":
+        from .web_search import WebSearch
+        return WebSearch(block)
+    if t == "python":
+        # plugin: module:function, function(query, k) -> list of Evidence dicts
+        mod, _, fn = block["entry"].partition(":")
+        if base_dir not in sys.path:
+            sys.path.insert(0, os.path.abspath(base_dir))
+        f = getattr(importlib.import_module(mod), fn)
+
+        class Plugin:
+            def __init__(self): self.id = block["id"]; self.label = block["label"]
+            def search(self, q, k):
+                out = []
+                for e in f(q, k):
+                    e = {**Evidence(), **e}
+                    e["source"] = self.id
+                    out.append(e)
+                return out
+        return Plugin()
+    raise ValueError("Unknown connector " + t)

customchat/connectors/http_json.py

@@ -0,0 +1,46 @@
+"""Generic REST connector: GET a URL template, map JSON fields to evidence. Covers most search APIs."""
+import json, os, urllib.parse, urllib.request
+from .base import Evidence
+
+
+def dig(obj, path):
+    for part in [p for p in (path or "").split(".") if p]:
+        if isinstance(obj, list):
+            try: obj = obj[int(part)]
+            except (ValueError, IndexError): return None
+        elif isinstance(obj, dict):
+            obj = obj.get(part)
+        else:
+            return None
+    return obj
+
+
+class HttpJson:
+    """block: url (with {query} and {k}), results_path, fields {title,text,url,authors,year,venue},
+    optional header_env: {Header-Name: ENV_VAR} so keys never live in the file."""
+    def __init__(self, block):
+        self.id, self.label, self.b = block["id"], block["label"], block
+
+    def search(self, query, k=6):
+        url = self.b["url"].replace("{query}", urllib.parse.quote(query)).replace("{k}", str(k))
+        headers = {"User-Agent": "customchat/0.1"}
+        for h, env in (self.b.get("header_env") or {}).items():
+            if os.environ.get(env):
+                headers[h] = os.environ[env]
+        with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=self.b.get("timeout", 20)) as r:
+            raw = r.read().decode("utf-8", "replace")
+        try:
+            data = json.loads(raw)
+        except ValueError:
+            # Broken or non-JSON reply: keep it as one text passage so the model can still read it.
+            return [Evidence(self.b.get("name", "Source reply"), raw[:4000], url, [], "", "", self.id, 1.0)] if raw.strip() else []
+        f = self.b.get("fields", {})
+        out = []
+        for i, item in enumerate(dig(data, self.b.get("results_path", "")) or []):
+            a = dig(item, f.get("authors", "authors")) or []
+            if isinstance(a, list):
+                a = [x.get("name", "") if isinstance(x, dict) else str(x) for x in a]
+            out.append(Evidence(str(dig(item, f.get("title", "title")) or ""), str(dig(item, f.get("text", "abstract")) or ""),
+                                str(dig(item, f.get("url", "url")) or ""), a, dig(item, f.get("year", "year")) or "",
+                                str(dig(item, f.get("venue", "venue")) or ""), self.id, 1.0 / (i + 1)))
+        return out[:k]