chatify-app 0.2.0__py3-none-any.whl

chatify/mysqlstore.py

@@ -0,0 +1,88 @@
+"""Optional MySQL/MariaDB backend for Store. SQLite stays the default and the fallback.
+Turned on by the CHATIFY_DB_URL environment variable, for example mysql://user:pass@host:3306/dbname.
+The URL is read from the environment only, so a password never sits in a file in the repo.
+Needs the PyMySQL package (pip install -r requirements-mysql.txt). If anything is missing or
+unreachable, Store falls back to SQLite and says so."""
+import os, re
+from urllib.parse import urlparse, unquote
+
+LONG = {"answer", "evidence", "ledger", "text", "payload", "summary", "standalone", "question"}
+
+
+def parse(url):
+    u = urlparse(url)
+    if u.scheme not in ("mysql", "mariadb") or not u.hostname:
+        raise ValueError("CHATIFY_DB_URL must look like mysql://user:password@host:3306/dbname")
+    return dict(host=u.hostname, port=u.port or 3306, user=unquote(u.username or ""), password=unquote(u.password or ""),
+                database=(u.path or "/").lstrip("/") or "chatify")
+
+
+def sql(s):
+    """Translate the SQLite statements the app uses into MySQL."""
+    s = s.replace("%", "%%").replace("?", "%s")
+    s = re.sub(r"(?i)^\s*INSERT OR REPLACE", "REPLACE", s)
+    s = re.sub(r"(?<![`\w])key(?![`\w])", "`key`", s)
+    return s
+
+
+def ddl(stmt):
+    stmt = stmt.strip()
+    if not stmt:
+        return None
+    stmt = re.sub(r"(?i)^CREATE INDEX IF NOT EXISTS", "CREATE INDEX", stmt)
+    stmt = re.sub(r"(?i)\bREAL\b", "DOUBLE", stmt)
+    stmt = re.sub(r"(?i)\bBLOB\b", "LONGBLOB", stmt)
+    def col(m):
+        n = m.group(1)
+        return n + (" LONGTEXT" if n.lower() in LONG else " VARCHAR(190)")
+    stmt = re.sub(r"(?i)\b(\w+) TEXT\b", col, stmt)
+    stmt = re.sub(r"(?i)(LONGTEXT) DEFAULT ''", r"\1", stmt)
+    stmt = re.sub(r"(?<![`\w])key(?![`\w])", "`key`", stmt)
+    return stmt
+
+
+class Conn:
+    """Looks enough like a sqlite3 connection for Store and Accounts."""
+    def __init__(self, cfg, driver=None):
+        if driver is None:
+            import pymysql
+            driver = pymysql
+        self.db = driver.connect(charset="utf8mb4", autocommit=False, connect_timeout=8,
+                                 cursorclass=driver.cursors.DictCursor, **cfg)
+
+    def execute(self, q, args=()):
+        cur = self.db.cursor()
+        cur.execute(sql(q), tuple(args))
+        return cur
+
+    def executescript(self, script):
+        for part in script.split(";"):
+            d = ddl(part)
+            if not d:
+                continue
+            try:
+                self.execute(d)
+            except Exception as e:
+                if "1061" in str(e) or "Duplicate key name" in str(e):
+                    continue
+                raise
+        self.db.commit()
+
+    def commit(self):
+        self.db.commit()
+
+    def close(self):
+        try:
+            self.db.close()
+        except Exception:
+            pass
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *a):
+        if a[0] is None:
+            self.db.commit()
+        else:
+            self.db.rollback()
+        return False

chatify/nerdload.py

@@ -0,0 +1,179 @@
+"""Review portable Nerds without importing code, then launch in a separate local workspace."""
+import base64,hashlib,io,json,os,re,shutil,subprocess,sys,threading,zipfile
+from pathlib import Path,PurePosixPath
+import yaml
+from . import schema
+from .portable import _check_values,ExportError
+MAX_UPLOAD=8*1024*1024
+MAX_EXPANDED=32*1024*1024
+_ALLOWED={'.yaml','.yml','.json','.md','.txt','.csv','.pdf','.py'}
+class NerdLoader:
+ def __init__(self,folder):
+  self.root=Path(folder)/'nerds';self.pending={};self.children=[];self.workspaces={};self.lock=threading.Lock()
+  if self.root.is_dir():
+   for folder in self.root.glob('nerd-*'):
+    if folder.is_symlink() or not folder.is_dir():continue
+    try:
+     config=yaml.safe_load((folder/'app.yaml').read_text());title=str(config['app']['title'])[:120]
+     self.workspaces[folder.name]={'folder':folder,'process':None,'title':title,'url':''}
+    except (OSError,ValueError,TypeError,KeyError):pass
+ def review(self,name,encoded):
+  try:data=base64.b64decode(encoded,validate=True)
+  except Exception:raise ValueError('Invalid upload') from None
+  if sum(p.poll() is None for p in self.children)>=4:raise ValueError('Up to four Nerds can run here. Stop a running workspace before loading more.')
+  if len(data)>MAX_UPLOAD:raise ValueError('Bundle limit is 8 MB')
+  files={}
+  if name.lower().endswith('.zip'):
+   try:
+    with zipfile.ZipFile(io.BytesIO(data)) as z:
+     if len(z.infolist())>500:raise ValueError('Too many files')
+     # Validate paths and private names before ignoring benign metadata.
+     for info in z.infolist():
+      path=PurePosixPath(info.filename)
+      if path.is_absolute() or '..' in path.parts or '\\' in info.filename or re.match(r'^[A-Za-z]:',info.filename):raise ValueError('Unsafe path in bundle: '+info.filename)
+      if (info.external_attr>>16)&0o170000==0o120000:raise ValueError('Symlinks are not allowed: '+info.filename)
+      if any(re.match(r'^\.env(?:$|[._-])',part,re.I) or part.lower() in ('.ssh','.aws','.azure','.gnupg','.npmrc','.pypirc','.netrc') for part in path.parts):raise ValueError('Private file is not allowed: '+info.filename)
+     names={str(PurePosixPath(i.filename).name) for i in z.infolist() if not i.is_dir()}
+     if 'pyproject.toml' in names and any('/chatify/__init__.py' in '/'+i.filename for i in z.infolist()):raise ValueError('This is the Chatify source ZIP, not a Nerd bundle. Install it to run Chatify. To share a Nerd, use Download app ZIP in Configuration.')
+     total=0
+     for info in z.infolist():
+      path=PurePosixPath(info.filename)
+      if path.is_absolute() or '..' in path.parts or '\\' in info.filename or re.match(r'^[A-Za-z]:',info.filename):raise ValueError('Unsafe path in bundle')
+      if (info.external_attr>>16)&0o170000==0o120000:raise ValueError('Symlinks are not allowed')
+      if info.is_dir():continue
+      benign={'.gitignore','.gitattributes','.editorconfig','.DS_Store','.github','.git','__MACOSX'}
+      hidden=[x for x in path.parts if x.startswith('.') or x=='__MACOSX']
+      if hidden:
+       if all(x in benign or x.startswith('._') for x in hidden):continue
+       raise ValueError('Unsupported hidden file: '+info.filename)
+      if re.search(r'(secret|credential|password|token|chatify\.db|accounts|sessions)',info.filename,re.I):raise ValueError('Private file is not allowed: '+info.filename)
+      if len(info.filename)>240:raise ValueError('Path too long')
+      if path.suffix.lower() not in _ALLOWED:raise ValueError('Unsupported file: '+info.filename)
+      total+=info.file_size
+      if total>MAX_EXPANDED:raise ValueError('Expanded bundle is too large')
+      if info.filename in files:raise ValueError('Duplicate path')
+      files[info.filename]=z.read(info)
+   except zipfile.BadZipFile:raise ValueError('Invalid ZIP') from None
+  elif name.lower().endswith(('.yaml','.yml','.json')):files['app.yaml']=data
+  else:raise ValueError('Choose a Nerd ZIP or YAML/JSON app file')
+  apps=[n for n in files if PurePosixPath(n).name in ('app.yaml','app.yml','app.json')]
+  if len(apps)!=1:raise ValueError('Bundle must contain exactly one app.yaml (or app.json)')
+  prefix=str(PurePosixPath(apps[0]).parent)
+  if prefix!='.':
+   if any(not n.startswith(prefix+'/') for n in files):raise ValueError('All files must be inside the app folder')
+   files={n[len(prefix)+1:]:v for n,v in files.items()}
+  appname=next(n for n in files if PurePosixPath(n).name in ('app.yaml','app.yml','app.json'))
+  try:raw=yaml.safe_load(files[appname].decode('utf-8'))
+  except Exception:raise ValueError('Invalid app YAML/JSON') from None
+  if isinstance(raw,dict) and 'articles' in raw and 'app' not in raw and 'provider' not in raw and 'sources' not in raw:
+   raise ValueError('This looks like an API response, not a Nerd app. In Code helpers, open Search connector, then Match a real response and paste this JSON there. It will not be imported or run.')
+  from .keybundle import extract
+  raw,imported_keys=extract(raw)
+  try:cfg=schema.validate(raw);_check_values(cfg)
+  except (schema.ConfigError,ExportError) as e:raise ValueError(str(e)) from None
+  files[appname]=yaml.safe_dump(cfg,sort_keys=False).encode()
+  for name,content in files.items():
+   if name.endswith('.json') and name!=appname:
+    try:_check_values(json.loads(content))
+    except (ValueError,ExportError):raise ValueError('Invalid or private JSON file: '+name) from None
+  if cfg['auth']['mode']!='none':raise ValueError('Local imported Nerds must use auth.mode: none. Set up hosted authentication separately.')
+  for name in ('data/theme.json','data/prompts.json'):
+   if name in files:
+    from . import theme,prompts
+    d=json.loads(files[name])
+    if name.endswith('theme.json'):files[name]=json.dumps(theme.clean(d)).encode()
+    elif not isinstance(d,dict) or set(d)-{'text','on'} or not isinstance(d.get('text',{}),dict) or not isinstance(d.get('on',{}),dict) or any(k not in prompts.STAGES or not isinstance(v,str) or len(v)>prompts.MAX_LEN for k,v in d.get('text',{}).items()) or any(k not in prompts.STAGES or type(v)!=bool for k,v in d.get('on',{}).items()):raise ValueError('Invalid prompt settings')
+  empty_sources=[]
+  for src in cfg['sources']:
+   for field in ('path','module'):
+    v=src.get(field)
+    if not v:continue
+    p=PurePosixPath(v)
+    if p.is_absolute() or '..' in p.parts or '\\' in v:raise ValueError('Sources must stay inside the bundle')
+   if src['type']=='local_files':
+    path=src.get('path','docs').rstrip('/')
+    if not any(n.startswith(path+'/') for n in files):empty_sources.append((src['label'],path))
+   if src['type']=='python':
+    mod=src.get('entry','').partition(':')[0]
+    if not re.fullmatch(r'[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*',mod):raise ValueError('Invalid Python module entry')
+    if mod.replace('.','/')+'.py' not in files:raise ValueError('Python connector module is missing from bundle')
+  for src in cfg['sources']:
+   if src['type']=='python':src['execution']='bounded'
+  cfg['storage']={'path':'data/chatify.db'};cfg['server']['host']='127.0.0.1';cfg['actions']={'allow':[],'writes':False}
+  files.pop(appname);files['app.yaml']=yaml.safe_dump(cfg,sort_keys=False).encode()
+  code=[n for n in files if n.endswith('.py')]
+  if sum(len(files[n]) for n in code)>200000:raise ValueError('Connector code is too large to review here')
+  envs=sorted({src.get('api_key_env') for src in cfg['sources'] if src.get('api_key_env')}|({cfg['provider']['api_key_env']} if cfg['provider'].get('api_key_env') else set()))
+  from .readiness import check
+  readiness=check(cfg,files,raw)
+  for label,path in empty_sources:readiness['rows'].append({'kind':'setup','item':label,'message':'No documents included for '+path+'. An empty source folder will be created. Add documents in the new workspace before asking source-backed questions.'})
+  if not readiness['can_load']:raise ValueError('Python connector does not compile. Fix it before loading.')
+  envs=readiness['required_env']
+  token=hashlib.sha256(data).hexdigest()
+  with self.lock:
+   if len(self.pending)>=4:self.pending.pop(next(iter(self.pending)))
+   self.pending[token]=(files,cfg,code,imported_keys)
+  from . import theme
+  look=theme.clean(json.loads(files['data/theme.json'])) if 'data/theme.json' in files else theme.clean(cfg.get('appearance',{}))
+  preview={'title':look.get('txt_title') or cfg['app']['title'],'tagline':look.get('txt_tagline') or cfg['app']['tagline'],'mode':look.get('mode','light'),'colors':look.get('dark' if look.get('mode')=='dark' else 'light',{}),'examples':cfg['app'].get('examples',[])[:3],'files':sorted(files),'static':True}
+  from . import bundle_review
+  return {'auto_review':bundle_review.review(cfg,files),'preview':preview,'readiness':readiness,'token':token,'title':cfg['app']['title'],'tagline':cfg['app']['tagline'],'provider':cfg['provider']['type'],'model':cfg['provider']['model'],'sources':[{'id':s['id'],'label':s.get('label',s['id']),'type':s['type']} for s in cfg['sources']],'required_env':envs,'imported_key_count':len(imported_keys),'code':[{'path':n,'text':files[n].decode('utf-8')} for n in code],'warnings':['Python uses a bounded process, NOT a sandbox. Host files/network remain accessible; code review required.','Separate local workspace. Current app is not overwritten.',str(len(imported_keys))+' API key entries will be placed in this workspace private key storage. Values are not shown. Accounts and chat history are not imported.','Imported local apps disable write actions. Review remote sources before asking a question.']}
+ def load(self,token,allow_code=False):
+  with self.lock:item=self.pending.get(token)
+  if not item:raise ValueError('Review the bundle first')
+  files,cfg,code,imported_keys=item
+  if sum(p.poll() is None for p in self.children)>=4:raise ValueError('Maximum four running Nerds. Stop a running workspace before loading more.')
+  if code and not allow_code:raise ValueError('Review Python code and explicitly allow it before loading')
+  from .launcher import pick_port
+  import tempfile
+  self.root.mkdir(parents=True,exist_ok=True)
+  folder=Path(tempfile.mkdtemp(prefix='nerd-',dir=str(self.root)))
+  for name,data in files.items():
+   p=folder/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(data)
+  for src in cfg['sources']:
+   if src['type']=='local_files':(folder/src.get('path','docs')).mkdir(parents=True,exist_ok=True)
+  port=pick_port(8090)
+  cfg['server']['port']=port;(folder/'app.yaml').write_text(yaml.safe_dump(cfg,sort_keys=False))
+  if imported_keys:
+   from .secrets import SecretStore
+   key_store=SecretStore(str(folder/'data'))
+   for name,value in imported_keys.items():key_store.set(name,value)
+  log=open(folder/'run.log','wb')
+  declared_env={cfg['provider'].get('api_key_env','')}|{src.get('api_key_env','') for src in cfg['sources']}
+  child=subprocess.Popen([sys.executable,'-m','chatify','run',str(folder/'app.yaml'),'--no-browser'],stdout=log,stderr=subprocess.STDOUT,cwd=str(Path(__file__).resolve().parents[1]),env={k:v for k,v in os.environ.items() if not re.search(r'(KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL)',k,re.I) or k in declared_env})
+  log.close();self.children.append(child);self.workspaces[folder.name]={'folder':folder,'process':child,'title':cfg['app']['title'],'url':'http://127.0.0.1:'+str(port)}
+  import time,urllib.request
+  url='http://127.0.0.1:'+str(port)
+  for _ in range(40):
+   if child.poll() is not None:raise ValueError('Nerd could not start. Check dependencies and connector setup in '+str(folder/'run.log'))
+   try:
+    with urllib.request.urlopen(url+'/api/health',timeout=.5):return {'url':url,'title':cfg['app']['title'],'workspace':str(folder)}
+   except OSError:time.sleep(.1)
+  child.terminate();raise ValueError('Nerd did not become ready; current app is unchanged')
+ def list(self):
+  with self.lock:
+   return [{'id':key,'title':v['title'],'url':v['url'],'running':v['process'] is not None and v['process'].poll() is None} for key,v in self.workspaces.items()]
+ def stop(self,key):
+  with self.lock:item=self.workspaces.get(key)
+  if not item:raise ValueError('Unknown imported workspace')
+  proc=item['process']
+  if proc and proc.poll() is None:
+   proc.terminate()
+   try:proc.wait(timeout=3)
+   except subprocess.TimeoutExpired:proc.kill();proc.wait()
+  return {'stopped':True}
+ def remove(self,key,confirmed):
+  if confirmed is not True:raise ValueError('Confirm removal of this imported workspace')
+  with self.lock:item=self.workspaces.get(key)
+  if not item:raise ValueError('Unknown imported workspace')
+  self.stop(key);folder=item['folder']
+  if folder.is_symlink() or folder.parent.resolve()!=self.root.resolve():raise ValueError('Workspace path changed; removal stopped')
+  shutil.rmtree(folder)
+  with self.lock:self.workspaces.pop(key,None)
+  return {'removed':True}
+ def close(self):
+  for child in self.children:
+   if child.poll() is None:
+    child.terminate()
+    try:child.wait(timeout=3)
+    except subprocess.TimeoutExpired:child.kill();child.wait()

chatify/ollama_setup.py

@@ -0,0 +1,138 @@
+"""Opt-in local Ollama installer. Fixed official sources, no user shell commands."""
+import hashlib, os, platform, secrets, shutil, subprocess, tempfile, threading, time, urllib.request
+from pathlib import Path
+from urllib.parse import urlparse
+from . import hardware
+SOURCES = {'Windows': 'https://ollama.com/download/OllamaSetup.exe', 'Darwin': 'https://ollama.com/download/Ollama.dmg', 'Linux': 'https://ollama.com/install.sh'}
+PAGES = {'Windows': 'https://ollama.com/download/windows', 'Darwin': 'https://ollama.com/download/mac', 'Linux': 'https://ollama.com/download/linux'}
+BASE = 'http://127.0.0.1:11434'
+LIMIT = 2 * 1024 ** 3
+
+def detect():
+    system = platform.system()
+    binary = shutil.which('ollama')
+    if not binary and system == 'Darwin':
+        p = Path('/Applications/Ollama.app/Contents/Resources/ollama')
+        if p.is_file(): binary = str(p)
+    if not binary and system == 'Windows':
+        p = Path(os.environ.get('LOCALAPPDATA', '')) / 'Programs/Ollama/ollama.exe'
+        if p.is_file(): binary = str(p)
+    running = hardware.installed(BASE) is not None
+    return {'system': system, 'arch': platform.machine(), 'state': 'running' if running else 'stopped' if binary else 'not_detected',
+            'binary': bool(binary), 'supported': system in SOURCES, 'source': SOURCES.get(system, ''), 'page': PAGES.get(system, ''),
+            'disk_free_gb': round(shutil.disk_usage(tempfile.gettempdir()).free / 1024 ** 3, 1),
+            'host_note': 'The computer running Chatify, not the browser or phone.', '_binary': binary}
+
+class OfficialRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        if urlparse(newurl).scheme != 'https' or urlparse(newurl).hostname not in ('ollama.com', 'github.com', 'objects.githubusercontent.com', 'release-assets.githubusercontent.com'):
+            raise ValueError('Installer redirected outside the official download hosts')
+        return super().redirect_request(req, fp, code, msg, headers, newurl)
+
+def download(url, folder, cancelled=None, progress=None):
+    if url not in SOURCES.values(): raise ValueError('Unknown installer source')
+    target = Path(folder) / url.rsplit('/', 1)[1]
+    opener = urllib.request.build_opener(OfficialRedirect)
+    with opener.open(url, timeout=60) as response, target.open('wb') as out:
+        total = int(response.headers.get('Content-Length') or 0)
+        if total > LIMIT: raise ValueError('Installer exceeds the 2 GB download limit')
+        used = 0
+        while True:
+            if cancelled and cancelled.is_set(): raise ValueError('Download cancelled')
+            chunk = response.read(1024 * 1024)
+            if not chunk: break
+            used += len(chunk)
+            if used > LIMIT: raise ValueError('Installer exceeds the 2 GB download limit')
+            out.write(chunk)
+            if progress:progress(used,total)
+    if not used: raise ValueError('Empty installer download')
+    return target
+
+class OllamaSetup:
+    def __init__(self):
+        self.lock = threading.Lock(); self.pending = {}; self.jobs = {}; self.processes = []; self.cancel_events = {}; self.job_processes = {}
+    def status(self):
+        data = detect(); data.pop('_binary', None)
+        return data
+    def prepare(self, action):
+        if action not in ('install', 'start'): raise ValueError('Choose install or start')
+        data = detect(); data.pop('_binary', None)
+        if not data['supported']: raise ValueError('Use the official Ollama installer on this platform')
+        if action == 'install' and data['state'] != 'not_detected': raise ValueError('Ollama is already detected. Start it or recheck instead.')
+        if action == 'start' and data['state'] != 'stopped': raise ValueError('Start is available only for a detected, stopped Ollama')
+        if action == 'install' and data['disk_free_gb'] < 5: raise ValueError('At least 5 GB free disk space is required before installation. Models need more.')
+        ticket = secrets.token_urlsafe(24)
+        with self.lock:
+            self.pending = {ticket: (action, time.monotonic(), data['system'])}
+        data.update(ticket=ticket, action=action, download_size='Not known until download; limit 2 GB',
+                    changes='Install Ollama on this host. OS installer may require local interaction or permission. No models or cloud accounts added.' if action == 'install' else 'Start the detected local Ollama service.',
+                    instructions={'Darwin': 'In the disk image, drag Ollama to Applications, then open it. Approve the CLI link only in the local OS prompt.',
+                                  'Windows': 'Complete the official installer on this computer. Ollama should run in the background.',
+                                  'Linux': 'The official script may require sudo. Chatify never asks for a password. If permission is needed, follow the official page in a local terminal.'}[data['system']])
+        return data
+    def execute(self, ticket, confirmed):
+        if confirmed is not True: raise ValueError('Review and confirm this installation first')
+        with self.lock:
+            item = self.pending.pop(ticket, None)
+            if not item or time.monotonic() - item[1] > 600: raise ValueError('Review the installation again; approval expired')
+            if any(x['state'] in ('downloading', 'starting', 'waiting') for x in self.jobs.values()): raise ValueError('A setup is already in progress')
+            action, _, system = item
+            if system != platform.system(): raise ValueError('Host changed; review again')
+            current = detect()
+            if action == 'install' and current['state'] != 'not_detected': raise ValueError('Ollama is now detected; recheck instead of installing again')
+            if action == 'start' and current['state'] != 'stopped': raise ValueError('Service state changed; recheck')
+            jid = secrets.token_urlsafe(18)
+            self.cancel_events[jid]=threading.Event()
+            self.jobs = {jid: {'id': jid, 'state': 'starting' if action == 'start' else 'downloading', 'message': 'Starting Ollama' if action == 'start' else 'Downloading the official installer', 'action': action}}
+        threading.Thread(target=self._work, args=(jid, action, system), daemon=True).start()
+        return {'id': jid}
+    def job(self, jid):
+        with self.lock: result = dict(self.jobs.get(jid, {}))
+        if not result: raise ValueError('Unknown setup')
+        return result
+    def _update(self, jid, **values):
+        with self.lock:
+            if self.jobs[jid]['state']!='cancelled':self.jobs[jid].update(values)
+    def cancel(self, jid):
+        with self.lock:
+            job=self.jobs.get(jid)
+            if not job:raise ValueError('Unknown setup')
+            if job['state'] in ('ready','failed','needs_manual','cancelled'):return dict(job)
+            self.cancel_events[jid].set()
+            proc=self.job_processes.get(jid)
+            if proc and proc.poll() is None:
+                try:proc.terminate()
+                except OSError:pass
+            job.update(state='cancelled',message='Chatify stopped its setup work. An OS wizard or child installer may still be open; close it on the host. Installed changes are not rolled back. Retry requires a new review; installer downloads restart, not resume.')
+            return dict(job)
+    def _work(self, jid, action, system):
+        try:
+            if action == 'start':
+                binary = detect().get('_binary')
+                if not binary: raise ValueError('Ollama executable no longer detected; recheck')
+                cmd = [binary, 'serve']
+            else:
+                # Keep mounted disk images available until the OS wizard is finished.
+                folder = tempfile.mkdtemp(prefix='chatify-ollama-')
+                target = download(SOURCES[system], folder, self.cancel_events.get(jid), lambda used,total:self._update(jid,downloaded_bytes=used,total_bytes=total,progress_pct=round(100*used/total) if total else None))
+                digestor = hashlib.sha256()
+                with target.open('rb') as downloaded:
+                    for chunk in iter(lambda: downloaded.read(1024 * 1024), b''): digestor.update(chunk)
+                digest = digestor.hexdigest()
+                self._update(jid, state='starting', message='Opening the official installer on this computer', sha256=digest)
+                cmd = [str(target)] if system == 'Windows' else ['open', str(target)] if system == 'Darwin' else ['sh', str(target)]
+            with self.lock:
+                if self.cancel_events.get(jid) and self.cancel_events[jid].is_set(): return
+                proc = subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+                self.processes.append(proc);self.job_processes[jid]=proc
+            self._update(jid, state='waiting', message='Complete any OS installer or permission prompt on the host, then recheck. No password is collected here.')
+            for _ in range(300):
+                if self.cancel_events.get(jid) and self.cancel_events[jid].is_set():return
+                if hardware.installed(BASE) is not None:
+                    self._update(jid, state='ready', message='Local Ollama service verified. Choose a model separately.'); return
+                if system == 'Linux' and proc.poll() not in (None, 0):
+                    self._update(jid, state='needs_manual', message='Installer could not complete non-interactively. Use the official install page in a local terminal; then recheck.', page=PAGES[system]); return
+                time.sleep(2)
+            self._update(jid, state='needs_manual', message='Local service is not reachable yet. Complete the OS setup or use the official instructions, then recheck.', page=PAGES[system])
+        except Exception:
+            self._update(jid, state='failed', message='Setup could not complete. No success claimed. Use the official install page, then recheck.', page=PAGES.get(system, 'https://ollama.com'))

chatify/pdfread.py

@@ -0,0 +1,97 @@
+"""Pull plain text out of a PDF. Uses pypdf when installed; otherwise a small built-in reader
+that handles ordinary text PDFs (Flate-compressed streams with Tj/TJ text operators).
+Scanned PDFs (images only) have no text to extract and raise a clear error."""
+import re, zlib
+
+MAX_BYTES = 8_000_000
+
+
+def _unescape(s):
+    s = re.sub(rb"\\([nrtbf()\\])", lambda m: {b"n": b"\n", b"r": b"\r", b"t": b"\t", b"b": b"", b"f": b""}.get(m.group(1), m.group(1)), s)
+    return re.sub(rb"\\([0-7]{1,3})", lambda m: bytes([int(m.group(1), 8) & 255]), s)
+
+
+def _builtin(data):
+    out = []
+    for m in re.finditer(rb"stream\r?\n(.*?)\r?\nendstream", data, re.S):
+        raw = m.group(1)
+        try:
+            raw = zlib.decompress(raw)
+        except Exception:
+            pass
+        if b"BT" not in raw:
+            continue
+        for blk in re.findall(rb"BT(.*?)ET", raw, re.S):
+            for arr in re.finditer(rb"\[(.*?)\]\s*TJ|\((.*?)(?<!\\)\)\s*(?:Tj|'|\")", blk, re.S):
+                line = []
+                if arr.group(1) is not None:
+                    parts = re.findall(rb"\(((?:\\.|[^\\)])*)\)|(-?\d+\.?\d*)", arr.group(1))
+                    for t, k in parts:
+                        if t or not k:
+                            line.append(_unescape(t).decode("latin-1"))
+                        elif float(k) < -200:
+                            line.append(" ")
+                else:
+                    line.append(_unescape(arr.group(2)).decode("latin-1"))
+                if line:
+                    out.append("".join(line))
+    return "\n".join(out)
+
+
+def extract(data):
+    if not data.startswith(b"%PDF"):
+        raise ValueError("That file is not a PDF")
+    if len(data) > MAX_BYTES:
+        raise ValueError("PDF is over 8 MB")
+    text = ""
+    try:
+        import io
+        from pypdf import PdfReader
+        text = "\n\n".join((p.extract_text() or "") for p in PdfReader(io.BytesIO(data)).pages)
+    except ImportError:
+        text = _builtin(data)
+    except Exception:
+        text = _builtin(data)
+    text = re.sub(r"[ \t]+", " ", text).strip()
+    if len(text) < 20:
+        raise ValueError("No readable text in this PDF (it may be a scan)")
+    return text[:150_000]
+
+
+def pages(data, ocr=False, max_pages=100):
+    """Page-preserving extraction; OCR is explicit and local, never a web upload."""
+    if not data.startswith(b"%PDF") or len(data) > MAX_BYTES:
+        raise ValueError("PDF must be valid and at most 8 MB")
+    try:
+        import io
+        from pypdf import PdfReader
+    except ImportError:
+        raise ValueError("Page anchors need pypdf. Install chatify[documents].") from None
+    reader = PdfReader(io.BytesIO(data))
+    if len(reader.pages) > max_pages:
+        raise ValueError("PDF exceeds the 100-page local extraction limit")
+    out = []
+    for number, page in enumerate(reader.pages, 1):
+        text = (page.extract_text() or "").strip()
+        if len(text) < 20 and ocr:
+            text = _ocr_page(data, number)
+        if text: out.append({"page": number, "text": text[:150000], "ocr": len((page.extract_text() or "").strip()) < 20 and ocr})
+    if not out: raise ValueError("No readable text. Set ocr: true and install local OCR tools for scans.")
+    return out
+
+
+def _ocr_page(data, number):
+    import shutil, subprocess, tempfile
+    from pathlib import Path
+    if not shutil.which("pdftoppm") or not shutil.which("tesseract"):
+        raise ValueError("OCR needs pdftoppm (Poppler) and Tesseract installed on this computer")
+    with tempfile.TemporaryDirectory(prefix="chatify-ocr-") as folder:
+        root = Path(folder); source = root / "document.pdf"; source.write_bytes(data)
+        try:
+            subprocess.run(["pdftoppm", "-f", str(number), "-l", str(number), "-scale-to", "2000", "-singlefile", "-png", str(source), str(root / "page")],
+                           check=True, timeout=30, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+            result = subprocess.run(["tesseract", str(root / "page.png"), "stdout"], check=True, timeout=30,
+                                    stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+        except (subprocess.SubprocessError, OSError):
+            raise ValueError("Local OCR failed or timed out; no text from that page was indexed") from None
+        return result.stdout.decode("utf-8", errors="replace").strip()

chatify/permissions.py

@@ -0,0 +1,21 @@
+"""Explicit source/document readers. Missing policy is public, empty readers deny all."""
+import copy
+
+def restricted(cfg):return any('read_users' in s or s.get('document_users') for s in cfg['sources'])
+def source(cfg,sid):return next((s for s in cfg['sources'] if s['id']==sid),None)
+def allowed(cfg,owner,sid,document=None):
+    s=source(cfg,sid)
+    if s is None:return not restricted(cfg) or sid=='uploads'  # own uploads are retrieved through the owner store only
+    if 'read_users' in s and owner not in s['read_users']:return False
+    rules=s.get('document_users',{})
+    return document not in rules or owner in rules[document]
+def documents(cfg,owner,sid,conn):
+    if not allowed(cfg,owner,sid):return []
+    s=source(cfg,sid)
+    if not s or not s.get('document_users'):return None
+    return sorted({d.get('document') for d in conn.docs if d.get('document') and allowed(cfg,owner,sid,d['document'])})
+def view(cfg,owner,row):
+    row=copy.deepcopy(row)
+    if any(not allowed(cfg,owner,e.get('source'),e.get('document')) for e in row.get('evidence',[])):
+        row.update(answer='This saved answer is no longer available under your document permissions.',evidence=[],ledger=[],standalone='',followups=[],scope=None)
+    return row