reedach-server 0.1.2 → 0.2.0

package/README.md

@@ -110,9 +110,10 @@ docker run -d -p 3000:3000 -e REEDACH_KEYS="длинный-ключ-сайта=h
 | `REEDACH_KEYS` | — (обязательно) | ключи доступа |
 | `PORT` | `3000` | порт |
 | `HOST` | `0.0.0.0` | адрес |
-| `REEDACH_MAX_BYTES` | `1000000` | максимальный размер запроса |
+| `REEDACH_MAX_BYTES` | `200000` | максимальный размер запроса в байтах (200 КБ — около 100 тысяч знаков русского текста) |
 | `REEDACH_CONCURRENCY` | `2` | сколько проверок идёт одновременно |
 | `REEDACH_QUEUE` | `20` | сколько запросов может ждать; остальные получат 503 |
+| `REEDACH_TIMEOUT` | `20000` | сколько миллисекунд может идти одна проверка; дольше — 503 |
 | `REEDACH_DEMO` | — | `1` — показывать страницу-пример по адресу `/demo/` |
 
 ## Как устроена орфография

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "reedach-server",
-  "version": "0.1.2",
+  "version": "0.2.0",
   "description": "Сервер Reedach: орфография (hunspell), морфология (Az.js + OpenCorpora) и подсказки по стилю для русского текста — библиотекой или по HTTP",
   "license": "MIT",
   "author": "Dima Faller <dima@dimafaller.ru>",
@@ -41,10 +41,10 @@
     "az": "0.2.3",
     "dictionary-en": "4.0.0",
     "dictionary-ru": "3.0.0",
-    "reedach": "0.1.2"
+    "reedach": "0.2.0"
   },
   "peerDependencies": {
-    "reedach-widget": "^0.1.2"
+    "reedach-widget": "^0.2.0"
   },
   "peerDependenciesMeta": {
     "reedach-widget": {

package/src/check.mjs

@@ -13,7 +13,8 @@ const identity=b=>({text:b.text,kind:['h','li'].includes(b.kind)?b.kind:'p',map:
 
 // input: {markdown | text | blocks:[{text,kind}], ignore:[own words], disable:[rule ids or categories], checks:[...]}
 // Markdown and text hints point into the source string; block hints carry `block` and point into that block's text.
-export async function check(input,{suggester:sg=suggester(),limits={}}={}){
+// Options for running elsewhere: `suggester` and `misspelled` replace the hunspell calls (the HTTP server's workers ask its main thread).
+export async function check(input,{suggester:sg=suggester(),misspelled:ms=misspelled,limits={}}={}){
  const lim={...LIMITS,...limits},{markdown,text,blocks:raw}=input??{};
  let blocks,blocksInput=false;
  if(typeof markdown==='string')blocks=sourceBlocks(markdown);
@@ -23,7 +24,7 @@ export async function check(input,{suggester:sg=suggester(),limits={}}={}){
  const list=(v,name)=>{if(v==null)return [];if(!Array.isArray(v)||v.some(x=>typeof x!=='string'))throw bad('Поле '+name+' должно быть списком строк');return v.slice(0,5000);};
  const ignore=list(input.ignore,'ignore'),disable=new Set(list(input.disable,'disable')),checks=new Set(input.checks==null?CHECKS:list(input.checks,'checks'));
  const [spelling,morph]=await Promise.all([
-  checks.has('spelling')?findSpelling(blocks,ignore,sg,lim):[],
+  checks.has('spelling')?findSpelling(blocks,ignore,sg,lim,ms):[],
   checks.has('morph')?morphCheck(blocks).catch(()=>[]):[],
  ]);
  const local=checks.has('style')?checkStyle(blocks):[];
@@ -32,8 +33,8 @@ export async function check(input,{suggester:sg=suggester(),limits={}}={}){
 }
 
 // Misspelled words minus own words; suggestions for the first few within a time budget, the rest get null (ask /v1/suggest).
-async function findSpelling(blocks,ignore,sg,lim){
- const found=(await misspelled(words(blocks.map(b=>b.text).join('\n')))).filter(w=>!known(w,ignore));
+async function findSpelling(blocks,ignore,sg,lim,ms){
+ const found=(await ms(words(blocks.map(b=>b.text).join('\n')))).filter(w=>!known(w,ignore));
  const result=found.map(word=>({word,suggestions:null}));
  const deadline=new Promise(resolve=>setTimeout(resolve,lim.suggestBudget).unref());
  await Promise.race([deadline,Promise.all(result.slice(0,lim.suggestWords).map(item=>sg.suggest(item.word).then(s=>{item.suggestions=s;},()=>{})))]);
@@ -52,4 +53,7 @@ export function rules(){
  return {categories:{...CATEGORIES,spelling:'Орфография'},rules:[...RULES.map(r=>pick(r.id,r)),...Object.entries(BUILTIN).map(([id,r])=>pick(id,r)),...Object.entries(MORPH_RULES).map(([id,r])=>pick(id,r))]};
 }
 
+// hunspell for the HTTP server's workers, run in the main thread.
+export const spellHost={misspelled:list=>misspelled(list),suggest:word=>suggester().suggest(word)};
+
 export function closeChecks(){shared?.close();shared=null;}

package/src/cli.mjs

@@ -6,7 +6,7 @@ const env=process.env,num=(name,fallback)=>{const v=Number(env[name]);return Num
 let keys,server;
 try{
  keys=parseKeys(env.REEDACH_KEYS);if(!keys.length)throw new Error('Задайте REEDACH_KEYS: ключ=https://ваш-сайт.ru; второй-ключ');
- server=createServer({keys,maxBytes:num('REEDACH_MAX_BYTES',1_000_000),concurrency:num('REEDACH_CONCURRENCY',2),queue:num('REEDACH_QUEUE',20),demo:env.REEDACH_DEMO==='1'});
+ server=createServer({keys,maxBytes:num('REEDACH_MAX_BYTES',200_000),concurrency:num('REEDACH_CONCURRENCY',2),queue:num('REEDACH_QUEUE',20),timeout:num('REEDACH_TIMEOUT',20000),demo:env.REEDACH_DEMO==='1'});
 }catch(e){console.error('reedach: '+e.message);process.exit(1);}
 const port=num('PORT',3000),host=env.HOST||'0.0.0.0';
 server.listen(port,host,()=>console.log('reedach: слушаю '+host+':'+port+', ключей: '+keys.length+(env.REEDACH_DEMO==='1'?', пример: /demo/':'')));

package/src/http.mjs

@@ -2,7 +2,8 @@ import http from 'node:http';
 import {readFileSync} from 'node:fs';
 import {fileURLToPath} from 'node:url';
 import {createHash,timingSafeEqual} from 'node:crypto';
-import {check,suggest,rules} from './check.mjs';
+import {Worker} from 'node:worker_threads';
+import {suggest,rules,spellHost} from './check.mjs';
 // HTTP API. Every /v1 call needs a key. A key may list the sites whose pages can call it from the browser;
 // calls without an Origin header (from a site's own server) work with any valid key.
 // The server keeps nothing: texts are not stored and not logged.
@@ -31,9 +32,35 @@ function demoFiles(){
  return new Map(Object.entries(map).map(([url,p])=>[url,{body:readFileSync(file(p)),type:types[p.split('.').pop()]}]));
 }
 
-export function createServer({keys,maxBytes=1_000_000,concurrency=2,queue=20,demo=false}={}){
+// Checks run in worker threads, at most one per slot: the server keeps answering while a long text is checked,
+// and a check running longer than `timeout` ms is stopped together with its worker; the next call starts a fresh one.
+// Workers ask this thread for hunspell (see worker.mjs); an answer to a stopped worker is dropped.
+function workers(timeout){
+ const answer=(w,{id,op,arg})=>Promise.resolve().then(()=>op==='suggest'?spellHost.suggest(arg):spellHost.misspelled(arg))
+  .then(value=>({id,value}),e=>({id,error:{message:e.message,status:e.status,code:e.code}}))
+  .then(reply=>{try{w.postMessage({answer:reply});}catch{}});
+ const free=[],all=new Set();
+ const spawn=()=>{const w=new Worker(new URL('./worker.mjs',import.meta.url));w.unref();all.add(w);w.once('exit',()=>all.delete(w));return w;};
+ return {
+  run(input){
+   const w=free.pop()||spawn();
+   return new Promise((resolve,reject)=>{
+    const end=(keep,fn,value)=>{clearTimeout(timer);w.off('message',onMessage).off('error',onError).off('exit',onExit);if(keep)free.push(w);else w.terminate();fn(value);};
+    const onMessage=m=>m.ask?answer(w,m.ask):m.error?end(true,reject,Object.assign(new Error(m.error.message),m.error)):end(true,resolve,m.result);
+    const onError=e=>end(false,reject,e),onExit=()=>end(false,reject,fail(500,'internal','Проверка прервалась'));
+    const timer=setTimeout(()=>end(false,reject,fail(503,'timeout','Текст проверяется слишком долго')),timeout);
+    w.on('message',onMessage).on('error',onError).on('exit',onExit);
+    // Copying to the worker throws at once on JSON nested thousands deep; the worker stays idle and goes back to the pool.
+    try{w.postMessage({input});}catch{end(true,reject,fail(422,'validation','Слишком глубокая вложенность JSON'));}
+   });
+  },
+  close(){for(const w of all)w.terminate();},
+ };
+}
+
+export function createServer({keys,maxBytes=200_000,concurrency=2,queue=20,timeout=20000,demo=false}={}){
  if(!keys?.length)throw new Error('Нужен хотя бы один ключ доступа');
- const pages=demo?demoFiles():null;
+ const pages=demo?demoFiles():null,pool=workers(timeout);
  const entries=keys.map(k=>({...k,hash:digest(k.key)})),siteOrigins=new Set(entries.flatMap(k=>k.origins));
  const find=token=>{const h=digest(token);let hit=null;for(const k of entries)if(timingSafeEqual(h,k.hash))hit=k;return hit;};
  let active=0;const waiting=[];
@@ -61,7 +88,7 @@ export function createServer({keys,maxBytes=1_000_000,concurrency=2,queue=20,dem
    if(!key)throw fail(401,'unauthorized','Нужен ключ доступа');
    if(origin&&!key.origins.includes(origin))throw fail(403,'origin','Этот ключ не работает на этом сайте');
    if(url.pathname==='/v1/rules'&&req.method==='GET')return send(200,rules());
-   const route=url.pathname==='/v1/check'?b=>check(b):url.pathname==='/v1/suggest'?b=>suggest(b?.words):null;
+   const route=url.pathname==='/v1/check'?b=>pool.run(b):url.pathname==='/v1/suggest'?b=>suggest(b?.words):null;
    if(!route)throw fail(404,'not_found','Нет такого адреса');
    if(req.method!=='POST')throw fail(405,'method','Используйте POST');
    if(!/^application\/json\b/i.test(req.headers['content-type']||''))throw fail(415,'content_type','Нужен JSON');
@@ -74,7 +101,7 @@ export function createServer({keys,maxBytes=1_000_000,concurrency=2,queue=20,dem
    send(status,{error:{code:e.code||'internal',message:status===500?'Внутренняя ошибка':e.message}},e.code==='busy'?{'retry-after':'5'}:status===413?{connection:'close'}:{});
   }
  });
- server.requestTimeout=30000;server.headersTimeout=10000;
+ server.requestTimeout=30000;server.headersTimeout=10000;server.on('close',()=>pool.close());
  return server;
 }
 

package/src/morph.mjs

@@ -5,9 +5,11 @@ const require=createRequire(import.meta.url),Az=require('az'),DICTS=path.join(pa
 let ready=null;
 const init=()=>ready??=new Promise((resolve,reject)=>Az.Morph.init(DICTS,error=>error?reject(error):resolve()));
 const cache=new Map();
+// Russian words are shorter than 40 letters; Az.js spends seconds on a made-up word of thousands, so those are not parsed.
+const MAX_WORD=40;
 function parse(word){
  const key=word.toLowerCase();let hit=cache.get(key);
- if(!hit){let p;try{p=Az.Morph(key)[0];}catch{}hit={pos:p?.tag?.POS||'',cas:p?.tag?.CAse||'',lemma:String(p?.normalize()?.word||key),score:p?.score||0};if(cache.size>50000)cache.clear();cache.set(key,hit);}
+ if(!hit){let p;try{if(key.length<=MAX_WORD)p=Az.Morph(key)[0];}catch{}hit={pos:p?.tag?.POS||'',cas:p?.tag?.CAse||'',lemma:String(p?.normalize()?.word||key),score:p?.score||0};if(cache.size>50000)cache.clear();cache.set(key,hit);}
  return hit;
 }
 const RULES={

package/src/spell.mjs

@@ -11,7 +11,8 @@ const spellError=(code,message)=>Object.assign(new Error(message),{status:503,co
 // Words worth checking: letters with inner hyphens; links, e-mails, abbreviations, words with digits
 // and English contractions (isn't, couldn’t) are skipped.
 export function words(text,{limit=20000}={}){
- const seen=new Set(),clean=String(text).replace(/https?:\/\/\S+|www\.\S+|[\w.+-]+@[\w-]+\.[\w.-]+/g,' ');
+ // An e-mail starts only where its run of characters starts: otherwise a long word is rescanned from every letter.
+ const seen=new Set(),clean=String(text).replace(/https?:\/\/\S+|www\.\S+|(?<![\w.+-])[\w.+-]+@[\w-]+\.[\w.-]+/g,' ');
  for(const [w] of clean.matchAll(/(?<![\w-])[A-Za-zА-Яа-яЁё]+(?:[-'’][A-Za-zА-Яа-яЁё]+)*(?![\w-])/g))if(w.length>1&&!/['’]/.test(w)&&w!==w.toUpperCase()&&seen.size<limit)seen.add(w);
  return [...seen];
 }

package/src/worker.mjs

@@ -0,0 +1,17 @@
+import {parentPort} from 'node:worker_threads';
+import {check} from './check.mjs';
+// One check per message, in a worker thread: a heavy text stalls only this worker, never the HTTP server.
+// hunspell is asked for from the main thread: a process started here would stay a zombie when the worker
+// is stopped on timeout, because only the thread that started a process collects its exit.
+let seq=0;const asked=new Map();
+const ask=(op,arg)=>new Promise((resolve,reject)=>{const id=++seq;asked.set(id,{resolve,reject});parentPort.postMessage({ask:{id,op,arg}});});
+const spell={suggester:{suggest:word=>ask('suggest',word)},misspelled:list=>ask('misspelled',list)};
+parentPort.on('message',async m=>{
+ if(m.answer){
+  const {id,value,error}=m.answer,wait=asked.get(id);asked.delete(id);
+  if(error)wait?.reject(Object.assign(new Error(error.message),error));else wait?.resolve(value);
+  return;
+ }
+ try{parentPort.postMessage({result:await check(m.input,spell)});}
+ catch(e){parentPort.postMessage({error:{message:e.message,status:e.status,code:e.code}});}
+});