Verify durable AI worker recovery after real process termination

2026-09-06 12:48:29 +03:00
parent fa28fb5f85
commit f2b33749a7
3 changed files with 97 additions and 0 deletions
@@ -0,0 +1,83 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {spawn} from 'node:child_process';
+import {mkdtempSync,rmSync} from 'node:fs';
+import {tmpdir} from 'node:os';
+import {join} from 'node:path';
+const storeUrl=new URL('../server/ai-job-store.mjs',import.meta.url).href;
+const runnerUrl=new URL('../server/ai-job-runner.mjs',import.meta.url).href;
+const sourceUrl=new URL('../server/ai-job-source.mjs',import.meta.url).href;
+const analysisUrl=new URL('../server/analysis-job.mjs',import.meta.url).href;
+const source='Kesintiden önce kaydedilen laboratuvar şartnamesi. '.repeat(4);
+const imports=`
+ import assert from 'node:assert/strict';
+ import {DatabaseSync} from 'node:sqlite';
+ import * as jobs from ${JSON.stringify(storeUrl)};
+ import {startAiJobRunner} from ${JSON.stringify(runnerUrl)};
+ import {resolveAiJobInput} from ${JSON.stringify(sourceUrl)};
+ import {executeAnalysisJob} from ${JSON.stringify(analysisUrl)};
+`;
+async function childAtCheckpoint(dir,code,kill) {
+ const child=spawn(process.execPath,['--input-type=module','-e',imports+code],{env:{...process.env,IHALEIST_USER_DATA_DIR:dir},stdio:['ignore','ignore','pipe','ipc']});
+ let errors='',timer;child.stderr.on('data',chunk=>errors+=chunk);
+ const exit=new Promise((resolve,reject)=>{child.once('error',reject);child.once('exit',(code,signal)=>resolve({code,signal}));});
+ try {
+  const checkpoint=await new Promise((resolve,reject)=>{
+   child.once('message',resolve);child.once('error',reject);
+   child.once('exit',(code,signal)=>reject(new Error(`Before checkpoint: ${code}/${signal} ${errors}`)));
+   timer=setTimeout(()=>reject(new Error('AI worker checkpoint timeout: '+errors)),10000);
+  });
+  if(kill)child.kill('SIGKILL');
+  const ended=await exit;
+  if(kill)assert.equal(ended.signal,'SIGKILL');else assert.equal(ended.code,0,errors);
+  return checkpoint;
+ }finally {clearTimeout(timer);if(child.exitCode===null&&child.signalCode===null)child.kill('SIGKILL');}
+}
+for(const operation of ['analiz','teklif-taslak']) {
+ test(`real worker SIGKILL preserves ${operation} source and requires explicit retry`,async()=>{
+  const dir=mkdtempSync(join(tmpdir(),'ihaleist-ai-crash-'));
+  try {
+   const first=await childAtCheckpoint(dir,`
+    const operation=${JSON.stringify(operation)},text=${JSON.stringify(source)};
+    const created=jobs.createAiJob('owner','crash-request',operation,{tender:{ikn:'2026/1'},text});
+    const hold=async()=>{process.send({id:created.id,input:jobs.getAiJob('owner',created.id).input});await new Promise(()=>{});};
+    const runner=startAiJobRunner(async(job,{signal})=>{
+     if(operation==='analiz')return executeAnalysisJob(job,{signal,pin:jobs.pinAiJobInput,analyze:hold});
+     await resolveAiJobInput(job,async()=>({text,kaynak:'test source A'}),jobs.pinAiJobInput,signal);return hold();
+    });
+    runner.wake();
+   `,true);
+   assert.equal(first.input.text,source);
+   const recovered=await childAtCheckpoint(dir,`
+    const id=${JSON.stringify(first.id)},operation=${JSON.stringify(operation)};
+    let calls=0,fetches=0;
+    assert.equal(jobs.getAiJob('owner',id).state,'running');
+    const runner=startAiJobRunner(async(job,{signal})=>{
+     calls++;
+     if(operation==='analiz')return executeAnalysisJob(job,{signal,pin:jobs.pinAiJobInput,analyze:async input=>({result:{text:input.text},model:{label:'test'}})});
+     const input=await resolveAiJobInput(job,async()=>{fetches++;return {text:'Source B must not replace source A'};},jobs.pinAiJobInput,signal);
+     return {success:true,result:{text:input.text}};
+    },{intervalMs:100000});
+    runner.wake();assert.equal(calls,0,'unexpired lease must not replay');
+    // Advance only lease assessment; no wall-clock delay or production setting change.
+    assert.equal(jobs.claimAiJob(Date.now()+61000),null);
+    assert.equal(jobs.getAiJob('owner',id).state,'interrupted');
+    runner.wake();assert.equal(calls,0,'interrupted work must not replay');
+    assert.equal(jobs.getAiJob('other-owner',id),null);
+    assert.equal(jobs.retryAiJob('owner',id),true);runner.wake();
+    const poll=setInterval(()=>{
+     const saved=jobs.getAiJob('owner',id);
+     if(saved.state==='completed'){
+      clearInterval(poll);runner.stop();
+      const db=new DatabaseSync(process.env.IHALEIST_USER_DATA_DIR+'/ai-jobs.sqlite');
+      const integrity=db.prepare('PRAGMA integrity_check').get().integrity_check;db.close();
+      process.send({input:saved.input,result:saved.result,calls,fetches,integrity});process.disconnect();
+     }
+    },10);
+   `,false);
+   assert.deepEqual(recovered.input,first.input);
+   assert.equal(recovered.result.result.text,source);
+   assert.equal(recovered.calls,1);assert.equal(recovered.fetches,0);assert.equal(recovered.integrity,'ok');
+  }finally {rmSync(dir,{recursive:true,force:true});}
+ });
+}