feat(retry): AiWorker unreachable/timeout 무한 retry — 15분 cap (#2 v0.2.3)
This commit is contained in:
@@ -228,21 +228,21 @@ describe('AiWorker telemetry emit', () => {
|
||||
expect(succeeded!.payload.durationMs).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
it('emits ai_failed with reason=unreachable on network error', async () => {
|
||||
it('unreachable error — ai_failed NOT emitted (infinite retry, no markAiFailed)', async () => {
|
||||
const { id } = repo.create({ rawText: '메모' });
|
||||
const provider = makeProvider({
|
||||
generate: vi.fn(async () => { throw new Error('fetch failed: ECONNREFUSED 11434'); })
|
||||
});
|
||||
const w = new AiWorker(repo, provider, {
|
||||
backoffsMs: [0, 0, 0],
|
||||
unreachableBackoffsMs: [10, 10, 10, 10, 10, 10],
|
||||
telemetry: collectingTelemetry
|
||||
});
|
||||
await w.enqueue(id);
|
||||
await w.drain();
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
const failed = events.find((e) => e.kind === 'ai_failed');
|
||||
expect(failed).toBeDefined();
|
||||
expect(failed!.payload.reason).toBe('unreachable');
|
||||
expect(failed!.payload.attempts).toBe(3);
|
||||
expect(failed).toBeUndefined();
|
||||
expect(repo.findById(id)!.aiStatus).toBe('pending');
|
||||
});
|
||||
|
||||
it('emits ai_failed with reason=schema on zod failure', async () => {
|
||||
@@ -304,3 +304,119 @@ describe('AiWorker — deletedAt guard (v0.2.3 #4)', () => {
|
||||
expect(repo.findById(id)!.aiStatus).toBe('pending');
|
||||
});
|
||||
});
|
||||
|
||||
describe('AiWorker — unreachable/timeout infinite retry (v0.2.3 #2)', () => {
|
||||
let db: Database.Database;
|
||||
let repo: NoteRepository;
|
||||
|
||||
beforeEach(() => {
|
||||
db = new Database(':memory:');
|
||||
runMigrations(db);
|
||||
repo = new NoteRepository(db);
|
||||
});
|
||||
|
||||
it('unreachable — markAiFailed 안 호출, attempts 증가 안 함', async () => {
|
||||
const provider = makeProvider({
|
||||
generate: vi.fn(async () => { throw new Error('ECONNREFUSED'); })
|
||||
});
|
||||
const w = new AiWorker(repo, provider, {
|
||||
backoffsMs: [0, 30_000, 120_000],
|
||||
unreachableBackoffsMs: [10, 10, 10, 10, 10, 10]
|
||||
});
|
||||
const { id } = repo.create({ rawText: 'x' });
|
||||
await w.enqueue(id);
|
||||
// 무한 retry — drain() 은 끝나지 않음. 짧게 대기 후 검증.
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
expect(repo.findById(id)!.aiStatus).toBe('pending');
|
||||
expect(provider.generate).toHaveBeenCalled();
|
||||
expect((provider.generate as any).mock.calls.length).toBeGreaterThanOrEqual(2);
|
||||
const job = repo.getAllPendingJobs().find((j) => j.noteId === id)!;
|
||||
expect(job.attempts).toBe(0);
|
||||
});
|
||||
|
||||
it('timeout — unreachable 동일 (Q2=A)', async () => {
|
||||
const provider = makeProvider({
|
||||
generate: vi.fn(async () => { throw new Error('Request timeout'); })
|
||||
});
|
||||
const w = new AiWorker(repo, provider, {
|
||||
backoffsMs: [0, 30_000, 120_000],
|
||||
unreachableBackoffsMs: [10, 10, 10, 10, 10, 10]
|
||||
});
|
||||
const { id } = repo.create({ rawText: 'x' });
|
||||
await w.enqueue(id);
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
expect(repo.findById(id)!.aiStatus).toBe('pending');
|
||||
expect((provider.generate as any).mock.calls.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it('schema fail max 3 — markAiFailed + ai_failed emit (reason=schema)', async () => {
|
||||
const { ZodError } = await import('zod');
|
||||
const provider = makeProvider({
|
||||
generate: vi.fn(async () => {
|
||||
throw new ZodError([{ code: 'custom', message: 'bad', path: [] } as any]);
|
||||
})
|
||||
});
|
||||
const events: any[] = [];
|
||||
const w = new AiWorker(repo, provider, {
|
||||
backoffsMs: [0, 0, 0],
|
||||
telemetry: { emit: async (e) => { events.push(e); } }
|
||||
});
|
||||
const { id } = repo.create({ rawText: 'x' });
|
||||
await w.enqueue(id);
|
||||
await w.drain();
|
||||
expect(repo.findById(id)!.aiStatus).toBe('failed');
|
||||
expect((provider.generate as any).mock.calls.length).toBe(3);
|
||||
const failed = events.find((e) => e.kind === 'ai_failed');
|
||||
expect(failed).toBeDefined();
|
||||
expect(failed.payload.reason).toBe('schema');
|
||||
});
|
||||
|
||||
it('other fail max 3 — markAiFailed + ai_failed emit (reason=other)', async () => {
|
||||
const provider = makeProvider({
|
||||
generate: vi.fn(async () => { throw new Error('something weird'); })
|
||||
});
|
||||
const events: any[] = [];
|
||||
const w = new AiWorker(repo, provider, {
|
||||
backoffsMs: [0, 0, 0],
|
||||
telemetry: { emit: async (e) => { events.push(e); } }
|
||||
});
|
||||
const { id } = repo.create({ rawText: 'x' });
|
||||
await w.enqueue(id);
|
||||
await w.drain();
|
||||
expect(repo.findById(id)!.aiStatus).toBe('failed');
|
||||
const failed = events.find((e) => e.kind === 'ai_failed');
|
||||
expect(failed.payload.reason).toBe('other');
|
||||
});
|
||||
|
||||
it('unreachable backoff schedule — nextBackoffMs(step) cap at index 5 (15분)', async () => {
|
||||
const w = new AiWorker(repo, makeProvider(), {
|
||||
backoffsMs: [0, 30_000, 120_000],
|
||||
unreachableBackoffsMs: [30_000, 60_000, 120_000, 240_000, 480_000, 900_000]
|
||||
});
|
||||
expect((w as any).nextBackoffMs(0)).toBe(30_000);
|
||||
expect((w as any).nextBackoffMs(2)).toBe(120_000);
|
||||
expect((w as any).nextBackoffMs(5)).toBe(900_000);
|
||||
expect((w as any).nextBackoffMs(10)).toBe(900_000); // cap
|
||||
});
|
||||
|
||||
it('success 후 unreachableBackoffStep reset', async () => {
|
||||
let callCount = 0;
|
||||
const provider = makeProvider({
|
||||
generate: vi.fn(async (): Promise<AiResponse> => {
|
||||
callCount += 1;
|
||||
if (callCount <= 2) throw new Error('ECONNREFUSED');
|
||||
return { title: 't', summary: 's', tags: [], dueDate: null };
|
||||
})
|
||||
});
|
||||
const w = new AiWorker(repo, provider, {
|
||||
backoffsMs: [0, 0, 0],
|
||||
unreachableBackoffsMs: [10, 10, 10, 10, 10, 10]
|
||||
});
|
||||
const { id } = repo.create({ rawText: 'x' });
|
||||
await w.enqueue(id);
|
||||
await w.drain();
|
||||
expect(repo.findById(id)!.aiStatus).toBe('done');
|
||||
expect(callCount).toBe(3);
|
||||
expect((w as any).unreachableBackoffStep).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user