Codewalkers/apps/server/scripts/backfill-metrics.test.ts

/**
 * Tests for the backfill-metrics script.
 *
 * Uses an in-memory test database to verify that backfillMetrics correctly
 * accumulates counts from agent_log_chunks and upserts into agent_metrics.
 */

import { describe, it, expect, beforeEach } from 'vitest';
import { createTestDatabase } from '../db/repositories/drizzle/test-helpers.js';
import type { DrizzleDatabase } from '../db/index.js';
import { agentLogChunks, agentMetrics } from '../db/index.js';
import { backfillMetrics } from './backfill-metrics.js';
import { nanoid } from 'nanoid';
import { eq } from 'drizzle-orm';

async function insertChunk(db: DrizzleDatabase, agentId: string, content: object | string) {
  await db.insert(agentLogChunks).values({
    id: nanoid(),
    agentId,
    agentName: 'test-agent',
    sessionNumber: 1,
    content: typeof content === 'string' ? content : JSON.stringify(content),
    createdAt: new Date(),
  });
}

describe('backfillMetrics', () => {
  let db: DrizzleDatabase;

  beforeEach(() => {
    db = createTestDatabase();
  });

  it('AskUserQuestion chunks — questionsCount correct', async () => {
    await insertChunk(db, 'agent-a', { type: 'tool_use', name: 'AskUserQuestion', input: { questions: [{}, {}] } });
    await insertChunk(db, 'agent-a', { type: 'tool_use', name: 'AskUserQuestion', input: { questions: [{}] } });

    await backfillMetrics(db);

    const rows = await db.select().from(agentMetrics).where(eq(agentMetrics.agentId, 'agent-a'));
    expect(rows).toHaveLength(1);
    expect(rows[0].questionsCount).toBe(3);
    expect(rows[0].subagentsCount).toBe(0);
    expect(rows[0].compactionsCount).toBe(0);
  });

  it('Agent tool chunks — subagentsCount correct', async () => {
    await insertChunk(db, 'agent-b', { type: 'tool_use', name: 'Agent' });
    await insertChunk(db, 'agent-b', { type: 'tool_use', name: 'Agent' });

    await backfillMetrics(db);

    const rows = await db.select().from(agentMetrics).where(eq(agentMetrics.agentId, 'agent-b'));
    expect(rows).toHaveLength(1);
    expect(rows[0].questionsCount).toBe(0);
    expect(rows[0].subagentsCount).toBe(2);
    expect(rows[0].compactionsCount).toBe(0);
  });

  it('Compaction chunks — compactionsCount correct', async () => {
    await insertChunk(db, 'agent-c', { type: 'system', subtype: 'init', source: 'compact' });

    await backfillMetrics(db);

    const rows = await db.select().from(agentMetrics).where(eq(agentMetrics.agentId, 'agent-c'));
    expect(rows).toHaveLength(1);
    expect(rows[0].questionsCount).toBe(0);
    expect(rows[0].subagentsCount).toBe(0);
    expect(rows[0].compactionsCount).toBe(1);
  });

  it('Irrelevant chunk type — no metrics row created', async () => {
    await insertChunk(db, 'agent-d', { type: 'text', text: 'hello' });

    await backfillMetrics(db);

    const rows = await db.select().from(agentMetrics).where(eq(agentMetrics.agentId, 'agent-d'));
    expect(rows).toEqual([]);
  });

  it('Malformed JSON chunk — skipped, no crash', async () => {
    await insertChunk(db, 'agent-e', 'not-valid-json');
    await insertChunk(db, 'agent-e', { type: 'tool_use', name: 'Agent' });

    await expect(backfillMetrics(db)).resolves.not.toThrow();

    const rows = await db.select().from(agentMetrics).where(eq(agentMetrics.agentId, 'agent-e'));
    expect(rows).toHaveLength(1);
    expect(rows[0].subagentsCount).toBe(1);
  });

  it('Multiple agents — counts isolated per agent', async () => {
    await insertChunk(db, 'agent-f', { type: 'tool_use', name: 'AskUserQuestion', input: { questions: [{}, {}, {}] } });
    await insertChunk(db, 'agent-f', { type: 'tool_use', name: 'AskUserQuestion', input: { questions: [{}, {}, {}] } });
    await insertChunk(db, 'agent-g', { type: 'tool_use', name: 'Agent' });

    await backfillMetrics(db);

    const rowsF = await db.select().from(agentMetrics).where(eq(agentMetrics.agentId, 'agent-f'));
    expect(rowsF).toHaveLength(1);
    expect(rowsF[0].questionsCount).toBe(6);
    expect(rowsF[0].subagentsCount).toBe(0);
    expect(rowsF[0].compactionsCount).toBe(0);

    const rowsG = await db.select().from(agentMetrics).where(eq(agentMetrics.agentId, 'agent-g'));
    expect(rowsG).toHaveLength(1);
    expect(rowsG[0].questionsCount).toBe(0);
    expect(rowsG[0].subagentsCount).toBe(1);
    expect(rowsG[0].compactionsCount).toBe(0);
  });

  it('Empty database — completes without error', async () => {
    await expect(backfillMetrics(db)).resolves.not.toThrow();

    const rows = await db.select().from(agentMetrics);
    expect(rows).toEqual([]);
  });

  it('Re-run idempotency note — second run doubles counts', async () => {
    // Documented behavior: run only once against a fresh agent_metrics table
    await insertChunk(db, 'agent-h', { type: 'tool_use', name: 'Agent' });

    await backfillMetrics(db);
    const rowsAfterFirst = await db.select().from(agentMetrics).where(eq(agentMetrics.agentId, 'agent-h'));
    expect(rowsAfterFirst[0].subagentsCount).toBe(1);

    await backfillMetrics(db);
    const rowsAfterSecond = await db.select().from(agentMetrics).where(eq(agentMetrics.agentId, 'agent-h'));
    expect(rowsAfterSecond[0].subagentsCount).toBe(2);
  });
});