Function createRunner

Returns {
    answerKeyHealth: (
        suiteKey: string,
        today?: Date,
    ) => Promise<AnswerKeyHealth<CaseRecord>>;
    attackBoards: (
        suiteKey: string,
    ) => Promise<
        {
            caseResults: Record<string, { models: number; passed: number }>;
            cases: number;
            liveRunId: string | null;
            provisional: number;
            route: Route;
            rows: BoardRow[];
            suite: AnySuite;
        }[],
    >;
    board: (
        suiteKey: string,
    ) => Promise<
        {
            caseResults: Record<string, { models: number; passed: number }>;
            cases: number;
            liveRunId: string | null;
            provisional: number;
            route: Route;
            rows: BoardRow[];
            suite: AnySuite;
        },
    >;
    casesFor: (suite: string, set: EvalSet) => Promise<CaseRecord[]>;
    casesInInstructions: (suiteKey: string) => Promise<CaseInInstructions[]>;
    compare: (
        suiteKey: string,
        a: string,
        b: string,
    ) => Promise<
        | {
            a: RunRecord;
            b: RunRecord;
            caveats: string[];
            comparison: Comparison;
            disagreements: Disagreement[];
            suite: AnySuite;
        }
        | null,
    >;
    createRun: (
        args: {
            fresh?: boolean;
            maxOutputTokens?: number | null;
            model: string;
            reasoningEffort?: ReasoningEffort | null;
            repeats?: number;
            set?: EvalSet;
            startedBy?: string | null;
            suite: string;
            temperature?: number | null;
        },
    ) => Promise<
        {
            estimate: {
                answers: number;
                cases: number;
                costUsd: number
                | null;
                reused: number;
            };
            run: RunRecord;
        },
    >;
    createTest: (
        args: {
            fresh?: boolean;
            maxOutputTokens?: number | null;
            model: string;
            reasoningEffort?: ReasoningEffort | null;
            repeats?: number;
            set?: EvalSet;
            startedBy?: string | null;
            suite: string;
            temperature?: number | null;
        },
    ) => Promise<
        {
            estimate: {
                answers: number;
                cases: number;
                costUsd: number
                | null;
                reused: number;
            };
            runs: RunRecord[];
        },
    >;
    draftFromActivity: (
        suiteKey: string,
        o?: { limit?: number; since?: Date; today?: Date },
    ) => Promise<ActivityDraftResult>;
    estimateRun: (
        suiteKey: string,
        model: string,
        set: EvalSet,
        repeats: number,
        o?: {
            fresh?: boolean;
            maxOutputTokens?: number | null;
            reasoningEffort?: ReasoningEffort | null;
            temperature?: number | null;
        },
    ) => Promise<
        {
            answers: number;
            cases: number;
            costUsd: number
            | null;
            reused: number;
        },
    >;
    estimateTest: (
        suiteKey: string,
        model: string,
        set: EvalSet,
        repeats: number,
        o?:
            | {
                fresh?: boolean;
                maxOutputTokens?: number
                | null;
                reasoningEffort?: ReasoningEffort | null;
                temperature?: number | null;
            }
            | undefined,
    ) => Promise<
        {
            answers: number;
            cases: number;
            costUsd: number
            | null;
            reused: number;
        },
    >;
    gradeOutput: (
        suite: AnySuite,
        c: CaseRecord,
        output: unknown,
        onCost?: (usd: number) => void,
    ) => Promise<CheckResult[]>;
    modelsFor: (operation: string) => string[];
    rescoreRecent: (suiteKey: string, limit?: number) => Promise<void>;
    rescoreRun: (runId: string) => Promise<void>;
    retireEasy: (suiteKey: string) => Promise<number>;
    retireStaleDrafts: (suiteKey: string, today?: Date) => Promise<number>;
    runDriftCheck: (suiteKey: string) => Promise<string | null>;
    runEval: (runId: string, o?: RunEvalOptions) => Promise<void>;
    startDriftCheck: (suiteKey: string) => Promise<string | null>;
    stopStuckRuns: (suiteKey: string, now?: number) => Promise<void>;
    suiteOrThrow: (key: string) => AnySuite;
}

• answerKeyHealth: (suiteKey: string, today?: Date) => Promise<AnswerKeyHealth<CaseRecord>>

• attackBoards: (
      suiteKey: string,
  ) => Promise<
      {
          caseResults: Record<string, { models: number; passed: number }>;
          cases: number;
          liveRunId: string | null;
          provisional: number;
          route: Route;
          rows: BoardRow[];
          suite: AnySuite;
      }[],
  >

• board: (
      suiteKey: string,
  ) => Promise<
      {
          caseResults: Record<string, { models: number; passed: number }>;
          cases: number;
          liveRunId: string | null;
          provisional: number;
          route: Route;
          rows: BoardRow[];
          suite: AnySuite;
      },
  >

• casesFor: (suite: string, set: EvalSet) => Promise<CaseRecord[]>

• casesInInstructions: (suiteKey: string) => Promise<CaseInInstructions[]>

• compare: (
      suiteKey: string,
      a: string,
      b: string,
  ) => Promise<
      | {
          a: RunRecord;
          b: RunRecord;
          caveats: string[];
          comparison: Comparison;
          disagreements: Disagreement[];
          suite: AnySuite;
      }
      | null,
  >

• createRun: (
      args: {
          fresh?: boolean;
          maxOutputTokens?: number | null;
          model: string;
          reasoningEffort?: ReasoningEffort | null;
          repeats?: number;
          set?: EvalSet;
          startedBy?: string | null;
          suite: string;
          temperature?: number | null;
      },
  ) => Promise<
      {
          estimate: {
              answers: number;
              cases: number;
              costUsd: number
              | null;
              reused: number;
          };
          run: RunRecord;
      },
  >

• createTest: (
      args: {
          fresh?: boolean;
          maxOutputTokens?: number | null;
          model: string;
          reasoningEffort?: ReasoningEffort | null;
          repeats?: number;
          set?: EvalSet;
          startedBy?: string | null;
          suite: string;
          temperature?: number | null;
      },
  ) => Promise<
      {
          estimate: {
              answers: number;
              cases: number;
              costUsd: number
              | null;
              reused: number;
          };
          runs: RunRecord[];
      },
  >

• draftFromActivity: (
      suiteKey: string,
      o?: { limit?: number; since?: Date; today?: Date },
  ) => Promise<ActivityDraftResult>

• estimateRun: (
      suiteKey: string,
      model: string,
      set: EvalSet,
      repeats: number,
      o?: {
          fresh?: boolean;
          maxOutputTokens?: number | null;
          reasoningEffort?: ReasoningEffort | null;
          temperature?: number | null;
      },
  ) => Promise<
      {
          answers: number;
          cases: number;
          costUsd: number
          | null;
          reused: number;
      },
  >

• estimateTest: (
      suiteKey: string,
      model: string,
      set: EvalSet,
      repeats: number,
      o?:
          | {
              fresh?: boolean;
              maxOutputTokens?: number
              | null;
              reasoningEffort?: ReasoningEffort | null;
              temperature?: number | null;
          }
          | undefined,
  ) => Promise<
      {
          answers: number;
          cases: number;
          costUsd: number
          | null;
          reused: number;
      },
  >

• gradeOutput: (
      suite: AnySuite,
      c: CaseRecord,
      output: unknown,
      onCost?: (usd: number) => void,
  ) => Promise<CheckResult[]>

• modelsFor: (operation: string) => string[]

• rescoreRecent: (suiteKey: string, limit?: number) => Promise<void>

• rescoreRun: (runId: string) => Promise<void>

• retireEasy: (suiteKey: string) => Promise<number>

• retireStaleDrafts: (suiteKey: string, today?: Date) => Promise<number>

• runDriftCheck: (suiteKey: string) => Promise<string | null>

• runEval: (runId: string, o?: RunEvalOptions) => Promise<void>

• startDriftCheck: (suiteKey: string) => Promise<string | null>

• stopStuckRuns: (suiteKey: string, now?: number) => Promise<void>

• suiteOrThrow: (key: string) => AnySuite

const runner = createRunner({
  store: prismaStore(prisma),
  suites: [ticketTriage],
  models: MODELS,
  promptVersion: (op) => PROMPT_VERSIONS[op] ?? "unregistered",
  route: settings.route,
  candidates: () => ({ models: Object.keys(MODELS) }),
});
const { run } = await runner.createRun({ suite: "ticket_triage", model: "claude-haiku-5-5" });
await runner.runEval(run.id);
const board = await runner.board("ticket_triage");
Copy