@sanity/agent-evals

A Vitest-style evaluation framework for testing AI agents.

Installation

pnpm add @sanity/agent-evals

Quick Start

1. Create a config file

// evals.config.ts
import { defineConfig } from "@sanity/agent-evals/config";

export default defineConfig({
  include: ["**/*.eval.ts"],
  exclude: ["node_modules/**"],

  judge: {
    model: anthropic("claude-sonnet-4-5"),
  },

  agent: (contextKey) => ({
    async run(prompt) {
      // Your agent implementation
      return {
        text: "",
        toolCalls: [],
        createdDocuments: [],
        updatedDocuments: [],
        duration: 0,
      };
    },
  }),

  client: (contextKey) =>
    createClient({
      /* ... */
    }),
});

2. Write evals

// query.eval.ts
import { describe, expect, it } from "@sanity/agent-evals";

describe("Document Query", () => {
  it("finds posts by author", async ({ agent }) => {
    const result = await agent.prompt("Find all posts by Knut");

    await expect(result).toSatisfy("returns a list of posts");
    expect(result).toHaveUsedTools(["groq-query"]);
  });
});

3. Run evals

npx sanity-evals

CLI Options

npx sanity-evals --help

License

MIT