@assay-ai/vitest

Custom Vitest matchers for LLM evaluation with Assay

Documentation · Metrics · API Reference

Installation

pnpm add -D @assay-ai/core @assay-ai/vitest     # pnpm
npm install -D @assay-ai/core @assay-ai/vitest   # npm
yarn add -D @assay-ai/core @assay-ai/vitest      # Yarn

Quick Start

import { beforeAll, describe, expect, test } from "vitest";
import { setupAssayMatchers } from "@assay-ai/vitest";

beforeAll(() => {
  setupAssayMatchers();
});

describe("Customer Support Chatbot", () => {
  test("answers are relevant", async () => {
    await expect({
      input: "What is your return policy?",
      actualOutput: "You can return items within 30 days of purchase.",
      retrievalContext: [
        "Our return policy allows returns within 30 days of purchase.",
      ],
    }).toBeRelevant({ threshold: 0.8 });
  });

  test("does not hallucinate", async () => {
    await expect({
      input: "What is your return policy?",
      actualOutput: "You can return items within 30 days of purchase.",
      context: [
        "Our return policy allows returns within 30 days of purchase.",
      ],
    }).toNotHallucinate();
  });
});

Part of the Assay monorepo