media-evals

Minimal image evaluation library for generative media.

Install

npm install -D tsx

Usage

import type { ImageSample } from "./packages/core";
import { LocalRunner } from "./packages/runner-local";
import { CLIPSimilarityMetric, AestheticScoreMetric } from "./packages/metrics";

const samples: ImageSample[] = [
  {
    id: "sample-001",
    input: { prompt: "A serene mountain landscape at sunset" },
    output: { image: "/path/to/image.png" },
  },
];

const runner = new LocalRunner();
const results = await runner.run(samples, [
  new CLIPSimilarityMetric(),
  new AestheticScoreMetric(),
]);

console.log(results);

Run Example

npx tsx examples/basic-image-eval.ts

Structure

packages/
  core/           # Core types (ImageSample, ImageMetric, EvalResult, EvalRunner)
  runner-local/   # LocalRunner - sequential execution
  metrics/        # Built-in metrics (CLIP, Aesthetic)
examples/
  basic-image-eval.ts

Core Types

ImageSample

interface ImageSample {
  id: string;
  input?: { prompt?: string; image?: string };
  output: { image: string };
  metadata?: Record<string, unknown>;
}

ImageMetric

interface ImageMetric {
  name: string;
  evaluate(sample: ImageSample): Promise<number | Record<string, number>>;
}

EvalResult

interface EvalResult {
  sampleId: string;
  metric: string;
  score: number | Record<string, number>;
}

Metrics

| Metric | Description | |--------|-------------| | CLIPSimilarityMetric | Text-image similarity (mock) | | AestheticScoreMetric | Aesthetic quality score (mock) |

Custom Metrics

import type { ImageMetric, ImageSample } from "./packages/core";

class MyMetric implements ImageMetric {
  readonly name = "my_metric";

  async evaluate(sample: ImageSample): Promise<number> {
    // Your evaluation logic
    return 0.5;
  }
}

License

MIT