@camera.ui/rust-decoder

A high-performance Rust-based image processing library with GPU acceleration using WebGPU, optimized for real-time video processing and computer vision applications.

Features

• 🚀 GPU Acceleration - Hardware-accelerated processing using WebGPU
• 🎥 Video Format Support - YUV420, NV12, RGB, RGBA, and Grayscale
• ⚡ SIMD Optimization - CPU-based SIMD acceleration for fallback operations
• 🖼️ Image Processing - Resize, crop, blur, and format conversion
• 🔧 Multi-Backend Support - Vulkan, Metal, DirectX12, and OpenGL
• 📊 Real-time Performance - Optimized for streaming and live video processing
• 🎯 Zero-Copy Operations - Efficient memory management
• 🔄 Format Conversion - Seamless conversion between color spaces

Installation

npm install @camera.ui/rust-decoder

Quick Start

Basic Image Processing

import { convertYuvToRgb, gaussianBlur, resizeImage } from '@camera.ui/rust-decoder';

// Convert YUV to RGB
const rgbData = convertYuvToRgb(yuvBuffer, width, height);

// Apply Gaussian blur
const blurredData = gaussianBlur(rgbData, width, height, 5);

// Resize image
const resizedData = resizeImage(
  rgbData, 
  originalWidth, 
  originalHeight, 
  3, // RGB channels
  newWidth, 
  newHeight
);

GPU-Accelerated Processing

import { GpuImageProcessor } from '@camera.ui/rust-decoder';

const processor = new GpuImageProcessor();

// Initialize with preferred GPU backend
const adapterDetails = await processor.initialize('vulkan');
console.log(`Using GPU: ${adapterDetails.name} (${adapterDetails.backend})`);

// Process NV12 video frame with GPU acceleration
const processedFrame = processor.processImage(nv12Buffer, {
  inputWidth: 1920,
  inputHeight: 1080,
  inputFormat: 12, // NV12
  outputFormat: ImageFormat.RGB,
  cropLeft: 100,
  cropTop: 100,
  cropWidth: 800,
  cropHeight: 600,
  resizeWidth: 640,
  resizeHeight: 480,
  blurRadius: 3
});

Real-time Video Processing Pipeline

import { ImageFormat, GpuImageProcessor, convertYuvToRgb } from '@camera.ui/rust-decoder';

class VideoProcessor {
  private gpuProcessor?: GpuImageProcessor;

  async initialize(preferredBackend: GpuBackend = 'auto') {
    this.gpuProcessor = new GpuImageProcessor();
    const adapter = await this.gpuProcessor.initialize(preferredBackend);
    
    console.log(`Initialized GPU processing on ${adapter.name}`);
    return adapter;
  }

  processFrame(nv12Data: Buffer, width: number, height: number) {
    if (!this.gpuProcessor?.initialized) {
      // Fallback to CPU processing
      return convertYuvToRgb(nv12Data, width, height);
    }

    // GPU-accelerated processing with resize and blur
    return this.gpuProcessor.processImage(nv12Data, {
      inputWidth: width,
      inputHeight: height,
      inputFormat: 12, // NV12
      outputFormat: ImageFormat.RGB,
      resizeWidth: Math.floor(width / 2),
      resizeHeight: Math.floor(height / 2),
      blurRadius: 2
    });
  }

  cleanup() {
    this.gpuProcessor?.resetState();
  }
}

API Reference

Core Functions

Format Conversion

// YUV420 to RGB conversion
convertYuvToRgb(frame: Buffer, width: number, height: number): Uint8Array

// YUV420 to RGBA conversion
convertYuvToRgba(frame: Buffer, width: number, height: number): Uint8Array

// YUV420 to Grayscale (Y-plane extraction)
convertYuvToGrayscale(frame: Buffer, width: number, height: number): Uint8Array

Image Operations

// Gaussian blur filter
gaussianBlur(
  frame: Uint8Array, 
  width: number, 
  height: number, 
  kernelSize: number
): Uint8Array

// Image resizing with bilinear interpolation
resizeImage(
  frame: Uint8Array,
  inputWidth: number,
  inputHeight: number,
  channels: number,
  outputWidth: number,
  outputHeight: number
): Uint8Array

// Image cropping
cropImage(
  frame: Uint8Array,
  inputWidth: number,
  channels: number,
  top: number,
  left: number,
  width: number,
  height: number
): Uint8Array

// Combined resize and crop operation
resizeAndCrop(
  inputFrame: Uint8Array,
  inputWidth: number,
  inputHeight: number,
  channels: number,
  cropLeft: number,
  cropTop: number,
  cropWidth: number,
  cropHeight: number,
  outputWidth: number,
  outputHeight: number
): Uint8Array

All-in-One Processing

// Combined YUV conversion, crop, resize, and blur
processImage(
  inputFrame: Uint8Array,
  inputWidth: number,
  inputHeight: number,
  channels: number,
  cropTop: number,
  cropLeft: number,
  cropWidth: number,
  cropHeight: number,
  resizeWidth: number,
  resizeHeight: number,
  blurRadius: number
): Uint8Array

GPU Image Processor

Initialization

class GpuImageProcessor {
  readonly initialized: boolean;
  
  // Initialize GPU with preferred backend
  initialize(preferredBackend?: GpuBackend): Promise<AdapterDetails>;
  
  // Reset GPU state and free resources
  resetState(): void;
}

GPU Processing

// GPU-accelerated image processing
processImage(
  nv12Data: Buffer, 
  params: ProcessingParams
): Uint8Array;

Configuration Types

Processing Parameters

interface ProcessingParams {
  inputWidth: number;
  inputHeight: number;
  inputFormat: number;          // 1=GRAY, 3=RGB, 4=RGBA, 12=NV12
  outputFormat?: number;        // 1=GRAY, 3=RGB, 4=RGBA
  cropTop?: number;
  cropLeft?: number;
  cropWidth?: number;
  cropHeight?: number;
  resizeWidth?: number;
  resizeHeight?: number;
  blurRadius?: number;
}

GPU Backend Options

type GpuBackend = 'vulkan' | 'metal' | 'directx12' | 'opengl' | 'auto';

// Get available GPU backends
getGpuBackends(): Omit<GpuBackend, 'auto'>[];

Image Formats

enum ImageFormat {
  GRAY = 1,
  RGB = 3,
  RGBA = 4,
  NV12 = 12
}

Platform Support

Supported Platforms

• Windows - x64 (DirectX12, Vulkan, OpenGL)
• macOS - Universal (Metal, Vulkan via MoltenVK)
• Linux - x64, ARM64 (Vulkan, OpenGL)
• FreeBSD - x64 (Vulkan, OpenGL)

GPU Backend Availability

| Platform | Vulkan | Metal | DirectX12 | OpenGL | |----------|--------|-------|-----------|--------| | Windows | ✅ | ❌ | ✅ | ✅ | | macOS | ✅* | ✅ | ❌ | ✅ | | Linux | ✅ | ❌ | ❌ | ✅ | | FreeBSD | ✅ | ❌ | ❌ | ✅ |

*Vulkan on macOS requires MoltenVK

Requirements

• Modern GPU with WebGPU support (recommended)
• Platform-specific requirements:
  • Windows: DirectX 12 or Vulkan drivers
  • macOS: Metal support (macOS 10.14+)
  • Linux: Vulkan or OpenGL drivers

Contributing

Contributions are welcome! Please read our contributing guidelines and submit pull requests to our repository.

License

MIT

Part of the camera.ui ecosystem - A comprehensive camera management solution.