YAML Loader

A comprehensive utility for loading and parsing YAML/JSON files with advanced $ref resolution, type safety, and performance optimizations.

Install

Install with NPM:

npm install @northern/di

or Yarn:

yarn add @northern/di

Table of Contents

• Overview
• Installation
• Basic Usage
• Type Safety
• Reference Resolution
• Configuration Options
• Error Handling
• Performance Features
• Security Features
• Advanced Usage
• API Reference
• Migration Guide
• Troubleshooting

Overview

The YAML loader module provides:

• Multi-format support: YAML and JSON file parsing
• Advanced reference resolution: Internal and external $ref resolution with circular detection
• Type safety: Full TypeScript generic support
• Performance: LRU caching and optimized algorithms
• Security: Configurable access controls and path validation
• Debugging: Comprehensive debug information and error reporting
• Extensibility: Custom resolver plugin system

Basic Usage

import loadYaml from '@northern/yaml-loader';

Simple YAML Loading

// Basic usage - returns any type
const config = loadYaml('config.yaml');
console.log(config.name); // Access properties with basic autocomplete

Loading JSON Files

// JSON files are supported automatically
const data = loadYaml('data.json');

Loading Nested Structures

// Complex nested YAML structures
const app = loadYaml('app.yaml');

// Access nested properties
const serverConfig = app.server;
const dbConfig = app.database;

Type Safety

Generic Types

interface AppConfig {
  name: string;
  version: number;
  server: {
    host: string;
    port: number;
    ssl?: boolean;
  };
  database: {
    url: string;
    pool: {
      min: number;
      max: number;
    };
  };
  features: string[];
}

// Full type inference and compile-time checking
const config = loadYaml<AppConfig>('app.yaml');

// TypeScript will catch type errors
const host: string = config.server.host; // ✅ Valid
const port: string = config.server.port; // ❌ TypeScript error: Type 'number' is not assignable to 'string'
config.invalidProperty; // ❌ TypeScript error: Property does not exist

Complex Nested Types

interface User {
  id: number;
  name: string;
  profile: {
    email: string;
    avatar?: string;
  };
  roles: Array<{
    name: string;
    permissions: string[];
  }>;
}

interface UsersConfig {
  users: User[];
  metadata: {
    created: Date;
    version: string;
    tags: string[];
  };
}

const usersConfig = loadYaml<UsersConfig>('users.yaml');

// Full type safety throughout
usersConfig.users.forEach(user => {
  console.log(user.profile.email); // ✅ Type-safe access
  user.roles.forEach(role => {
    console.log(role.permissions.join(', ')); // ✅ Type-safe array operations
  });
});

Union Types and Discriminated Unions

type DatabaseConfig = 
  | { type: 'postgres'; host: string; port: number; database: string; }
  | { type: 'mongodb'; url: string; collection: string; };

interface ServiceConfig {
  name: string;
  database: DatabaseConfig;
}

const config = loadYaml<ServiceConfig>('service.yaml');

// TypeScript will help you handle different database types
if (config.database.type === 'postgres') {
  console.log(config.database.host); // ✅ TypeScript knows this is postgres config
  // config.database.url; // ❌ TypeScript error: Property 'url' does not exist
}

Reference Resolution

Internal References

# config.yaml
definitions:
  server:
    host: localhost
    port: 3000
  database:
    url: postgresql://localhost:5432/mydb
    
services:
  api:
    # Internal reference to definitions
    $ref: "#/definitions/server"
  database:
    $ref: "#/definitions/database"

const config = loadYaml('config.yaml');
// Result will have fully resolved structure
// {
//   definitions: { server: {...}, database: {...} },
//   services: {
//     api: { host: 'localhost', port: 3000 },
//     database: { url: 'postgresql://localhost:5432/mydb' }
//   }
// }

External File References

# shared/database.yaml
url: postgresql://localhost:5432/mydb
pool:
  min: 5
  max: 20

# config.yaml
database:
  $ref: "./shared/database.yaml"
  
api:
  version: v1
  port: 8080

Combined External and Pointer References

# schemas/user.yaml
definitions:
  User:
    type: object
    properties:
      id:
        type: integer
      name:
        type: string
      email:
        type: string
        format: email

# api.yaml
userSchema:
  $ref: "./schemas/user.yaml#/definitions/User"
  
endpoints:
  - path: /users
    schema:
      $ref: "./schemas/user.yaml#/definitions/User"

Nested Reference Chains

# level3.yaml
finalValue: "Deeply nested reference resolved"

# level2.yaml
data:
  $ref: "./level3.yaml"

# level1.yaml
nested:
  $ref: "./level2.yaml"

# main.yaml
result:
  $ref: "./level1.yaml"

Array References

# item.yaml
type: shared-item
properties:
  name: string
  value: number

# main.yaml
items:
  - name: inline1
    type: local
  - $ref: "./item.yaml"
  - name: inline2
    type: local
  - $ref: "./item.yaml"

JSON Pointer Features

# data.yaml
"a/b path":
  value: "path with slash"
"a~b":
  value: "path with tilde"
items:
  - first
  - second
  - third

# ref.yaml
# Reference with escaped characters
slashExample:
  $ref: "./data.yaml#/a~1b path"
tildeExample:
  $ref: "./data.yaml#/a~0b"
arrayItem:
  $ref: "./data.yaml#/items/1"

Configuration Options

Basic Configuration

import { YamlLoaderOptions } from './yaml-loader';

const options: YamlLoaderOptions = {
  maxCacheSize: 50,
  allowExternalAccess: false,
  strictMode: true
};

const config = loadYaml('config.yaml', options);

Cache Configuration

// Large cache for enterprise applications
const enterpriseOptions: YamlLoaderOptions = {
  maxCacheSize: 500 // Cache up to 500 files
};

// Small cache for simple applications
const simpleOptions: YamlLoaderOptions = {
  maxCacheSize: 10 // Cache only 10 files
};

Security Configuration

// Default: prevent directory traversal
const secureOptions: YamlLoaderOptions = {
  allowExternalAccess: false
};

// Allow external access for trusted environments
const openOptions: YamlLoaderOptions = {
  allowExternalAccess: true
};

Strict Mode

// Enable strict validation
const strictOptions: YamlLoaderOptions = {
  strictMode: true
};

// Disable for legacy compatibility
const lenientOptions: YamlLoaderOptions = {
  strictMode: false
};

Error Handling

Basic Error Handling

import { YamlLoaderError } from './yaml-loader';

try {
  const config = loadYaml('config.yaml');
  console.log('Loaded successfully:', config);
} catch (error) {
  if (error instanceof YamlLoaderError) {
    console.error(`YAML Error (${error.type}): ${error.message}`);
    if (error.path) {
      console.error('Path:', error.path);
    }
    if (error.refChain) {
      console.error('Reference chain:', error.refChain.join(' -> '));
    }
  } else {
    console.error('Unexpected error:', error);
  }
}

Error Types

// Circular reference error
try {
  loadYaml('circular.yaml');
} catch (error) {
  if (error instanceof YamlLoaderError && error.type === 'circular_ref') {
    console.log('Circular reference detected');
    console.log('Chain:', error.refChain);
  }
}

// File not found error
try {
  loadYaml('missing.yaml');
} catch (error) {
  if (error instanceof YamlLoaderError && error.type === 'file_not_found') {
    console.log('File not found:', error.path);
  }
}

// Invalid JSON pointer
try {
  loadYaml('invalid-pointer.yaml');
} catch (error) {
  if (error instanceof YamlLoaderError && error.type === 'invalid_pointer') {
    console.log('Invalid pointer path:', error.path);
  }
}

// Parse error
try {
  loadYaml('invalid.yaml');
} catch (error) {
  if (error instanceof YamlLoaderError && error.type === 'parse_error') {
    console.log('Parse error:', error.message);
  }
}

Validation Mode

import { validateYamlReferences } from './yaml-loader';

// Validate without full resolution
const validation = validateYamlReferences('config.yaml');

if (validation.isValid) {
  console.log('All references are valid');
  const config = loadYaml('config.yaml');
} else {
  console.log('Validation failed:');
  validation.errors.forEach(error => {
    console.error(`- ${error.type}: ${error.message}`);
  });
  validation.warnings.forEach(warning => {
    console.warn(`Warning: ${warning}`);
  });
}

Performance Features

LRU Cache

// Configure cache size based on application needs
const options: YamlLoaderOptions = {
  maxCacheSize: 100 // Default: 100 files
};

// Cache automatically handles:
// - Most recently used files stay in memory
// - Least recently used files are evicted when limit reached
// - Multiple references to same file use cached version

const config = loadYaml('main.yaml', options);
// If main.yaml references shared.yaml multiple times, it's loaded once

Debug Mode

import { loadYamlWithDebug } from './yaml-loader';

const { result, debug } = loadYamlWithDebug('complex.yaml');

console.log(`Resolution completed in ${debug.resolutionTime}ms`);
console.log(`Processed ${debug.refChain.length} references`);
console.log(`Cached ${debug.fileCache.size} files`);

// Analyze cache contents
for (const [file, type] of debug.fileCache) {
  console.log(`${file}: ${type}`);
}

Performance Monitoring

// Create a performance wrapper
function loadWithMetrics<T = any>(filename: string): T {
  const start = Date.now();
  const { result, debug } = loadYamlWithDebug(filename);
  const duration = Date.now() - start;
  
  console.log(`Load time: ${duration}ms`);
  console.log(`Cache hits: ${debug.fileCache.size}`);
  console.log(`References resolved: ${debug.refChain.length}`);
  
  return result;
}

const config = loadWithMetrics('config.yaml');

Security Features

Directory Traversal Protection

// Default: prevents access outside base directory
const secureConfig = loadYaml('config.yaml');
// This will fail if config.yaml contains: $ref: "../secret.yaml"

Allow External Access

// Enable external access for trusted environments
const options: YamlLoaderOptions = {
  allowExternalAccess: true
};

const config = loadYaml('config.yaml', options);
// Now allows: $ref: "../shared/config.yaml"

Path Validation

// Custom path validation
function loadWithValidation(filename: string, allowedPaths: string[]) {
  try {
    return loadYaml(filename, { allowExternalAccess: true });
  } catch (error) {
    if (error instanceof YamlLoaderError && error.type === 'file_not_found') {
      // Check if path is in allowed list
      const isAllowed = allowedPaths.some(path => 
        error.path?.includes(path)
      );
      if (!isAllowed) {
        throw new Error('Access denied to external file');
      }
    }
    throw error;
  }
}

Advanced Usage

Builder Pattern

import { YamlLoaderBuilder } from './yaml-loader';

// Simple builder
const loader = new YamlLoaderBuilder()
  .withCache(50)
  .withStrictMode(true)
  .withExternalAccess(false)
  .build();

const config = loader('config.yaml');

// Generic builder with type safety
interface AppConfig {
  name: string;
  version: number;
}

const typedLoader = new YamlLoaderBuilder()
  .withCache(25)
  .buildGeneric<AppConfig>();

const appConfig = typedLoader('app.yaml');
// Full type inference with AppConfig interface

Custom Resolvers

import { YamlLoaderBuilder } from './yaml-loader';

// Environment variable resolver
const loader = new YamlLoaderBuilder()
  .withCustomResolver('env:', (ref) => {
    const varName = ref.replace('env:', '');
    return process.env[varName] || '';
  })
  .build();

// Usage in YAML:
# config.yaml
database:
  url: env:DATABASE_URL
  password: env:DB_PASSWORD

const config = loader('config.yaml');
// config.database.url will contain the environment variable value

HTTP Resolver

import axios from 'axios';

// HTTP-based resolver for remote schemas
const httpLoader = new YamlLoaderBuilder()
  .withCustomResolver('http:', async (ref) => {
    const response = await axios.get(ref);
    return response.data;
  })
  .withCustomResolver('https:', async (ref) => {
    const response = await axios.get(ref);
    return response.data;
  })
  .build();

// Usage in YAML:
# config.yaml
schema:
  $ref: "https://example.com/schemas/user.json"

Template Resolver

// Template-based resolver for dynamic values
const templateLoader = new YamlLoaderBuilder()
  .withCustomResolver('template:', (ref) => {
    const templateName = ref.replace('template:', '');
    const templates = {
      'user-service': {
        port: 3000,
        endpoints: ['/users', '/users/{id}']
      },
      'auth-service': {
        port: 3001,
        endpoints: ['/login', '/logout', '/refresh']
      }
    };
    return templates[templateName];
  })
  .build();

// Usage in YAML:
# config.yaml
userService:
  $ref: "template:user-service"
authService:
  $ref: "template:auth-service"

Conditional Loading

function loadConfig(configPath: string, isProduction: boolean) {
  const options: YamlLoaderOptions = {
    maxCacheSize: isProduction ? 500 : 50,
    allowExternalAccess: !isProduction,
    strictMode: isProduction
  };
  
  return loadYaml(configPath, options);
}

const devConfig = loadConfig('config.yaml', false);
const prodConfig = loadConfig('config.yaml', true);

Plugin System

// Create a plugin loader
interface Plugin {
  name: string;
  resolver: (ref: string) => any;
}

class ExtensibleYamlLoader {
  private builder = new YamlLoaderBuilder();
  
  addPlugin(plugin: Plugin): this {
    this.builder.withCustomResolver(`${plugin.name}:`, plugin.resolver);
    return this;
  }
  
  build<T = any>() {
    return this.builder.buildGeneric<T>();
  }
}

// Usage
const loader = new ExtensibleYamlLoader()
  .addPlugin({
    name: 'env',
    resolver: (ref) => process.env[ref.replace('env:', '')] || ''
  })
  .addPlugin({
    name: 'secret',
    resolver: (ref) => {
      // Load from secret manager
      return getSecret(ref.replace('secret:', ''));
    }
  })
  .build();

API Reference

Core Functions

loadYaml<T>(filename: string, options?: YamlLoaderOptions): T

Loads and parses a YAML file with reference resolution.

Parameters:

• filename: string - Absolute path to the YAML/JSON file
• options?: YamlLoaderOptions - Configuration options (optional)

Returns:

• T - Parsed and resolved content with type inference

Example:

const config = loadYaml<Config>('config.yaml');

loadYamlWithDebug<T>(filename: string, options?: YamlLoaderOptions): { result: T; debug: DebugInfo }

Loads YAML with debug information for troubleshooting.

Returns:

• result: T - The loaded configuration
• debug: DebugInfo - Debug information including cache stats and timing

validateYamlReferences(filename: string, options?: YamlLoaderOptions): ValidationResult

Validates references without full resolution.

Returns:

• isValid: boolean - Whether all references are valid
• errors: YamlLoaderError[] - List of validation errors
• warnings: string[] - List of warnings

Classes

YamlLoaderBuilder

Builder pattern for creating configured loaders.

Methods:

• withCache(size: number): this - Set cache size
• withStrictMode(enabled: boolean): this - Enable/disable strict mode
• withExternalAccess(enabled: boolean): this - Allow/deny external access
• withCustomResolver(prefix: string, resolver: (ref: string) => any): this - Add custom resolver
• build(): (filename: string) => any - Build configured loader
• buildGeneric<T>(): (filename: string) => T - Build typed loader

YamlLoaderError

Enhanced error class for YAML loading issues.

Properties:

• type: 'circular_ref' | 'file_not_found' | 'invalid_pointer' | 'parse_error' - Error category
• path?: string - File path or reference path
• refChain?: string[] - Chain of references that led to the error

Interfaces

YamlLoaderOptions

Configuration options for YAML loading.

interface YamlLoaderOptions {
  maxCacheSize?: number;        // Maximum files to cache (default: 100)
  allowExternalAccess?: boolean; // Allow directory traversal (default: false)
  customResolvers?: Map<string, (ref: string) => any>; // Custom resolvers
  strictMode?: boolean;         // Enable strict validation (default: false)
}

DebugInfo

Debug information from loadYamlWithDebug.

interface DebugInfo {
  refChain: string[];                    // Reference resolution chain
  fileCache: Map<string, string>;        // Cached files and their types
  resolutionTime: number;                 // Time in milliseconds
}

Migration Guide

From Basic Usage

Before:

const yamlLoader = require('./yaml-loader');
const config = yamlLoader('config.yaml');

After:

import { loadYaml } from './yaml-loader';
const config = loadYaml('config.yaml');

Adding Type Safety

Before:

const config = loadYaml('config.yaml');
// No type checking
const port = config.port; // Could be anything

After:

interface Config {
  port: number;
  host: string;
}
const config = loadYaml<Config>('config.yaml');
const port = config.port; // TypeScript knows it's a number

Adding Configuration

Before:

const config = loadYaml('config.yaml');

After:

const config = loadYaml('config.yaml', {
  maxCacheSize: 50,
  allowExternalAccess: true,
  strictMode: false
});

Using Builder Pattern

Before:

const config = loadYaml('config.yaml');

After:

const loader = new YamlLoaderBuilder()
  .withCache(50)
  .withStrictMode(true)
  .build();

const config = loader('config.yaml');

Troubleshooting

Common Issues

Circular References

Error: Circular reference detected: A -> B -> A

Solution:

• Check reference chains in YAML files
• Use validateYamlReferences() to detect issues early
• Consider restructuring to avoid circular dependencies

File Not Found

Error: Failed to load file: /path/to/missing.yaml

Solutions:

• Verify file paths are correct
• Check if allowExternalAccess: true is needed for external files
• Use absolute paths for reliable resolution

Type Errors

Error: TypeScript compilation errors

Solutions:

• Define proper interfaces for your YAML structure
• Use generic type parameter: loadYaml<MyInterface>()
• Check for optional properties with ? in interfaces

Performance Issues

Symptoms: Slow loading times

Solutions:

• Increase cache size with maxCacheSize
• Use debug mode to identify bottlenecks
• Consider simplifying reference chains

Debug Techniques

Using Debug Mode

const { result, debug } = loadYamlWithDebug('complex.yaml');

console.log('Performance Analysis:');
console.log(`- Total time: ${debug.resolutionTime}ms`);
console.log(`- References: ${debug.refChain.length}`);
console.log(`- Cache size: ${debug.fileCache.size}`);

console.log('Reference Chain:');
debug.refChain.forEach((ref, index) => {
  console.log(`${index + 1}. ${ref}`);
});

Validation Before Loading

const validation = validateYamlReferences('config.yaml');
if (!validation.isValid) {
  console.log('Issues found:');
  validation.errors.forEach(error => {
    console.error(`- ${error.type}: ${error.message}`);
  });
  return;
}

// Only load if validation passes
const config = loadYaml('config.yaml');

File Analysis

// Create a file usage analyzer
function analyzeFileUsage(filename: string) {
  const { debug } = loadYamlWithDebug(filename);
  
  const fileUsage = new Map<string, number>();
  
  // Count file usage from reference chain
  debug.refChain.forEach(ref => {
    const filePath = ref.split('#')[0];
    fileUsage.set(filePath, (fileUsage.get(filePath) || 0) + 1);
  });
  
  console.log('File Usage:');
  for (const [file, count] of fileUsage) {
    console.log(`${file}: ${count} references`);
  }
}

analyzeFileUsage('config.yaml');

Performance Optimization

Cache Tuning

// For small applications
const smallAppOptions = { maxCacheSize: 10 };

// For medium applications
const mediumAppOptions = { maxCacheSize: 50 };

// For large applications
const largeAppOptions = { maxCacheSize: 200 };

Reference Optimization

# Instead of many small references
api:
  user:
    $ref: "./schemas/user.yaml"
  product:
    $ref: "./schemas/product.yaml"
  order:
    $ref: "./schemas/order.yaml"

# Consider consolidating
api:
  schemas:
    $ref: "./schemas/all.yaml"

Lazy Loading

// Load only what you need
function loadSection<T>(filename: string, pointer: string): T {
  const fullPath = `${filename}#${pointer}`;
  return loadYaml<T>(fullPath);
}

const userConfig = loadSection('config.yaml', '#/services/user');
const dbConfig = loadSection('config.yaml', '#/database');