@nestamai/ai-3d-renderer

A secure, AI-powered Three.js 3D rendering library with sandboxed execution for educational applications.

🚀 Features

• 🔒 Security First: Web Worker sandboxing with AST-based code validation
• 🤖 AI-Powered: Generate 3D scenes from natural language queries
• ⚡ Performance: OffscreenCanvas rendering with 60fps target
• 🛡️ Error-Proof: Comprehensive fallback system with safe default scenes
• 🎯 Educational: Perfect for interactive learning visualizations
• 📱 React Ready: Easy integration with React applications

📦 Installation

npm install @nestamai/ai-3d-renderer

🎯 Quick Start

Basic Usage

import React from 'react';
import { AI3DRenderer } from '@nestamai/ai-3d-renderer';

function App() {
  return (
    <div>
      <h1>AI 3D Visualizations</h1>
      <AI3DRenderer 
        query="Create a rotating cube with physics simulation"
        width={800}
        height={600}
        fallbackText="Generating your 3D visualization..."
        onError={(error) => console.error('Render error:', error)}
        onLoad={() => console.log('Scene loaded successfully')}
      />
    </div>
  );
}

Interactive Example with AI Integration

import React, { useState, useRef } from 'react';
import { AI3DRenderer } from '@nestamai/ai-3d-renderer';

// Define the ref type for accessing renderer methods
interface AI3DRendererRef {
  renderScene: (code: string) => Promise<void>;
}

export function InteractiveExample() {
  const [query, setQuery] = useState('Create a rotating cube with physics simulation');
  const [isLoading, setIsLoading] = useState(false);
  const [error, setError] = useState<string | null>(null);
  const rendererRef = useRef<AI3DRendererRef>(null);

  // Mock AI API call - replace with your actual AI service
  const callMockAI = async (query: string): Promise<string> => {
    // Simulate API delay
    await new Promise(resolve => setTimeout(resolve, 1000));
    
    // Return mock Three.js code based on query
    if (query.toLowerCase().includes('cube')) {
      return `
// Rotating cube scene
console.log('Loading rotating cube scene');

// Clear existing scene
while (scene.children.length > 0) {
  scene.remove(scene.children[0]);
}

// Create a rotating cube
const geometry = new THREE.BoxGeometry(1, 1, 1);
const material = new THREE.MeshStandardMaterial({ 
  color: 0x4CAF50,
  metalness: 0.1,
  roughness: 0.8
});
const cube = new THREE.Mesh(geometry, material);
scene.add(cube);

// Add lighting
const ambientLight = new THREE.AmbientLight(0x404040, 0.4);
scene.add(ambientLight);

const directionalLight = new THREE.DirectionalLight(0xffffff, 0.8);
directionalLight.position.set(1, 1, 1);
scene.add(directionalLight);

// Animate
function animate() {
  cube.rotation.x += 0.01;
  cube.rotation.y += 0.01;
  renderer.render(scene, camera);
  requestAnimationFrame(animate);
}

animate();
console.log('Rotating cube scene loaded');
`;
    } else if (query.toLowerCase().includes('particle')) {
      return `
// Particle system scene
console.log('Loading particle system scene');

// Clear existing scene
while (scene.children.length > 0) {
  scene.remove(scene.children[0]);
}

// Create particle system
const particleCount = 100;
const particles = new THREE.BufferGeometry();
const positions = new Float32Array(particleCount * 3);
const colors = new Float32Array(particleCount * 3);

for (let i = 0; i < particleCount; i++) {
  positions[i * 3] = (Math.random() - 0.5) * 10;
  positions[i * 3 + 1] = (Math.random() - 0.5) * 10;
  positions[i * 3 + 2] = (Math.random() - 0.5) * 10;
  
  const color = new THREE.Color();
  color.setHSL(Math.random(), 0.7, 0.6);
  colors[i * 3] = color.r;
  colors[i * 3 + 1] = color.g;
  colors[i * 3 + 2] = color.b;
}

particles.setAttribute('position', new THREE.BufferAttribute(positions, 3));
particles.setAttribute('color', new THREE.BufferAttribute(colors, 3));

const particleMaterial = new THREE.PointsMaterial({
  size: 0.1,
  vertexColors: true,
  transparent: true,
  opacity: 0.8,
});

const particleSystem = new THREE.Points(particles, particleMaterial);
scene.add(particleSystem);

// Add lighting
const ambientLight = new THREE.AmbientLight(0x000020, 0.2);
scene.add(ambientLight);

// Animate
let time = 0;
function animate() {
  time += 0.01;
  particleSystem.rotation.y = time * 0.1;
  renderer.render(scene, camera);
  requestAnimationFrame(animate);
}

animate();
console.log('Particle system scene loaded');
`;
    } else {
      // Default scene
      return `
// Default scene
console.log('Loading default scene');

// Clear existing scene
while (scene.children.length > 0) {
  scene.remove(scene.children[0]);
}

// Create a simple rotating cube
const geometry = new THREE.BoxGeometry(1, 1, 1);
const material = new THREE.MeshStandardMaterial({ color: 0x4CAF50 });
const cube = new THREE.Mesh(geometry, material);
scene.add(cube);

// Add lighting
const ambientLight = new THREE.AmbientLight(0x404040, 0.4);
scene.add(ambientLight);

const directionalLight = new THREE.DirectionalLight(0xffffff, 0.8);
directionalLight.position.set(1, 1, 1);
scene.add(directionalLight);

// Animate
function animate() {
  cube.rotation.x += 0.01;
  cube.rotation.y += 0.01;
  renderer.render(scene, camera);
  requestAnimationFrame(animate);
}

animate();
console.log('Default scene loaded');
`;
    }
  };

  const handleGenerateScene = async () => {
    if (!rendererRef.current) return;

    setIsLoading(true);
    setError(null);

    try {
      // Call your AI API (replace with actual AI service)
      const code = await callMockAI(query);
      
      // Pass the generated code to the renderer
      await rendererRef.current.renderScene(code);
      
    } catch (error) {
      console.error('Failed to generate scene:', error);
      setError(error instanceof Error ? error.message : 'Failed to generate scene');
    } finally {
      setIsLoading(false);
    }
  };

  return (
    <div style={{ padding: '20px', fontFamily: 'Arial, sans-serif' }}>
      <h1>AI 3D Renderer - Interactive Example</h1>
      
      <div style={{ marginBottom: '20px' }}>
        <label htmlFor="query-input" style={{ display: 'block', marginBottom: '8px' }}>
          Describe your 3D scene:
        </label>
        <div style={{ display: 'flex', gap: '10px' }}>
          <input
            id="query-input"
            type="text"
            value={query}
            onChange={(e) => setQuery(e.target.value)}
            style={{
              flex: 1,
              padding: '12px',
              fontSize: '16px',
              border: '1px solid #ccc',
              borderRadius: '4px',
            }}
            placeholder="e.g., Create a solar system with planets orbiting the sun"
            onKeyPress={(e) => e.key === 'Enter' && handleGenerateScene()}
          />
          <button
            onClick={handleGenerateScene}
            disabled={isLoading}
            style={{
              padding: '12px 24px',
              fontSize: '16px',
              backgroundColor: isLoading ? '#ccc' : '#4CAF50',
              color: 'white',
              border: 'none',
              borderRadius: '4px',
              cursor: isLoading ? 'not-allowed' : 'pointer',
            }}
          >
            {isLoading ? 'Generating...' : 'Generate Scene'}
          </button>
        </div>
      </div>

      {error && (
        <div style={{
          padding: '12px',
          backgroundColor: '#ffebee',
          color: '#c62828',
          borderRadius: '4px',
          marginBottom: '20px',
        }}>
          <strong>Error:</strong> {error}
        </div>
      )}

      <div style={{
        border: '1px solid #ddd',
        borderRadius: '8px',
        overflow: 'hidden',
        backgroundColor: '#f5f5f5',
      }}>
        <AI3DRenderer
          ref={rendererRef}
          width={800}
          height={600}
          fallbackText="Generating your 3D visualization..."
          onError={(error) => setError(error.message)}
          onLoad={() => {
            setError(null);
            setIsLoading(false);
          }}
          onInteraction={(type, data) => {
            console.log('Interaction:', type, data);
          }}
          style={{
            width: '100%',
            height: '600px',
          }}
        />
      </div>

      <div style={{ marginTop: '20px', fontSize: '14px', color: '#666' }}>
        <h3>Example Queries:</h3>
        <ul>
          <li>"Create a rotating cube" - Shows a spinning green cube</li>
          <li>"Generate a particle system" - Shows colorful floating particles</li>
          <li>"Any other query" - Shows a default rotating cube</li>
        </ul>
        
        <div style={{ marginTop: '20px', padding: '15px', backgroundColor: '#e3f2fd', borderRadius: '4px' }}>
          <h4>💡 Note:</h4>
          <p>This example uses mock AI responses. In a real application, you would:</p>
          <ol>
            <li>Replace <code>callMockAI()</code> with your actual AI service (Azure AI, OpenAI, etc.)</li>
            <li>Handle retries and error cases</li>
            <li>Add proper loading states and user feedback</li>
          </ol>
          <p>See <code>CLIENT_USAGE.md</code> for complete integration examples.</p>
        </div>
      </div>
    </div>
  );
}

📚 API Reference

AI3DRenderer Props

| Prop | Type | Default | Description | |------|------|---------|-------------| | query | string | - | Required. Natural language description of the 3D scene to generate | | width | number | 800 | Canvas width in pixels | | height | number | 600 | Canvas height in pixels | | fallbackText | string | "Generating visualization..." | Loading message shown while generating scene | | onError | (error: Error) => void | - | Callback fired when rendering fails | | onLoad | () => void | - | Callback fired when scene loads successfully | | onInteraction | (type: string, data: any) => void | - | Callback fired on user interactions | | className | string | - | CSS class name for the container | | style | React.CSSProperties | - | Inline styles for the container |

useRenderer Hook

For more advanced usage, you can use the useRenderer hook directly:

import { useRenderer } from '@nestamai/ai-3d-renderer';

function CustomRenderer() {
  const {
    canvasRef,
    isLoading,
    error,
    isInitialized,
    fallbackActive,
    handleCanvasClick,
    renderScene,
  } = useRenderer({
    query: "Show me a solar system animation",
    width: 1000,
    height: 800,
  });

  return (
    <div>
      {isLoading && <div>Loading...</div>}
      {error && <div>Error: {error.message}</div>}
      {fallbackActive && <div>Using fallback scene</div>}
      <canvas
        ref={canvasRef}
        onClick={handleCanvasClick}
        width={1000}
        height={800}
      />
    </div>
  );
}

🔧 Configuration

AI Backend Integration

The library expects an AI backend that can generate scene configurations. You'll need to implement the AI API integration:

// Example AI API integration
async function generateSceneConfig(query: string): Promise<SceneConfig> {
  const response = await fetch('/api/ai/generate-scene', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ query }),
  });
  
  return response.json();
}

Scene Configuration Schema

The AI backend should return a scene configuration matching this schema:

interface SceneConfig {
  metadata: {
    title: string;
    description: string;
    version: string;
    author?: string;
  };
  settings: {
    backgroundColor: string;
    camera: {
      position: [number, number, number];
      target: [number, number, number];
      fov: number;
      near: number;
      far: number;
    };
    lighting: {
      ambient: {
        color: string;
        intensity: number;
      };
      directional?: {
        color: string;
        intensity: number;
        position: [number, number, number];
      };
    };
  };
  objects: SceneObject[];
  animations?: AnimationDefinition[];
  interactions?: InteractionDefinition[];
}

🛡️ Security

The library implements multiple layers of security:

1. Web Worker Sandboxing

• All rendering code runs in a Web Worker
• No access to DOM, window, or network APIs
• Complete isolation from the main thread

2. AST-Based Code Validation

• Static analysis of generated code
• Deny-list of dangerous JavaScript constructs
• Allow-list of safe Three.js APIs

3. Safe API Wrapper

• Limited subset of Three.js functionality
• No direct access to dangerous methods
• Controlled execution environment

Blocked Operations

• eval, Function, setTimeout, setInterval
• window, document, localStorage, sessionStorage
• fetch, XMLHttpRequest, WebSocket
• import, require, module, exports
• Direct console access (uses safe logger)

🎨 Fallback System

When AI generation fails, the library automatically falls back to safe, pre-built scenes:

• Default: Simple rotating cube
• Physics: Falling objects simulation
• Geometric: Animated geometric patterns
• Particle: Particle system visualization

🧪 Testing

# Run tests
npm test

# Run tests with coverage
npm run test:coverage

# Run tests in UI mode
npm run test:ui

🏗️ Development

# Install dependencies
npm install

# Start development server
npm run dev

# Build for production
npm run build

# Lint code
npm run lint

# Type checking
npm run type-check

📊 Performance

• Target FPS: 60fps
• Memory Usage: < 100MB
• Load Time: < 2s
• Worker Startup: < 500ms

🤝 Contributing

1. Fork the repository
2. Create a feature branch
3. Make your changes
4. Add tests
5. Submit a pull request

📄 License

MIT License - see LICENSE file for details.

🆘 Support

• 📧 Email: [email protected]
• 🐛 Issues: GitHub Issues
• 📖 Docs: Documentation

🗺️ Roadmap

• [ ] Physics engine integration (Cannon-es)
• [ ] Scene serialization and persistence
• [ ] AR/VR support (WebXR)
• [ ] Real-time collaboration
• [ ] Advanced animation system
• [ ] Performance profiling tools

Built with ❤️ by the NestamAI team for educational applications.