@vocoweb/portability

Production-ready GDPR Article 20 data portability with schema scanning and OAuth2 direct transfer

Features

• Data Discovery Engine: Automatically scans Postgres schemas via information_schema
• Anonymization Layer: Protects shared resources when exporting user data
• OAuth2 Direct Transfer: Seamless data transfer between services
• Multiple Export Formats: Support for CSV, JSON, and XML
• 30-Day Compliance Timer: Automatic expiration of export links
• RLS Policy Awareness: Respects Row Level Security policies

Installation

npm install @vocoweb/portability
# or
yarn add @vocoweb/portability
# or
pnpm add @vocoweb/portability

Quick Start

1. Export User Data

import { exportUserData } from '@vocoweb/portability/server';

// Export as JSON
const jsonData = await exportUserData('user-123', {
  format: 'json',
  anonymizeShared: true,
  includeSchema: true,
  includeRLS: true
});

// Export as CSV
const csvData = await exportUserData('user-123', {
  format: 'csv',
  tables: ['users', 'orders', 'subscriptions']
});

// Export as XML
const xmlData = await exportUserData('user-123', {
  format: 'xml'
});

2. Discover User Data Schema

import { discoverUserData } from '@vocoweb/portability/server';

const discovery = await discoverUserData(db, 'user-123');

console.log(discovery);
// {
//   tables: ['users', 'profiles', 'orders', 'subscriptions'],
//   userTables: ['users', 'profiles'],
//   sharedTables: ['orders', 'subscriptions'],
//   piiFields: ['email', 'name', 'phone', 'address'],
//   estimatedRows: 150
// }

3. Request Data Export (Background Job)

import { requestDataExport } from '@vocoweb/portability/server';

// Request export (creates background job)
const exportId = await requestDataExport('user-123', {
  format: 'json',
  anonymizeShared: true,
  expiresAt: new Date(Date.now() + 30 * 24 * 60 * 60 * 1000) // 30 days
});

// Returns export ID for tracking
console.log(exportId);

4. Process Pending Exports

import { processPendingExports } from '@vocoweb/portability/server';

// Process pending exports (call from cron job)
const processed = await processPendingExports(10);
console.log(`Processed ${processed} exports`);

5. OAuth2 Direct Transfer

import { transferDataDirectly } from '@vocoweb/portability/server';

// Transfer data directly to another service
await transferDataDirectly('user-123', {
  targetUrl: 'https://api.example.com/import',
  accessToken: 'oauth-token-123',
  format: 'json',
  verifyTransfer: true
});

API Reference

Server Functions

import {
  exportUserData,
  requestDataExport,
  processPendingExports,
  discoverUserData,
  transferDataDirectly,
  getExportStatus,
  cancelExport,
  downloadExport
} from '@vocoweb/portability/server';

// Export user data
await exportUserData(
  userId: string,
  options: {
    format?: 'json' | 'csv' | 'xml';
    anonymizeShared?: boolean;
    includeSchema?: boolean;
    includeRLS?: boolean;
    tables?: string[];
  }
): Promise<string | Buffer>;

// Request data export (background job)
await requestDataExport(
  userId: string,
  options?: {
    format?: 'json' | 'csv' | 'xml';
    anonymizeShared?: boolean;
    expiresAt?: Date;
  }
): Promise<string>; // Returns export ID

// Process pending exports
await processPendingExports(limit?: number): Promise<number>; // Returns count

// Discover user data schema
await discoverUserData(
  db: Database,
  userId: string
): Promise<DataDiscovery>;

// Transfer data directly via OAuth2
await transferDataDirectly(
  userId: string,
  config: {
    targetUrl: string;
    accessToken: string;
    format?: 'json' | 'csv' | 'xml';
    verifyTransfer?: boolean;
  }
): Promise<TransferResult>;

// Get export status
await getExportStatus(exportId: string): Promise<ExportStatus>;

// Cancel export
await cancelExport(exportId: string): Promise<boolean>;

// Download export
await downloadExport(exportId: string): Promise<Buffer>;

Data Discovery

import { discoverUserData, scanDatabase } from '@vocoweb/portability/server';

// Discover all user data
const discovery = await discoverUserData(db, 'user-123');

// Returns:
// {
//   tables: string[];           // All tables with user data
//   userTables: string[];       // Tables owned by user
//   sharedTables: string[];     // Tables with shared data
//   piiFields: string[];        // PII fields found
//   estimatedRows: number;      // Estimated rows to export
//   relationships: {            // Table relationships
//     foreignKeys: ForeignKey[];
//     dependencies: string[];
//   };
// }

// Scan database schema
const schema = await scanDatabase(db);
// Returns: { tables, columns, foreignKeys, indexes }

Anonymization

import { exportUserData, anonymizeData } from '@vocoweb/portability/server';

// Export with anonymization of shared resources
const data = await exportUserData('user-123', {
  format: 'json',
  anonymizeShared: true
});

// Shared data will be anonymized:
// {
//   users: [{ id: 'user-123', email: '[email protected]' }],
//   orders: [
//     { id: 'order-1', userId: 'user-123', ... },     // User's data
//     { id: 'order-2', userId: 'user-456', ... }      // Anonymized
//   ]
// }

// Anonymize specific data
const anonymized = await anonymizeData(data, {
  fields: ['email', 'name', 'phone'],
  strategy: 'hash'
});

OAuth2 Direct Transfer

import {
  transferDataDirectly,
  verifyTransfer,
  getTransferHistory
} from '@vocoweb/portability/server';

// Transfer data to another service
const result = await transferDataDirectly('user-123', {
  targetUrl: 'https://api.example.com/data-import',
  accessToken: 'Bearer eyJhbGc...',
  format: 'json',
  verifyTransfer: true,
  retryCount: 3,
  timeout: 30000
});

// Returns:
// {
//   success: true,
//   transferId: 'transfer-123',
//   bytesTransferred: 1024000,
//   recordsTransferred: 150
// }

// Verify transfer completed
const verified = await verifyTransfer('transfer-123');
// Returns: { verified: true, confirmedAt: '2024-01-15T10:30:00Z' }

// Get transfer history
const history = await getTransferHistory('user-123');

Export Formats

// JSON Format
await exportUserData('user-123', { format: 'json' });
// Returns structured JSON with all tables and relationships

// CSV Format
await exportUserData('user-123', { format: 'csv' });
// Returns CSV files (one per table) as ZIP archive

// XML Format
await exportUserData('user-123', { format: 'xml' });
// Returns XML compliant with data portability standards

Database Schema

-- Data exports
CREATE TABLE data_exports (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    user_id UUID NOT NULL,
    status TEXT NOT NULL DEFAULT 'pending',
    requested_at TIMESTAMPTZ DEFAULT NOW(),
    completed_at TIMESTAMPTZ,
    expires_at TIMESTAMPTZ NOT NULL,
    data JSONB,
    options JSONB,
    download_count INTEGER DEFAULT 0,
    last_downloaded_at TIMESTAMPTZ,
    error TEXT,
    export_format TEXT NOT NULL,
    file_size BIGINT,
    checksum TEXT
);

-- Data transfers
CREATE TABLE data_transfers (
    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
    user_id UUID NOT NULL,
    target_url TEXT NOT NULL,
    status TEXT NOT NULL DEFAULT 'pending',
    requested_at TIMESTAMPTZ DEFAULT NOW(),
    completed_at TIMESTAMPTZ,
    verified_at TIMESTAMPTZ,
    transfer_format TEXT NOT NULL,
    bytes_transferred BIGINT DEFAULT 0,
    records_transferred INTEGER DEFAULT 0,
    error TEXT,
    retry_count INTEGER DEFAULT 0,
    access_token_hash TEXT
);

-- Indexes for efficient queries
CREATE INDEX idx_data_exports_user ON data_exports(user_id);
CREATE INDEX idx_data_exports_status ON data_exports(status);
CREATE INDEX idx_data_exports_expires_at ON data_exports(expires_at);
CREATE INDEX idx_data_transfers_user ON data_transfers(user_id);
CREATE INDEX idx_data_transfers_status ON data_transfers(status);

Configuration

import { configurePortability } from '@vocoweb/portability/server';

configurePortability({
  // Export settings
  export: {
    defaultFormat: 'json',
    maxFileSize: 100 * 1024 * 1024, // 100MB
    compressionEnabled: true,
    includeMetadata: true
  },

  // Anonymization
  anonymization: {
    enabled: true,
    strategy: 'hash',
    fields: ['email', 'name', 'phone', 'address']
  },

  // Transfer settings
  transfer: {
    timeout: 30000,
    maxRetries: 3,
    retryDelay: 1000,
    verifyTransfers: true
  },

  // Retention
  retention: {
    exportExpiresDays: 30,
    transferHistoryDays: 90
  },

  // Storage
  storage: {
    provider: 's3',
    bucket: 'data-exports',
    region: 'eu-central-1'
  }
});

Best Practices

1. Use Background Jobs: Always use requestDataExport for large datasets
2. Anonymize Shared Data: Protect other users' privacy in shared tables
3. Set Expiration Dates: Comply with 30-day limit for export availability
4. Verify Transfers: Confirm successful data transfer for OAuth2 operations
5. Monitor Process Queue: Regularly process pending exports from cron jobs

Legal Compliance

This package helps comply with:

• GDPR Article 20: Right to data portability
• CCPA: Right to access and transfer personal information
• UK GDPR: Data portability requirements

License

MIT

Made with ❤️ by VocoWeb