Union Stack

Handling Uploads via Script

const client = UnionStack.init({
  apiKey: process.env.UNION_STACK_API_KEY,
  fetch: (url, options = {}) => {
    return fetch(url, {
      ...options,
      headers: {
        ...(options.headers || {}),
        Origin: 'http://localhost:3000', // URL allowed in your Union Stack CORS policy
      },
    });
  }
});

const MIME_TO_EXT = {
  'application/pdf': 'pdf',
  'image/jpeg': 'jpg',
  'image/png': 'png',
  'image/avif': 'avif',
  'image/webp': 'webp',
  'video/mp4': 'mp4',
  'text/csv': 'csv',
  'application/xml': 'xml',
  'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'pptx',
  'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'xlsx',
  'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'docx',

  // ... expand this based on your analysis first based on the files present in the database
};


function resolveExtension(mime, rowId) {
  if (MIME_TO_EXT[mime]) return MIME_TO_EXT[mime];

  console.error(`unknown mime: ${mime} for row ${rowId}`);

  fs.appendFileSync('./unknown-mimes.log', JSON.stringify({ rowId, mime }) + '\n')

  throw new Error(`couldn't resolve file type`);
}

async function withRetry(fn, retries = 3) {
  let lastErr;
  for(let i = 0; i < retries; i++) {
    try {
      return await fn();
    } catch(error) {
      lastErr = error;
    }
  }
  throw lastErr;
}

async function getFinalUrl(row) {
  try {
    const response = await withRetry(() => fetch(row.url, {
      method: 'GET'
    }));

    if(!response.ok) {
      throw new Error("Failed to fetch the resource");
    }

    const mime = response.headers
      .get('content-type')
      ?.split(';')[0]
      ?.trim() || 'application/octet-stream';

    const ext = resolveExtension(mime, row.id);

    // if (contentEncoding === 'br') {
    //   // decompression logic, before enabling this block, check for a single filetype to figure out if brotliCompression was ever done in the first place on files, on FileStack from my experience, there isn't any brotliCompression even if the tag is br.
    //   finalStrema = nodeStream.pipe(createBrotliDecompress());
    // }
    // const filepath = `./temp/file_${row.id}.${ext}`;

    // await pipeline(
    //   nodeStream,
    //   fs.createWriteStream(filepath)
    // );

    // const buffer = fs.readFileSync(filepath);

    const contentLength = response.headers.get('content-length');
    
    const buffer = Buffer.from(await response.arrayBuffer());

    if(contentLength && buffer.length !== Number(contentLength)) {
      throw new Error(`size mismatch: exp=> ${contentLength}, got=> ${buffer.length}`);
    }

    const blob = new Blob([buffer], {
      fileName: `file_${row.id}.${ext}`,
      type: mime
    });

    console.log(blob);
    const uploadResponse = await withRetry(() => client.upload(blob, {
      filename: `file_${row.id}.${ext}`,
      mimeType: mime
    }));

    if(uploadResponse.status !== 'Stored') {
      throw new Error("Failed to store uploaded file to UnionStack");
    }

    return uploadResponse.url;
  } catch (error) {
    throw new Error(error.message);
  }
}