Iterate through multiple files or streams at the same time line by line without loading into memory.

Install

npm install iterate-multiple-files

Code Example

iterate

import imf from 'iterate-multiple-files';

let runningSum = 0;

//The function will receive an array of lines in the same order as the input file paths
function operation(lines){
  // The return statement will ensure that the value gets appended to the final
  // array returned either in the promise or callback. If you do not return anything
  // then nothing will be saved to the array - this is good if dealing with large 
  // files and running out of memory.

  // Do any operation in here
  const lineSum = parseInt(lines[0]) + parseInt(lines[1]);
  runningSum += lineSum;
  return lineSum
}

//Using a promise:
//You can pass in all files, all Readable streams or a combination of both in the same array
imf.iterate(['path-to-file1','path-to-file2', fs.createReadStream('path-to-file3')],operation)
  .then(function(result){
    //Do something with the result
    //Result is an array of each line's sum
  })
  .catch(function(err){
    //Handle the error
  });

Using a callback

//Using a callback:
//You can pass in all files, all Readable streams or a combination of both in the same array
imf.iterate(['path-to-file1','path-to-file2',fs.createReadStream('path-to-file3')],operation, function(err, result){
  if(err){
    //Handle the error
  }
  //Do something with result
});

Iterator functionality

Will come in a TBD version

Generator functionality

Will come in a TBD version

Motivation

Perform operations line by line on files too large to load in memory.

Contributors

If you are interested in contributing please contact [email protected]

License

MIT