j-hash-node
v0.0.3
Published
File hashing algorithm using SHA256 and Merkle trees
Downloads
2
Readme
j-hash-node
Node.js bindings for the J-hash C library.
J-hash is a file hashing algorithm based on SHA256 and merkle trees. It has the unique property that proofs can be generated for any arbitrary substring of a file that prove the substring authentic without having to see the rest of the file.
Usage
1. Simple hashing
const { JHashCtx } = require('j-hash-node');
const ctx = new JHashCtx;
const buffer = Buffer.from('Hello, world!', 'utf8');
ctx.update(buffer);
const hash = ctx.final();
console.log(hash); // 'jh1024:1160344149:OsPiEahBd2VBEL6h6s6wOkRYrCRK0tUBM+YZFo5SaKA='
2. Hashing a file & writing all intermediate hashes to an output file
(async() => {
const f_in = await fs.open('file.mov', 'r');
const f_out = await fs.open('file.mov.jhash', 'w');
const buf_in = new Buffer.alloc(16384);
const buf_out = new Buffer.alloc(65536);
const THRESHOLD = buf_out.length - buf_in.length; // How full the output buffer becomes before we flush it
const ctx = new JHashCtx(buf_out);
while (true) {
const { bytesRead } = await f_in.read(buf_in, 0, buf_in.length);
if (bytesRead === 0) {
break;
}
ctx.update(buf_in, bytesRead);
if (ctx.outputBufferSize >= THRESHOLD) {
await f_out.write(buf_out, 0, ctx.outputBufferSize);
ctx.outputBufferFlush();
}
}
const hash = ctx.final();
console.log('Final hash', hash);
await f_out.write(buf_out, 0, jhash.outputBufferSize);
f_out.close();
f_in.close();
})();
The resulting .jhash
file will contain all intermediate hashes that constitute the file's merkle tree and is needed when generating J-proofs. The file will be 6.25% the size of the original file.
The output buffer you pass to the context should be sufficiently large to not overflow during a jhash.update()
. On incredibly large inputs (500GB+) a single jhash.update()
call can hypothetically produce 40 hashes in addition to one hash per 1024 bytes of input.
4. Generating a J-proof for a byte range of a file
const { JProofGenerateCtx } = require('j-hash-node');
async function getFileMetadata(filename) {
// ...
return {
filesize: // ...
hashFilename: // ...
};
}
async function readFileRanges(filename, ranges) {
// Imagine this function to take an array of index ranges, it makes a request for the file's content
// at the specified ranges and returns a Buffer for each range.
}
async function fileRequest(filename, rangeFrom, rangeLength) {
const { filesize, hashFilename } = await getFileMetadata(filename);
const ctx = new JProofGenerateCtx(filesize, rangeFrom, rangeFrom + rangeLength);
const request = ctx.getRequest();
/*
{
head: [ 455704576, 874 ], <-- The offset and length of the head
tail: [ 477129638, 90 ], <-- The offset and length of the tail
hashes: [
[ 16777152, 32 ], [ 25165728, 32 ], [ 27262848, 32 ], [ 28311392, 32 ],
[ 28442432, 32 ], [ 28475168, 32 ], [ 28479232, 32 ], [ 28481248, 32 ],
[ 29820288, 32 ], [ 29820576, 32 ], [ 29822688, 32 ], [ 29826784, 32 ], <-- The offset and length of the hashes we need
[ 29834976, 32 ], [ 29851360, 32 ], [ 29884128, 32 ], [ 30408512, 32 ],
[ 31457088, 32 ], [ 33554240, 32 ], [ 67108768, 32 ], [ 101003040, 32 ]
],
payloadLength: 1604 <-- Size of the byte payload of the proof
}
*/
// Request one range from the file
const rangeIncludingHeadAndTail = [ request.head[0], request.head[1] + rangeLength + request.tail[1] ];
const [byteRangeData] = await readFileRanges(filename, [rangeIncludingHeadAndTail]);
// Request all the hashes from the hash file
const hashes = await readFileRanges(hashFilename, request.hashes);
// Prepare our file data
const head = byteRangeData.slice(0, request.head[1]);
const content = byteRangeData.slice(request.head[1], request.head[1] + rangeLength);
const tail = byteRangeData.slice(request.head[1] + rangeLength);
const proofPayload = Buffer.concat([head, tail, ...hashes]);
const proof = ctx.generate(proofPayload);
return { content, proof }; // Return the requested content with its proof
}
5. Verifying a J-proof
Given a Buffer byteRangeData
, a string proof
, and the expected fileHash
, we can verify the byte range as follows:
const { JProofVerifyCtx } = require('j-hash-node');
function verifyRange(byteRangeData, proof, fileHash) {
const ctx = new JProofVerifyCtx(proof); // Pass in the "jp1024:..." proof string
ctx.update(byteRangeData); // Process all the data
const hash = ctx.final(); // Receive the final hash
if (hash && hash === fileHash) {
return true; // Byte range & proof produce correct hash
} else {
return true; // Failure :(
}
}
Byte range data can also be streamed in:
function verifyRange(byteStream, proof, fileHash, callback) {
const ctx = new JProofVerifyCtx(proof);
if (ctx.hasError()) {
callback(false); // proof parse error
}
stream.on('data', chunk => {
jproof.update(chunk);
if (jproof.hasError()) {
stream.close();
callback(false); // received more data than expected
}
});
stream.on('end', () => {
callback(jproof.final());
});
}