Distributed Systems Framework

A framework for building intuitive distributed applications as if it were a regular application.

Creating a Distributed Application

Setting Up Your Project

>  npm init -y
>  npm install dsf-toolkit

Building Your Stack- Run the same application on different machines with different peer IDs but same topic(communication stack, your unique string)

1) Identity Service- Provides a unique identity to a peer

import Identity from "dsf-toolkit/servicesLayer/identity/identity.js";

const identity = new Identity({
    static:true, //same identity on restart, false- new identity each time,
    id:"A" //you can override id
}).getIdentity()

console.log(identity)

>  node app.js
{ id: 'A' }

2) Communication Stack- For communicating between peers via messages

import Communication from "dsf-toolkit/messagingLayer/communication/hyperswarm/hyperswarm.js";
import Queue from "dsf-toolkit/messagingLayer/queuing/bypassQueue/bypassQueue.js";
import Router from "dsf-toolkit/protocolLayer/routing/routing.js";

//defining custom topology
let topology = {
  "A": ["B"],
  "B": ["A", "C", "D"],
  "C": ["B", "E"],
  "D": ["B", "E"],
  "E": ["C", "D", "F"],
  "F": ["E"]
}

//communication adapter
let communication = new Communication({
    identity:identity,
    topic:"yourUniqueId",
    allowedNeighbours:new Set(topology[identity.id]) //optional
})

//queuing adapter
let queue = new Queue({
    adapter:communication
})

//routing adapter
let router = new Router({
    identity:identity,
    adapter:queue
})

3) Leader Election- electing k leaders

import Election from "dsf-toolkit/servicesLayer/election/election.js"

let election = new Election({
    identity:identity,
    adapter:router
})

//electing 3 leaders
setTimeout(async()=>{
  let elected = await election.elect(3);
  console.log("[APP] initiator sees result ",elected)
},30000) //we wait for the peers to connect

>  node app.js
initiator sees result  {
  electionId: '01KE4WTG8PNXNMNQSSH18E0T0Y',
  elected: [ 'F', 'E', 'B' ]
}

4) Load Balancer- Peer to peer load balancing

import LoadBalancer from "dsf-toolkit/servicesLayer/loadBalancer/loadBalancer.js";

let loadBalancer = new LoadBalancer({
    identity: identity,
    callback:async(request)=>{
        //simulating a server that takes 10 seconds to process a request
        return new Promise((res, rej)=>{
            setTimeout(()=>{
                res({message:"some response"})
            },10000)
        })
    },
    adapter:router
});

setInterval(async()=>{
        let response = await loadBalancer.send({
            message:"some request"
        })
        console.log("[APP] response",response)

        if(response.status == "FAILURE"){
            console.log("[APP] request was dropped")
        }else{
            console.log("[APP] response", response)
        }
        
    },50) 

5) RateLimiter- drops excess messages

import RateLimiter from "dsf-toolkit/servicesLayer/rateLimiter/rateLimiter.js";

let rateLimiter = new RateLimiter({
    adapter:loadBalancer,
    rate:15 //allow 15 requests per second
})

setInterval(async()=>{
        let response = await rateLimiter.send({
            message:"some request"
        })

        console.log("[APP] response",response)

        if(response){
            if(response.status == "FAILURE"){
                console.log("[APP] request was dropped")
            }else{
                console.log("[APP] response", response)
            }
        }else{
            console.log("[APP] request was rate limited and dropped")
        }
    },50) //an interval of 50 milliseconds results in 20 req/sec

6) Throttler- queues excess messages

import Throttler from "dsf-toolkit/servicesLayer/throttler/throttler.js"

let throttler = new Throttler({
    adapter:loadBalancer,
    rate:15 //allows 15 requests per second
})

setInterval(async()=>{
        let response = await throttler.send({
            message:"some request"
        })
        console.log("[APP] response",response)

        if(response.status == "FAILURE"){
            console.log("[APP] request was dropped")
        }else{
            console.log("[APP] response", response)
        }
    },50) //an interval of 50 milliseconds results in 20 req/sec

7) Local Storage- A key-value storage

import LocalStorage from "dsf-toolkit/servicesLayer/localStorage/localStorage.js";

const localStorage = new LocalStorage({
    identifier: "USERS" // define scope/context
});

localStorage.set("alice", {
    name: "Alice Wonderland",
    age: 28,
    email: "[email protected]",
    role: "admin",
    registered: true
});

console.log(localStorage.get("alice"));

Upcoming Services

1. local compute
2. distributed storage
3. distributed compute
4. cache
5. firewall
6. container orchestration