Trooptravel Google Cloud utilities

A few utilities to standardise certain code blocks within the team. utilities include:

• Google Cloud Trace Instancing
• Google Stackdriver Logger enhanced by Bunyan and Google Error Reporting
• Firebase instancing and general CRUD Commands
• MySQL instancing and general CRUD Commands
• Centralised Error Handling in Express (via middleware with manual override: i.e. in routes)

Installing

Using npm:

# install
$ npm install tt-cloud-utilities

#update
$ npm i tt-cloud-utilities@latest

Examples

Google Cloud Tracing

In order to gain the TypeScript typings (for intellisense / autocomplete) while using CommonJS imports with require() use the following approach:

const { Tracer } = require('tt-cloud-utilities');

let tracerInstance = new Tracer({
    projectId: 'YOUR-GOOGLE-PROJECT-ID',
    keyPath: 'PATH/TO/SERVICE-ACCOUNT.JSON',
    plugins: {
        http: true, // boolean: to include in constructor of NodeTracerProvider
        https: true,
        express: false,
        mysql: false
    }
});

let tracer = tracerInstance.createTracer();
let api = tracerInstance.getApi();

Propagate trace between services

This will forward 'traceparent' in header to subsequent services/projects

const axios = require("axios");

function clientDemoRequest() {
    console.log("Starting client demo request");

    const span = tracer.startSpan("clientDemoRequest()", {
        parent: tracer.getCurrentSpan(),
        kind: api.SpanKind.SERVER
    });

    tracer.withSpan(span, async () => {
        span.setAttribute('prop', 'value');
        span.setAttributes({
            prop1: 'value',
            prop2: 'value'
        });
        span.addEvent('sending request');
        await axios.get("request/url")
        .then(results => {
            
        })
        span.setStatus({ code: api.CanonicalCode.OK });

        span.end();

        // The process must remain alive for the duration of the exporter flush
        // timeout or spans might be dropped
        console.log("Client request complete, waiting to ensure spans flushed...");
        setTimeout(() => {
            console.log("Done 🎉");
        }, 2000);
    });
}

clientDemoRequest();

Resources:

• Open Telemetry

Google Cloud Logger

In order to gain the TypeScript typings (for intellisense / autocomplete) while using CommonJS imports with require() use the following approach:

const { Logger } = require('tt-cloud-utilities');
 
let loggerInstance = new Logger({
    projectId: 'YOUR-GOOGLE-PROJECT-ID',
    keyPath: 'PATH/TO/SERVICE-ACCOUNT.JSON',
    serviceName: 'default',
    projectVersion: '1.0.8'
});

const constructorOptions = {
    level: "info",
    // add more option according to API reference
}
 
let logger = loggerInstance.createLogger(constructorOptions); // look at https://www.npmjs.com/package/bunyan#levels for log levels
let reporter = loggerInstance.createReporter();
 
logger.error(error); // submits error to Stackdriver
reporter.report(error); // reports error to Error Reporting
logger.error(new Error(error)); // submits error to both Stackdriver and Error Reporting

let labelObject = {
    key: key,
    clientId: clientId,
    tripId: tripId,
    originId: originId,
    destinationId: destinationId
};

// If you wish to view log entries inline with trace spans in the Stackdriver Trace Viewer. This means log entry shows in Cloud Logging, Error Reporting and Cloud Trace
logger.error(await loggerInstance.getLoggerKey(tracer.getCurrentSpan().spanContext.traceId, labelObject), new Error("Error now logged inline with trace span"));  

MySQL Client for Google Cloud SQL

In order to gain the TypeScript typings (for intellisense / autocomplete) while using CommonJS imports with require() use the following approach:

const { MySql } = require('tt-cloud-utilities');

// All connection configurations from the mysql repo is available
const connection = new MySql({
    host: 'localhost',
    user: 'root',
    database: 'accomodation',
    password: '',
    connectionLimit: 5
})

// Example 1: Promised based execution
const mysql = connection.connect(true, true)
async function executeQuery(){
    // simple query
    await mysql.query('SELECT * FROM `hp_accomodation` LIMIT 1')
        .then( ([results, fields]) => {
            console.log(results); 
        })
        .catch( error => {
            
        })

    console.log('finished')
}
executeQuery()

// Example 2: Promised based await queries
const mysql = connection.connect(true, true)
async function executeWithAwait(){
    let results = await mysql.query('SELECT * FROM `hp_accomodation` LIMIT 1');
    console.log(results)
    console.log('finished') 
}
executeWithAwait()

// Example 3: Promised based Pool Connections
const mysql = connection.pool(true)
console.log('created pool')
async function executeQueryInPool(){
    await asyncForEach([1, 2], async (num) => {
        let results = await mysql.query(`SELECT * FROM hp_accomodation LIMIT ?`,
            [num]
        );
        console.log(results)
    })
    console.log('finished')  
    mysql.end() 
}
executeQueryInPool()

async function asyncForEach(array, callback) {
    for (let index = 0; index < array.length; index++) {
        await callback(array[index], index, array);
    }
}

// Example 4: Async queries
const mysql = connection.connect(false, true)
async function executeWithAwait(){
    mysql.query('SELECT * FROM `hp_accomodation`');
    console.log('query was deployed and connection will automatically end() when finished')  
}
executeWithAwait()

// Example 5: Pool based async queries in loop
const mysql = connection.pool(false)
console.log('created pool')
async function executeQueryInPool(){
    await asyncForEach([1, 2], async (num) => {
        mysql.query(`SELECT * FROM hp_accomodation LIMIT ?`,
            [num]
        );
        console.log('query was deployed')  
    })
    console.log('pool connection will close when becoming stale')  
}
executeQueryInPool()

async function asyncForEach(array, callback) {
    for (let index = 0; index < array.length; index++) {
        await callback(array[index], index, array);
    }
}

// With the above example, you can query the state of your deployed queries and close the pool on finalisation in order to preserve concurrency
// Remember, calling connection.pool(false) which equals createPool() from the repo, will establish a connection in itself
setTimeout(function(){ 
    console.log(`All Connections ${mysql._allConnections.length}`);
    console.log(`Acquiring Connections ${mysql._acquiringConnections.length}`);
    console.log(`Free Connections ${mysql._freeConnections.length}`);
    console.log(`Queue Connections ${mysql._connectionQueue.length}`);
}, 3000);

Centralised Error Handling

In order to gain the TypeScript typings (for intellisense / autocomplete) while using CommonJS imports with require() use the following approach:

When used as middleware:

const { ErrorMiddleware } = require('tt-cloud-utilities')
const apiName = require('./package.json').name;
const projectVersion = require('./package.json').version;
let errorMiddleware = new ErrorMiddleware(apiName, projectVersion);

// strictly to be used last in line before app.listin()
app.use((err, req, res, next) => {
    errorMiddleware.errorResponse(err, req, res, next)
});

app.listen(process.env.PORT || 8080);

// -- automatically catch error in route and format it when passed to express error handler
const { catchAsync } = require('tt-cloud-utilities')
app.use('/places', catchAsync(async (req, res, next) => {
    new Error('Oeps, there is an error on this line...')
});

When used manually:

const { AppError } = require('tt-cloud-utilities')
app.all('*', (req, res, next) => {
    next(
        new AppError(
            `Can't find ${req.originalUrl} on this server!`, // Error Message
            404 // specified error code, else will default to 500
        )
    );
});

Google Cloud Tasks

In order to gain the TypeScript typings (for intellisense / autocomplete) while using CommonJS imports with require() use the following approach:

Setup TasksClient and send with a one-liner:

const { TasksClient } = require('tt-cloud-utilities');
 
// CLOOUDTASK CLIENT
const cloudTasksClient = new TasksClient({
    context: tracer, // if needed
    loggerInstance: loggerInstance // if needed
    projectId: 'YOUR-GOOGLE-PROJECT-ID',
    keyPath: 'PATH/TO/SERVICE-ACCOUNT.JSON',
});

// content of the task
const payload = {
    prop: 'test-from-repo'
}

const time_in_seconds_from_now = (Math.round(new Date() / 1000)) + 300 // 5 minutes from now

cloudTasksClient.sendTask({
    method: 'POST', 
    url: `https://url.com`, 
    body: payload, 
    queue: 'my-queue', 
    location: 'europe-west1'
    spanName: 'custom span name', // specify if you want to execute underneath its own span
    headers: {
        key: 'value'
    },
    scheduleTime: time_in_seconds_from_now // specify if you want to schedule the task in the future (must be in seconds)
})

TT Workflow Manager

In order to gain the TypeScript typings (for intellisense / autocomplete) while using CommonJS imports with require() use the following approach:

Setup TasksClient and send with a one-liner:

const { TasksClient, Workflow } = require('tt-cloud-utilities');
 
let tracer = new Tracer({
    projectId: 'YOUR-GOOGLE-PROJECT-ID',
    keyPath: 'PATH/TO/SERVICE-ACCOUNT.JSON',
    plugins: {
        http: true, // boolean: to include in constructor of NodeTracerProvider
        https: true,
        express: false
    }
}).createTracer()

var express = require("express");
var app = express();

app.post("/", async function(req, res, next) {

    const workflow = new Workflow({
        context: tracer, // tracer will be passed on to CloudTask construction (if needed)
        loggerInstance: loggerInstance // if needed
        projectId: 'YOUR-GOOGLE-PROJECT-ID',
        keyPath: 'PATH/TO/SERVICE-ACCOUNT.JSON',
    })
    
    // content of the task
    const payload = {
        prop: 'test-from-repo'
    }

    const time_in_seconds_from_now = (Math.round(new Date() / 1000)) + 300 // 5 minutes from now
    
    workflow.kickChampion([
        {
    
            service: 'http://example.com',
            spanName: 'GET call 1', // specify if you want to execute underneath its own span
            operation: {
                method: 'GET', 
                body: payload, // in case of POST
                queue: 'tt-hotel-api', 
                location: 'europe-west1',
                scheduleTime: time_in_seconds_from_now // specify if you want to schedule the task in the future (must be in seconds)
            }
        },
        {
            service: 'http://example.com',  
            spanName: 'GET call 2', // specify if you want to execute underneath its own span
            operation: {
                method: 'GET', 
                body: payload, // in case of POST
                queue: 'tt-hotel-api', 
                location: 'europe-west1'
            }
        }
    ])

    res.send("done");

    setInterval(function(){ 
        console.log(workflow.getWorkflowQueue())
    }, 5000);
});

var listener = app.listen(8080, function() {
  console.log("Listening on port " + listener.address().port);
});

TT PubSub Client

In order to gain the TypeScript typings (for intellisense / autocomplete) while using CommonJS imports with require() use the following approach:

Setup PubSubClient:

const { PubSubClient } = require('tt-cloud-utilities');
 
let pubsub = new Tracer({
    projectId: 'YOUR-GOOGLE-PROJECT-ID',
    keyPath: 'PATH/TO/SERVICE-ACCOUNT.JSON',
}).init()

pubsub 
    .topic(topicName).publish(dataBuffer)
    .then(messageId => {
        console.log(`Message ${messageId} published.`);
    })
    .catch(err => {
        console.error('ERROR:', err);
    });

Internal management

Provide your authentication credentials

• gcloud init && git config --global credential.https://source.developers.google.com.helper gcloud.sh

Init Git & commands in directory

• git init
• git add .
• git remote add origin remote repository URL
• git commit -m "Initial commit"
• npm version patch // to increment your package version
• git push origin master
• git remote -v // list repo destinations - can be used in '$ git remote add google'

Push to this GSC repo

• git remote add google https://source.developers.google.com/p/tt-hotel-api/r/tt-cloud-utilities // Add your Cloud Repository as a remote - if not created via GCP
• git push --all google // Push from your local Git repository
• npm publish // publish/update to npm
• npm unpublish // remove package from npm (there are some rules you need to adhere to...)

** Workflow for local git repo and how to push to remote **

// checkout branch from local master
git checkout -b new-feature master

// Edit some files
git add <file>
git commit -m "Start a feature"

// Edit some files
git add <file>
git commit -m "Finish a feature"

- Merge in the new-feature branch
git checkout master
git merge new-feature
git branch -d new-feature

- delete remote branch
git push origin --delete test

- push local master to remote master
npm version patch
git push --all goole // push to master mbranch and build the new release

- remote branch management
git checkout -b test // creates and checks out branch locally
git push origin --delete test // push local branch to remote