@nmg/k8s

NMG helpers for kubernetes workloads

Installation

$ npm install @nmg/k8s

Usage

Cron:

shouldRun(crontab, timezone)

Returns Boolean if crontab should run

Example

const { cron } = require('@nmg/k8s');
const crontab = '* * * * *';

cron.shouldRun(crontab) ? true : false;

Data Warehouse:

queryDataWarehouse(options, [headers])

Example

const { queryDataWarehouse } = require('@nmg/k8s');
queryDataWarehouse(
  {
    url,
    variables,
    query,
  }, 
  headers
);

Dependent Environment Variables

• process.env.NMG_K8S_DATA_WAREHOUSE_TOKEN

Email:

sendEmail(send_email_params, [credentials])

Example

const { sendEmail } = require('@nmg/k8s');
sendEmail(
  {
    to,
    subject,
    text,
    html,
    attachments,
  }, 
  { user, password }
); // auth vaules optional

Recommended Environment Variables

• process.env.NMG_K8S_EMAIL_USER
• process.env.NMG_K8S_EMAIL_PASSWORD

Encryption:

Encrypt

encrypt(value)

Example

const { encrypt } = require('@nmg/k8s');
encrypt(value);

• process.env.NMG_K8S_SECRET

Decypt

decrypt(encrypted_value)

Example

const { encrypt } = require('@nmg/k8s');
encrypt(value);

Dependent Environment Variables

• process.env.NMG_K8S_SECRET

Image:

cropImage(input_filepath, output_filepath, [options])

Example

const { cropImage } = require('@nmg/k8s');
cropImage(
  'path/to/original.png', 
  'path/to/new.png', 
  { color_bands: false }
);

IP:

getIp([debug])

debug: Boolean - if true, will log response status code

Will return an IPv4 address or null if there is an error

Example

const ip_helper = require('@nmg/k8s');
ip_helper.getIp().then((ip) => console.log(ip));

Log:

log(log_params)

Formats a log to show up in nmg-big-data.logging.logs BigQuery table

Example

const { log } = require('@nmg/k8s');
log(
  {
    level,
    error_message,
    message,
  }
);

MongoDB:

connect()

Sets up the connection to the mongo db.

getCollection(collection_name)

Gets the collection object for the name passed in.

create(collection_name, data, account) (recommended)

Creates a collection with data and account information.

insert(collection_name, data)

Inserts raw data into collection.

update(collection_name, data, account)

Updates inserted data.

read(collection_name, options, return_cursor)

Reads from collection based on filter and returns either cursor or data.

remove(collection_name, filter, account)

Removes data from collection based on filter.

disconnect()

Closes the connection to the mongo db.

Example

const MongoDbClient = require('@nmg/k8s');
const client = new MongoDBClient(
  {
    db_name,
    connection_url,
    user,
    password,
    options,
  }
);
const db = await client.connect();
await client.disconnect();

Recommended Environment Variables

• process.env.MONGODB_DB_NAME
• process.env.MONGODB_CONNECTION_URL
• process.env.MONGODB_USER
• process.env.MONGODB_PASSWORD

Mysql Helper:

setupDB()

Sets the DB connection info and returns it

query(_query)

runs a query and returns the results

getTableStructure(table)

describes the given table

structureToSchema(structure)

writeMysqlRowsToBigQueryJsonFile(table_config, where_mysql, output_filepath, mysql_structure)

Example

const MongoDbClient = require('@nmg/k8s');
setupDB(
  {
    host,
    user,
    password,
    database,
    port,
  }
);
const description = await getTableStructure('table_name');

Recommended Environment Variables

• process.env.MYSQL_HOST
• process.env.MYSQL_USER
• process.env.MYSQL_PASS
• process.env.MYSQL_NAME
• process.env.MYSQL_PORT

Netsuite:

netsuite.executeScript(options, [env_overrides])

Execute NetSuite RESTlets using URL

Example

const { netsuite } = require('@nmg/k8s');
const env_overrides = {}
const restlet_response = await netsuite.executeScript(
  {
    url: 'https://5000005.restlets.api.netsuite.com/app/site/hosting/restlet.nl?script=999&deploy=1',
    data: {
      saved_search_id: 'customsearch1',
    },
  },
  env_overrides
)

options will be passed to the request({ ...options }) function with the authentication defaults

Recommended Env Variables

• process.env.NETSUITE_REALM
• process.env.NETSUITE_CUSTOMER_KEY
• process.env.NETSUITE_CUSTOMER_SECRET
• process.env.NETSUITE_TOKEN_KEY
• process.env.NETSUITE_TOKEN_SECRET

Product Search (soon to be obsolete)

class ProductSearch({ gcs_keyfile })

Example

const { ProductSearch } = require('@nmg/k8s')
const product_search = new ProductSearch({
  keyfile: path.resolve(process.env.GOOGLE_APPLICATION_CREDENTIALS),
})
await product_search.setupCache();

const searches_results = await product_search.multiProductSearch({
  searches: [
    { pn_search: 'GSS25GMHES' },
    { pn_search: 'HBLP651RUC' },
    { pn_search: 'GSS25GGHWW' },
  ],
  num_possible_results: 1,
  cutoff: 0.9,
})

Environment Variables

• process.env.NMG_K8S_DATA_WAREHOUSE_TOKEN

Queue:

Queue(size)

Example

const { Queue } = require('@nmg/k8s');
const queue = new Queue(10);

for (const page of response.pageRanges) {
  const _page = page;
  console.log(`QUEUE - ${_page.index}`);
  queue.push(
    async () => {
      console.log(`RUN - ${_page.index}`);
      return executeScript({
        ...options,
        page_index: _page.index,
      }).then((page_results) => {
        console.log(`RETURNED - ${_page.index}`)
        results.push(...page_results)
      });
    }
  );
}
queue.start()
await new Promise((resolve, reject) => {
  const resolveIfEmpty = () => {
    if (queue.isEmpty()) {
      resolve();
    }
    else {
      setTimeout(resolveIfEmpty, 100)
    }
  }
  resolveIfEmpty();
})
queue.stop();
}

Rate Limiting:

RateLimit(options)

Example

const { RateLimit } = require('@nmg/k8s');

const rate_limiter = new RateLimiter({
  shouldRequeueOnError(error) {
    if (error.message === 'THIS NEEDS EXPONENTIAL BACKOFF') {
      rate_limiter.exponentiallyBackoff()
    }
    return /(Quota|exponential)/ig.test(error.message)
  }
});
rate_limiter.on('request_processed', () => {
  rate_limiter.resetExponentialBackoff(); //always reset
  console.log(`REMAINING REQUESTS - ${rate_limiter.requests.length}`)
});
rate_limiter.queueRequest(
  async () => {
    return Promise.resolve();
  }
);
rate_limiter.startProcessing();
await new Promise((resolve) => {
  rate_limiter.once('queue_empty', resolve));
}

options

limits = [
  // limit to 3 requests for 1000ms
  {
    type: 'RATE_LIMIT',
    requests: 3,
    requests_timespan: 1000, // milliseconds
  },
  // AND limit to 1000 requests for 100000ms
  {
    type: 'RATE_LIMIT',
    requests: 1000,
    requests_timespan: 100000, // milliseconds
  },
  // AND limit to 5 concurrent requests
  {
    type: 'CONCURRENCY',
    requests: 5,
  },
],
requests = [], // intialize with requests
log = false,
shouldRequeueOnError = null, // return true to retry

constants

returns a list of constants used in NMG projects

Example

const { nmg_constants } = require('@nmg/k8s');

//gets the collection name for member_sale_outs
nmg_constants.collections.member_sale_outs;

Tests

The Jest testing framework has been implemented. To run these tests run: npm run test

Contributing

Automated deploy is set up on the pipelines/npm branch