Geojson Feature Reader

Reads large GeoJSON files and returns valid features one by one. Useful when read time does not matter.

Built to provide a solution to json streams backpressure overflowing when processing geometries before writing

NB blocks event loop incrementally

The callback emits a feature when found. The promise is resolved when alldata is read, returns the feature count.

Rate Average

30mb file - 1000 features per second using +-150mb memory for simple polygons.

• avg 1 min to process 30k features.

90mb file - 200-300 features per second using +-200mb memory for larger complex polygons.

• avg 10min to process 120k features.

300mb file > 20-50 features per second using +-250mb memory for larger complex polygons.

• avg 2hours to process 400k features

Read speed increases as the process runs

Install

npm install geojson-feature-reader

Usage

Base Example

const read_stream = fs.createReadStream;
const geojsonString = "{...}"
const feature_reader = require('geojson-feature-reader')
const options = {}

    //with custom options
feature_reader(read_stream || geojsonString, options, feature_callback)
    .then(count => console.log(`${count} features processed`))
    .catch(e => console.error(`Unable to read data`))
    
    //with default options
feature_reader(read_stream || geojsonString, feature_callback)
    .then(count => console.log(`${count} features processed`))
    .catch(e => console.error(`Unable to read data`))
    
    //feature callback
function feature_callback(feature){
    feature // string
}

Options

 let options = {
    
// number
// The amount of features to process before waiting.
// Less features opens the event loop more often.
  pause_feature_counter : 1000,
  
// number or function
// A. the wait time after the features counter is reached.
// B. a function to indicate when more features can be read.
// > opens event loop
  pause_feature_timeout : 100,
  
// number
// Expand geometrycollections to individual features.
// Counts towards `pause_feature_counter`
  expand_geom_collection : true 
}

Basic Usage

let feature_reader = require('geojson-feature-reader')
let read_stream = fs.createReadStream('/path/data.geojson')

feature_reader(read_stream, feature_callback)
    .then(count => console.log(`${count} features processed`))
    .catch(e => console.error(`Unable to read data`))
    
function feature_callback(feature){
    typeof feature === 'string' //true
    
    console.log(feature) 
    // { "type": "Feature", "properties": { ... }, "geometry": { ... }
    // or
    // { "type": "feature", "properties": { ... }, "geometry": { ...} }
}

With options

let feature_reader = require('geojson-feature-reader')
let read_stream = fs.createReadStream('/path/data.geojson')
let options = {
    pause_feature_counter : 1000,
    pause_feature_timeout : 10,
    expand_geom_collection : false
}

feature_reader(read_stream, options, feature_callback)
    .then(count => console.log(`${count} features processed`))
    .catch(e => console.error(`Unable to read data`))
    
function feature_callback(feature){
    typeof feature === 'string' //true
    
    console.log(feature)
    // { "type": "Feature", "properties": { ... }, "geometry": { ... }
    // or
    // { "type": "feature", "properties": { ... }, "geometry": { ...} }
}

Handle readiness

let feature_reader = require('geojson-feature-reader')
let read_stream = fs.createReadStream('/path/data.geojson')
let options = {
    pause_feature_counter : 1000, 
    pause_feature_timeout : continue_read, //wait until function returns true
    expand_geom_collection : true 
}

feature_reader(read_stream, feature_callback)
    .then(count => console.log(`${count} features processed`))
    .catch(e => console.error(`Unable to read data`))
    
function feature_callback(feature){
    typeof feature === 'string' //true
    
    console.log(feature)
    // { "type": "Feature", "properties": { ... }, "geometry": { ... }
    // or
    // { "type": "feature", "properties": { ... }, "geometry": { ...} }
}   

let ready_for_next = false;
function continue_read(){
    
    // file reading resumes
    // function is tested every 100ms once reading is pasued
    return ready_for_next
}

//
setTimeout(() =>{
    ready_for_next = true
},1000)

Provide complete featurecollection as string

let feature_reader = require('geojson-feature-reader')
let read_stream = fs.createReadStream('/path/data.geojson','utf8')
let allData = ''

read_stream
    .on('data', data => allData += data) 
    .on('error', error => console.error('read fail'))
    .on('end', () => extract_features(allData))

let options = {
    pause_feature_counter : 1000, 
    pause_feature_timeout : 10, 
    expand_geom_collection : true
}

function extract_features(data){
    feature_reader(data, feature_callback)
        .then(count => console.log(`${count} features processed`))
        .catch(e => console.error(`Unable to read data`))
}
    
function feature_callback(feature){
    typeof feature === 'string' //true
    
    console.log(feature)
    // { "type": "Feature", "properties": { ... }, "geometry": { ... }
    // or
    // { "type": "feature", "properties": { ... }, "geometry": { ...} }
}

Dependencies

• none

Built With

• Node ReadStream - Native Node file reader

Authors

• startkev - gitLab

License

This project is licensed under the MIT License