Skip to content

Instantly share code, notes, and snippets.

@pineapplemachine

pineapplemachine/lib.csv-stringifiers.abstract-concat.js

Created August 21, 2018 15:17
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save pineapplemachine/ec5f2356b6470729084f022441d0954c to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save pineapplemachine/ec5f2356b6470729084f022441d0954c to your computer and use it in GitHub Desktop.
Download ZIP
csv-writer performance and memory use comparison
Raw
lib.csv-stringifiers.abstract-concat.js
// Put this file in `lib/csv-stringifiers` alongside `abstract.js`
'use strict';
const RECORD_DELIMITER = '\n';
class AbstractCsvStringifier {
constructor(params) {
this._fieldStringifier = params.fieldStringifier;
this._fieldDelimiter = params.fieldDelimiter;
}
getHeaderString() {
const headerRecord = this._getHeaderRecord();
return headerRecord ? this.stringifyRecords([headerRecord]) : null;
}
stringifyRecords(records) {
let output = '';
for (let record of records) {
output += this._getCsvLine(this._getRecordAsArray(record));
output += RECORD_DELIMITER;
}
return output;
}
/* istanbul ignore next */_getRecordAsArray(_record) {
throw new Error('Must be overridden in subclasses');
}
/* istanbul ignore next */_getHeaderRecord() {
throw new Error('Must be overridden in subclasses');
}
_getCsvLine(record) {
return record
.map(fieldValue => this._fieldStringifier.stringify(fieldValue))
.join(this._fieldDelimiter);
}
}
module.exports = AbstractCsvStringifier;
Raw
lib.csv-stringifiers.abstract-join-eager.js
// Put this file in `lib/csv-stringifiers` alongside `abstract.js`
'use strict';
const RECORD_DELIMITER = '\n';
class AbstractCsvStringifier {
constructor(params) {
this._fieldStringifier = params.fieldStringifier;
this._fieldDelimiter = params.fieldDelimiter;
}
getHeaderString() {
const headerRecord = this._getHeaderRecord();
return headerRecord ? this.stringifyRecords([headerRecord]) : null;
}
stringifyRecords(records) {
const array = [];
for (let record of records) {
array.push(this._getCsvLine(this._getRecordAsArray(record)));
}
array.push('');
return array.join(RECORD_DELIMITER);
}
/* istanbul ignore next */_getRecordAsArray(_record) {
throw new Error('Must be overridden in subclasses');
}
/* istanbul ignore next */_getHeaderRecord() {
throw new Error('Must be overridden in subclasses');
}
_getCsvLine(record) {
return record
.map(fieldValue => this._fieldStringifier.stringify(fieldValue))
.join(this._fieldDelimiter);
}
}
module.exports = AbstractCsvStringifier;
Raw
lib.csv-stringifiers.abstract-original.js
// Put this file in `lib/csv-stringifiers` alongside `abstract.js`
'use strict';
const RECORD_DELIMITER = '\n';
class AbstractCsvStringifier {
constructor(params) {
this._fieldStringifier = params.fieldStringifier;
this._fieldDelimiter = params.fieldDelimiter;
}
getHeaderString() {
const headerRecord = this._getHeaderRecord();
return headerRecord ? this.stringifyRecords([headerRecord]) : null;
}
stringifyRecords(records) {
const csvLines = Array.from(records)
.map(record => this._getRecordAsArray(record))
.map(record => this._getCsvLine(record));
return csvLines.join(RECORD_DELIMITER) + RECORD_DELIMITER;
}
/* istanbul ignore next */_getRecordAsArray(_record) {
throw new Error('Must be overridden in subclasses');
}
/* istanbul ignore next */_getHeaderRecord() {
throw new Error('Must be overridden in subclasses');
}
_getCsvLine(record) {
return record
.map(fieldValue => this._fieldStringifier.stringify(fieldValue))
.join(this._fieldDelimiter);
}
}
module.exports = AbstractCsvStringifier;
Raw
profile-instance.js
# Put in same directory as `profile.js`
const createObjectCsvStringifier = require("./index").createObjectCsvStringifier;
const totalRows = 5000;
const totalIterations = 100;
function* enumerateRecords(){
for(let i = 0; i < totalRows; i++){
yield {"a": i, "b": 2 * i, "c": 3 * i};
}
}
let highestHeapTotal = 0;
let highestHeapUsed = 0;
let highestExternal = 0;
let highestResidentSetSize = 0;
let lastOutputLength = 0;
const startTime = (new Date()).getTime();
for(let i = 0; i < totalIterations; i++){
const csvWriter = createObjectCsvStringifier({header: [
{id: "a", title: "Column A"},
{id: "b", title: "Column B"},
{id: "c", title: "Column C"},
]});
const csvContent = csvWriter.stringifyRecords(
enumerateRecords()
);
lastOutputLength = (csvContent && csvContent.length) || 0;
const usage = process.memoryUsage();
if(usage.heapTotal > highestHeapTotal){
highestHeapTotal = usage.heapTotal;
}
if(usage.heapUsed > highestHeapUsed){
highestHeapUsed = usage.heapUsed;
}
if(usage.external > highestExternal){
highestExternal = usage.external;
}
if(usage.rss > highestResidentSetSize){
highestResidentSetSize = usage.rss;
}
}
const endTime = (new Date()).getTime();
console.log(`Number of rows in CSV: ${totalRows} rows.`);
console.log(`Number of iterations: ${totalIterations} times.`);
console.log(`Time taken: ${endTime - startTime} ms.`);
console.log(`Peak heap total: ${highestHeapTotal} bytes.`);
console.log(`Peak heap used: ${highestHeapUsed} bytes.`);
console.log(`Peak external: ${highestExternal} bytes.`);
console.log(`Peak RSS: ${highestResidentSetSize} bytes.`);
console.log(`CSV output length: ${lastOutputLength}.`);
Raw
profile.js
// Main test file. Place it in the root csv-writer directory alongside `index.js`. Run `node profile.js`
const fs = require("fs");
const abstractPath = "./lib/csv-stringifiers/";
const abstractNames = [
"abstract-original.js",
"abstract-concat.js",
"abstract-join-eager.js",
];
for(let abstractName of abstractNames){
const sourcePath = abstractPath + abstractName;
const targetPath = abstractPath + "abstract.js";
const source = fs.readFileSync(sourcePath, "utf-8").toString();
fs.writeFileSync(targetPath, source);
console.log(`Testing: ${abstractName}`);
require("./profile-instance");
for(let key in require.cache){
delete require.cache[key];
}
console.log();
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment