Skip to content

Instantly share code, notes, and snippets.

@james-turner

james-turner/json-to-glue-common.js

Last active July 29, 2020 15:33
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save james-turner/71028537fd106de907cf18206ca10075 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save james-turner/71028537fd106de907cf18206ca10075 to your computer and use it in GitHub Desktop.
Download ZIP
Convert a json object example to cloudformation glue table column definition format [CAVEAT: data types are not perfect, you need a good value if you want the right type otherwise it defaults to string]
Raw
json-to-glue-common.js
const fs = require('fs');
const cliArgs = process.argv.slice(2);
const data = fs.readFileSync(cliArgs[0], 'UTF-8');
const tree = JSON.parse(data);
const dfs = (todo, acc) => {
if(todo.length === 0) return acc;
const [name, node] = todo[0];
todo = todo.slice(1);
if(node !== null && Array.isArray(node)){
return acc + [`${normalizeName(name)}array<${node.map(n => dfs([['',n]], ''))}>`, dfs(todo)].filter(_=>_).join(',');
} else if(node !== null && typeof node === 'object'){
const keys = Object.keys(node);
keys.forEach(k => {
todo.unshift([k, node[k]]);
});
return acc + `${normalizeName(name)}struct<${dfs(todo, '')}>`;
} else {
return [`${normalizeName(name)}${getType(node)}`, dfs(todo, '')].filter(_=>_).join(',');
}
};
const normalizeName = (name) => {
if(name !== '') {
return `${name.toLowerCase()}:`
}
return '';
};
const getType = (n) => {
if(n === null) return 'string';
else {
switch(typeof n){
case 'number': return 'int'; // could be a float so far to tell - maybe provide better parsing here.
case 'boolean': return 'boolean';
default: return 'string'; // without any other info this might be the safest option for us.
}
}
};
const keys = Object.keys(tree);
console.log("Columns:")
keys.forEach(k => {
console.log(` - Name: ${k.toLowerCase()}`);
console.log(` Type: ` + dfs([['',tree[k]]],''));
console.log(` Comment: FILL ME IN PLS`);
});
@james-turner
Copy link
Author

james-turner commented Jul 29, 2020

Example output:

Columns:
  - Name: hookname
    Type: string
    Comment: FILL ME IN PLS
  - Name: time
    Type: string
    Comment: FILL ME IN PLS
  - Name: hookid
    Type: string
    Comment: FILL ME IN PLS
  - Name: data
    Type: struct<identities:string,previousuids:string,lastlogindate:string,signupdate:string,likescount:int,boardscount:int,pinscount:int,webprofiles:string,profileimageurls:string,gravatarimageurl:string,company:string,subscription:string,privategists:int,publicgists:int,gistsurl:string,starredurl:string,currency:string,totalprivaterepository:int,memberurlresources:string,badges:string,suggestions:string,jobbookmarks:string,publications:string,placeslived:string,lruserid:string,kloutscore:string,relatedprofileviews:string,provideraccesscredential:string,professionalheadline:string,favoritethings:string,patents:string,age:string,repositoryurl:string,hireable:boolean,publicrepository:string,agerange:string,books:string,movies:string,mutualfriends:string,televisionshow:string,family:string,games:string,projects:string,languages:string,recommendationsreceived:string,volunteer:string,courses:string,certifications:string,currentstatus:string,skills:string,awards:string,honors:string,numrecommenders:int,associations:string,totalstatusescount:int,isgeoenabled:string,friendscount:int,followerscount:int,httpsimageurl:string,inspirationalpeople:string,sports:string,political:string,religion:string,interests:string,interestedin:string,quote:string,quota:string,relationshipstatus:string,isprotected:boolean,firstlogin:boolean,profilecountry:string,localcountry:string,profilecity:string,localcity:string,profilemodifieddate:string,modifieddate:string,createddate:string,created:string,mainaddress:string,addresses:array<struct<country:string,region:string,postalcode:string,state:string,city:string,address2:string,address1:string,type:string>>,imaccounts:string,phonenumbers:string,educations:string,positions:string,updatedtime:string,verified:string,language:string,tagline:string,coverphoto:string,locallanguage:string,timezone:string,about:string,industry:string,city:string,state:string,hometown:string,profileurl:string,favicon:string,imageurl:string,thumbnailimageurl:string,country:string,email:array<struct<value:string,type:string>>,website:string,gender:string,birthdate:string,profilename:string,nickname:string,fullname:string,suffix:string,lastname:string,middlename:string,firstname:string,prefix:string,provider:string,id:string,isrequiredfieldsfilledonce:boolean,externalids:string,privacypolicy:struct<acceptdatetime:string,acceptsource:string,version:string,issecurepassword:boolean,unverifiedemail:string,iscustomuid:boolean,registrationsource:string,lastloginlocation:string,loginlockedtype:int,isloginlocked:boolean,registrationprovider:string,externaluserloginid:string,phoneidverified:boolean,phoneid:string,nooflogins:int,username:string,isemailsubscribed:boolean,customfields:struct<organizationid:string,uid:string,isdeleted:boolean,isactive:boolean,emailverified:boolean,lastpasswordchangetoken:string,passwordexpirationdate:string,lastpasswordchangedate:string,password:string>>>
    Comment: FILL ME IN PLS

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment