Last active
August 29, 2015 14:20
-
-
Save tonydieu/10a711799e7f24c96b44 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| var express = require('express'), | |
| path = require('path'), | |
| favicon = require('serve-favicon'), | |
| logger = require('morgan'), | |
| cookieParser = require('cookie-parser'), | |
| bodyParser = require('body-parser'), | |
| request = require('request'), | |
| CronJob = require('cron').CronJob, | |
| _ = require('lodash'), | |
| uuid = require('node-uuid'), | |
| fs = require('fs'), | |
| request = require("request"), | |
| Q = require("q"), | |
| feed = require('feed-read'); | |
| //... | |
| // example RSS Feeds | |
| var urls = [ | |
| ['gamespot', 'http://www.gamespot.com/feeds/news/'], | |
| ['ign', 'http://feeds.ign.com/ign/news'], | |
| ['n4g', 'http://n4g.com/rss/news?channel=&sort=latest'], | |
| ['pcgamer', 'http://www.pcgamer.com/rss/'], | |
| ['gamesradar', 'http://www.gamesradar.com/all-platforms/news/rss/'], | |
| ['polygon', 'http://www.polygon.com/rss/index.xml'] | |
| ]; | |
| // temporary fix for source name | |
| var sourceNames = { | |
| 'GameSpot News': 'gamespot', | |
| 'IGN News': 'ign', | |
| 'GamesRadar - Updates': 'gamesradar', | |
| 'Polygon - All': 'polygon', | |
| 'N4G: news feed': 'n4g', | |
| 'PC Gamer latest stories': 'pcgamer' | |
| }; | |
| /** | |
| * seconds: 0-59 | |
| * minutes: 0-59 | |
| * hours: 0-23 | |
| * day of Month: 1-31 | |
| * months: 0-11 | |
| * day of Week: 0-6 | |
| */ | |
| var job = new CronJob('* 20 * * * *', function() { | |
| var collection = db.get('rss'); | |
| /** | |
| * findCollection | |
| * @param {string} sourceName | |
| * @param {int} limit | |
| * @param {int} sort | |
| */ | |
| function findCollection(sourceName, limit, sort) { | |
| var deferred = Q.defer(); | |
| collection.find({ | |
| source: sourceName | |
| }, { | |
| limit: limit, | |
| sort: { | |
| _id: sort | |
| } | |
| }, function(err, docs) { | |
| if (err) { | |
| deferred.reject(err); | |
| } | |
| deferred.resolve(docs); | |
| }); | |
| return deferred.promise; | |
| } | |
| /** | |
| * checkingDuplicate | |
| * @param {string} a new feed | |
| * @param {int} b existing feeds | |
| * @param {int} filter | |
| * @param {callback} to be inserted | |
| */ | |
| function checkingDuplicate(a, b, filter, callback) { | |
| var bLen = b.length; | |
| if (bLen == 0 && !(_.isEmpty(a))) { | |
| console.log('New first insert...') | |
| callback(a); | |
| } else { | |
| if (!(_.isEmpty(a))) { | |
| console.log('New insert...') | |
| var b = b[0].feed; | |
| for (var i = 0; i < b.length; i++) { | |
| var d = _.difference(_.values(_.omit(a.article, filter)), _.values(_.uniq(b)[i].article)); | |
| if (d.length === 0) { | |
| // found a duplicate | |
| console.log('DUPLICATE'); | |
| break; | |
| } | |
| if (i === b.length - 1) { | |
| // did not found a duplicate | |
| console.log('NEW INSERT') | |
| callback(a); | |
| } | |
| } | |
| }; | |
| } | |
| } | |
| // loop through our list of RSS feed urls | |
| for (var j = 0; j < urls.length; j++) { | |
| feed(urls[j][1], function(err, articles) { | |
| if (err) { | |
| throw err; | |
| } else { | |
| var temporaryName = articles[0].feed.name; | |
| var mappedSourceName = sourceNames[temporaryName]; | |
| var articleCont = [{ | |
| source: mappedSourceName, | |
| feed: [] | |
| }]; | |
| // promises for finding old collection | |
| Q.fcall(findCollection, mappedSourceName, 1, -1) | |
| .then(function(oldCollections) { | |
| var oldCollections = oldCollections; | |
| console.log('oldCollections', oldCollections); | |
| // loop through the list of articles returned | |
| for (var i = 0; i < articles.length; i++) { | |
| var newArticle = { | |
| uuid: uuid.v1(), | |
| isActive: true, | |
| tags: [], | |
| type: [], | |
| article: { | |
| title: articles[i].title, | |
| link: articles[i].link, | |
| published: articles[i].pubDate || articles[i].published | |
| } | |
| }; | |
| checkingDuplicate(newArticle, oldCollections, ['published'], function(res) { | |
| articleCont[0].feed.push(res); | |
| }); | |
| if (i === articles.length - 1) { | |
| console.log('DONE'); | |
| } | |
| }; // end inner for loop | |
| if (articleCont[0].feed.length > 0) { | |
| collection.insert( | |
| articleCont, | |
| function(err) { | |
| if (err) { | |
| // if it failed, return error | |
| console.log('There was a problem adding the information to the database.'); | |
| } else { | |
| console.log('INSERT SUCCESS'); | |
| } | |
| }); // end of collection insert | |
| } else { | |
| console.log('Nothing to insert'); | |
| } | |
| }); // end of promise | |
| } | |
| }); // end call to feed (feed-read) method | |
| } | |
| }, null, true, 'Europe/Copenhagen'); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "_id": { | |
| "$oid": "552e2d776418bd03008e54a8" | |
| }, | |
| "source": "ign", | |
| "feed": [ | |
| { | |
| "uuid": "b8338330-e350-11e4-afb2-010690af1d1b", | |
| "isActive": true, | |
| "tags": [], | |
| "type": [], | |
| "article": { | |
| "title": "Report: Wildstar Pulled From Australian Stores", | |
| "link": "http://feeds.ign.com/~r/ign/news/~3/sdSEgylm218/report-wildstar-pulled-from-australian-stores", | |
| "published": { | |
| "$date": "2015-04-15T08:54:27.000Z" | |
| } | |
| } | |
| } | |
| ] | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "_id": { | |
| "$oid": "552e2d796418bd03008e54ab" | |
| }, | |
| "source": "ign", | |
| "feed": [ | |
| { | |
| "uuid": "b95769c0-e350-11e4-afb2-010690af1d1b", | |
| "isActive": true, | |
| "tags": [], | |
| "type": [], | |
| "article": { | |
| "title": "Report: Wildstar Pulled From Australian Stores", | |
| "link": "http://feeds.ign.com/~r/ign/news/~3/sdSEgylm218/report-wildstar-pulled-from-australian-stores", | |
| "published": { | |
| "$date": "2015-04-15T08:54:27.000Z" | |
| } | |
| } | |
| } | |
| ] | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "_id": { | |
| "$oid": "552e1f306418bd03008e5435" | |
| }, | |
| "source": "n4g", | |
| "feed": [ | |
| { | |
| "uuid": "35ca5340-e348-11e4-afb2-010690af1d1b", | |
| "isActive": true, | |
| "tags": [], | |
| "type": [], | |
| "article": { | |
| "title": "Call of Duty Black Ops 3 Teaser Whats Good and Whats Bad", | |
| "link": "http://n4g.com/news/1708846/call-of-duty-black-ops-3-teaser-whats-good-and-whats-bad", | |
| "published": "" | |
| } | |
| }, | |
| { | |
| "uuid": "35ca7a50-e348-11e4-afb2-010690af1d1b", | |
| "isActive": true, | |
| "tags": [], | |
| "type": [], | |
| "article": { | |
| "title": "Can We Take a Moment and Realize How Weird These 10 Games Are?", | |
| "link": "http://n4g.com/news/1708968/can-we-take-a-moment-and-realize-how-weird-these-10-games-are", | |
| "published": "" | |
| } | |
| }, | |
| { | |
| "uuid": "35ca7a51-e348-11e4-afb2-010690af1d1b", | |
| "isActive": true, | |
| "tags": [], | |
| "type": [], | |
| "article": { | |
| "title": "Dyscourse Shows Me Why I Should Never Be in Charge", | |
| "link": "http://n4g.com/news/1708984/dyscourse-shows-me-why-i-should-never-be-in-charge", | |
| "published": "" | |
| } | |
| }, | |
| { | |
| "uuid": "35ca7a52-e348-11e4-afb2-010690af1d1b", | |
| "isActive": true, | |
| "tags": [], | |
| "type": [], | |
| "article": { | |
| "title": "Super Smash Bros. for 3DS Update 1.0.6 Now Available", | |
| "link": "http://n4g.com/news/1709272/super-smash-bros-for-3ds-update-1-0-6-now-available", | |
| "published": "" | |
| } | |
| }, | |
| { | |
| "uuid": "35ca7a53-e348-11e4-afb2-010690af1d1b", | |
| "isActive": true, | |
| "tags": [], | |
| "type": [], | |
| "article": { | |
| "title": "Titan Souls Review | AOTF", | |
| "link": "http://n4g.com/news/1709260/titan-souls-review-aotf", | |
| "published": "" | |
| } | |
| }, | |
| { | |
| "uuid": "35ca7a54-e348-11e4-afb2-010690af1d1b", | |
| "isActive": true, | |
| "tags": [], | |
| "type": [], | |
| "article": { | |
| "title": "PSGamecast 11 James Marsden / David Jaffe / LEGO Dimensions / Daredevil", | |
| "link": "http://n4g.com/news/1708960/psgamecast-11-james-marsden-david-jaffe-lego-dimensions-daredevil", | |
| "published": "" | |
| } | |
| }, | |
| { | |
| "uuid": "35ca7a55-e348-11e4-afb2-010690af1d1b", | |
| "isActive": true, | |
| "tags": [], | |
| "type": [], | |
| "article": { | |
| "title": "Are Games Better Now Than They Used To Be", | |
| "link": "http://n4g.com/news/1709194/are-games-better-now-than-they-used-to-be", | |
| "published": "" | |
| } | |
| }, | |
| { | |
| "uuid": "35ca7a56-e348-11e4-afb2-010690af1d1b", | |
| "isActive": true, | |
| "tags": [], | |
| "type": [], | |
| "article": { | |
| "title": "Get a PS4 console for just £279.99 with our exclusive voucher code", | |
| "link": "http://n4g.com/news/1708770/get-a-ps4-console-for-just-279-99-with-our-exclusive-voucher-code", | |
| "published": "" | |
| } | |
| }, | |
| { | |
| "uuid": "35ca7a57-e348-11e4-afb2-010690af1d1b", | |
| "isActive": true, | |
| "tags": [], | |
| "type": [], | |
| "article": { | |
| "title": "David Jaffe Would do a more open world 'God of War' like Zelda If He Would Return To GoW", | |
| "link": "http://n4g.com/news/1709197/david-jaffe-would-do-a-more-open-world-god-of-war-like-zelda-if-he-would-return-to-gow", | |
| "published": "" | |
| } | |
| }, | |
| { | |
| "uuid": "35ca7a58-e348-11e4-afb2-010690af1d1b", | |
| "isActive": true, | |
| "tags": [], | |
| "type": [], | |
| "article": { | |
| "title": "Review: Slender: The Arrival (DarkZero)", | |
| "link": "http://n4g.com/news/1708842/review-slender-the-arrival-darkzero", | |
| "published": "" | |
| } | |
| } | |
| ] | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment