Last active
June 15, 2018 09:49
-
-
Save Jonas74/3d87b4c5f140222e4b76ffe39f989fd3 to your computer and use it in GitHub Desktop.
Query Wikipedia with List of Titles
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Use nodejs to query | |
| ```javascript | |
| const fetch = require("isomorphic-fetch") | |
| const files = require("fs") | |
| const slowy = (ms) => new Promise(r => setTimeout(r,ms)) | |
| const askWikipedia = async (wirks,file) => { | |
| console.time("> Done asking wikipedia") | |
| const wikipedia = {} | |
| const items = Object.keys(wirks).map(key => { | |
| return { code:key, name: wirks[key] } | |
| }) | |
| const stats = { | |
| total: items.length, | |
| okays: 0, | |
| error: 0 | |
| } | |
| for(const item of items) { | |
| const url = `https://de.wikipedia.org/w/api.php?action=query&titles=${item.name}&format=json` | |
| await fetch(url).then(raw => raw.json()).then((json) => { | |
| const pageId = Object.keys(json.query.pages).pop() | |
| if(pageId != '-1') { | |
| wikipedia[item.code] = `https://de.wikipedia.org/?curid=${pageId}` | |
| stats.okays++ | |
| } | |
| else{ | |
| stats.error++ | |
| } | |
| }) | |
| .then(() => { | |
| console.log(`> ${item.code} ${item.name}`) | |
| return slowy(1000) | |
| }) | |
| .catch(err => { | |
| console.log(err) | |
| }) | |
| } | |
| files.writeFileSync(file, JSON.stringify(wikipedia,null,2) ) | |
| console.log(`> Total: ${stats.total} Okays: ${stats.okays} Error: ${stats.error}`) | |
| console.time("> Done asking wikipedia") | |
| } | |
| // wirks {Object} | |
| // { | |
| // // Json File replaces "wiki" with "https://de.wikipedia.org/?curid=5553" | |
| // frage1: "hello" | |
| // } | |
| askWikipedia( wirks, './docs/wikipedia.json' ) | |
| ``` |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment