env vars:
SLACK_WEBHOOK_URL
CRAIGSLIST_SEARCH_URL=https://sfbay.craigslist.org/search/sfc/apa?query=glen+park&hasPic=1&postedToday=1&availabilityMode=0&sale_date=all+dates
SLACK_ICON_URL
Created
May 22, 2019 16:34
-
-
Save chasestarr/d81e92b238adad6e74cb111adaa6246a to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| const p = require('puppeteer'); | |
| const request = require('request-promise-native'); | |
| async function postToSlack(listing) { | |
| const options = { | |
| method: 'POST', | |
| uri: process.env.SLACK_WEBHOOK_URL, | |
| body: { | |
| icon_url: process.env.SLACK_ICON_URL, | |
| attachments: [ | |
| { | |
| title: listing.title, | |
| title_link: listing.href, | |
| fallback: listing.title, | |
| thumb_url: listing.img, | |
| }, | |
| ], | |
| }, | |
| headers: { | |
| 'Content-type': 'application/json', | |
| }, | |
| json: true, | |
| }; | |
| try { | |
| await request(options); | |
| } catch (error) { | |
| console.error(`failed to post ${listing.href} to slack.`); | |
| console.log(error); | |
| } | |
| } | |
| async function getDetails(listing) { | |
| const img = await listing.$eval('img', img => img.getAttribute('src')); | |
| const titleElement = await listing.$('.result-title'); | |
| const title = await (await titleElement.getProperty('textContent')).jsonValue(); | |
| const href = await (await titleElement.getProperty('href')).jsonValue(); | |
| return { img, title, href }; | |
| } | |
| async function main() { | |
| const browser = await p.launch({ | |
| args: ['--no-sandbox'], | |
| defaultViewport: { width: 1024, height: 720 }, | |
| }); | |
| const page = await browser.newPage(); | |
| await page.goto(process.env.CRAIGSLIST_SEARCH_URL); | |
| const rows = await page.$$('.rows>*'); | |
| const headingIndex = rows.findIndex(r => r._remoteObject.description === 'h4.ban.nearby'); | |
| const relevant = rows.slice(0, headingIndex); | |
| const listings = await Promise.all(relevant.map(getDetails)); | |
| await Promise.all(listings.map(postToSlack)); | |
| await browser.close(); | |
| } | |
| exports.main = main; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "name": "glen-park-homes", | |
| "version": "1.0.0", | |
| "description": "", | |
| "main": "index.js", | |
| "scripts": { | |
| "test": "echo \"Error: no test specified\" && exit 1" | |
| }, | |
| "keywords": [], | |
| "author": "", | |
| "license": "ISC", | |
| "dependencies": { | |
| "@google-cloud/pubsub": "^0.24.1", | |
| "puppeteer": "^1.12.2", | |
| "request": "^2.88.0", | |
| "request-promise-native": "^1.0.5" | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment