Last active
November 19, 2025 10:11
-
-
Save dpw1/65f4b2ce1505ee6d24e20daa12c38939 to your computer and use it in GitHub Desktop.
Script shopiy scraper
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| /* | |
| run in the browser at https://community.shopify.com/ | |
| */ | |
| /* | |
| PROMPT: | |
| I'm a shopify developer looking for ideas to record youtube videos and write technical blogs for merchants who do not know coding. | |
| Basically I code code snippets which can be pasted in a Shopify theme. based on the attached JSON file, which is scraped data from Shopify forum and Shopify reddit, I want you to: Give me ideas of what blogs/youtube videos to make and why. | |
| Add the idea with best potential at the top of the list. A post which has a lot of likes and views and is recent, has good potential. If there are many people complaining about the same problem, then it's a perfect idea. | |
| The format should be like this: '1. [blog idea] - (based on X comments, like: I wish to there was an app to do Y etc ...' | |
| And also add the URL for this specific post, so I can see it. | |
| */ | |
| window.ezfy = window.ezfy || {}; | |
| ezfy = (function () { | |
| const TOTAL_PAGES_EACH_URL = 2; | |
| const AUTO_DOWNLOAD = true; | |
| const DOWNLOAD_FILENAME = "shopify-community.json"; | |
| const URLS = [ | |
| "https://community.shopify.com/c/technical-qa/211.json?page=1", | |
| "https://community.shopify.com/c/store-design/133.json?page=1" | |
| ]; | |
| function buildPageUrl(url, page) { | |
| const pageParamRegex = /([?&]page=)\d+/; | |
| if (pageParamRegex.test(url)) { | |
| return url.replace(pageParamRegex, `$1${page}`); | |
| } | |
| const separator = url.includes("?") ? "&" : "?"; | |
| return `${url}${separator}page=${page}`; | |
| } | |
| function fetchPages(pages = TOTAL_PAGES_EACH_URL) { | |
| const normalizedPages = Number(pages); | |
| const pageCount = Number.isFinite(normalizedPages) && normalizedPages > 0 | |
| ? Math.floor(normalizedPages) | |
| : TOTAL_PAGES_EACH_URL; | |
| const requests = []; | |
| URLS.forEach(baseUrl => { | |
| for (let page = 1; page <= pageCount; page += 1) { | |
| const targetUrl = buildPageUrl(baseUrl, page); | |
| requests.push( | |
| fetch(targetUrl) | |
| .then(res => res.json()) | |
| .catch(err => { | |
| console.error("EZFY fetch error", targetUrl, err); | |
| return null; | |
| }) | |
| ); | |
| } | |
| }); | |
| return Promise.all(requests).then(results => results.filter(Boolean)); | |
| } | |
| function formatDate(isoString) { | |
| if (!isoString) return null; | |
| const date = new Date(isoString); | |
| if (Number.isNaN(date.getTime())) return isoString; | |
| return date.toLocaleString(undefined, { | |
| year: "numeric", | |
| month: "short", | |
| day: "numeric", | |
| hour: "numeric", | |
| minute: "2-digit" | |
| }); | |
| } | |
| function cleanObject(item) { | |
| if (item.pinned_globally) return null; | |
| return { | |
| fancy_title: item.fancy_title, | |
| posts_count: item.posts_count, | |
| reply_count: item.reply_count, | |
| excerpt: item.excerpt, | |
| views: item.views, | |
| has_accepted_answer: item.has_accepted_answer, | |
| created_at: formatDate(item.created_at), | |
| last_posted_at: formatDate(item.last_posted_at), | |
| url: `https://community.shopify.com/t/${item.slug}/${item.id}` | |
| }; | |
| } | |
| function cleanList(list) { | |
| return list | |
| .slice() | |
| .sort((a, b) => { | |
| const dateA = new Date(a.created_at).getTime(); | |
| const dateB = new Date(b.created_at).getTime(); | |
| return dateB - dateA; | |
| }) | |
| .map(cleanObject) | |
| .filter(Boolean); | |
| } | |
| function downloadJSON(data, filename = "shopify-community.json") { | |
| try { | |
| const blob = new Blob([JSON.stringify(data, null, 2)], { type: "application/json" }); | |
| const url = URL.createObjectURL(blob); | |
| const anchor = document.createElement("a"); | |
| anchor.href = url; | |
| anchor.download = filename; | |
| document.body.appendChild(anchor); | |
| anchor.click(); | |
| document.body.removeChild(anchor); | |
| URL.revokeObjectURL(url); | |
| } catch (error) { | |
| console.error("EZFY download error", error); | |
| } | |
| } | |
| function start() { | |
| return fetchPages(TOTAL_PAGES_EACH_URL).then(responses => { | |
| const topics = responses.flatMap(response => response?.topic_list?.topics || []); | |
| const cleaned = cleanList(topics); | |
| console.log(cleaned); | |
| if (AUTO_DOWNLOAD) { | |
| downloadJSON(cleaned, DOWNLOAD_FILENAME); | |
| } | |
| return cleaned; | |
| }); | |
| } | |
| function init() { | |
| start(); | |
| } | |
| return { | |
| init, | |
| start, | |
| downloadJSON | |
| }; | |
| })(); | |
| ezfy.init(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment