Skip to content

Instantly share code, notes, and snippets.

@rothnic

rothnic/reddit-fetcher.ts

Created March 14, 2026 14:52
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save rothnic/47c3d98855fdc552b14f0a9359310e6a to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save rothnic/47c3d98855fdc552b14f0a9359310e6a to your computer and use it in GitHub Desktop.
Download ZIP
Reddit Upvoted/Saved Posts Fetcher - Bun/TypeScript with tracking
Raw
reddit-fetcher.ts
/**
* Reddit Data Fetcher
* Fetches upvoted posts and saved bookmarks from Reddit
* Tracks processed items to avoid duplicates
*
* Usage:
* bun run reddit-fetcher.ts
*
* Requires:
* - Reddit app credentials (client_id, client_secret)
* - Reddit username/password
* - Bun runtime
*/
interface RedditPost {
id: string;
title: string;
url: string;
subreddit: string;
author: string;
created_utc: number;
score: number;
selftext?: string;
permalink: string;
}
interface ProcessedTracker {
upvoted: Set<string>;
saved: Set<string>;
lastRun: number;
}
// Configuration - Replace with your credentials
const REDDIT_CONFIG = {
clientId: process.env.REDDIT_CLIENT_ID || "YOUR_CLIENT_ID",
clientSecret: process.env.REDDIT_CLIENT_SECRET || "YOUR_CLIENT_SECRET",
username: process.env.REDDIT_USERNAME || "YOUR_USERNAME",
password: process.env.REDDIT_PASSWORD || "YOUR_PASSWORD",
userAgent: "clawd-reddit-reader/1.0 (by /u/YOUR_USERNAME)",
};
// File to track processed posts
const TRACKER_FILE = "./reddit-processed.json";
/**
* Load tracker of processed posts
*/
async function loadTracker(): Promise<ProcessedTracker> {
try {
const file = await Bun.file(TRACKER_FILE).text();
const data = JSON.parse(file);
return {
upvoted: new Set(data.upvoted || []),
saved: new Set(data.saved || []),
lastRun: data.lastRun || 0,
};
} catch {
return {
upvoted: new Set(),
saved: new Set(),
lastRun: 0,
};
}
}
/**
* Save tracker of processed posts
*/
async function saveTracker(tracker: ProcessedTracker): Promise<void> {
await Bun.write(
TRACKER_FILE,
JSON.stringify(
{
upvoted: Array.from(tracker.upvoted),
saved: Array.from(tracker.saved),
lastRun: Date.now(),
},
null,
2
)
);
}
/**
* Get Reddit OAuth token
*/
async function getRedditToken(): Promise<string> {
const auth = Buffer.from(
`${REDDIT_CONFIG.clientId}:${REDDIT_CONFIG.clientSecret}`
).toString("base64");
const response = await fetch("https://www.reddit.com/api/v1/access_token", {
method: "POST",
headers: {
Authorization: `Basic ${auth}`,
"Content-Type": "application/x-www-form-urlencoded",
},
body: new URLSearchParams({
grant_type: "password",
username: REDDIT_CONFIG.username,
password: REDDIT_CONFIG.password,
}),
});
if (!response.ok) {
throw new Error(`Auth failed: ${response.status} ${await response.text()}`);
}
const data = await response.json();
return data.access_token;
}
/**
* Fetch Reddit API with pagination
*/
async function* fetchRedditData(
endpoint: string,
token: string
): AsyncGenerator<RedditPost[]> {
let after: string | null = null;
let count = 0;
while (count < 100) {
// Limit to 1000 posts max
const url = new URL(`https://oauth.reddit.com${endpoint}`);
url.searchParams.set("limit", "100");
if (after) url.searchParams.set("after", after);
const response = await fetch(url.toString(), {
headers: {
Authorization: `Bearer ${token}`,
"User-Agent": REDDIT_CONFIG.userAgent,
},
});
if (!response.ok) {
console.error(`Fetch failed: ${response.status}`);
break;
}
const data = await response.json();
const posts: RedditPost[] = data.data.children.map((child: any) => ({
id: child.data.id,
title: child.data.title,
url: child.data.url,
subreddit: child.data.subreddit,
author: child.data.author,
created_utc: child.data.created_utc,
score: child.data.score,
selftext: child.data.selftext,
permalink: `https://reddit.com${child.data.permalink}`,
}));
if (posts.length === 0) break;
yield posts;
after = data.data.after;
count += posts.length;
if (!after) break; // No more pages
// Rate limiting - Reddit allows 60 req/min
await new Promise((resolve) => setTimeout(resolve, 1000));
}
}
/**
* PLACEHOLDER: Add post to knowledge base
* Replace this with your actual KB integration
*/
async function addToKnowledgeBase(
post: RedditPost,
type: "upvoted" | "saved"
): Promise<void> {
// TODO: Replace with your KB integration
// Examples:
// - Write to SQLite database
// - Append to markdown file
// - Send to API endpoint
// - Add to vector store
console.log(`[KB ADD] ${type}: ${post.title.substring(0, 60)}...`);
// Example: Write to daily markdown file
const date = new Date().toISOString().split("T")[0];
const entry = `
## Reddit ${type}: ${post.title}
- **Subreddit**: r/${post.subreddit}
- **Author**: u/${post.author}
- **URL**: ${post.url}
- **Permalink**: ${post.permalink}
- **Score**: ${post.score}
- **Date**: ${new Date(post.created_utc * 1000).toISOString()}
${post.selftext ? `- **Content**: ${post.selftext.substring(0, 200)}...` : ""}
`;
// Append to daily file (or your preferred storage)
const filename = `./kb/reddit-${date}.md`;
await Bun.write(filename, entry, { append: true });
}
/**
* Main function: Fetch and process Reddit data
*/
async function main() {
console.log("🔍 Starting Reddit fetcher...");
// Load tracker
const tracker = await loadTracker();
console.log(
`📊 Previously processed: ${tracker.upvoted.size} upvoted, ${tracker.saved.size} saved`
);
// Get OAuth token
console.log("🔑 Authenticating...");
const token = await getRedditToken();
console.log("✅ Authenticated");
// Fetch upvoted posts
console.log("⬆️ Fetching upvoted posts...");
let newUpvoted = 0;
for await (const posts of fetchRedditData("/user/me/upvoted", token)) {
for (const post of posts) {
if (tracker.upvoted.has(post.id)) continue;
await addToKnowledgeBase(post, "upvoted");
tracker.upvoted.add(post.id);
newUpvoted++;
}
}
console.log(`✅ Processed ${newUpvoted} new upvoted posts`);
// Fetch saved posts
console.log("🔖 Fetching saved posts...");
let newSaved = 0;
for await (const posts of fetchRedditData("/user/me/saved", token)) {
for (const post of posts) {
if (tracker.saved.has(post.id)) continue;
await addToKnowledgeBase(post, "saved");
tracker.saved.add(post.id);
newSaved++;
}
}
console.log(`✅ Processed ${newSaved} new saved posts`);
// Save tracker
await saveTracker(tracker);
console.log(`💾 Tracker saved: ${TRACKER_FILE}`);
console.log("\n🎉 Done!");
console.log(` Total upvoted: ${tracker.upvoted.size}`);
console.log(` Total saved: ${tracker.saved.size}`);
}
// Run if called directly
if (import.meta.main) {
main().catch((err) => {
console.error("❌ Error:", err);
process.exit(1);
});
}
export {
fetchRedditData,
addToKnowledgeBase,
loadTracker,
saveTracker,
type RedditPost,
type ProcessedTracker,
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment