Created
February 22, 2025 06:59
-
-
Save steveblue/a4f6db7dc6e8f79eb1f16d0a06781845 to your computer and use it in GitHub Desktop.
Save Bluesky User Feed To JSON
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| require("dotenv").config(); | |
| const { BskyAgent } = require("@atproto/api"); | |
| const fs = require("fs/promises"); | |
| const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); | |
| // Configuration | |
| const RATE_LIMIT_DELAY = 500; // milliseconds between requests | |
| const BATCH_SIZE = 100; // posts per request | |
| const OUTPUT_FILE = "bluesky-feed.json"; | |
| // Initialize Bluesky agent | |
| const agent = new BskyAgent({ | |
| service: "https://bsky.social", | |
| }); | |
| // Process a post to extract relevant information | |
| function processPost(model) { | |
| const postData = { | |
| uri: model.post.uri, | |
| cid: model.post.cid, | |
| author: model.post.author.handle, | |
| text: model.post.record.text, | |
| createdAt: model.post.record.createdAt, | |
| images: [], | |
| embeds: [], | |
| }; | |
| // Process embedded content | |
| if (model.post.embed) { | |
| if (model.post.embed.images) { | |
| postData.images = model.post.embed.images.map((img) => ({ | |
| alt: img.alt, | |
| path: img.fullsize, | |
| thumb: img.thumb, | |
| })); | |
| } | |
| if (model.post.embed.external) { | |
| postData.embeds.push({ | |
| uri: model.post.embed.external.uri, | |
| title: model.post.embed.external.title, | |
| description: model.post.embed.external.description, | |
| }); | |
| } | |
| } | |
| return postData; | |
| } | |
| async function fetchAllPosts() { | |
| if (!process.env.BLUESKY_USERNAME || !process.env.BLUESKY_PASSWORD) { | |
| throw new Error( | |
| "Please set BLUESKY_USERNAME and BLUESKY_PASSWORD environment variables" | |
| ); | |
| } | |
| try { | |
| // Login to Bluesky | |
| await agent.login({ | |
| identifier: process.env.BLUESKY_USERNAME, | |
| password: process.env.BLUESKY_PASSWORD, | |
| }); | |
| const allPosts = []; | |
| let cursor = undefined; | |
| let totalFetched = 0; | |
| console.log("Starting to fetch posts..."); | |
| while (true) { | |
| try { | |
| const response = await agent.getAuthorFeed({ | |
| actor: process.env.BLUESKY_USERNAME, | |
| limit: BATCH_SIZE, | |
| cursor: cursor, | |
| }); | |
| const posts = response.data.feed; | |
| if (!posts || posts.length === 0) { | |
| break; | |
| } | |
| const processedPosts = posts.map((post) => processPost(post)); | |
| allPosts.push(...processedPosts); | |
| totalFetched += posts.length; | |
| console.log(`Fetched ${totalFetched} posts so far...`); | |
| if (!response.data.cursor) { | |
| break; | |
| } | |
| cursor = response.data.cursor; | |
| // Rate limiting | |
| await delay(RATE_LIMIT_DELAY); | |
| } catch (error) { | |
| console.error("Error fetching batch:", error); | |
| // If we hit an error, wait longer before retrying | |
| await delay(RATE_LIMIT_DELAY * 5); | |
| } | |
| } | |
| console.log(`Finished fetching ${totalFetched} posts. Saving to file...`); | |
| // Save results to file | |
| await fs.writeFile( | |
| OUTPUT_FILE, | |
| JSON.stringify({ posts: allPosts }, null, 2) | |
| ); | |
| console.log(`Results saved to ${OUTPUT_FILE}`); | |
| } catch (error) { | |
| console.error("Fatal error:", error); | |
| process.exit(1); | |
| } | |
| } | |
| // Run the script | |
| fetchAllPosts(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment