Skip to content

Instantly share code, notes, and snippets.

@morganabc

morganabc/patreon_dl.rs

Last active March 13, 2025 02:25
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save morganabc/5948b1075fa663e980e51569b69744b0 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save morganabc/5948b1075fa663e980e51569b69744b0 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
patreon_dl.rs
use axum::{Router, routing::{get, post}};
use axum::extract::Json;
use axum::response::{Response, IntoResponse};
use axum::http::StatusCode;
use serde_json::json;
use serde::{Serialize, Deserialize};
use tokio::sync::mpsc::{channel, Sender};
use regex::Regex;
use tower::ServiceBuilder;
use tower_http::cors::{CorsLayer, Any};
use std::net::SocketAddr;
use indicatif::ProgressBar;
#[derive(Serialize, Deserialize, Debug)]
struct Artist {
id: u64,
name: String,
}
#[derive(Serialize, Deserialize, Debug)]
struct DownloadItem {
url: String,
id: String,
post: String,
name: String,
ext: String,
}
#[derive(Serialize, Deserialize, Debug)]
struct DownloadRequest {
artist: String,
data: Vec<DownloadItem>,
}
const JS: &str = include_str!("script.js");
async fn handle_gadget() -> impl IntoResponse {
Response::builder()
.status(StatusCode::OK)
.body(axum::body::Body::from(JS))
.unwrap()
}
async fn handle_user(Json(artist): Json<Artist>) -> impl IntoResponse {
println!("{:#?}", artist);
(StatusCode::OK, Json(json!({"status": "success"})))
}
async fn handle_download(Json(download_request): Json<DownloadRequest>, tx: Sender<DownloadRequest>) -> impl IntoResponse {
match tx.send(download_request).await {
Ok(_) => (StatusCode::OK, Json(json!({"status": "success"}))),
Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, Json(json!({"status": "error"}))),
}
}
async fn worker(mut rx: tokio::sync::mpsc::Receiver<DownloadRequest>) {
while let Some(download_request) = rx.recv().await {
let artist = download_request.artist.clone();
let progress_bar = ProgressBar::new(download_request.data.len() as u64);
let mut download_tasks = vec![];
for item in download_request.data {
let artist = artist.clone();
let url = item.url.clone();
let filename = format!("{}-{}-{}.{}", item.post, item.id, sanitize(&item.name), item.ext);
let filepath = format!("patreon/{}/{}", artist, filename);
let progress_bar = progress_bar.clone();
let handle = tokio::spawn(async move {
fetch_and_save(url, filepath).await;
progress_bar.inc(1);
});
download_tasks.push(handle);
}
for task in download_tasks {
match task.await {
Ok(_) => {},
Err(e) => eprintln!("Task failed: {:?}", e),
}
}
progress_bar.finish_with_message("Download complete");
}
}
async fn fetch_and_save(url: String, path: String) {
if std::path::Path::new(&path).exists() {
println!("Already exists");
return;
}
let res = match reqwest::get(&url).await {
Ok(response) => response,
Err(_) => {
println!("Error downloading {}", url);
return;
}
};
let bytes = match res.bytes().await {
Ok(b) => b,
Err(_) => {
println!("Error reading bytes from {}", url);
return;
}
};
if let Some(parent) = std::path::Path::new(&path).parent() {
if let Err(e) = tokio::fs::create_dir_all(parent).await {
println!("Error creating directories: {}", e);
return;
}
}
if let Err(e) = tokio::fs::write(&path, &bytes).await {
println!("Error writing to file: {}", e);
}
}
fn sanitize(filename: &str) -> String {
let re = Regex::new(r"[^\w\-.]").unwrap();
re.replace_all(filename, "_").to_string().chars().take(255).collect()
}
#[tokio::main]
async fn main() {
let cors = CorsLayer::new()
.allow_origin(Any)
.allow_methods(Any)
.allow_headers(Any);
let (tx, rx) = channel::<DownloadRequest>(100);
let app = Router::new()
.route("/gadget", get(handle_gadget))
.route("/user", post(handle_user))
.route("/download", post({
let tx = tx.clone();
move |Json(download_request)| handle_download(Json(download_request), tx.clone())
}))
.layer(ServiceBuilder::new().layer(cors));
tokio::spawn(worker(rx));
let addr = SocketAddr::from(([127, 0, 0, 1], 8080));
println!("Server running at http://{}", addr);
axum::Server::bind(&addr)
.serve(app.into_make_service())
.await
.unwrap();
}
Raw
script.js
(async () => {
const basePostURL = "https://www.patreon.com/api/posts?"
const campaignID = Number(window.patreon.bootstrap.creator.data.id)
const artistName = window.patreon.bootstrap.creator.data.attributes.name;
console.log("Sending Patreon information to patreon-dl...")
await fetch("http://localhost:8080/user", {
method: 'POST',
headers: {"content-type": "application/json"},
body: JSON.stringify({
id: campaignID,
name: window.patreon.bootstrap.creator.data.attributes.name
})
});
const initialQueryParams = new URLSearchParams({
"include": "images,media",
"fields[post]": "post_metadata",
"fields[media]": "id,image_urls,download_url,metadata,file_name",
"filter[campaign_id]": campaignID,
"filter[contains_exclusive_posts]": true,
"sort": "-published_at",
"json-api-version": "1.0"
})
let downloads = [];
let posts = [];
const initalPostRequest = await fetch(basePostURL + initialQueryParams.toString())
const parsedInital = await initalPostRequest.json()
let initialLength = 0;
if ("included" in parsedInital) {
initialLength = parsedInital.included.length
}
posts = posts.concat(parsedInital.data);
for (let i = 0; i < initialLength; i++) {
if(parsedInital.included[i].attributes.file_name === null) {
continue
}
const originalFilename = parsedInital.included[i].attributes.file_name.split(".")
const fileExtension = originalFilename.pop()
const newFilename = `${originalFilename.join(".")}-${parsedInital.included[i].id}.${fileExtension}`
downloads.push({file: parsedInital.included[i].attributes.file_name, id: parsedInital.included[i].id, url: parsedInital.included[i].attributes.download_url});
}
console.log(`Collected ${downloads.length} posts...`)
let nextURL = ""
if ("links" in parsedInital) {
nextURL = parsedInital.links.next;
}
while (nextURL !== "") {
const recursivePostRequest = await fetch(nextURL)
const parsedPosts = await recursivePostRequest.json()
posts = posts.concat(parsedPosts.data);
let includedLength = 0;
if ("included" in parsedPosts) {
includedLength = parsedPosts.included.length
}
for (let i = 0; i < includedLength; i++) {
downloads.push({file: parsedPosts.included[i].attributes.file_name, id: parsedPosts.included[i].id, url: parsedPosts.included[i].attributes.download_url});
}
if ("links" in parsedPosts) {
nextURL = parsedPosts.links.next;
} else {
nextURL = ""
}
console.log(`Collected ${downloads.length} posts...`)
}
let reverseLookup = {};
for (let p of posts){
for (let imgs of p.relationships.images.data) {
reverseLookup[imgs.id] = p.id;
}
for (let imgs of p.relationships.media.data) {
reverseLookup[imgs.id] = p.id;
}
}
let request = [];
function addFileDownload(filename, id, fileURL) {
let newFilename = ""
if(filename == null) {
console.log("Skipping image you don't have access to...")
return
} else if(fileURL == null) {
console.log("Skipping image you don't have access to...")
return
} else {
let originalFilename = filename.split(".")
if (reverseLookup[id] == undefined ) {
console.error(`Cannot find post of image ${id} for ${url}`)
return
}
let postId = reverseLookup[id]
let fileExtension = originalFilename.pop()
newFilename = originalFilename.join(".")
request.push({
url: fileURL,
id: id,
post: postId,
name: newFilename,
ext: fileExtension
})
}
}
for (let d of downloads) {
addFileDownload(d.file, d.id, d.url)
}
console.log(`Sending ${request.length} image links to patreon-dl...`)
await fetch("http://localhost:8080/download", {
method: 'POST',
headers: {"content-type": "application/json"},
body: JSON.stringify({artist: artistName, data: request})
});
console.log("patreon-dl is starting download...")
})();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment