Created
February 21, 2025 00:40
-
-
Save D221/814e8957537abc5e518bba4b442b367d to your computer and use it in GitHub Desktop.
twitch chat convert thing. TwitchDownloaderWPF to yt-dlp / chat-downloader like, for use on archive.ragtag.moe/player. not all badges are displayed, timestamp is little off
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python3 | |
| import argparse | |
| import json | |
| import os | |
| from collections import OrderedDict | |
| from datetime import datetime | |
| def parse_timestamp(timestamp_str): | |
| """ | |
| Parse an ISO 8601 timestamp (with 'Z' or a timezone offset) | |
| and return the Unix epoch time in microseconds. | |
| """ | |
| ts = timestamp_str.replace("Z", "+00:00") | |
| dt = datetime.fromisoformat(ts) | |
| return int(dt.timestamp() * 1_000_000) | |
| def format_time(seconds): | |
| """Convert seconds into a m:ss formatted string.""" | |
| minutes, sec = divmod(seconds, 60) | |
| return f"{minutes}:{sec:02d}" | |
| # --- Badge mapping configuration --- | |
| # https://www.streamdatabase.com/twitch/global-badges - all badge urls too much effort | |
| badge_mappings = { | |
| "premium": { | |
| "clickAction": "VISIT_URL", | |
| "clickURL": "https://gaming.amazon.com", | |
| "icons": [ | |
| { | |
| "height": 18, | |
| "id": "18x18", | |
| "url": "https://static-cdn.jtvnw.net/badges/v1/bbbe0db0-a598-423e-86d0-f9fb98ca1933/1", | |
| "width": 18, | |
| }, | |
| { | |
| "height": 36, | |
| "id": "36x36", | |
| "url": "https://static-cdn.jtvnw.net/badges/v1/bbbe0db0-a598-423e-86d0-f9fb98ca1933/2", | |
| "width": 36, | |
| }, | |
| { | |
| "height": 72, | |
| "id": "72x72", | |
| "url": "https://static-cdn.jtvnw.net/badges/v1/bbbe0db0-a598-423e-86d0-f9fb98ca1933/3", | |
| "width": 72, | |
| }, | |
| ], | |
| "name": "premium", | |
| "title": "Prime Gaming", | |
| "version": 1, | |
| }, | |
| "moderator": { | |
| "icons": [ | |
| { | |
| "height": 18, | |
| "id": "18x18", | |
| "url": "https://static-cdn.jtvnw.net/badges/v1/3267646d-33f0-4b17-b3df-f923a41db1d0/1", | |
| "width": 18, | |
| }, | |
| { | |
| "height": 36, | |
| "id": "36x36", | |
| "url": "https://static-cdn.jtvnw.net/badges/v1/3267646d-33f0-4b17-b3df-f923a41db1d0/2", | |
| "width": 36, | |
| }, | |
| { | |
| "height": 72, | |
| "id": "72x72", | |
| "url": "https://static-cdn.jtvnw.net/badges/v1/3267646d-33f0-4b17-b3df-f923a41db1d0/3", | |
| "width": 72, | |
| }, | |
| ], | |
| "name": "moderator", | |
| "title": "Moderator", | |
| "version": 1, | |
| }, | |
| } | |
| def transform_comment(comment): | |
| """ | |
| Transform one comment (from the original format) into the output message format, | |
| with keys ordered as: | |
| author, emotes, message, message_id, message_type, time_in_seconds, time_text, timestamp | |
| """ | |
| # Process commenter and message data. | |
| commenter = comment.get("commenter", {}) | |
| message = comment.get("message", {}) | |
| # Build the author object as an OrderedDict with desired order: | |
| # badges, colour, display_name, id, name. | |
| author = OrderedDict() | |
| badges = message.get("user_badges", []) | |
| author_badges = [] | |
| for badge in badges: | |
| badge_id = badge.get("_id") | |
| if badge_id in badge_mappings: | |
| author_badges.append(badge_mappings[badge_id]) | |
| if author_badges: | |
| author["badges"] = author_badges | |
| if message.get("user_color"): | |
| author["colour"] = message["user_color"] | |
| author["display_name"] = commenter.get("display_name") | |
| author["id"] = str(commenter.get("_id")) | |
| author["name"] = commenter.get("name") | |
| # Process emotes if any. | |
| emoticons = message.get("emoticons", []) | |
| emotes = [] | |
| if emoticons: | |
| fragments = message.get("fragments", []) | |
| for emo in emoticons: | |
| emo_id = emo.get("_id") | |
| begin = emo.get("begin", 0) | |
| fragment_text = None | |
| for frag in fragments: | |
| if ( | |
| frag.get("emoticon") | |
| and frag["emoticon"].get("emoticon_id") == emo_id | |
| ): | |
| fragment_text = frag.get("text", "") | |
| break | |
| if fragment_text is not None: | |
| fragment_length = len(fragment_text) | |
| else: | |
| fragment_length = emo.get("end", begin) - begin | |
| location_str = f"{begin}-{begin + fragment_length - 1}" | |
| images = [ | |
| { | |
| "height": 28, | |
| "id": "28x28-light", | |
| "url": f"https://static-cdn.jtvnw.net/emoticons/v2/{emo_id}/default/light/1.0", | |
| "width": 28, | |
| }, | |
| { | |
| "height": 56, | |
| "id": "56x56-light", | |
| "url": f"https://static-cdn.jtvnw.net/emoticons/v2/{emo_id}/default/light/2.0", | |
| "width": 56, | |
| }, | |
| { | |
| "height": 112, | |
| "id": "112x112-light", | |
| "url": f"https://static-cdn.jtvnw.net/emoticons/v2/{emo_id}/default/light/3.0", | |
| "width": 112, | |
| }, | |
| { | |
| "height": 28, | |
| "id": "28x28-dark", | |
| "url": f"https://static-cdn.jtvnw.net/emoticons/v2/{emo_id}/default/dark/1.0", | |
| "width": 28, | |
| }, | |
| { | |
| "height": 56, | |
| "id": "56x56-dark", | |
| "url": f"https://static-cdn.jtvnw.net/emoticons/v2/{emo_id}/default/dark/2.0", | |
| "width": 56, | |
| }, | |
| { | |
| "height": 112, | |
| "id": "112x112-dark", | |
| "url": f"https://static-cdn.jtvnw.net/emoticons/v2/{emo_id}/default/dark/3.0", | |
| "width": 112, | |
| }, | |
| ] | |
| emotes.append( | |
| { | |
| "id": emo_id, | |
| "images": images, | |
| "locations": location_str, | |
| "name": fragment_text if fragment_text is not None else "", | |
| } | |
| ) | |
| # Build the top-level message object as an OrderedDict in the desired order. | |
| new_msg = OrderedDict() | |
| new_msg["author"] = author | |
| if emotes: # Only add "emotes" if there is at least one. | |
| new_msg["emotes"] = emotes | |
| new_msg["message"] = message.get("body") | |
| new_msg["message_id"] = comment.get("_id") | |
| new_msg["message_type"] = "text_message" | |
| time_in_seconds = comment.get("content_offset_seconds", 0) | |
| new_msg["time_in_seconds"] = time_in_seconds | |
| new_msg["time_text"] = format_time(time_in_seconds) | |
| created_at = comment.get("created_at") | |
| new_msg["timestamp"] = parse_timestamp(created_at) if created_at else 0 | |
| return new_msg | |
| def main(): | |
| parser = argparse.ArgumentParser( | |
| description="Convert JSON format from one structure to another" | |
| ) | |
| parser.add_argument("input_file", help="Input JSON file to convert") | |
| args = parser.parse_args() | |
| input_file = args.input_file | |
| base, ext = os.path.splitext(input_file) | |
| if not ext: | |
| ext = ".json" | |
| output_file = f"{base}_conv{ext}" | |
| with open(input_file, "r", encoding="utf-8") as infile: | |
| data = json.load(infile) | |
| comments = data.get("comments", []) | |
| output_messages = [transform_comment(comment) for comment in comments] | |
| with open(output_file, "w", encoding="utf-8") as outfile: | |
| json.dump(output_messages, outfile, indent=4) | |
| print(f"Conversion complete. Output written to {output_file}") | |
| if __name__ == "__main__": | |
| main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment