Skip to content

Instantly share code, notes, and snippets.

@fqborges

fqborges/cleanup-roms.sh

Last active March 13, 2026 08:53
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save fqborges/bab702def885869d841aea697ffc5218 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save fqborges/bab702def885869d841aea697ffc5218 to your computer and use it in GitHub Desktop.
Download ZIP
rsync.myrient.erista.me/files/No-Intro
Raw
cleanup-roms.sh
#!/bin/sh
mkdir -p ./trash
find . -maxdepth 1 -name '*.zip' \
| grep -vF -f .meta/roms-selected.txt \
| xargs --no-run-if-empty -d '\n' -l1 mv -v -t ./trash
Raw
select-roms.sh
#!/bin/bash
function setup_system() {
SYSTEMS=(
'Nintendo - Game Boy'
'Nintendo - Game Boy Advance'
'Nintendo - Game Boy Color'
'Nintendo - Nintendo 64 (BigEndian)'
'Nintendo - Nintendo Entertainment System (Headered)'
'Nintendo - Super Nintendo Entertainment System'
'Sega - Mega Drive - Genesis'
'Sega - Master System - Mark III'
);
echo "Setting up rom sources"
echo "Please select a system, or ctrl+c to cancel"
select SYSTEM in "${SYSTEMS[@]}"
do
if [ -z "$SYSTEM" ]; then
:
else
echo "Selected '$SYSTEM', saving to .meta/system.txt"
break;
fi
done
echo "$SYSTEM" > .meta/system.txt
}
function default_allowlist_contents() {
cat << EOF > $1
# remove this line to allow all, add one filter per line to allow specific games
EOF
}
function default_blocklist_contents() {
cat << EOF > $1
(Sample)
(Program)
(Pirate)
(Demo)
(Aftermarket)
(Proto)
(Proto 1)
(Proto 2)
(Beta)
(Beta 1)
(Beta 2)
(Beta 3)
[BIOS]
EOF
}
function default_ranking_contents() {
cat << EOF > $1
Brazil
USA
World
Europe
Collection)
EOF
}
function get_available_rom_list() {
SYSTEM=$1
URL="rsync://rsync.myrient.erista.me/files/No-Intro/${SYSTEM}/";
rsync -nvr "${URL}" . \
| grep '.zip$'
}
function filter_rom_allowlist() {
ROM_LIST=$1
FILTER_FILE=$2
grep --fixed-strings --ignore-case --word-regexp \
-f $FILTER_FILE \
< $ROM_LIST
}
function filter_rom_blocklist() {
ROM_LIST=$1
FILTER_FILE=$2
grep --fixed-strings --ignore-case --word-regexp \
--invert-match \
-f $FILTER_FILE \
< $ROM_LIST
}
function game_names_from_rom_list() {
FILE_LIST=$1
while read file; do echo ${file%% (*}; done \
< ${FILE_LIST} \
| uniq
}
function pattern_from_game_name() {
pattern=$1
pattern=${pattern//[/\\\[}
pattern="^$pattern ("
echo $pattern
}
function list_roms_for_game() {
game=$1
rom_list=$2
pattern=$(pattern_from_game_name "$game")
grep "$pattern" $rom_list
}
# setup temp dir
rm -rf .temp && mkdir -p .temp
mkdir -p .temp/games
# setup meta dir
mkdir -p .meta
if [ ! -f .meta/allowlist.txt ]; then default_allowlist_contents .meta/allowlist.txt; fi
if [ ! -f .meta/blocklist.txt ]; then default_blocklist_contents .meta/blocklist.txt; fi
if [ ! -f .meta/ranking.txt ]; then default_ranking_contents .meta/ranking.txt; fi
if [ ! -f .meta/system.txt ]; then setup_system; fi
read -r SYSTEM < .meta/system.txt
get_available_rom_list "$SYSTEM" > .meta/roms-available.txt
echo "filtering rom list"
cp -f .meta/roms-available.txt .temp/roms-filtered.txt
if [ -s .meta/allowlist.txt ]; then
cp -f .temp/roms-filtered.txt .temp/roms-to-filter.txt
filter_rom_allowlist .temp/roms-to-filter.txt .meta/allowlist.txt \
> .temp/roms-filtered.txt
fi
if [ -s .meta/blocklist.txt ]; then
cp -f .temp/roms-filtered.txt .temp/roms-to-filter.txt
filter_rom_blocklist .temp/roms-to-filter.txt .meta/blocklist.txt \
> .temp/roms-filtered.txt
fi
echo "getting game names from rom list"
game_names_from_rom_list .temp/roms-filtered.txt \
> .temp/games.txt
echo "listing roms by games"
while read game
do
list_roms_for_game "$game" .temp/roms-filtered.txt \
> ".temp/games/$game"
done < .temp/games.txt
echo "select roms for games"
while read game
do
file=".temp/games/$game"
count=$(wc -l < "$file")
if [[ "$count" == "1" ]]; then continue; fi
# read lines raw, avoiding trimming leading whitespace and interpreting backslash sequences
# https://stackoverflow.com/a/1521498
while IFS="" read -r TAG || [ -n "$TAG" ]
do
mv "$file" "$file.temp"
grep -F -e "$TAG" "$file.temp" > "$file"
count=$(wc -l < "$file")
if [[ "$count" == "0" ]]; then
# filtered too much: revert filter
mv "$file.temp" "$file"
elif [[ "$count" == "1" ]]; then
# got expected answer: stop
break;
fi
done < .meta/ranking.txt
rm -f "$file.temp"
done < .temp/games.txt
echo "checking games with more than one rom or none"
while read game
do
count=$(wc -l < ".temp/games/$game")
if [[ "$count" > "1" ]]; then
echo "[$count]" "$game"
cat ".temp/games/$game"
fi
done < .temp/games.txt
echo "generating final rom list"
dest=.meta/roms-selected.txt
rm -f "$dest"
touch "$dest"
while read game
do
cat ".temp/games/$game" >> "$dest"
done < .temp/games.txt
echo "final rom list:"
echo total roms: $(wc -l < "$dest")
if [ ! -s "$dest" ]; then
echo "====================="
echo "= NO ROMS SELECTED! ="
echo "====================="
echo "filters on .meta/allowlist.txt or .meta/blocklist.txt are to restritive."
echo '---.meta/allowlist.txt'
cat .meta/allowlist.txt
echo '---'
echo '---.meta/blocklist.txt'
cat .meta/blocklist.txt
echo '---'
echo "====================="
else
echo '---'
head "$dest"
echo '...'
tail "$dest"
echo '---'
fi
# cleanup
rm -rf .temp
Raw
sync-roms.sh
#!/bin/sh
if [ ! -f .meta/system.txt ];
then echo missing .meta/system.txt, exiting;
exit -1
fi
if [ ! -f .meta/roms-selected.txt ];
then echo missing .meta/roms-selected.txt, exiting;
exit -1
fi
read -r SYSTEM < .meta/system.txt
rsync --stats -hir --delete --files-from=.meta/roms-selected.txt "rsync://rsync.myrient.erista.me/files/No-Intro/$SYSTEM/" .
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment