-
-
Save Winterhuman/21d7b148db40ff041f397b07a7aafb83 to your computer and use it in GitHub Desktop.
| #!/usr/bin/env -S unshare --pid --mount-proc --kill-child --map-root-user /bin/sh | |
| # Licensed under the Zero-Clause BSD terms: https://opensource.org/license/0bsd | |
| ## Requires: find, oxipng, and gifsicle | |
| ## Optional: pngquant, cwebp & gif2webp, gif2apng, and (perl-image-)exiftool | |
| ## -C: Fail if redirects try to overwrite an existing file. | |
| ## -e: Fail if any command fails (with exceptions). | |
| ## -u: Fail if an unset variable tries to be expanded. | |
| ## -f: No glob expansion. | |
| set -Ceuf | |
| perr() { printf "\a\033[1m\033[31m%b\033[0;39m" "$1" >&2; } | |
| clear_prog() { printf "\033]9;4;0\007"; } | |
| exit_clear() { clear_prog; exit 1; } | |
| osc777() { printf "\a\033]777;notify;Sisyphus;%b\033\\" "$1"; } | |
| nexit() { osc777 "$1"; exit_clear; } | |
| pquit() { perr "$1"; nexit "$1"; } | |
| pstat() { printf "\033[1m\033[34m%b\033[0;39m\033[1m%b\033[0;39m" "$1" "$2"; } | |
| ## `kill` handles background jobs, but `exit` is required for normal processes. | |
| ### The second trap handles unexpected signals, where a notification IS desired. | |
| #### Trying to trap SIGTERM leads to a "Segmentation fault" error | |
| trap 'kill "$$"; exit_clear' INT | |
| trap 'kill "$$"; nexit "SIGQUIT or SIGABRT received! Was operating on: $1"' QUIT ABRT | |
| help() { | |
| cat <<"HELP"; exit 0 | |
| Usage: sisyphus [OPTION]... SRC DEST | |
| Losslessly optimise PNGs & GIFs by all known means. | |
| If DEST is omitted, DEST is set to '{SRC wo/ext}.new.ext'. Possible output formats | |
| include: PNG, GIF, WEBP, and APNG. | |
| Options: | |
| -a, --all-oxi <bool> Optionally takes a boolean value. Controls whether to always, | |
| or never, try all OxiPNG variations. If unset, heuristically | |
| determine whether to try every OxiPNG variation. | |
| -f, --force Overwrite existing destination files. | |
| -m, --max-procs <int> Takes an integer greater than zero. Limits the number of | |
| simultaneous processes. The default is '8'. | |
| -n, --no-webp Do not use CWebP nor GIF2WebP. | |
| -N, --no-apng Do not use APNG, aka. 'gif2apng'. | |
| -q, --quiet Do not print messages to STDOUT, nor send notifications. | |
| -r, --results <arg> Optionally takes either a boolean value or a string. | |
| Controls how the results list is printed, if at all. | |
| If <arg> is an integer, only the top <arg> trials will | |
| be printed. | |
| -s, --size <int> Takes an integer in bytes. Sets the minimum qualifying size. | |
| If the best encoding is greater or equal to the minimum size, | |
| then DEST is not created. | |
| -h, --help Display this help message, and then exit. | |
| Warning: | |
| HDR SRC images will NOT be losslessly optimised! | |
| HELP | |
| } | |
| # Setup a private tmpfs for this script to use, which is what 'unshare' is for. | |
| ## 'nr_inodes' must be >= to 'max number of files + 1' | |
| work="/tmp" | |
| mount -t tmpfs -o nosuid,nodev,noexec,size=50%,nr_inodes=75776 sisyphus "$work"/ || | |
| pquit "Failed to overmount '$work/'!\n" | |
| # Argument parsing | |
| ALL_OXI="" FORCE="" MAX_PROCS="8" NO_WEBP="" NO_APNG="" | |
| QUIET="" RESULTS="0" SIZE="" INTERNAL="" SKIP="" | |
| ARG_COUNT="0" | |
| while [ "$ARG_COUNT" -lt "$#" ]; do | |
| if [ -z "$SKIP" ]; then | |
| case "$1" in | |
| -a|--all-oxi) | |
| case "${2:-}" in | |
| 0|no|false) ALL_OXI="0"; shift ;; | |
| 1|yes|true) ALL_OXI="1"; shift ;; | |
| *) ALL_OXI="1" ;; | |
| esac | |
| ;; | |
| -f|--force) FORCE="1" ;; | |
| -m|--max-procs) | |
| case "${2:-}" in | |
| [1-9]*) MAX_PROCS="$2"; shift ;; | |
| *) pquit "No positive integer greater than zero was given for '-m|--max-procs'!\n" ;; | |
| esac | |
| ;; | |
| -n|--no-webp) NO_WEBP="1" ;; | |
| -N|--no-apng) NO_APNG="1" ;; | |
| -q|--quiet) | |
| ## '--quiet' already inhibits the list; however, | |
| ## '--results 0' also saves doing other steps | |
| QUIET="1" RESULTS="0" | |
| ## Functions referencing each other need to be | |
| ## defined again to be updated. | |
| ### This `perr()` omits `\a`. "$@" handles args | |
| perr() { printf "\033[1m\033[31m%b\033[0;39m" "$1" >&2; } | |
| nexit() { : "$@"; exit_clear; } | |
| pquit() { perr "$1"; exit_clear; } | |
| pstat() { : "$@"; } | |
| ;; | |
| -r|--results) | |
| case "${2:-}" in | |
| no|false|hide|none) shift ;; | |
| yes|true|show|all) RESULTS="ALL"; shift ;; | |
| [0-9]*) RESULTS="$2"; shift ;; | |
| *) pquit "An unknown value was given for '-r|--results'!\n" ;; | |
| esac | |
| ;; | |
| -s|--size) | |
| case "${2:-}" in | |
| [0-9]*) SIZE="$2"; shift ;; | |
| *) pquit "No positive integer was given for '-s|--size'!\n" ;; | |
| esac | |
| ;; | |
| -h|--help) help ;; | |
| --_internal) | |
| ## DO NOT USE MANUALLY. For the to-GIF function | |
| INTERNAL="1" | |
| ;; | |
| --) SKIP="1" ;; | |
| -*) pquit "'$1' is not a known option!\n" ;; | |
| *) set -- "$@" "$1"; ARG_COUNT="$(( ARG_COUNT + 1 ))" ;; | |
| esac | |
| else set -- "$@" "$1"; ARG_COUNT="$(( ARG_COUNT + 1 ))" | |
| fi | |
| shift | |
| done | |
| # Validate arguments | |
| ## `[ cond ] && cmd` will carry over non-zero exit codes; always use `||` | |
| [ "$#" -le 2 ] || pquit "Too many arguments given!\n" | |
| ## Don't try WebP if `cwebp` & `gif2webp` aren't available | |
| command -v cwebp >/dev/null || NO_WEBP="1" | |
| command -v gif2webp >/dev/null || NO_WEBP="1" | |
| ## Don't try APNG if `gif2apng` isn't available | |
| command -v gif2apng >/dev/null || NO_APNG="1" | |
| ## Don't try `pngquant` if it's not available | |
| NO_QUANT="" | |
| command -v pngquant >/dev/null || NO_QUANT="1" | |
| ## Dont' try anything if `find` isn't available. | |
| command -v find >/dev/null || pquit "The 'find' command is required!\n" | |
| ## SRC | |
| src_real="${1:?$(pquit "No source file was specified!\n")}" | |
| [ -s "$src_real" ] || pquit "'$src_real' is not a non-empty file!\n" | |
| ### Assign the SRC a file descriptor | |
| exec 3<"$src_real" | |
| src="/proc/self/fd/3" | |
| src_real="$(realpath -- "$src_real")" | |
| ### Size detection | |
| find_size() { | |
| ## Handle non-existent files by giving them arbitrarily huge sizes | |
| if [ -f "$1" ]; then | |
| wc -c <"$1" | |
| else | |
| printf "99999999999999" | |
| fi | |
| } | |
| src_size="$(find_size "$src")" | |
| ### Minimum size target validation | |
| check_number() { | |
| [ "$1" != "" ] || return 0 | |
| num="$1" oldnum="$num" option="$2" | |
| ## Strip leading zeros and avoid integer overflow | |
| num="$(printf "%s" "$num" | sed "s/^0*//")" | |
| ## If `sed` trimmed '0' to nothing, undo that | |
| num="${num:-"0"}" | |
| if ! printf "%d" "$num" >/dev/null 2>&1; then | |
| perr "The value given for '$option' is not an integer, or is far beyond the integer limit!\nValue: $oldnum\n" | |
| return 1 | |
| fi | |
| if [ "$num" -ne "$(printf "%s" "$num" | cut -c -12)" ]; then | |
| perr "The value given for '$option' is over the integer limit!\nValue: $oldnum\n" | |
| return 1 | |
| fi | |
| printf "%d" "$num" | |
| } | |
| SIZE="$(check_number "$SIZE" "-s|--size")" | |
| ### Max processes validation | |
| MAX_PROCS="$(check_number "$MAX_PROCS" "-m|--max-procs")" | |
| ## Results validation | |
| [ "$RESULTS" = "ALL" ] || RESULTS="$(check_number "$RESULTS" "-r|--results")" | |
| ### Mimetype detection | |
| mime="$(file --dereference --brief --mime-type -- "$src")" | |
| src_height="$(file --dereference --brief -- "$src" | | |
| sed -n "s/.*x \([[:digit:]]\+\).*/\1/p")" | |
| src_height="$(check_number "$src_height" "\$src_height")" | |
| ## DEST | |
| dest_path="${2:-}" | |
| dest_path="${dest_path%.*}" | |
| dest_path="${dest_path:-${src_real%.*}.new}" | |
| dest_path="$(realpath -- "$dest_path")" | |
| dest_exists() { | |
| [ -z "$FORCE" ] || return 0 | |
| ## Allow globbing for just this for-loop | |
| set +f | |
| for similar in "$1"*; do | |
| [ -e "$similar" ] || continue | |
| if [ "${similar%.*}" = "$1" ]; then | |
| perr "'$similar' shares a filename with '$1'!\n" | |
| return 1 | |
| fi | |
| done | |
| set -f | |
| } | |
| dest_exists "$dest_path" || | |
| nexit "'$similar' shares a filename with '$dest_path'!\n" | |
| pstat "Input:\t\t\t" "$src_real \033[37m($mime)\033[0;39m\n" | |
| pstat "Output template:\t" "$dest_path.???\n" | |
| [ -z "$SIZE" ] || pstat "Size target:\t\t" "$SIZE bytes\n" | |
| # Optimisation | |
| ## In case of PNG | |
| create_png_list() { | |
| ## `--brute-{level,lines}` only applies to the brute filter (`-f 9`). | |
| zc="1" | |
| while [ "$zc" -le 12 ]; do | |
| pzc="$(printf "%-2s" "$zc")" | |
| cat <<-ZC | |
| --zc=$pzc --filters=0-8 | |
| --zc=$pzc --filters=0-8 --nb | |
| --zc=$pzc --filters=0-8 --nc | |
| --zc=$pzc --filters=0-8 --nb --nc | |
| --zc=$pzc --filters=0-8 --ng | |
| --zc=$pzc --filters=0-8 --nb --ng | |
| --zc=$pzc --filters=0-8 --nc --ng | |
| --zc=$pzc --filters=0-8 --nb --nc --ng | |
| --zc=$pzc --filters=0-8 --np | |
| --zc=$pzc --filters=0-8 --nb --np | |
| --zc=$pzc --filters=0-8 --nc --np | |
| --zc=$pzc --filters=0-8 --nb --nc --np | |
| --zc=$pzc --filters=0-8 --ng --np | |
| --zc=$pzc --filters=0-8 --nb --ng --np | |
| --zc=$pzc --filters=0-8 --nc --ng --np | |
| --zc=$pzc --filters=0-8 --nb --nc --ng --np | |
| ZC | |
| zc="$(( zc + 1 ))" | |
| done | |
| cat <<-ZOP | |
| --zopfli --zi=1024 --ziwi=16 --filters=0-8 | |
| --zopfli --zi=1024 --ziwi=16 --filters=0-8 --nb | |
| --zopfli --zi=1024 --ziwi=16 --filters=0-8 --nc | |
| --zopfli --zi=1024 --ziwi=16 --filters=0-8 --nb --nc | |
| --zopfli --zi=1024 --ziwi=16 --filters=0-8 --ng | |
| --zopfli --zi=1024 --ziwi=16 --filters=0-8 --nb --ng | |
| --zopfli --zi=1024 --ziwi=16 --filters=0-8 --nc --ng | |
| --zopfli --zi=1024 --ziwi=16 --filters=0-8 --nb --nc --ng | |
| --zopfli --zi=1024 --ziwi=16 --filters=0-8 --np | |
| --zopfli --zi=1024 --ziwi=16 --filters=0-8 --nb --np | |
| --zopfli --zi=1024 --ziwi=16 --filters=0-8 --nc --np | |
| --zopfli --zi=1024 --ziwi=16 --filters=0-8 --nb --nc --np | |
| --zopfli --zi=1024 --ziwi=16 --filters=0-8 --ng --np | |
| --zopfli --zi=1024 --ziwi=16 --filters=0-8 --nb --ng --np | |
| --zopfli --zi=1024 --ziwi=16 --filters=0-8 --nc --ng --np | |
| --zopfli --zi=1024 --ziwi=16 --filters=0-8 --nb --nc --ng --np | |
| ZOP | |
| ## `--brute-lines` only makes sense up to the height of the SRC image | |
| blin_max="$(( | |
| src_height > 16 | |
| ? 16 | |
| : src_height | |
| ))" | |
| blev="1" | |
| while [ "$blev" -le 12 ]; do | |
| blin="2" | |
| while [ "$blin" -le "$blin_max" ]; do | |
| ## Ensure numbers like '1' and '12' are the same length | |
| pblev="$(printf "%-2s" "$blev")" | |
| pblin="$(printf "%-2s" "$blin")" | |
| zc="1" | |
| while [ "$zc" -le 12 ]; do | |
| pzc="$(printf "%-2s" "$zc")" | |
| ## `${pblin% *}` prevents "... lines=N .png" | |
| cat <<-ZCF9 | |
| --zc=$pzc --filters=9 --brute-level=$pblev --brute-lines=${pblin% *} | |
| --zc=$pzc --filters=9 --brute-level=$pblev --brute-lines=$pblin --nb | |
| --zc=$pzc --filters=9 --brute-level=$pblev --brute-lines=$pblin --nc | |
| --zc=$pzc --filters=9 --brute-level=$pblev --brute-lines=$pblin --nb --nc | |
| --zc=$pzc --filters=9 --brute-level=$pblev --brute-lines=$pblin --ng | |
| --zc=$pzc --filters=9 --brute-level=$pblev --brute-lines=$pblin --nb --ng | |
| --zc=$pzc --filters=9 --brute-level=$pblev --brute-lines=$pblin --nc --ng | |
| --zc=$pzc --filters=9 --brute-level=$pblev --brute-lines=$pblin --nb --nc --ng | |
| --zc=$pzc --filters=9 --brute-level=$pblev --brute-lines=$pblin --np | |
| --zc=$pzc --filters=9 --brute-level=$pblev --brute-lines=$pblin --nb --np | |
| --zc=$pzc --filters=9 --brute-level=$pblev --brute-lines=$pblin --nc --np | |
| --zc=$pzc --filters=9 --brute-level=$pblev --brute-lines=$pblin --nb --nc --np | |
| --zc=$pzc --filters=9 --brute-level=$pblev --brute-lines=$pblin --ng --np | |
| --zc=$pzc --filters=9 --brute-level=$pblev --brute-lines=$pblin --nb --ng --np | |
| --zc=$pzc --filters=9 --brute-level=$pblev --brute-lines=$pblin --nc --ng --np | |
| --zc=$pzc --filters=9 --brute-level=$pblev --brute-lines=$pblin --nb --nc --ng --np | |
| ZCF9 | |
| zc="$(( zc + 1 ))" | |
| done | |
| ## Don't duplicate the `--zopfli` lines per `--zc` level | |
| cat <<-ZOPF9 | |
| --zopfli --zi=1024 --ziwi=16 --filters=9 --brute-level=$pblev --brute-lines=${pblin% *} | |
| --zopfli --zi=1024 --ziwi=16 --filters=9 --brute-level=$pblev --brute-lines=$pblin --nb | |
| --zopfli --zi=1024 --ziwi=16 --filters=9 --brute-level=$pblev --brute-lines=$pblin --nc | |
| --zopfli --zi=1024 --ziwi=16 --filters=9 --brute-level=$pblev --brute-lines=$pblin --nb --nc | |
| --zopfli --zi=1024 --ziwi=16 --filters=9 --brute-level=$pblev --brute-lines=$pblin --ng | |
| --zopfli --zi=1024 --ziwi=16 --filters=9 --brute-level=$pblev --brute-lines=$pblin --nb --ng | |
| --zopfli --zi=1024 --ziwi=16 --filters=9 --brute-level=$pblev --brute-lines=$pblin --nc --ng | |
| --zopfli --zi=1024 --ziwi=16 --filters=9 --brute-level=$pblev --brute-lines=$pblin --nb --nc --ng | |
| --zopfli --zi=1024 --ziwi=16 --filters=9 --brute-level=$pblev --brute-lines=$pblin --np | |
| --zopfli --zi=1024 --ziwi=16 --filters=9 --brute-level=$pblev --brute-lines=$pblin --nb --np | |
| --zopfli --zi=1024 --ziwi=16 --filters=9 --brute-level=$pblev --brute-lines=$pblin --nc --np | |
| --zopfli --zi=1024 --ziwi=16 --filters=9 --brute-level=$pblev --brute-lines=$pblin --nb --nc --np | |
| --zopfli --zi=1024 --ziwi=16 --filters=9 --brute-level=$pblev --brute-lines=$pblin --ng --np | |
| --zopfli --zi=1024 --ziwi=16 --filters=9 --brute-level=$pblev --brute-lines=$pblin --nb --ng --np | |
| --zopfli --zi=1024 --ziwi=16 --filters=9 --brute-level=$pblev --brute-lines=$pblin --nc --ng --np | |
| --zopfli --zi=1024 --ziwi=16 --filters=9 --brute-level=$pblev --brute-lines=$pblin --nb --nc --ng --np | |
| ZOPF9 | |
| blin="$(( blin + 1 ))" | |
| done | |
| blev="$(( blev + 1 ))" | |
| done | |
| } | |
| create_png_oxi() { | |
| cat <<-"OXI" | |
| ## `noexec` on `$work` prevents having `#!/bin/sh` here instead | |
| set -Ceuf | |
| ## Redirect STDERR to FD 9. This will be reverted later. | |
| ### This doesn't mean processes won't try to write to FD 2 though | |
| 2>&9 | |
| perr() { | |
| printf "\a\033[1m\033[31m%b\033[0;39m" "$1" | |
| printf "\033]777;notify;Sisyphus;%b\033\\" "$1" | |
| } | |
| ## `exit 255` prevents `xargs` from processing any more line batches (the | |
| ## batches from `--max-procs`) | |
| pquit() { perr "$1"; exit 255; } | |
| options="$1" safe="$2" quant_src="$3" smallest_known="$4" | |
| total_lines="$5" work="$6" oxi_baseline_dest="$7" | |
| src="/proc/self/fd/3" tmp_list="/proc/self/fd/4" proginfo="/proc/self/fdinfo/8" | |
| ## Print the progress. | |
| ### Print it now in case something ends early (e.g. baseline output skip) | |
| progress() { | |
| line_index="$(sed "\|$options|q" "$tmp_list" | wc --lines)" | |
| fd_index="$(sed --quiet "/pos/s/.*[[:space:]]//p;q" "$proginfo")" | |
| index_diff="$(( line_index - fd_index ))" | |
| ### Only advance '$fd_index' if there's been forward progress | |
| [ "$index_diff" -gt 0 ] || return 0 | |
| ### By reading FD 8, the position value increases; it will never decrease | |
| dd bs="$index_diff" count=1 <&8 >/dev/null 2>&1 || return 0 | |
| ### An equivalent to '$progress_diff' wouldn't account for extra rounding | |
| progress="$(( ( fd_index * 100 ) / total_lines ))" | |
| next_progress="$(( ( line_index * 100 ) / total_lines ))" | |
| ### Don't update the progress bar if it probably won't change its percent | |
| [ "$(( next_progress - progress ))" -gt 0 ] || return 0 | |
| printf "\033]9;4;1;%d\007" "$progress" | |
| } | |
| progress ||: | |
| ## If '$quant_src' exists (meaning the variable is set), handle its variants | |
| if [ -n "${quant_src:-}" ]; then | |
| ## If "quant" is in '$options', use '$quant_src' as '$src' | |
| [ "$options" = "${options#*quant}" ] || src="$quant_src" | |
| fi | |
| ## Construct the output path based on the options. | |
| ### "... /zop /g" ensures "zop" and "zc1" are the same length/spacing | |
| out="$work/$( | |
| printf "%b" "$options" | sed \ | |
| -e "s/--zc=/zc/g" \ | |
| -e "s/--zopfli.*--ziwi=16/zop /g" \ | |
| -e "s/--filters=/--f/g" \ | |
| -e "s/brute-lines/lines/g" \ | |
| -e "s/brute-level/level/g" \ | |
| -e "s/[[:space:]]--/ \+ /g" | |
| ).png" | |
| ## Check if the output (e.g. '$oxi_baseline_dest') already exists | |
| [ ! -f "$out" ] || exit 0 | |
| ## Execute `oxipng` with additional options & arguments. | |
| ### For whatever reason, `env` doesn't accept `oxipng "$options"` | |
| env --split-string \ | |
| "oxipng $options" \ | |
| --alpha \ | |
| --strip "$safe" \ | |
| --out "$out" \ | |
| -- "$src" >/dev/null 2>&1 || | |
| pquit "'oxipng $options --alpha --strip $safe --out $out -- $src' failed!\n" | |
| ## Check if the output exists now | |
| [ -f "$out" ] | |
| ## If the size of '$out' isn't less than '$smallest_known', then fake its size | |
| ## so `wc` can read it (but, it'll take no space) | |
| discard() { | |
| trial="$(realpath -- "$1")" | |
| baseline="$(realpath -- "$2")" | |
| [ -s "$trial" ] || return 0 | |
| [ -s "$baseline" ] || | |
| perr "'$baseline' does not exist, or has no content, but should!\n" | |
| if cmp "$1" "$2" >/dev/null; then | |
| ## This handles otherwise truncated trials which are | |
| ## ordered before '$baseline' in the final list | |
| ln --force --symbolic "$baseline" -- "$trial" \ | |
| 2>/dev/null || | |
| perr "Couldn't replace '$trial' with a symlink!\n" | |
| else | |
| trial_size="$(wc -c <"$trial")" | |
| if [ "$smallest_known" -le "$trial_size" ]; then | |
| truncate --size=0 -- "$trial" | |
| truncate --size="$trial_size" -- "$trial" | |
| fi | |
| fi | |
| } | |
| ## If '$quant_src' is set, use it for the comparison instead of '$src' | |
| if [ -n "${quant_src:-}" ]; then | |
| discard "$out" "$quant_src" | |
| else | |
| discard "$out" "$oxi_baseline_dest" | |
| fi | |
| OXI | |
| } | |
| try_oxi_vars() { | |
| # Generate the command list | |
| ## `pngquant` strips APNGs; therefore, skip it if '$safe' is being used | |
| quant_src="$work/quant.png" | |
| if [ "$safe" = "all" ] && [ -z "$NO_QUANT" ]; then | |
| ## `--quality 100-100` means '$quant_src' won't exist if it can't | |
| ## be created losslessly | |
| pngquant --quality 100-100 --speed 1 --strip \ | |
| --output "$quant_src" -- "$src" || | |
| perr "Failed to create '$quant_src'!\n" | |
| ## If '$src' was already quantised beforehand, '$quant_src' will | |
| ## be identical to '$src'. Therefore, check & remove duplicates | |
| ! cmp -- "$src" "$quant_src" >/dev/null || | |
| rm -- "$quant_src" || | |
| perr "Failed to remove '$quant_src'!\n" | |
| fi & | |
| ## Generate a list of all the commands to be passed to `xargs`. | |
| ### This is hidden later; see the comment after `[ -n "${quant_src:-}" ]` | |
| tmp_list="$work/list" | |
| create_png_list >>"$tmp_list" | |
| ## Wait for `pngquant` to finish if it's still being executed | |
| wait | |
| ## If '$quant_src' wasn't created, unset its variable | |
| [ -f "$quant_src" ] || unset quant_src | |
| ## If '$quant_src' is set, append the list with the `pngquant` variants | |
| if [ -n "${quant_src:-}" ]; then | |
| quant_list="$work/quant_list" | |
| ## `sed "..." file >>file` leads to input buffering issues. | |
| ### `cat` always prints in argument order; STDIN will be appended | |
| sed "s|^|quant > |" "$tmp_list" | | |
| cat -- "$tmp_list" - >"$quant_list" | |
| mv -- "$quant_list" "$tmp_list" | |
| fi | |
| ## `mv` can't handle file descriptors; so instead, hide the file now. | |
| ### If `exec` is ran before `mv`, the old '$tmp_list' contents are left | |
| ### accessible at FD 4 | |
| exec 4<"$tmp_list" | |
| rm -- "$tmp_list" | |
| tmp_list_real="$tmp_list" | |
| tmp_list="/proc/self/fd/4" | |
| ## '$total_lines' is read by `$tmp_oxi`; it skips repetitive calculations. | |
| total_lines="$(wc --lines <"$tmp_list")" | |
| ## The read position of FD 8 tracks the progress | |
| progress_file="$work/progress" | |
| truncate --size="$total_lines" -- "$progress_file" | |
| exec 8<"$progress_file" | |
| rm -- "$progress_file" | |
| ## Check if the list was sufficiently populated | |
| [ "$total_lines" -gt 12 ] || pquit "'$tmp_list_real' was truncated!\n" | |
| # Create the `xargs` script | |
| tmp_oxi="$work/oxi" | |
| :>"$tmp_oxi" | |
| exec 5<"$tmp_oxi" | |
| rm -- "$tmp_oxi" | |
| tmp_oxi="/proc/self/fd/5" | |
| ## Avoid using `sh -c ''` inside `xargs` by creating an external script. | |
| ### Heredocs can only be read once before their buffers are emptied; | |
| ### therefore, write the script to a hidden file | |
| create_png_oxi >|"$tmp_oxi" | |
| # Pass each command in the list to '$tmp_oxi' | |
| ## `9>&2` redirects the script's STDERR to FD 2, while redirecting the | |
| ## STDERR of `xargs` to `/dev/null`. `exit 255` causes the STDERR logs | |
| xargs \ | |
| --max-procs "$MAX_PROCS" \ | |
| --delimiter "\n" \ | |
| --replace="%" \ | |
| -- sh "$tmp_oxi" \ | |
| "%" \ | |
| "$safe" \ | |
| "${quant_src:-}" \ | |
| "$smallest_known" \ | |
| "$total_lines" \ | |
| "$work" \ | |
| "$oxi_baseline_dest" \ | |
| <"$tmp_list" 9>&2 2>/dev/null || | |
| exit_clear | |
| } | |
| src_is_png() { | |
| # Create the CWebP & baseline OxiPNG outputs for the heuristics | |
| cwebp_dest="$work/cwebp.webp" | |
| ## `--brute-{level,lines}` only applies to the brute filter (9). | |
| ## Also, `--brute-lines` values greater than the SRC height aren't useful | |
| baseline_max_blin="$(( | |
| src_height > 8 | |
| ? 8 | |
| : src_height | |
| ))" | |
| oxi_baseline_dest="$work/zc12 + f9 + level=5 + lines=$baseline_max_blin.png" | |
| ## `$safe` = "safe" is required when SRC is an APNG image. | |
| ### Controlled by the `--_internal` option that's set when self-executing | |
| safe="all" | |
| [ -z "$INTERNAL" ] || safe="safe" | |
| if [ "$NO_WEBP" != 1 ]; then | |
| ## Specifying the `-q` or `-m` options disables lossless mode | |
| cwebp -quiet -mt -z 9 -alpha_filter best -o "$cwebp_dest" \ | |
| -- "$src" || | |
| perr "Passing '$src_real' through 'cwebp' failed!\n" | |
| fi & | |
| oxipng --filters 9 --brute-level=5 --brute-lines="$baseline_max_blin" \ | |
| --strip "$safe" --alpha --out "$oxi_baseline_dest" -- "$src" \ | |
| >/dev/null 2>&1 || | |
| perr "Passing '$src_real' through 'oxipng' (zc12 + f9 + level=5 + lines=$baseline_max_blin) failed!\n" | |
| wait | |
| ## Skip trying the OxiPNG variants if explicitly requested | |
| [ "$ALL_OXI" != "0" ] || return 0 | |
| # Heuristics | |
| oxi_baseline_size="$(find_size "$oxi_baseline_dest")" | |
| cwebp_size="$(find_size "$cwebp_dest")" | |
| smallest_heuristic="$(( | |
| oxi_baseline_size < cwebp_size | |
| ? oxi_baseline_size | |
| : cwebp_size | |
| ))" | |
| ## '$smallest_known' gets read by `try_oxi_vars` | |
| smallest_known="$(( | |
| smallest_heuristic < src_size | |
| ? smallest_heuristic | |
| : src_size | |
| ))" | |
| ## If '$SIZE' was given, factor it into the smallest-known size value | |
| [ -z "$SIZE" ] || | |
| smallest_known="$(( | |
| smallest_heuristic < SIZE | |
| ? smallest_heuristic | |
| : SIZE | |
| ))" | |
| # Try the OxiPNG variants if: | |
| # 1. Explicitly requested. | |
| # 2. Else, if the baseline OxiPNG's size is less or equal to 1KiB. | |
| # 3. Else, if the input image is still the smallest known encoding. | |
| if [ "$ALL_OXI" = 1 ] || | |
| [ "$oxi_baseline_size" -le 256 ] || | |
| [ "$src_size" -le "$smallest_heuristic" ]; then | |
| try_oxi_vars | |
| fi | |
| } | |
| ## In case of GIF | |
| create_gif_list() { | |
| ## `gif2apng` doesn't have an output option; it has to be handled later | |
| cat <<-GIFLIST | |
| gif2webp -quiet -mt -min_size -m 6 -q 100 -metadata none -o "$gif2webp_dest" | |
| gifsicle --optimize=3 --optimize=keep-empty --threads="$nproc" -o "${gifsicle_prefix}3.gif" | |
| gifsicle --optimize=2 --optimize=keep-empty --threads="$nproc" -o "${gifsicle_prefix}2.gif" | |
| gifsicle --optimize=1 --optimize=keep-empty --threads="$nproc" -o "${gifsicle_prefix}1.gif" | |
| gif2apng -i20 -z0 | |
| gif2apng -i20 -z1 | |
| gif2apng -i20 -z2 | |
| gif2apng -i20 -z0 -kp | |
| gif2apng -i20 -z1 -kp | |
| gif2apng -i20 -z2 -kp | |
| GIFLIST | |
| } | |
| create_gif_oxi() { | |
| cat <<-"OXI" | |
| ## `noexec` on '$work' prevents using `#!/bin/sh` here | |
| set -Ceuf | |
| ## Redirect STDERR to FD 9. This will be reverted later | |
| 2>&9 | |
| perr() { | |
| printf "\a\033[1m\033[31m%b\033[0;39m" "$1" | |
| printf "\033]777;notify;Sisyphus;%b\033\\" "$1" | |
| } | |
| ## `exit 255` prevents `xargs` from processing any more line batches (the | |
| ## batches from `--max-procs`) | |
| pquit() { perr "$1"; exit 255; } | |
| cmdline="$1" smallest_known="$2" total_lines="$3" work="$4" | |
| src="/proc/self/fd/3" tmp_list="/proc/self/fd/4" proginfo="/proc/self/fdinfo/8" | |
| ## Print the progress. | |
| ### Print it now in case something ends early | |
| progress() { | |
| line_index="$(sed "\|$options|q" "$tmp_list" | wc --lines)" | |
| fd_index="$(sed --quiet "/pos/s/.*[[:space:]]//p;q" "$proginfo")" | |
| index_diff="$(( line_index - fd_index ))" | |
| ### Only advance '$fd_index' if there's been forward progress | |
| [ "$index_diff" -gt 0 ] || return 0 | |
| ### By reading FD 8, the position value increases; it will never decrease | |
| dd bs="$index_diff" count=1 <&8 >/dev/null 2>&1 || return 0 | |
| ### An equivalent to '$progress_diff' wouldn't account for extra rounding | |
| progress="$(( ( fd_index * 100 ) / total_lines ))" | |
| next_progress="$(( ( line_index * 100 ) / total_lines ))" | |
| ### Don't update the progress bar if it probably won't change its percent | |
| [ "$(( next_progress - progress ))" -gt 0 ] || return 0 | |
| printf "\033]9;4;1;%d\007" "$progress" | |
| } | |
| progress ||: | |
| ## If '$cmdline' is for `gif2apng`, specify an output file | |
| out="$( | |
| if [ "$cmdline" != "${cmdline%%gif2apng*}" ]; then | |
| printf "%s.apng" "$cmdline" | | |
| sed \ | |
| -e "s|.*-z|$work/gif2apng + z|" \ | |
| -e "s/[[:space:]]-/ \+ /g" | |
| fi | |
| )" | |
| ## Execute '$cmdline' with additional arguments | |
| if [ -z "$out" ]; then | |
| env --split-string "$cmdline" -- "$src" || | |
| pquit "'$cmdline -- $src' failed!\n" | |
| else | |
| ## `gif2apng` needs both SRC & DEST as arguments. | |
| ### It also only takes relative paths. Source: | |
| ### https://sourceforge.net/p/gif2apng/discussion/1022150/thread/8ec5e7e288 | |
| src_rel="$(realpath --relative-to="$PWD" -- "$src")" | |
| out_rel="$(realpath --relative-to="$PWD" -- "$out")" | |
| env --split-string "$cmdline" -- "$src_rel" "$out_rel" || | |
| pquit "'$cmdline -- $src $out' failed!\n" | |
| ## Strip leftover metadata from `gif2apng` | |
| exiftool -overwrite_original_in_place -all= -- "$out" 2>&1 ||: | |
| fi >/dev/null | |
| ## Print the progress. | |
| line_index="$(sed "\|$cmdline|q" "$tmp_list" | wc --lines)" | |
| fd_index="$(sed --quiet "/pos/s/.*[[:space:]]//p;q" "$proginfo")" | |
| index_diff="$(( line_index - fd_index ))" | |
| ### Only advance '$fd_index' if there's been forward progress | |
| [ "$index_diff" -gt 0 ] || exit 0 | |
| ### By reading FD 8, the position value increases; it will never decrease | |
| dd bs="$index_diff" count=1 <&8 >/dev/null 2>&1 || exit 0 | |
| ### An equivalent to '$progress_diff' wouldn't account for the double rounding | |
| progress="$(( ( fd_index * 100 ) / total_lines ))" | |
| next_progress="$(( ( line_index * 100 ) / total_lines ))" | |
| ### Don't update the progress bar if it probably won't change its percentage | |
| [ "$(( next_progress - progress ))" -gt 0 ] || exit 0 | |
| printf "\033]9;4;1;%d\007" "$progress" | |
| OXI | |
| } | |
| src_is_gif() { | |
| # Generate the command list | |
| gif2webp_dest="$work/gif2webp.webp" | |
| gifsicle_prefix="$work/gifsicle + o" | |
| tmp_list="$work/list" | |
| nproc="$(nproc)" | |
| smallest_known="$src_size" | |
| [ -z "$SIZE" ] || smallest_known="$SIZE" | |
| create_gif_list >>"$tmp_list" | |
| ## Skip `gif2webp` and or `gif2apng` if requested. | |
| ### `sed --in-place` can't operate on file descriptor paths | |
| [ -z "$NO_WEBP" ] || sed --in-place "/^gif2webp/d" "$tmp_list" | |
| [ -z "$NO_APNG" ] || sed --in-place "/^gif2apng/d" "$tmp_list" | |
| ## `src_is_png()` overrides the file descriptors as needed | |
| exec 4<"$tmp_list" | |
| rm -- "$tmp_list" | |
| tmp_list_real="$tmp_list" | |
| tmp_list="/proc/self/fd/4" | |
| ## '$total_lines' is read by `$work/oxi`; it skips repetitive calculations | |
| total_lines="$(wc --lines <"$tmp_list")" | |
| ## The read position of FD 8 tracks the progress | |
| progress_file="$work/progress" | |
| truncate --size="$total_lines" -- "$progress_file" | |
| exec 8<"$progress_file" | |
| rm -- "$progress_file" | |
| ## Not `-gt 9`, since WebP and APNG can both be excluded | |
| [ "$total_lines" -gt 2 ] || pquit "'$tmp_list_real' was truncated!\n" | |
| # Create the `xargs` script | |
| tmp_oxi="$work/oxi" | |
| :>"$tmp_oxi" | |
| exec 5<"$tmp_oxi" | |
| rm -- "$tmp_oxi" | |
| tmp_oxi="/proc/self/fd/5" | |
| ## Avoid using `sh -c ''` inside `xargs` by creating an external script. | |
| ### Heredocs can only be read once before their buffers are emptied; | |
| ### therefore, write the script to a hidden file | |
| create_gif_oxi >|"$tmp_oxi" | |
| # Pass each command in the list to '$tmp_oxi' | |
| ## `9>&2` redirects the script's STDERR to FD 2, while redirecting the | |
| ## STDERR of `xargs` to `/dev/null`. `exit 255` causes the STDERR logs | |
| xargs \ | |
| --max-procs "$MAX_PROCS" \ | |
| --delimiter "\n" \ | |
| --replace="%" \ | |
| -- sh "$tmp_oxi" \ | |
| "%" \ | |
| "$smallest_known" \ | |
| "$total_lines" \ | |
| "$work" \ | |
| <"$tmp_list" 9>&2 2>/dev/null || | |
| exit_clear | |
| ## Clear the progress bar now that we have the initial APNG images | |
| clear_prog | |
| # Optimise the `gif2apng` outputs with this script | |
| ## ...unless `gif2apng` was explicitly excluded | |
| [ -z "$NO_APNG" ] || return 0 | |
| ## Allow globbing for just these for-loops | |
| set +f | |
| ## De-duplicate the APNGs beforehand | |
| for dup in /tmp/gif2apng*; do | |
| [ -f "$dup" ] || continue | |
| [ ! -L "$dup" ] || continue | |
| for dup2 in /tmp/gif2apng*; do | |
| [ "$dup" != "$dup2" ] || continue | |
| ! cmp "$dup" "$dup2" >/dev/null || | |
| ln --force --symbolic "$dup" "$dup2" 2>/dev/null || | |
| perr "Failed to symlink '$dup2' to '$dup'!\n" | |
| done | |
| done | |
| for apngsrc in /tmp/gif2apng*; do | |
| ## Don't optimise duplicate SRC images; the output is identical | |
| if [ -L "$apngsrc" ]; then | |
| apngsrc_sym="$(readlink -- "$apngsrc")" | |
| pstat ":: " "'$apngsrc' and '$apngsrc_sym' are identical. Skipping.\n" | |
| continue | |
| fi | |
| [ -f "$apngsrc" ] || continue | |
| apngdest="${apngsrc%.*}.oxipng.apng" | |
| apngsrc_real="$apngsrc" | |
| apngdest_real="$apngdest" | |
| :>"$apngdest" | |
| ## Each `sisyphus` instance has its own '$work' directory; | |
| ## therefore, pass the input & output as file descriptors | |
| exec 6<"$apngsrc" | |
| exec 7<"$apngdest" | |
| apngsrc="/proc/self/fd/6" | |
| apngdest="/proc/self/fd/7" | |
| pstat ":: " "Optimising '$apngsrc_real'...\n" | |
| ## `--force` is required to write to '$apngdest' | |
| "$0" --_internal --quiet --force --max-procs "$MAX_PROCS" \ | |
| --no-webp --all-oxi "${ALL_OXI:-"1"}" -- \ | |
| "$apngsrc" "$apngdest" ||: | |
| ## If '$apngdest' is still empty, substitute it with '$apngsrc' | |
| if [ -f "$apngdest_real" ] && [ ! -s "$apngdest_real" ]; then | |
| ln --force --symbolic "$apngsrc_real" -- "$apngdest_real" \ | |
| 2>/dev/null || | |
| perr "Failed to symlink '$apngdest_real' to '$apngsrc_real'!\n" | |
| fi | |
| done | |
| set -f | |
| } | |
| # File selection | |
| case "$mime" in | |
| "image/png") src_is_png ;; | |
| "image/gif") src_is_gif ;; | |
| *) pquit "Mimetype of '$src_real' is neither PNG nor GIF!\n" ;; | |
| esac | |
| # Output listing & selection | |
| ## Repurpose '$tmp_list' as the output list. | |
| ### This variable isn't set if `try_oxi_vars` never happens (e.g. `-a 0`) | |
| tmp_list="${tmp_list:-"$work/output"}" | |
| ## 0. Gather all the results, while ignoring any possible empty files that can | |
| ## be created by failed APNG optimisations. | |
| ## 1. Prefix lines with their length followed by a space (e.g. '20 8 bytes ...'). | |
| ## 2. Sort the lines by their length values (aka. `sort -V`). | |
| ## 3. Use space as the delimiter (aka. `-d " "`), and remove everything from | |
| ## before the second field (e.g. '20 8 bytes ...' -> '8 bytes ...'). | |
| ## 4. Sort numerically, but for cases where the byte count is the same, preserve | |
| ## the ordering from the previous steps (aka. `sort -sn`). | |
| ### Attempting this command inside '$smallest', using `tee` to split the output, | |
| ### will sometimes yield truncated output. It's not worth it. | |
| find "$work"/ -maxdepth 1 -follow -type f -size +0c -printf "%s\t%f\n" | | |
| awk '{ print length, $0 }' | | |
| sort --version-sort | | |
| cut --delimiter=" " --fields=2- | | |
| sort --numeric-sort --stable \ | |
| >|"$tmp_list" | |
| smallest="$work/$( | |
| head --lines=1 "$tmp_list" | cut --delimiter=" " --fields=2- | |
| )" | |
| smallest_size="$(find_size "$smallest")" | |
| if [ "$RESULTS" != "0" ]; then | |
| pstat "Size order:\n" "" | |
| if [ "$RESULTS" = "ALL" ]; then | |
| sed "s/\t/ bytes\t/g ; s/^/\t/g" "$tmp_list" | |
| else | |
| head --lines "$RESULTS" -- "$tmp_list" | | |
| sed "s/\t/ bytes\t/g ; s/^/\t/g" | |
| fi | |
| ## If `-q -r N` is specified, don't print an extra newline | |
| [ -n "$QUIET" ] || printf "\n" | |
| fi | |
| ## Check for an improvement | |
| [ "$smallest_size" -le "${SIZE:-"$smallest_size"}" ] || | |
| pquit "'$smallest' was equal to, or larger than, the minimum size target ($SIZE bytes).\n" | |
| if [ "$smallest_size" -eq "$src_size" ]; then | |
| pstat ":: " "'$src_real' is already optimal.\n" | |
| [ -n "$QUIET" ] || osc777 "'$src_real' is already optimal." | |
| exit 0 | |
| fi | |
| [ "$smallest_size" -le "$src_size" ] || | |
| pquit "'$smallest' was larger than '$src_real'.\n" | |
| ## Symlinks also have an apparent size of '0'; exclude them from this check | |
| if [ "$(stat --format "%b" -- "$smallest")" -le 0 ] && [ ! -L "$smallest" ]; then | |
| pquit "'$smallest' was truncated at some point; it cannot be used!\n"; fi | |
| ## Set '$final_dest' and check the path for existing files | |
| final_dest="$dest_path.${smallest##*.}" | |
| if [ -e "$final_dest" ] && [ -z "$FORCE" ]; then | |
| pquit "'$final_dest' already exists!\n"; fi | |
| ## If `--_internal` is used, then '$final_dest' must be a file descriptor | |
| if [ -z "$INTERNAL" ]; then | |
| cp -- "$smallest" "$final_dest" | |
| else | |
| [ -L "$2" ] || | |
| pquit "'--_internal' was given, but the destination ('$2') is not | |
| a file descriptor or doesn't exist!\n" | |
| ## You can't `cp` to a file descriptor; write to it directly | |
| cat "$smallest" >|"$2" | |
| fi | |
| ## Print the final statistics. | |
| ### "> ": as in the "quant > ..." prefix of the filename | |
| method="${smallest##*"> "}" | |
| method="${smallest##*"/"}" | |
| method="${method%.*}" | |
| pstat "Final output:\t" "$final_dest \033[37m($method)\033[0;39m\n" | |
| pstat "Size diff:\t" \ | |
| "$src_size bytes \033[37m->\033[0;39m \033[1m$smallest_size bytes\n" | |
| clear_prog | |
| [ -n "$QUIET" ] || | |
| osc777 "'$final_dest' is finished! Size diff: $src_size bytes -> $smallest_size bytes ($method)" |
I'm not able to test it since it's an EXE, but, since I think it's still using OptiPNG under the hood, I would guess my script is better for now (OptiPNG always faired worse than OxiPNG from my testing). The script currently tries every valid, unordered permutation of:
[pngquant --quality 100-100 --speed 1 --strip |,] oxipng [--zopfli --zi=255,--zc={0..12}] --n[bcgp] --opt max --strip all --alpha
If, when FileOptimizer switches to OxiPNG, it additonally executes oxipng with some other options beyond these (which I'm not aware of currently), or it implements a palette randomiser itself (which is what pngquant functions as here), it could beat this script.
Per the current source @ https://sourceforge.net/p/nikkhokkho/code/HEAD/tree/trunk/FileOptimizer/Source/cppMain.cpp#l2087, FO currently uses "apngopt, pngquant, PngOptimizer, TruePNG, pngout, oxipng, pngwolf, Leanify, ect, pingo, advpng, deflopt, defluff, deflopt" (not a typo, runs deflopt 2× based on testing) in the PNG toolchain, w/ configurable options via GUI & INI.
FO reportedly works fine under WINE & emulators. Perhaps you could compare that way?
Managed to use Bottles to run FO, and... inconclusive? Here's what I'm noticing:
- Sisyphus can beat FO, and vice-versa, so neither is guaranteed to win unfortunately.
- From that
cppMain.cppfile you linked, I'm noticing thatoxipng -Zis always being used, however, Zopfli can actually lose to non-Zopfli compression (and also, there's no option permutation like I'm doing here, which can also help). pingois able to beatcwebpsometimes, which is unfortunate sincepingois also a Windows-only executable, so this script is definitely missing out in those cases.
I haven't tested GIFs or APNGs yet, and I also need to update this script with my new APNG brute-forcing pipeline, but from the above results, I'm guessing that's going to be a mixed bag as well.
EDIT: I've updated Sisyphus with the new APNG stuff, along with some other changes.
Thanks for the detailed comparison! If you've test files you'd be willing to make available that Sisyphus beats FO, that'd be much appreciated. 🙇🏾♂️
Pinging @javiergutierrezchamorro (the FO dev) to take note of above feedback.
Here's some samples:
Sisyphus: 1824 bytes (zop.png. I threw these first two samples in to show that FO can beat Sisyphus)
FO: 1823 bytes
Sisyphus: 479 bytes (quant > zop + nb.png)
FO: 474 bytes
[FO >] Sisyphus: 413 bytes (zop.png. It's probably --opt max that makes the difference with these zop.png results)
FO: 414 bytes
Sisyphus: 534 bytes (zop.png)
FO: 537 bytes
FO > Sisyphus: 536 bytes (zop.png)
[FO >] Sisyphus: 246 bytes (quant > zop.png. Regular zop.png matched FO, so pngquant did something here)
FO: 247 bytes
Sisyphus: 3396 bytes (zop.png)
FO: 3398 bytes
I forgot to set the optimisation level to 9 in FO... one moment.
EDIT: Done
Just to make sure I understand, just the last example beats FO (by 1 byte)? 🤔
Yeah, I had some other images going before I realised I forgot the level 9 setting; will edit with some of the other examples when I have time.
EDIT: Okay, so bad news. Pretty much every time Sisyphus yields a smaller result, it's because it uses --zopfli and --opt max, which FO doesn't. Except for that quant > zop.png result, where regular zop.png matched FO, so the extra palette randomisation that pngquant introduces can help on rare occasions.
But yeah, ever since I set the level in FO's settings correctly, I've been struggling to beat it; it's usually a tie or loss save for these samples. Welp.
1 tip: I think you're pretty used to long processing times b/c of your script, but, if you've the spare processing power, multiple instances of FO work quite well on split file lists to speed up overall processing.
Also, does running Sisyphus before/after FO-processed images improve anything?
Found these (I'll edit the previous samples if I find anything for them too):
FO: 388 bytes
FO > Sisyphus: 387 bytes (zop + nb.png. Regular zop.png matches FO, and every permutation of zop + *.png beats it by a single byte, so it's not just --opt max here)
FO: 2987 bytes (FO > FO: Same size)
FO > Sisyphus: 2956 bytes (zop.png. Probably --opt max again)
FO > Sisyphus > FO: 2955 bytes (???)
Sometimes running FO multiple times on the same file (Shift+F5,


















@Winterhuman How does this compare to, e.g., FileOptimizer?