Last active
April 30, 2020 10:26
-
-
Save PrometheusPi/5ef848a72430dabd55dff61bcd15475c to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| # This is an adoptation of a script by Axel Huebl (ax3l) | |
| # to compress hdf5 files in PIConGPU simulations. | |
| # It is geared to run on SLURM systems with a recent version | |
| # of SLURM. | |
| #SBATCH --time=72:00:00 | |
| #SBATCH --nodes=1 | |
| #SBATCH --ntasks=24 | |
| #SBATCH --job-name=compress_hdf5 | |
| #SBATCH --chdir=. | |
| #set user rights to u=rwx;g=r-x;o=--- | |
| umask 0027 | |
| runs=$( find . -type d -iname simOutput -exec dirname '{}' \; | sort ) | |
| for i in ${runs[@]}; do | |
| dirName=`pwd`/${i}/simOutput | |
| if [ ! -d "$dirName" ] | |
| then | |
| echo "Missing simOutput in $dirName" 1>&2 | |
| continue | |
| fi | |
| if [ -e $dirName/../compressed ] | |
| then | |
| echo "Simulation already compressed in $dirName" 1>&2 | |
| continue | |
| fi | |
| echo "before: "`du -hs $dirName` | |
| if [ ! -e $dirName/../stdout ] | |
| then | |
| echo "Simulation not yet finished in $dirName" 1>&2 | |
| continue | |
| fi | |
| cd $dirName | |
| find $dirName -name "*.h5" | xargs -n1 -P24 -I{} \ | |
| sh -c 'echo "compress $1 ..." && \ | |
| h5repack -i $1 -o $1.gz -f GZIP=1 && mv $1.gz $1' _ {} | |
| touch ../compressed | |
| cd - | |
| done |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment