Created
October 3, 2025 16:41
-
-
Save mcejp/54cf9dbac1647cd035164dbb931cb423 to your computer and use it in GitHub Desktop.
NASA Earthdata download script patch to avoid re-downloading existing files
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- ../download.sh | |
| +++ ../download.sh | |
| @@ -64,13 +64,33 @@ | |
| fetch_urls() { | |
| if command -v curl >/dev/null 2>&1; then | |
| setup_auth_curl | |
| + row=0 | |
| while read -r line; do | |
| + row=$((row + 1)) | |
| + | |
| # Get everything after the last '/' | |
| filename="${line##*/}" | |
| # Strip everything after '?' | |
| stripped_query_params="${filename%%\?*}" | |
| + # If file already exists, check whether it's a valid zip. If invalid, remove and re-download. | |
| + if [ -f "$stripped_query_params" ]; then | |
| + #echo "Found existing file '$stripped_query_params'. Validating..." | |
| + if command -v unzip >/dev/null 2>&1; then | |
| + if unzip -t "$stripped_query_params" >/dev/null 2>&1; then | |
| + echo "$row: $stripped_query_params already downloaded" | |
| + continue | |
| + else | |
| + echo "$row: $stripped_query_params is invalid or corrupted; removing and re-downloading." | |
| + rm -f "$stripped_query_params" | |
| + fi | |
| + else | |
| + exit_with_error "unzip not found. Please install unzip" | |
| + fi | |
| + fi | |
| + | |
| + echo "$row: $stripped_query_params" | |
| curl -f -b "$cookiejar" -c "$cookiejar" -L --netrc-file "$netrc" -g -o $stripped_query_params -- $line && echo || exit_with_error "Command failed with error. Please retrieve the data manually." | |
| done; | |
| elif command -v wget >/dev/null 2>&1; then |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment