|
--- functions.bak 2025-11-27 02:42:21 |
|
+++ functions 2025-11-27 03:37:53 |
|
@@ -204,6 +204,16 @@ |
|
fi |
|
DLDEST="apt_dest" |
|
export APTSTATE DLDEST DEBFOR |
|
+ ;; |
|
+ parallel) |
|
+ if [ "${2-}" = "var-state" ]; then |
|
+ APTSTATE="var/state/apt" |
|
+ else |
|
+ APTSTATE="var/lib/apt" |
|
+ fi |
|
+ DLDEST="apt_dest" |
|
+ PARALLEL="1" |
|
+ export APTSTATE DLDEST DEBFOR PARALLEL |
|
;; |
|
*) |
|
error 1 BADDLOAD "unknown download style" |
|
@@ -835,8 +845,265 @@ |
|
echo >&5 ${leftover# } |
|
) |
|
} |
|
+ |
|
+# Collect URLs for parallel download |
|
+collect_download_urls () { |
|
+ local m pkgdest url_list |
|
+ m="$1" |
|
+ pkgdest="$2" |
|
+ url_list="$3" |
|
+ shift; shift; shift |
|
+ |
|
+ "$PKGDETAILS" PKGS "$m" "$pkgdest" "$@" | while read p ver arc mdup fil checksum size; do |
|
+ if [ "$ver" != "-" ]; then |
|
+ debcache="$(echo "${p}_${ver}_${arc}.deb" | sed 's/:/%3a/')" |
|
+ debdest="$($DLDEST deb "$p" "$ver" "$arc" "$m" "$fil")" |
|
+ # Format: url|destfile|debdest|pkgname|version|checksum|size |
|
+ echo "$m/$fil|$debcache|$debdest|$p|$ver|$checksum|$size" |
|
+ fi |
|
+ done >> "$url_list" |
|
+} |
|
+ |
|
+# Parallel download using aria2c |
|
+parallel_download_exec () { |
|
+ local url_list cache_dir aria2_input download_count |
|
+ url_list="$1" |
|
+ cache_dir="$2" |
|
+ aria2_input="$cache_dir/aria2_input.txt" |
|
|
|
+ test -s "$url_list" || return 0 |
|
+ |
|
+ # Create aria2c input file |
|
+ : > "$aria2_input" |
|
+ download_count=0 |
|
+ |
|
+ while IFS='|' read url debcache debdest pkgname pkgver checksum size; do |
|
+ # Skip if already exists in TARGET |
|
+ test -f "${TARGET}${debdest}" && continue |
|
+ echo "$url" |
|
+ echo " out=$debcache" |
|
+ download_count=$((download_count + 1)) |
|
+ done < "$url_list" >> "$aria2_input" |
|
+ |
|
+ if test ! -s "$aria2_input"; then |
|
+ info ALLCACHED "All packages found in cache" |
|
+ return 0 |
|
+ fi |
|
+ |
|
+ info TODOWNLOAD "Downloading %s packages..." "$download_count" |
|
+ |
|
+ # Parallel download count (default: 4 connections) |
|
+ PARALLEL_COUNT="${PARALLEL_DL_COUNT:-4}" |
|
+ |
|
+ if in_path aria2c; then |
|
+ info PARALLELDOWNLOAD "Parallel downloading with aria2c (%s connections)..." "$PARALLEL_COUNT" |
|
+ aria2c -x "$PARALLEL_COUNT" -s "$PARALLEL_COUNT" -j "$PARALLEL_COUNT" \ |
|
+ -d "$cache_dir" -i "$aria2_input" \ |
|
+ --continue=true --auto-file-renaming=false \ |
|
+ --console-log-level=notice --summary-interval=0 \ |
|
+ 2>&1 | while read line; do |
|
+ case "$line" in |
|
+ *"Download complete:"*) |
|
+ debfile="${line##*/}" |
|
+ pkgname="${debfile%%_*}" |
|
+ info DOWNLOADED "Downloaded %s" "$pkgname" |
|
+ ;; |
|
+ esac |
|
+ done |
|
+ # Note: pipe hides aria2c exit status, assume success if we get here |
|
+ else |
|
+ # Fallback: parallel wget with background jobs |
|
+ info PARALLELDOWNLOAD "Parallel downloading with wget (%s parallel jobs)..." "${PARALLEL_DL_COUNT:-4}" |
|
+ local job_count max_jobs |
|
+ job_count=0 |
|
+ max_jobs="${PARALLEL_DL_COUNT:-4}" |
|
+ while IFS='|' read url debcache debdest pkgname pkgver checksum size; do |
|
+ if test ! -f "${TARGET}${debdest}"; then |
|
+ (wget -q -O "$cache_dir/$debcache" "$url" && info DOWNLOADED "Downloaded %s" "$pkgname") & |
|
+ job_count=$((job_count + 1)) |
|
+ if test "$job_count" -ge "$max_jobs"; then |
|
+ wait |
|
+ job_count=0 |
|
+ fi |
|
+ fi |
|
+ done < "$url_list" |
|
+ wait |
|
+ fi |
|
+ |
|
+ return 0 |
|
+} |
|
+ |
|
+# Copy downloaded packages from cache to target and create debpaths |
|
+install_from_cache () { |
|
+ local url_list cache_dir leftover cache_file target_file |
|
+ url_list="$1" |
|
+ cache_dir="$2" |
|
+ leftover="" |
|
+ |
|
+ while IFS='|' read url debcache debdest pkgname pkgver checksum size; do |
|
+ cache_file="$cache_dir/$debcache" |
|
+ target_file="$TARGET$debdest" |
|
+ |
|
+ if test -f "$cache_file"; then |
|
+ # Verify checksum if provided |
|
+ if test -n "$checksum"; then |
|
+ if verify_checksum "$cache_file" "$checksum" "$size"; then |
|
+ info VALIDATING "Validated %s" "$pkgname" |
|
+ else |
|
+ warning BADCHECKSUM "Checksum mismatch for %s, removing" "$pkgname" |
|
+ rm -f "$cache_file" |
|
+ leftover="$leftover $pkgname" |
|
+ continue |
|
+ fi |
|
+ fi |
|
+ # Copy if locations differ |
|
+ if test "$cache_file" != "$target_file"; then |
|
+ mkdir -p "$(dirname "$target_file")" |
|
+ cp "$cache_file" "$target_file" |
|
+ fi |
|
+ echo >>"$TARGET/debootstrap/deburis" "$pkgname $pkgver $url" |
|
+ echo >>"$TARGET/debootstrap/debpaths" "$pkgname $debdest" |
|
+ else |
|
+ warning NOTINCACHE "Package %s not found in cache" "$pkgname" |
|
+ leftover="$leftover $pkgname" |
|
+ fi |
|
+ done < "$url_list" |
|
+ |
|
+ echo ${leftover# } |
|
+} |
|
+ |
|
+# Parallel version of download_release |
|
+download_release_parallel () { |
|
+ local m1 numdebs countdebs totaldebs leftoverdebs path pkgdest \ |
|
+ dloaddebs archs s c a m parallel_cache url_list url_count failed_pkgs |
|
+ m1="${MIRRORS%% *}" |
|
+ |
|
+ numdebs="$#" |
|
+ |
|
+ countdebs=0 |
|
+ progress "$countdebs" "$numdebs" SIZEDEBS "Finding package sizes" |
|
+ |
|
+ totaldebs=0 |
|
+ leftoverdebs="$*" |
|
+ |
|
+ # Fix possible duplicate package names |
|
+ leftoverdebs=$(printf "$leftoverdebs"|tr ' ' '\n'|sort -u|tr '\n' ' ') |
|
+ numdebs=$(printf "$leftoverdebs"|wc -w) |
|
+ |
|
+ archs="$ARCH" |
|
+ if [ $ARCH_ALL_SUPPORTED -eq 1 ]; then |
|
+ archs="all $ARCH" |
|
+ fi |
|
+ |
|
+ # Setup cache directory |
|
+ parallel_cache="${CACHE_DIR:-$TARGET/var/cache/apt/archives}" |
|
+ mkdir -p "$parallel_cache" |
|
+ |
|
+ # URL list file |
|
+ url_list="$TARGET/debootstrap/parallel_urls.txt" |
|
+ : > "$url_list" |
|
+ |
|
+ # Phase 1: Collect all package sizes |
|
+ for s in $SUITE $EXTRA_SUITES; do |
|
+ for a in $archs; do |
|
+ for c in $COMPONENTS; do |
|
+ if [ "$countdebs" -ge "$numdebs" ]; then break; fi |
|
+ |
|
+ path="dists/$s/$c/binary-$a/Packages" |
|
+ pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m1" "$path")" |
|
+ if [ ! -e "$pkgdest" ]; then continue; fi |
|
+ |
|
+ info CHECKINGSIZES "Checking component %s on %s..." "$c" "$m1" |
|
+ |
|
+ leftoverdebs="$(get_package_sizes "$m1" "$pkgdest" $leftoverdebs)" |
|
+ |
|
+ countdebs=$(($countdebs + ${leftoverdebs%% *})) |
|
+ leftoverdebs=${leftoverdebs#* } |
|
+ |
|
+ totaldebs=${leftoverdebs%% *} |
|
+ leftoverdebs=${leftoverdebs#* } |
|
+ |
|
+ progress "$countdebs" "$numdebs" SIZEDEBS "Finding package sizes" |
|
+ done |
|
+ done |
|
+ done |
|
+ |
|
+ if [ "$countdebs" -ne "$numdebs" ]; then |
|
+ error 1 LEFTOVERDEBS "Couldn't find these debs: %s" "$leftoverdebs" |
|
+ fi |
|
+ |
|
+ # Phase 2: Collect all download URLs |
|
+ info COLLECTURLS "Collecting download URLs..." |
|
+ pkgs_to_get="$*" |
|
+ for s in $SUITE $EXTRA_SUITES; do |
|
+ for a in $archs; do |
|
+ for c in $COMPONENTS; do |
|
+ path="dists/$s/$c/binary-$a/Packages" |
|
+ for m in $MIRRORS; do |
|
+ pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m" "$path")" |
|
+ if [ ! -e "$pkgdest" ]; then continue; fi |
|
+ collect_download_urls "$m" "$pkgdest" "$url_list" $pkgs_to_get |
|
+ break |
|
+ done |
|
+ done |
|
+ done |
|
+ done |
|
+ |
|
+ # Phase 3: Parallel download |
|
+ url_count=$(wc -l < "$url_list" | tr -d ' ') |
|
+ info PARALLELSTART "Starting parallel download of %s packages..." "$url_count" |
|
+ |
|
+ progress 0 "$totaldebs" DOWNDEBS "Downloading packages (parallel)" |
|
+ :>"$TARGET/debootstrap/debpaths" |
|
+ |
|
+ if ! parallel_download_exec "$url_list" "$parallel_cache"; then |
|
+ warning PARALLELFAIL "Parallel download failed, falling back to sequential" |
|
+ PARALLEL="" |
|
+ rm -f "$url_list" |
|
+ download_release "$@" |
|
+ return $? |
|
+ fi |
|
+ |
|
+ # Phase 4: Install from cache and record in debpaths |
|
+ info INSTALLFROMCACHE "Installing packages from cache..." |
|
+ failed_pkgs=$(install_from_cache "$url_list" "$parallel_cache") |
|
+ |
|
+ progress "$totaldebs" "$totaldebs" DOWNDEBS "Downloading packages (parallel)" |
|
+ |
|
+ if test -n "$failed_pkgs"; then |
|
+ warning RETRYFAILED "Retrying failed packages: %s" "$failed_pkgs" |
|
+ for s in $SUITE $EXTRA_SUITES; do |
|
+ for a in $archs; do |
|
+ for c in $COMPONENTS; do |
|
+ path="dists/$s/$c/binary-$a/Packages" |
|
+ for m in $MIRRORS; do |
|
+ pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m" "$path")" |
|
+ test -e "$pkgdest" || continue |
|
+ failed_pkgs="$(download_debs "$m" "$pkgdest" $failed_pkgs 5>&1 1>&6)" |
|
+ test -z "$failed_pkgs" && break |
|
+ done 6>&1 |
|
+ test -z "$failed_pkgs" && break |
|
+ done |
|
+ test -z "$failed_pkgs" && break |
|
+ done |
|
+ test -z "$failed_pkgs" && break |
|
+ done |
|
+ fi |
|
+ |
|
+ if test -n "$failed_pkgs"; then |
|
+ error 1 COULDNTDLPKGS "Couldn't download packages: %s" "$failed_pkgs" |
|
+ fi |
|
+ |
|
+ rm -f "$url_list" "$parallel_cache/aria2_input.txt" |
|
+} |
|
+ |
|
download_release () { |
|
+ # Use parallel download if enabled |
|
+ if [ "$PARALLEL" = "1" ]; then |
|
+ download_release_parallel "$@" |
|
+ return $? |
|
+ fi |
|
+ |
|
local m1 numdebs countdebs totaldebs leftoverdebs path pkgdest \ |
|
dloaddebs archs s c a m |
|
m1="${MIRRORS%% *}" |
