#!/bin/bash
#
# Fetch the bullseye-security binary packages that Debian's live archive no
# longer serves, from snapshot.debian.org.
#
# Debian 11 went end-of-life on 2026-08-31. On 2026-09-12 the security suite's
# Release file was re-signed WITHOUT a Valid-Until, so apt-get update works
# again -- but the pool behind that index has been reaped. The index still
# advertises packages whose .deb files are gone, so installing anything that
# resolves to a security version fails with a 404. snapshot.debian.org retains
# the archive as it stood before the reap.
#
# Excluded by default: Debian kernels including their source tarballs (OSMC ships
# its own kernel) and debug symbols.
#
# Safe to interrupt and re-run. Everything already present and checksum-clean
# is skipped, so a second run costs almost nothing.
#
set -u

BASE_DIR="$(cd "$(dirname "$0")" && pwd)"
SNAPSHOT="${SNAPSHOT:-https://snapshot.debian.org/archive/debian-security/20260901T000000Z}"
SUITE="bullseye-security"
COMPONENTS="main contrib non-free"
ARCHES="armhf all"

POOL_DIR="$BASE_DIR/pool"
SRC_DIR="$BASE_DIR/src"
META_DIR="$BASE_DIR/meta"
LOG="$BASE_DIR/fetch.log"
MANIFEST="$BASE_DIR/manifest.tsv"
SRC_MANIFEST="$BASE_DIR/manifest-src.tsv"
FAILED="$BASE_DIR/failed.tsv"

# Be polite: snapshot.debian.org is storage- and bandwidth-constrained and says so.
RATE_LIMIT="${RATE_LIMIT:-10M}"
SLEEP_BETWEEN="${SLEEP_BETWEEN:-0.25}"
RETRIES="${RETRIES:-3}"

# Source packages excluded wholesale: their binaries AND their sources. These are
# desktop/server software no OSMC device runs, and they dominate the source tree
# (one .orig tarball each, hundreds of MiB).
EXCLUDE_SOURCES="${EXCLUDE_SOURCES:-rustc-web thunderbird chromium firefox-esr nvidia-graphics-drivers libreoffice kicad krita ceph llvm-toolchain-19 llvm-toolchain-22 zfs-linux spl clamav nvidia-modprobe p7zip libclamunrar p7zip-rar nvidia-settings}"

DRY_RUN=0
PRUNE=0
SKIP_KERNELS=1
SKIP_DEBUG=1
WANT_SOURCES=0
BINARIES=1

usage() {
    cat <<EOF
Usage: $0 [options]

  -n, --dry-run        List what would be fetched; download nothing.
      --with-kernels   Include linux-image/headers/kbuild/support/source/config/doc
                     (default: excluded).
      --with-debug     Include -dbg and -dbgsym packages (default: excluded).
      --rate LIMIT     curl --limit-rate value (default: $RATE_LIMIT).
  -h, --help           This.

Environment: SNAPSHOT, RATE_LIMIT, SLEEP_BETWEEN, RETRIES.
EOF
}

while [ $# -gt 0 ]; do
    case "$1" in
        -n|--dry-run)   DRY_RUN=1 ;;
        --sources)      WANT_SOURCES=1 ;;
        --prune)        PRUNE=1 ;;
        --sources-only) WANT_SOURCES=1; BINARIES=0 ;;
        --with-kernels) SKIP_KERNELS=0 ;;
        --with-debug)   SKIP_DEBUG=0 ;;
        --rate)         RATE_LIMIT="$2"; shift ;;
        -h|--help)      usage; exit 0 ;;
        *)              echo "unknown option: $1" >&2; usage; exit 2 ;;
    esac
    shift
done

log() { printf '%s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" | tee -a "$LOG"; }
die() { log "FATAL: $*"; exit 1; }

for t in curl sha256sum gzip awk; do
    command -v "$t" >/dev/null 2>&1 || die "missing required tool: $t"
done

mkdir -p "$POOL_DIR" "$SRC_DIR" "$META_DIR" || die "cannot create directories under $BASE_DIR"
log "=== run starting (dry-run=$DRY_RUN, kernels=$((1-SKIP_KERNELS)), debug=$((1-SKIP_DEBUG)))"
log "excluded sources: $EXCLUDE_SOURCES"
log "snapshot: $SNAPSHOT"

# ---------------------------------------------------------------- indexes ---
fetch_indexes() {
    local comp arch url out
    for comp in $COMPONENTS; do
        for arch in $ARCHES; do
            out="$META_DIR/${comp}-${arch}.Packages"
            url="$SNAPSHOT/dists/$SUITE/updates/$comp/binary-$arch/Packages.gz"
            if [ -s "$out" ]; then
                log "index cached: $comp/$arch ($(grep -c '^Package: ' "$out") packages)"
                continue
            fi
            log "fetching index: $comp/$arch"
            if curl -sSfL --retry "$RETRIES" --retry-delay 2 --max-time 300 \
                    --limit-rate "$RATE_LIMIT" "$url" -o "$out.gz" 2>>"$LOG"; then
                gzip -dc "$out.gz" > "$out" 2>>"$LOG" && rm -f "$out.gz" \
                    || { log "WARN: could not decompress $comp/$arch"; rm -f "$out" "$out.gz"; }
                [ -s "$out" ] && log "  -> $(grep -c '^Package: ' "$out") packages"
            else
                log "WARN: no index for $comp/$arch (may not exist upstream)"
                rm -f "$out.gz"
            fi
        done
    done
    ls "$META_DIR"/*.Packages >/dev/null 2>&1 || die "no indexes fetched; cannot continue"
}

# --------------------------------------------------------------- manifest ---
# One row per wanted package: name <TAB> filename <TAB> size <TAB> sha256
build_manifest() {
    : > "$MANIFEST"
    awk -v skip_kern="$SKIP_KERNELS" -v skip_dbg="$SKIP_DEBUG" -v excl="$EXCLUDE_SOURCES" '
        BEGIN { n = split(excl, a, " "); for (i = 1; i <= n; i++) bad[a[i]] = 1 }
        function flush(   s) {
            if (pkg != "" && fn != "" && sha != "") {
                drop = 0
                if (skip_kern && (pkg ~ /^linux-(image|headers|kbuild|support|source|config|doc)/)) drop = 1
                if (skip_dbg  && (pkg ~ /(-dbg|-dbgsym)$/))                       drop = 1
                # "Source: name (version)" when the source version differs
                s = (src != "" ? src : pkg); sub(/ .*$/, "", s)
                if (s in bad) drop = 1
                if (!drop) printf "%s\t%s\t%s\t%s\n", pkg, fn, size, sha
            }
            pkg=""; fn=""; size=""; sha=""; src=""
        }
        /^Package: /  { flush(); pkg = $2; next }
        /^Source: /   { src  = $2; next }
        /^Filename: / { fn   = $2; next }
        /^Size: /     { size = $2; next }
        /^SHA256: /   { sha  = $2; next }
        /^$/          { flush(); next }
        END           { flush() }
    ' "$META_DIR"/*.Packages | sort -u -k2,2 > "$MANIFEST"

    local n bytes
    n=$(wc -l < "$MANIFEST")
    bytes=$(awk -F'\t' '{t+=$3} END{printf "%.0f", t+0}' "$MANIFEST")
    log "manifest: $n packages, $(awk -v b="$bytes" 'BEGIN{printf "%.1f MiB", b/1048576}')"
}

# --------------------------------------------------------------- download ---
verify() {  # $1 path, $2 expected sha256 -> 0 if good
    [ -f "$1" ] || return 1
    [ "$(sha256sum "$1" | cut -d' ' -f1)" = "$2" ]
}

download_all() {
    local name fn size sha dest url tmp
    local total done_ok skipped failed idx=0
    total=$(wc -l < "$MANIFEST"); done_ok=0; skipped=0; failed=0

    while IFS=$'\t' read -r name fn size sha; do
        idx=$((idx+1))
        dest="$POOL_DIR/$fn"

        if verify "$dest" "$sha"; then
            skipped=$((skipped+1))
            continue
        fi
        # present but wrong: a truncated or corrupt earlier attempt
        [ -f "$dest" ] && { log "re-fetching (checksum mismatch): $fn"; rm -f "$dest"; }

        if [ "$DRY_RUN" -eq 1 ]; then
            printf '  would fetch %s (%s bytes)\n' "$fn" "$size"
            continue
        fi

        mkdir -p "$(dirname "$dest")" || { log "ERR: mkdir failed for $fn"; continue; }
        url="$SNAPSHOT/$fn"
        tmp="$dest.part"

        # -C - resumes a partial .part across runs
        if curl -sSfL -C - --retry "$RETRIES" --retry-delay 2 --max-time 900 \
                --limit-rate "$RATE_LIMIT" "$url" -o "$tmp" 2>>"$LOG"; then
            if verify "$tmp" "$sha"; then
                mv -f "$tmp" "$dest"
                done_ok=$((done_ok+1))
                [ $((done_ok % 25)) -eq 0 ] && log "progress: $idx/$total ($done_ok fetched, $skipped already present)"
            else
                log "ERR: checksum mismatch after download: $fn"
                printf '%s\t%s\tchecksum\n' "$name" "$fn" >> "$FAILED"
                rm -f "$tmp"; failed=$((failed+1))
            fi
        else
            log "ERR: fetch failed: $fn"
            printf '%s\t%s\tfetch\n' "$name" "$fn" >> "$FAILED"
            rm -f "$tmp"; failed=$((failed+1))
        fi
        sleep "$SLEEP_BETWEEN"
    done < "$MANIFEST"

    log "=== complete: $done_ok fetched, $skipped already present, $failed failed (of $total)"
    [ "$failed" -gt 0 ] && log "failures listed in $FAILED — re-run to retry them"
    return 0
}

# ---------------------------------------------------------- source indexes ---
# Source stanzas are shaped differently from binary ones: the path is
# "Directory:" plus each filename listed in the multi-line "Checksums-Sha256:"
# block, and one source package yields several files (.dsc, .orig.tar.*,
# .debian.tar.*). Debug packages do not exist as sources, so only the kernel
# filter applies here.
fetch_source_indexes() {
    local comp url out
    for comp in $COMPONENTS; do
        out="$META_DIR/${comp}.Sources"
        url="$SNAPSHOT/dists/$SUITE/updates/$comp/source/Sources.gz"
        if [ -s "$out" ]; then
            log "source index cached: $comp ($(grep -c '^Package: ' "$out") sources)"
            continue
        fi
        log "fetching source index: $comp"
        if curl -sSfL --retry "$RETRIES" --retry-delay 2 --max-time 300 \
                --limit-rate "$RATE_LIMIT" "$url" -o "$out.gz" 2>>"$LOG"; then
            gzip -dc "$out.gz" > "$out" 2>>"$LOG" && rm -f "$out.gz" \
                || { log "WARN: could not decompress sources for $comp"; rm -f "$out" "$out.gz"; }
            [ -s "$out" ] && log "  -> $(grep -c '^Package: ' "$out") sources"
        else
            log "WARN: no source index for $comp (may not exist upstream)"
            rm -f "$out.gz"
        fi
    done
    ls "$META_DIR"/*.Sources >/dev/null 2>&1 || die "no source indexes fetched; cannot continue"
}

# Only the sources behind the binaries we actually keep -- that is what the
# licence obliges us to offer, and it avoids pulling sources for the kernels
# and debug packages we deliberately exclude.
wanted_source_names() {
    awk -v skip_kern="$SKIP_KERNELS" -v skip_dbg="$SKIP_DEBUG" -v excl="$EXCLUDE_SOURCES" '
        BEGIN { n = split(excl, a, " "); for (i = 1; i <= n; i++) bad[a[i]] = 1 }
        function flush() {
            if (pkg != "") {
                drop = 0
                if (skip_kern && (pkg ~ /^linux-(image|headers|kbuild|support|source|config|doc)/)) drop = 1
                if (skip_dbg  && (pkg ~ /(-dbg|-dbgsym)$/))                       drop = 1
                # "Source: name (version)" when the source version differs
                s = (src != "" ? src : pkg)
                sub(/ .*$/, "", s)
                if (s in bad) drop = 1
                if (!drop) print s
            }
            pkg=""; src=""
        }
        /^Package: / { flush(); pkg = $2; next }
        /^Source: /  { src = $2; next }
        /^$/         { flush(); next }
        END          { flush() }
    ' "$META_DIR"/*.Packages | sort -u
}

# One row per FILE: source <TAB> path <TAB> size <TAB> sha256
build_source_manifest() {
    wanted_source_names > "$META_DIR/wanted-sources.txt"
    log "source packages behind the kept binaries: $(wc -l < "$META_DIR/wanted-sources.txt")"

    : > "$SRC_MANIFEST"
    awk -v wantfile="$META_DIR/wanted-sources.txt" '
        BEGIN { while ((getline l < wantfile) > 0) want[l] = 1 }
        function flush() {
            if (pkg != "" && dir != "" && (pkg in want)) {
                for (i = 1; i <= nf; i++)
                    printf "%s\t%s/%s\t%s\t%s\n", pkg, dir, fname[i], fsize[i], fsha[i]
            }
            pkg=""; dir=""; nf=0; insums=0
        }
        /^Package: /   { flush(); pkg = $2; next }
        /^Directory: / { dir = $2; insums = 0; next }
        /^Checksums-Sha256:/ { insums = 1; next }
        /^[A-Za-z0-9-]+:/ { insums = 0 }
        {
            # continuation lines of the checksum block: " <sha> <size> <name>"
            if (insums && $0 ~ /^ /) { nf++; fsha[nf] = $1; fsize[nf] = $2; fname[nf] = $3 }
            next
        }
        /^$/ { flush(); next }
        END  { flush() }
    ' "$META_DIR"/*.Sources | sort -u -k2,2 > "$SRC_MANIFEST"

    local n bytes
    n=$(wc -l < "$SRC_MANIFEST")
    bytes=$(awk -F'\t' '{t+=$3} END{printf "%.0f", t+0}' "$SRC_MANIFEST")
    log "source manifest: $n files, $(awk -v b="$bytes" 'BEGIN{printf "%.1f MiB", b/1048576}')"
}

download_sources() {
    local name fn size sha dest url tmp
    local total done_ok skipped failed idx=0
    total=$(wc -l < "$SRC_MANIFEST"); done_ok=0; skipped=0; failed=0

    while IFS=$'\t' read -r name fn size sha; do
        idx=$((idx+1))
        dest="$SRC_DIR/$fn"

        if verify "$dest" "$sha"; then
            skipped=$((skipped+1))
            continue
        fi
        [ -f "$dest" ] && { log "re-fetching (checksum mismatch): $fn"; rm -f "$dest"; }

        if [ "$DRY_RUN" -eq 1 ]; then
            printf '  would fetch %s (%s bytes)\n' "$fn" "$size"
            continue
        fi

        mkdir -p "$(dirname "$dest")" || { log "ERR: mkdir failed for $fn"; continue; }
        url="$SNAPSHOT/$fn"
        tmp="$dest.part"

        if curl -sSfL -C - --retry "$RETRIES" --retry-delay 2 --max-time 900 \
                --limit-rate "$RATE_LIMIT" "$url" -o "$tmp" 2>>"$LOG"; then
            if verify "$tmp" "$sha"; then
                mv -f "$tmp" "$dest"
                done_ok=$((done_ok+1))
                [ $((done_ok % 25)) -eq 0 ] && log "source progress: $idx/$total ($done_ok fetched, $skipped already present)"
            else
                log "ERR: checksum mismatch after download: $fn"
                printf '%s\t%s\tchecksum-src\n' "$name" "$fn" >> "$FAILED"
                rm -f "$tmp"; failed=$((failed+1))
            fi
        else
            log "ERR: fetch failed: $fn"
            printf '%s\t%s\tfetch-src\n' "$name" "$fn" >> "$FAILED"
            rm -f "$tmp"; failed=$((failed+1))
        fi
        sleep "$SLEEP_BETWEEN"
    done < "$SRC_MANIFEST"

    log "=== sources complete: $done_ok fetched, $skipped already present, $failed failed (of $total)"
    return 0
}

# ------------------------------------------------------------------ prune ---
# Filters tighten over time (kernels, debug, whole source packages). Anything
# already downloaded that the current manifest no longer lists is dead weight,
# and worse, would still be published if the tree were handed to reprepro as-is.
prune_tree() {
    local tree="$1" manifest="$2" label="$3" col="$4"
    [ -d "$tree" ] || return 0

    local expected removed bytes
    expected=$(mktemp) || return 1
    # LC_ALL=C on BOTH sorts: comm requires identical collation, and these paths
    # are full of +, ~ and % which a UTF-8 locale orders differently.
    cut -f"$col" "$manifest" | sed "s|^|$tree/|" | LC_ALL=C sort -u > "$expected"

    removed=0; bytes=0
    # Every regular file, not a list of extensions: the manifests also contain
    # .diff.gz (old-format sources) and .asc (upstream signatures), and a partial
    # download leaves a .part. Anything under this tree that the manifest does
    # not list is by definition dead weight.
    while IFS= read -r file; do
        removed=$((removed+1))
        bytes=$((bytes + $(stat -c%s "$file" 2>/dev/null || echo 0)))
        if [ "$DRY_RUN" -eq 1 ]; then
            printf '  would remove %s\n' "${file#$tree/}"
        else
            rm -f "$file"
        fi
    done < <(find "$tree" -type f | LC_ALL=C sort -u | comm -23 - "$expected")

    rm -f "$expected"
    if [ "$removed" -gt 0 ]; then
        log "prune $label: $removed files, $(awk -v b="$bytes" 'BEGIN{printf "%.1f MiB", b/1048576}')$([ "$DRY_RUN" -eq 1 ] && echo ' (dry run)')"
        [ "$DRY_RUN" -eq 0 ] && find "$tree" -type d -empty -delete 2>/dev/null
    else
        log "prune $label: nothing to remove"
    fi
}

do_prune() {
    [ "$BINARIES" -eq 1 ]     && prune_tree "$POOL_DIR" "$MANIFEST"     "pool" 2
    [ "$WANT_SOURCES" -eq 1 ] && prune_tree "$SRC_DIR"  "$SRC_MANIFEST" "src"  2
    return 0
}

# Binary indexes are needed either way: the source set is derived from them.
fetch_indexes
build_manifest

if [ "$WANT_SOURCES" -eq 1 ]; then
    fetch_source_indexes
    build_source_manifest
fi

[ "$PRUNE" -eq 1 ] && do_prune

if [ "$DRY_RUN" -eq 1 ]; then
    log "dry run — nothing will be downloaded"
    if [ "$BINARIES" -eq 1 ]; then
        head -20 "$MANIFEST" | awk -F'\t' '{printf "  %-40s %10.2f KiB\n", $1, $3/1024}'
        log "(showing first 20 of $(wc -l < "$MANIFEST") binaries)"
    fi
    if [ "$WANT_SOURCES" -eq 1 ]; then
        head -20 "$SRC_MANIFEST" | awk -F'\t' '{printf "  %-52s %10.2f KiB\n", $2, $3/1024}'
        log "(showing first 20 of $(wc -l < "$SRC_MANIFEST") source files)"
    fi
    exit 0
fi

: > "$FAILED"   # one failure log per run, whichever modes are selected
[ "$BINARIES" -eq 1 ] && download_all
[ "$WANT_SOURCES" -eq 1 ] && download_sources

[ "$BINARIES" -eq 1 ]     && log "binary pool is at $POOL_DIR"
[ "$WANT_SOURCES" -eq 1 ] && log "source pool is at $SRC_DIR"
log "verify later with: $0 --dry-run [--sources]   (re-checks every checksum without downloading)"