TerslerThe little junk rig that could

Source files / scripts · raw

check-updates.sh

#!/usr/bin/env bash
# What changed upstream since the last check? Reports:
#   - new commits on the build fork beyond the pinned LLAMA_COMMIT (with log)
#   - new branches on the build fork (e.g. a new P100 branch)
#   - changes on the other watched repos (install/upstreams.list)
#   - new GitHub repos matching P100/Pascal llama.cpp searches
#   - driver / CUDA package updates (on the rig, via apt)
#   - model repo updates on Hugging Face (if MODEL_HF_REPO is set)
#
#   check-updates.sh            print a Markdown report
#   check-updates.sh -o FILE    also write it to FILE
#   check-updates.sh --no-save  don't update the "last seen" state
#
# Exit code: 0 = nothing new, 10 = something new, other = error.
# No root needed. Uses git ls-remote, a tree-less cache clone of the fork,
# and the unauthenticated GitHub/Hugging Face APIs (set GITHUB_TOKEN to raise
# the GitHub rate limit).
set -euo pipefail
here=$(dirname "$(readlink -f "$0")")
# shellcheck source=lib.sh
source "$here/lib.sh"
jr_load_config

OUT=""; SAVE=1
while (( $# )); do
    case $1 in
        -o) OUT=$2; shift 2 ;;
        --no-save) SAVE=0; shift ;;
        -h|--help) sed -n '2,18p' "$0"; exit 0 ;;
        *) jr_die "unknown option $1" ;;
    esac
done

UPSTREAMS=${UPSTREAMS:-}
for c in "$here/../install/upstreams.list" "$JR_ETC/upstreams.list"; do
    [[ -z $UPSTREAMS && -f $c ]] && UPSTREAMS=$c
done
STATE_DIR=${XDG_STATE_HOME:-$HOME/.local/state}/tersler
CACHE_DIR=${XDG_CACHE_HOME:-$HOME/.cache}/tersler
STATE=$STATE_DIR/seen.json
mkdir -p "$STATE_DIR" "$CACHE_DIR"
[[ -s $STATE ]] || echo '{}' > "$STATE"
NEW=0
report=""
say()  { report+="$*"$'\n'; }
seen() { jq -r --arg k "$1" '.[$k] // empty' "$STATE"; }
set_seen() { local tmp; tmp=$(mktemp); jq --arg k "$1" --arg v "$2" '.[$k]=$v' "$STATE" > "$tmp" && mv "$tmp" "$STATE"; }
gh_api() {
    local auth=(); [[ -n ${GITHUB_TOKEN:-} ]] && auth=(-H "Authorization: Bearer $GITHUB_TOKEN")
    curl -fsSL "${auth[@]}" -H 'Accept: application/vnd.github+json' "https://api.github.com/$1"
}

say "# tersler update check — $(date '+%Y-%m-%d %H:%M')"
say ""

# ---- 1. build fork vs pinned commit -----------------------------------------
say "## Build fork: ${LLAMA_REPO#https://github.com/} @ $LLAMA_BRANCH"
cache=$CACHE_DIR/fork.git
if [[ ! -d $cache ]]; then
    git clone -q --bare --filter=tree:0 --single-branch --branch "$LLAMA_BRANCH" "$LLAMA_REPO" "$cache"
else
    git -C "$cache" remote set-url origin "$LLAMA_REPO"
    git -C "$cache" fetch -q --filter=tree:0 origin "+refs/heads/$LLAMA_BRANCH:refs/heads/$LLAMA_BRANCH"
fi
head=$(git -C "$cache" rev-parse "$LLAMA_BRANCH")
pin=$(git -C "$cache" rev-parse --verify -q "$LLAMA_COMMIT^{commit}" || true)
if [[ -z $pin ]]; then
    say "- ⚠ pinned commit \`$LLAMA_COMMIT\` not found on this branch (rebased/force-pushed?)"
    NEW=1
elif [[ $pin == "$head" ]]; then
    say "- pinned \`${pin:0:9}\` is the branch head: up to date"
else
    n=$(git -C "$cache" rev-list --count "$pin..$head")
    nf=$(git -C "$cache" rev-list --count --first-parent "$pin..$head")
    say "- **$n new commit(s)** since pinned \`${pin:0:9}\` → head \`${head:0:9}\` ($nf on the fork's own line; the rest arrived via upstream merges):"
    say '```'
    say "$(git -C "$cache" log --first-parent --format='%h %ad %s' --date=short "$pin..$head" | sed -n 1,40p)"
    say '```'
    say "  Try it: \`LLAMA_COMMIT=${head:0:9} sudo -E install/install.sh build\`, then \`tersler-bench\` (see docs/keeping-up.md)"
    NEW=1
fi
branches=$(git ls-remote --heads "$LLAMA_REPO" | awk '{sub("refs/heads/","",$2); print $2}' | sort)
old_branches=$(seen "branches:$LLAMA_REPO")
if [[ -n $old_branches ]]; then
    added=$(comm -13 <(tr ' ' '\n' <<<"$old_branches" | sort) <(echo "$branches"))
    if [[ -n $added ]]; then say "- **new branches:** $(tr '\n' ' ' <<<"$added")"; NEW=1; fi
fi
(( SAVE )) && set_seen "branches:$LLAMA_REPO" "$(tr '\n' ' ' <<<"$branches")"
say ""

# ---- 2. other watched repos ---------------------------------------------------
say "## Watched repositories"
if [[ -n $UPSTREAMS ]]; then
    trim() { sed 's/^[[:space:]]*//; s/[[:space:]]*$//' <<<"$1"; }
    while IFS='|' read -r name url ref note; do
        [[ $name =~ ^[[:space:]]*(#|$) ]] && continue
        name=$(trim "$name"); url=$(trim "$url"); ref=$(trim "$ref"); note=$(trim "${note:-}")
        if [[ $ref == tags:* ]]; then
            val=$(git ls-remote --tags --refs "$url" 2>/dev/null | awk '{sub("refs/tags/","",$2); print $2}' \
                  | grep -E "${ref#tags:}" | sort -V | tail -1 || true)
        elif [[ $ref == HEAD ]]; then
            val=$(git ls-remote "$url" HEAD 2>/dev/null | awk '{print substr($1,1,9)}')
        else
            val=$(git ls-remote --heads "$url" "$ref" 2>/dev/null | awk '{print substr($1,1,9)}')
        fi
        if [[ -z $val ]]; then say "- $name: ⚠ unreachable ($url)"; continue; fi
        old=$(seen "repo:$url:$ref")
        if [[ -z $old ]];          then say "- $name: \`$val\` (first check) — $note"
        elif [[ $old != "$val" ]]; then say "- **$name: \`$old\` → \`$val\`** — $note ($url)"; NEW=1
        else                            say "- $name: unchanged (\`$val\`)"
        fi
        (( SAVE )) && set_seen "repo:$url:$ref" "$val"
    done < "$UPSTREAMS"
else
    say "- (no upstreams.list found)"
fi
say ""

# ---- 3. new P100 / Pascal projects on GitHub ---------------------------------
say "## GitHub search: P100 / Pascal llama.cpp projects"
for q in 'p100 llama' 'pascal llama.cpp' 'sm_60 llama' 'tesla p100 llm'; do
    enc=$(jq -rn --arg q "$q" '$q|@uri')
    gh_api "search/repositories?q=$enc&sort=updated&order=desc&per_page=15" 2>/dev/null \
        | jq -r '.items[]? | [.full_name, .pushed_at[0:10], .html_url, (.description // "" | .[0:80])] | @tsv' || true
done | sort -u -t$'\t' -k1,1 > "$CACHE_DIR/search.tsv"
if [[ ! -s $CACHE_DIR/search.tsv ]]; then
    say "- ⚠ search failed (rate limit? set GITHUB_TOKEN)"
else
    known=$(seen "search:repos")
    newrepos=0
    while IFS=$'\t' read -r full pushed url desc; do
        if [[ -n $known && " $known " != *" $full "* ]]; then
            say "- **new:** [$full]($url) (pushed $pushed) $desc"; newrepos=1; NEW=1
        fi
    done < "$CACHE_DIR/search.tsv"
    [[ -z $known ]] && say "- first run: recorded $(wc -l < "$CACHE_DIR/search.tsv") repos as known"
    [[ -n $known && $newrepos == 0 ]] && say "- nothing new"
    (( SAVE )) && set_seen "search:repos" "$(cut -f1 "$CACHE_DIR/search.tsv" | sort -u | tr '\n' ' ') $known"
fi
say ""

# ---- 4. driver / CUDA (only meaningful on the rig) ---------------------------
say "## Driver and CUDA packages"
if command -v apt-cache >/dev/null && apt-cache policy "$DRIVER_PKG" 2>/dev/null | grep Candidate >/dev/null; then
    inst=$(apt-cache policy "$DRIVER_PKG" | awk '/Installed:/ {print $2}')
    cand=$(apt-cache policy "$DRIVER_PKG" | awk '/Candidate:/ {print $2}')
    if [[ $inst != "$cand" ]]; then say "- **$DRIVER_PKG: $inst → $cand available** (held; \`apt-mark unhold\` to upgrade deliberately)"; NEW=1
    else say "- $DRIVER_PKG: $inst (current)"; fi
    newest12=$(apt-cache pkgnames cuda-toolkit-12- 2>/dev/null | grep -E '^cuda-toolkit-12-[0-9]+$' | sort -V | tail -1 || true)
    if [[ -n $newest12 && $newest12 != "$CUDA_PKG" ]] && [[ $(printf '%s\n%s\n' "$newest12" "$CUDA_PKG" | sort -V | tail -1) == "$newest12" ]]; then
        say "- **newer CUDA 12.x toolkit available: $newest12** (installed/pinned: $CUDA_PKG). CUDA 13+ has no Pascal support."; NEW=1
    else
        say "- $CUDA_PKG is the newest 12.x toolkit in the configured repos"
    fi
    if apt-cache pkgnames nvidia-driver- 2>/dev/null | grep -E "nvidia-driver-(5[89][0-9]|[6-9][0-9]{2})-server$" >/dev/null; then
        say "- note: newer driver branches exist; they don't support Pascal (R580 is the last). Stay on $DRIVER_BRANCH."
    fi
else
    say "- skipped ($DRIVER_PKG not known to apt here; run on the rig)"
fi
say ""

# ---- 5. model ------------------------------------------------------------------
say "## Model"
if [[ -n ${MODEL_HF_REPO:-} ]]; then
    if info=$(curl -fsSL "https://huggingface.co/api/models/$MODEL_HF_REPO" 2>/dev/null); then
        sha=$(jq -r '.sha[0:9]' <<<"$info"); mod=$(jq -r '.lastModified[0:10]' <<<"$info")
        old=$(seen "hf:$MODEL_HF_REPO")
        if [[ -n $old && $old != "$sha" ]]; then say "- **$MODEL_HF_REPO updated** ($old → $sha, $mod)"; NEW=1
        else say "- $MODEL_HF_REPO: \`$sha\` (modified $mod)"; fi
        (( SAVE )) && set_seen "hf:$MODEL_HF_REPO" "$sha"
    else
        say "- ⚠ could not query $MODEL_HF_REPO"
    fi
else
    say "- MODEL_HF_REPO not set; skipped"
fi
say ""
(( NEW )) && say "**Result: something new — review above.**" || say "Result: nothing new."

printf '%s' "$report"
if [[ -n $OUT ]]; then mkdir -p "$(dirname "$OUT")"; printf '%s' "$report" > "$OUT"; jr_log "report written to $OUT"; fi
(( NEW )) && exit 10 || exit 0