Source files / scripts · raw

server.sh

#!/usr/bin/env bash
# Start llama-server with the configured model and flags (reference recipe).
#   tersler-server [extra llama-server args...]
# Env overrides: LLAMA_BIN (build dir), MODEL (gguf path), LLAMA_HOST, LLAMA_PORT.
set -euo pipefail
# shellcheck source=lib.sh
source "$(dirname "$(readlink -f "$0")")/lib.sh"
# the config sets LLAMA_HOST/LLAMA_PORT too; callers' values (tersler-bench) win
env_host=${LLAMA_HOST:-} env_port=${LLAMA_PORT:-}
jr_load_config
LLAMA_HOST=${env_host:-$LLAMA_HOST} LLAMA_PORT=${env_port:-$LLAMA_PORT}

bin=${LLAMA_BIN:-$JR_PREFIX/current}/bin/llama-server
model=${MODEL:-$(jr_model_path)}
[[ -x $bin ]]   || jr_die "llama-server not found: $bin (run: sudo install/install.sh build)"
[[ -f $model ]] || jr_die "model not found: $model (set MODEL_URL, run: sudo install/install.sh model)"

extra=()
if [[ -n ${MMPROJ_FILE:-} ]]; then
    [[ -f $JR_MODEL_DIR/$MMPROJ_FILE ]] || jr_die "mmproj not found: $JR_MODEL_DIR/$MMPROJ_FILE (run: sudo install/install.sh model)"
    extra=(--mmproj "$JR_MODEL_DIR/$MMPROJ_FILE")
fi

envs=("${LLAMA_ENV[@]}")
if jr_p2p_enabled; then
    envs+=(GGML_CUDA_P2P=1)
else
    jr_warn "P2P off (iommu=pt not active or LLAMA_P2P=0); costs a few percent"
fi

jr_log "build: $(readlink -f "${LLAMA_BIN:-$JR_PREFIX/current}")"
jr_log "model: $model"
exec env "${envs[@]}" "$bin" -m "$model" "${extra[@]}" "${LLAMA_ARGS[@]}" \
    --host "$LLAMA_HOST" --port "$LLAMA_PORT" "$@"