#!/usr/bin/env bash
# corecp-panel-apply — the root helper behind `corecp-panel self-update`.
#
# A panel updates itself from its own package source (round 6, session
# r6-paneel-zelfupdate; docs/panel-self-update.md). This script does the work,
# and it is a script of its own rather than a function of the panel binary for
# one reason: it replaces that binary, and the step that puts the previous
# package back has to run precisely when the new one does not start.
#
# `corecp-panel self-update` starts it as the transient unit
# corecp-panel-self-update, so the work belongs to systemd and not to the
# terminal: a dropped ssh session does not interrupt a package swap halfway.
#
#   corecp-panel-apply update --to <ver> [--hold-contract] [--no-backup <reason>]
#   corecp-panel-apply finish
#   corecp-panel-apply rollback
#   corecp-panel-apply source --url https://packages.<domain> --suite <suite>
#   (every verb also takes --config <file> --actor <name> --run-id <id>)
#
# An update, in order — and what a failure at each step leaves behind:
#
#   1 preflight  installed from a package, the target newer, the target AND the
#                current version both in this panel's own source (no way back,
#                no update), nothing held from a previous update.
#                Failure: nothing changed.
#   2 backup     pgBackRest, incremental, of the panel database; the new backup
#                label is read back and must be younger than this run.
#                Failure: nothing changed. --no-backup <reason> skips it, loudly.
#   3 download   both packages, through apt, from this panel's own source only
#                (signature and hash checked by apt); the target's binary must
#                say it is the target version.
#   4 expand     the TARGET's binary applies the expand migrations while the
#                current version keeps serving — expand migrations never remove
#                what the running version reads.
#                Failure: the panel still runs the current version; the
#                migration that failed rolled back in its own transaction.
#   5 swap       the contract hold is set, then apt installs the target package.
#   6 health     the service restarted after the swap, serves the target
#                version, answers its health probe, and is still doing so ten
#                seconds later without having restarted again.
#                Failure at 5 or 6: the previous package is installed again and
#                must pass the same health check; the hold is lifted.
#   7 contract   the contract migrations (all that remain), unless
#                --hold-contract; after this there is no way back.
#                Failure: the panel runs the target; the hold stays; --finish
#                again once the cause is fixed.
#
# The journal (/var/lib/corecp-panel-apply/state.json) is written at every
# step, so a helper that is killed leaves a record of where.
set -uo pipefail
umask 022

DIR=/var/lib/corecp-panel-apply
HOLD=$DIR/contract-hold
STATE=$DIR/state.json
PKGDIR=$DIR/packages
WORK=$DIR/work
LOCK=/run/lock/corecp-panel-apply.lock
SOURCES=/etc/apt/sources.list.d/corecp-panel-self.sources
KEYRING=/usr/share/keyrings/corecp-panel-self.gpg
PKG=corecp-panel
STANZA=corecp-panel
SERVICE=corecp-panel.service
HEALTH_TIMEOUT=180
HEALTH_SETTLE=10

CONFIG=/etc/corecp-panel/panel.yaml
ACTOR=root
RUN_ID=""
TO=""
HOLD_CONTRACT=0
NO_BACKUP=""
URL=""
SUITE=""

say()  { printf '[self-update] %s\n' "$*"; }
warn() { printf '[self-update] !! %s\n' "$*" >&2; }

# Versions reach apt as one argument; anything that is not a package version is
# refused before it gets there.
valid_version() { [[ $1 =~ ^[0-9][A-Za-z0-9.+~]{0,62}$ ]]; }

VERB=${1:-}
[ -n "$VERB" ] && shift
while [ $# -gt 0 ]; do
  case "$1" in
    --to)            TO=${2:-}; shift 2 ;;
    --hold-contract) HOLD_CONTRACT=1; shift ;;
    --no-backup)     NO_BACKUP=${2:-}; shift 2 ;;
    --config)        CONFIG=${2:-}; shift 2 ;;
    --actor)         ACTOR=${2:-}; shift 2 ;;
    --run-id)        RUN_ID=${2:-}; shift 2 ;;
    --url)           URL=${2:-}; shift 2 ;;
    --suite)         SUITE=${2:-}; shift 2 ;;
    *) warn "unknown argument: $1"; exit 2 ;;
  esac
done
[ "$(id -u)" = 0 ] || { warn "runs as root"; exit 2; }
[[ $RUN_ID =~ ^[0-9a-f]{0,32}$ ]] || { warn "bad run id"; exit 2; }
[[ $ACTOR =~ ^[A-Za-z0-9._@-]{1,64}$ ]] || ACTOR=root
[ -r "$CONFIG" ] || { warn "no configuration at $CONFIG"; exit 2; }

install -d -m 0755 -o root -g root "$DIR" "$PKGDIR"

# ---------------------------------------------------------------------------
# The journal
# ---------------------------------------------------------------------------

# journal key=value … — merge into state.json, atomically. `new` starts a fresh
# journal for this run; `expanded` is a comma-separated list; `contract_ran` a
# boolean. Everything else is a string.
journal() {
  python3 - "$STATE" "$RUN_ID" "$@" <<'PY'
import json, os, sys, datetime
path, run_id, pairs = sys.argv[1], sys.argv[2], sys.argv[3:]
now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
state = {}
if "new=1" not in pairs:
    try:
        with open(path) as f:
            state = json.load(f)
    except (OSError, ValueError):
        state = {}
else:
    state = {"started_at": now, "contract_ran": False}
state["run_id"] = run_id
for p in pairs:
    k, _, v = p.partition("=")
    if k == "new":
        continue
    if k == "contract_ran":
        state[k] = v == "true"
    elif k == "expanded":
        state[k] = [x for x in v.split(",") if x]
    else:
        state[k] = v
state["updated_at"] = now
tmp = path + ".tmp"
with open(tmp, "w") as f:
    json.dump(state, f, indent=2)
    f.write("\n")
os.chmod(tmp, 0o644)
os.replace(tmp, path)
PY
}

# jget <key> — one field of the current journal, empty when absent.
jget() {
  python3 - "$STATE" "$1" <<'PY' 2>/dev/null
import json, sys
try:
    v = json.load(open(sys.argv[1])).get(sys.argv[2], "")
except (OSError, ValueError):
    v = ""
print(str(v).lower() if isinstance(v, bool) else v)
PY
}

# refuse <reason> — nothing changed; say so in the journal and stop.
refuse() {
  warn "refused: $1"
  journal result=refused "reason=$1"
  exit 3
}

# ---------------------------------------------------------------------------
# What is installed, what the source has, whether the panel is well
# ---------------------------------------------------------------------------

installed_version() {
  local line
  line=$(dpkg-query -W -f='${db:Status-Abbrev}|${Version}' "$PKG" 2>/dev/null) || return 1
  case "$line" in
    "ii "*"|"*) printf '%s' "${line#*|}" ;;
    *) return 1 ;;
  esac
}

# apt, reading this panel's own source and no other. The same options on the
# update, the lookup and the download: a version that only some other source
# carries is not a version this panel's own source serves.
self_apt() {
  apt-get -q -o Dir::Etc::sourcelist="$SOURCES" -o Dir::Etc::sourceparts=- \
    -o APT::Get::List-Cleanup=0 -o DPkg::Lock::Timeout=300 "$@"
}
self_cache() {
  apt-cache -o Dir::Etc::sourcelist="$SOURCES" -o Dir::Etc::sourceparts=- "$@"
}
source_has() {
  self_cache madison "$PKG" 2>/dev/null | awk -F'|' '{gsub(/ /, "", $2); print $2}' | grep -Fxq -- "$1"
}

# download <ver> — the package file from this panel's own source, into PKGDIR;
# prints its path. apt checks the source's signature and the file's hash.
download() {
  local ver=$1 f
  ( cd "$PKGDIR" && self_apt download "$PKG=$ver" >&2 ) || return 1
  for f in "$PKGDIR/${PKG}_"*.deb; do
    [ -f "$f" ] || continue
    if [ "$(dpkg-deb -f "$f" Package 2>/dev/null)" = "$PKG" ] &&
       [ "$(dpkg-deb -f "$f" Version 2>/dev/null)" = "$ver" ]; then
      printf '%s' "$f"
      return 0
    fi
  done
  return 1
}

# keep_only <deb>… — the package files of this run stay, older ones go.
keep_only() {
  local f k keep
  for f in "$PKGDIR/${PKG}_"*.deb; do
    [ -f "$f" ] || continue
    keep=0
    for k in "$@"; do [ "$f" = "$k" ] && keep=1; done
    [ "$keep" = 1 ] || rm -f -- "$f"
  done
}

panel_listen() {
  local l
  l=$(sed -n 's/^listen:[[:space:]]*//p' "$CONFIG" | head -1 | tr -d "\"' ")
  printf '%s' "${l:-127.0.0.1:8080}"
}

# healthy <ver> <since-epoch> — the service restarted at or after <since>,
# serves <ver>, answers the probe, and is still doing so HEALTH_SETTLE seconds
# later without another restart. Sets HEALTH_REASON on failure.
HEALTH_REASON=""
healthy() {
  local want=$1 since=$2 deadline listen got enter restarts
  listen=$(panel_listen)
  deadline=$(( $(date +%s) + HEALTH_TIMEOUT ))
  HEALTH_REASON="the service did not come back within ${HEALTH_TIMEOUT}s"
  while [ "$(date +%s)" -lt "$deadline" ]; do
    sleep 3
    if ! systemctl is-active --quiet "$SERVICE"; then
      HEALTH_REASON="$SERVICE is $(systemctl is-active "$SERVICE" 2>/dev/null)"
      continue
    fi
    enter=$(date -d "$(systemctl show -p ActiveEnterTimestamp --value "$SERVICE")" +%s 2>/dev/null || echo 0)
    if [ "$enter" -lt "$since" ]; then
      HEALTH_REASON="$SERVICE was not restarted after the package changed (still the process from before)"
      continue
    fi
    got=$(curl -fsS --max-time 5 "http://$listen/api/v1/meta/build" 2>/dev/null |
          python3 -c 'import json,sys; print(json.load(sys.stdin).get("version",""))' 2>/dev/null)
    if [ "$got" != "$want" ]; then
      HEALTH_REASON="the panel serves version '${got:-nothing}', not $want"
      continue
    fi
    if ! curl -fsS --max-time 5 -o /dev/null "http://$listen/api/v1/health"; then
      HEALTH_REASON="the panel's health probe does not answer"
      continue
    fi
    restarts=$(systemctl show -p NRestarts --value "$SERVICE")
    sleep "$HEALTH_SETTLE"
    if systemctl is-active --quiet "$SERVICE" &&
       [ "$(systemctl show -p NRestarts --value "$SERVICE")" = "$restarts" ] &&
       curl -fsS --max-time 5 -o /dev/null "http://$listen/api/v1/health"; then
      HEALTH_REASON=""
      return 0
    fi
    HEALTH_REASON="the panel came up on $want and did not stay up"
  done
  return 1
}

# install_package <deb> — apt installs a package file we already hold; its
# dependencies may come from any configured source.
install_package() {
  DEBIAN_FRONTEND=noninteractive apt-get install -y -q --allow-downgrades \
    -o DPkg::Lock::Timeout=300 -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold \
    "$1"
}

# go_back <from> <deb> <why> — the previous package, the same health check, the
# hold lifted. Exits.
go_back() {
  local from=$1 deb=$2 why=$3 since
  warn "$why — putting $from back"
  journal phase=rollback "reason=$why"
  since=$(date +%s)
  if install_package "$deb" && healthy "$from" "$since"; then
    rm -f "$HOLD"
    journal phase=done result=rolled_back "reason=$why; $from is running again"
    say "rolled back: $from is running again"
    exit 4
  fi
  journal phase=rollback result=broken \
    "reason=$why; going back to $from failed too: ${HEALTH_REASON:-apt could not install it}"
  warn "going back to $from failed too: ${HEALTH_REASON:-apt could not install it}"
  warn "the contract hold stays; the database backup is $(jget backup)"
  exit 5
}

# contract_step — the migrations that remain. Sets the journal and exits.
contract_step() {
  local to=$1 before after
  journal phase=contract result=running
  before=$(/usr/bin/corecp-panel migrate --pending --json --config "$CONFIG" 2>/dev/null)
  if /usr/bin/corecp-panel migrate --phase all --config "$CONFIG"; then
    rm -f "$HOLD"
    journal phase=done result=done contract_ran=true "reason="
    say "done: $to is running and its migrations are complete"
    exit 0
  fi
  after=$(/usr/bin/corecp-panel migrate --pending --json --config "$CONFIG" 2>/dev/null)
  # A contract migration that did run has removed something the previous
  # version reads: from then on the way back is closed, whatever else failed.
  local ran=false
  if python3 - "$before" "$after" <<'PY'
import json, sys
def contract(s):
    try:
        return {m["name"] for m in json.loads(s or "[]") if m.get("contract")}
    except ValueError:
        return None
b, a = contract(sys.argv[1]), contract(sys.argv[2])
sys.exit(0 if b is None or a is None or (b - a) else 1)
PY
  then ran=true; fi
  journal phase=contract result=held contract_ran=$ran \
    "reason=the contract migrations failed; $to is running, run corecp-panel self-update --finish once the cause is fixed"
  warn "the contract migrations failed; the hold stays"
  exit 6
}

exec 9>"$LOCK"
if ! flock -n 9; then
  warn "another self-update is running"
  exit 3
fi

case "$VERB" in
# ---------------------------------------------------------------------------
source)
  [[ $URL =~ ^https://[a-z0-9]([a-z0-9.-]{0,251}[a-z0-9])?$ ]] || { warn "not a source URL: $URL"; exit 2; }
  [[ $SUITE =~ ^[a-z][a-z0-9-]{0,31}$ ]] || { warn "not a suite: $SUITE"; exit 2; }
  tmp=$(mktemp -d)
  trap 'rm -rf "$tmp"' EXIT
  curl -fsSL --max-time 30 "$URL/corecp-archive-keyring.asc" -o "$tmp/key.asc" || { warn "no key at $URL/corecp-archive-keyring.asc"; exit 1; }
  gpg --batch --quiet --dearmor < "$tmp/key.asc" > "$tmp/key.gpg" || { warn "the key at $URL is not an OpenPGP key"; exit 1; }
  install -m 0644 "$tmp/key.gpg" "$KEYRING"
  cat > "$tmp/sources" <<EOF
# CoreCP — this panel's own package source, from which it updates itself.
# Written by corecp-panel self-update source; see docs/panel-self-update.md.
Types: deb
URIs: $URL
Suites: $SUITE
Components: main
Signed-By: $KEYRING
EOF
  install -m 0644 "$tmp/sources" "$SOURCES"
  say "apt source $SOURCES → $URL $SUITE"
  say "key: $(gpg --batch --show-keys --with-colons "$tmp/key.gpg" 2>/dev/null | awk -F: '/^fpr/ {print $10; exit}')"
  self_apt update >/dev/null || { warn "apt could not read $URL"; exit 1; }
  say "this panel's source serves: $(self_cache madison "$PKG" 2>/dev/null | awk -F'|' '{gsub(/ /,"",$2); printf "%s ", $2}')"
  exit 0
  ;;

# ---------------------------------------------------------------------------
update)
  valid_version "$TO" || { warn "not a version: $TO"; exit 2; }
  prior=$(jget result)
  if [ -e "$HOLD" ] || [ "$prior" = held ] || [ "$prior" = running ] || [ "$prior" = broken ]; then
    # Not journaled: the journal is the previous update's, and it is the record
    # somebody needs to act on.
    warn "the previous self-update ($(jget from) → $(jget to)) is ${prior:-unfinished}, or its contract hold is set:"
    warn "finish it (--finish) or roll it back (--rollback) first"
    exit 3
  fi
  FROM=$(installed_version) || FROM=""
  journal new=1 "from=$FROM" "to=$TO" phase=preflight result=running "actor=$ACTOR"
  [ -n "$FROM" ] || refuse "this panel is not installed from a package (it runs from a source deploy); self-update works on a package install"
  [ "$FROM" != "$TO" ] || refuse "this panel already runs $TO"
  dpkg --compare-versions "$TO" gt "$FROM" || refuse "$TO is older than $FROM; going back is corecp-panel self-update --rollback, which only works while the contract step has not run"
  [ -r "$SOURCES" ] || refuse "this panel has no source of its own configured ($SOURCES); run corecp-panel self-update source"
  say "reading this panel's own source"
  self_apt update >/dev/null 2>&1 || refuse "apt could not read this panel's own source"
  source_has "$TO" || refuse "$TO is not in this panel's own source"
  source_has "$FROM" || refuse "$FROM, the version running now, is no longer in this panel's own source, so there would be no way back"

  journal phase=backup
  if [ -n "$NO_BACKUP" ]; then
    warn "no database backup first — $NO_BACKUP"
    journal "backup_skipped=$NO_BACKUP"
  else
    if [ ! -r /etc/pgbackrest/pgbackrest.conf ] || ! grep -q "^\[$STANZA\]" /etc/pgbackrest/pgbackrest.conf; then
      refuse "pgBackRest is not configured for the panel database (stanza $STANZA); set it up or pass --no-backup with a reason"
    fi
    started=$(date +%s)
    say "database backup (pgBackRest, incremental)"
    runuser -u postgres -- pgbackrest --stanza="$STANZA" --type=incr --log-level-console=warn backup ||
      refuse "the database backup failed, so nothing was changed"
    label=$(runuser -u postgres -- pgbackrest --stanza="$STANZA" --output=json info 2>/dev/null | python3 -c '
import json, sys
started = int(sys.argv[1])
try:
    b = json.load(sys.stdin)[0]["backup"][-1]
except (ValueError, IndexError, KeyError):
    sys.exit(1)
if b["timestamp"]["stop"] < started:
    sys.exit(1)
print(b["label"])' "$started") || refuse "the database backup reported success but no backup newer than this run is in the repository"
    journal "backup=$label"
    say "backup $label"
  fi

  journal phase=download
  NEW_DEB=$(download "$TO") || refuse "apt could not download $TO from this panel's own source"
  OLD_DEB=$(download "$FROM") || refuse "apt could not download $FROM from this panel's own source, so there would be no way back"
  rm -rf "$WORK" && install -d -m 0700 "$WORK"
  keep_only "$NEW_DEB" "$OLD_DEB"
  dpkg-deb -x "$NEW_DEB" "$WORK/new" || refuse "the package for $TO does not unpack"
  said=$("$WORK/new/usr/bin/corecp-panel" version 2>/dev/null)
  [ "$said" = "corecp-panel $TO" ] || refuse "the package for $TO carries a binary that says '$said'"

  journal phase=expand
  say "expand migrations, from $TO, while $FROM keeps serving"
  if ! out=$("$WORK/new/usr/bin/corecp-panel" migrate --phase expand --config "$CONFIG" 2>&1); then
    printf '%s\n' "$out" | sed 's/^/   /'
    applied=$(printf '%s\n' "$out" | sed -n 's/^\[corecp\] applied //p' | paste -sd, -)
    last=$(printf '%s\n' "$out" | grep -v '^\[corecp\] applied' | tail -1)
    journal phase=expand result=failed "expanded=$applied" \
      "reason=an expand migration of $TO failed before the swap, so $FROM is still running: ${last:0:400}"
    rm -rf "$WORK"
    warn "an expand migration failed; $FROM is still running"
    exit 1
  fi
  printf '%s\n' "$out" | sed 's/^/   /'
  journal "expanded=$(printf '%s\n' "$out" | sed -n 's/^\[corecp\] applied //p' | paste -sd, -)"
  rm -rf "$WORK"

  journal phase=swap
  touch "$HOLD"
  since=$(date +%s)
  say "installing $TO"
  if ! install_package "$NEW_DEB"; then
    go_back "$FROM" "$OLD_DEB" "apt could not install $TO"
  fi
  journal phase=health
  say "waiting for $TO to serve and stay up"
  if ! healthy "$TO" "$since"; then
    go_back "$FROM" "$OLD_DEB" "$TO did not become healthy: $HEALTH_REASON"
  fi
  say "$TO is running and healthy"

  if [ "$HOLD_CONTRACT" = 1 ]; then
    journal phase=health result=held "reason=the contract migrations are held; corecp-panel self-update --finish runs them, --rollback puts $FROM back"
    say "held: the contract migrations wait for --finish; --rollback still puts $FROM back"
    exit 0
  fi
  contract_step "$TO"
  ;;

# ---------------------------------------------------------------------------
finish)
  [ "$(jget result)" = held ] || { warn "there is no held self-update to finish (last result: $(jget result))"; exit 3; }
  TO=$(jget to)
  valid_version "$TO" || { warn "the journal names no valid target"; exit 3; }
  [ "$(installed_version)" = "$TO" ] || { warn "the journal says $TO, but $(installed_version) is installed"; exit 3; }
  journal "actor=$ACTOR"
  contract_step "$TO"
  ;;

# ---------------------------------------------------------------------------
rollback)
  FROM=$(jget from); TO=$(jget to)
  [ "$(jget result)" = held ] || { warn "there is no held self-update to roll back (last result: $(jget result))"; exit 3; }
  [ "$(jget contract_ran)" = false ] || { warn "the contract migrations of $TO have run; there is no way back to $FROM"; exit 3; }
  valid_version "$FROM" && valid_version "$TO" || { warn "the journal names no valid versions"; exit 3; }
  [ "$(installed_version)" = "$TO" ] || { warn "the journal says $TO, but $(installed_version) is installed"; exit 3; }
  # Refusals here leave the journal as it is: it still describes the held
  # update, which is the thing somebody has to act on.
  self_apt update >/dev/null 2>&1 || { warn "apt could not read this panel's own source; nothing changed"; exit 3; }
  OLD_DEB=$(download "$FROM") || { warn "$FROM is no longer in this panel's own source; nothing changed"; exit 3; }
  journal "actor=$ACTOR"
  go_back "$FROM" "$OLD_DEB" "rolled back on request by $ACTOR"
  ;;

*)
  warn "usage: corecp-panel-apply update|finish|rollback|source …"
  exit 2
  ;;
esac
