fwt_software/scripts/fgc

#!/usr/bin/env bash
#
# fgc — run fire_gimbal_control detached from the terminal, in a tmux session.
#
# The tower is operated over ssh. Launched directly, the program is a child of
# the ssh session's shell and dies with it — mid-scan, and with the camera torn
# down uncleanly (which wedges it for the next run). Here tmux owns the pty, so
# disconnecting is invisible to the program.
#
#   scripts/fgc start [args...]   start it detached (args go to the binary)
#   scripts/fgc attach            open the dashboard; Ctrl-b d to leave it running
#   scripts/fgc status            is it running, since when, as which PID
#   scripts/fgc stop              clean shutdown (camera + motor teardown)
#   scripts/fgc peek              print what the pane shows right now
#   scripts/fgc restart [args...] stop, then start
#
# Detaching is Ctrl-b d. NOTE: the dashboard's own `q` key *exits the program*,
# it does not detach — that is the one thing to get right when attached.
#
# Only one instance may run. The binary enforces that itself with a lockfile
# (see src/core/SingleInstance.cpp), so a bare ./build/fire_gimbal_control is
# refused too, not just a second `fgc start`.
#
# Env: FGC_SESSION overrides the tmux session name (default "fgc").
set -euo pipefail

here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
repo="$(cd "$here/.." && pwd)"
session="${FGC_SESSION:-fgc}"
target="=$session"   # "=" makes tmux match the name exactly, not as a prefix

die() { echo "fgc: $*" >&2; exit 1; }

need_tmux() {
    command -v tmux >/dev/null 2>&1 || die "tmux is not installed (apt-get install tmux)"
}

session_exists() { tmux has-session -t "$target" 2>/dev/null; }

# "1" if the pane's program has exited, "0" if alive, "" if there is no session.
# remain-on-exit keeps a finished pane around, so a session existing is not on
# its own proof that the program is running.
pane_dead() { tmux list-panes -t "$target" -F '#{pane_dead}' 2>/dev/null | head -1; }

running() { [[ "$(pane_dead)" == "0" ]]; }

# PID of the pane's process: the `_run` wrapper shell, which forwards signals to
# the app and reports its exit status to tmux. For the app's own PID use
# app_pid() — the lockfile is written by the program itself.
pane_pid() { tmux list-panes -t "$target" -F '#{pane_pid}' 2>/dev/null | head -1; }

# PID of fire_gimbal_control itself, from the lockfile it writes at startup.
app_pid() {
    local f; f="$(lock_path)"
    [[ -e "$f" ]] || return 1
    local p; read -r p <"$f" 2>/dev/null || return 1
    [[ -n "$p" ]] && echo "$p"
}

# capture-pane needs a pane target; a session name alone gets "can't find pane".
# %N pane ids are unambiguous, so resolve one rather than guessing at "fgc:0.0".
pane_id() { tmux list-panes -t "$target" -F '#{pane_id}' 2>/dev/null | head -1; }

# Mirrors SingleInstance::defaultLockPath(). Kept env-independent for the same
# reason it is there: $XDG_RUNTIME_DIR is absent in a non-interactive ssh shell.
lock_path() {
    local uid; uid="$(id -u)"
    if [[ -d "/run/user/$uid" ]]; then
        echo "/run/user/$uid/fire_gimbal_control.lock"
    else
        echo "/tmp/fire_gimbal_control-$uid.lock"
    fi
}

# Is the lock actually held right now? Asked by trying to take it, not by
# checking whether the recorded PID exists: with remain-on-exit set, tmux does
# not reap the pane's process until the pane is destroyed, so a finished run
# leaves a zombie that `kill -0` happily reports as alive. The lock itself is
# the only thing that answers the question, and the kernel drops it on death.
lock_held() {
    local f="$1"
    [[ -e "$f" ]] || return 1
    ! flock -n "$f" true 2>/dev/null
}

# Vimba X needs the GenTL transport-layer dir on GENICAM_GENTL64_PATH or
# VmbStartup() fails with "Could not start Vimba X API". The SDK installs that
# via /etc/profile.d, which non-interactive ssh shells don't source. Resolving it
# *here* — i.e. inside the pane, at launch — rather than in the caller's shell
# matters: a tmux server left over from an earlier, differently-configured ssh
# session would otherwise hand the new session its stale environment.
resolve_vimba_env() {
    local s d
    # The SDK's profile.d script reads GENICAM_GENTL64_PATH before setting it, so
    # sourcing it under `set -u` aborts with "unbound variable" and the app never
    # launches. Relax nounset for the duration of the source only.
    set +u
    for s in /etc/profile.d/*Vimba*GenTL*.sh; do
        [[ -f "$s" ]] && . "$s"
    done
    set -u
    if [[ -z "${GENICAM_GENTL64_PATH:-}" ]]; then
        for d in "${VIMBA_CTI_PATH:-}" /opt/VimbaX/cti /opt/VimbaX_*/cti; do
            if [[ -n "$d" && -d "$d" ]]; then export GENICAM_GENTL64_PATH="$d"; break; fi
        done
    fi
}

cmd_start() {
    need_tmux
    if running; then
        die "already running (session '$session') — use 'fgc attach', or 'fgc restart'"
    fi
    if session_exists; then
        # A dead pane left over by remain-on-exit from a previous run. Clear it,
        # but say so — it is the only trace of how that run ended.
        echo "fgc: clearing the finished session from a previous run" >&2
        tmux kill-session -t "$target" 2>/dev/null || true
    fi

    # tmux hands the command to `sh -c`, so quote each forwarded argument rather
    # than relying on the split surviving the round trip. No `exec` here, and no
    # exec in `_run` either — see the comment there for why the wrapper shell has
    # to stay in place. `fgc stop` signals the wrapper, which forwards.
    local launch
    launch="$(printf '%q ' "$here/fgc" _run "$@")"

    # remain-on-exit=failed, not =on: a *failed* start must leave its output on
    # screen rather than destroying the session and looking like nothing
    # happened, but a clean exit must take the session with it. With =on, quitting
    # the dashboard left an operator staring at a frozen dead pane with no process
    # behind it — keys and Ctrl-C did nothing, and it read as a hung ssh session.
    #
    # It is a *window* option, and a pane that dies immediately takes the window
    # with it before the option can be set — so create the session on a
    # placeholder, set the option, then respawn the pane with the real command.
    # -c: the program resolves ./config.ini and the output dir from the cwd. The
    # pane runs this script's hidden `_run`, so the Vimba environment is resolved
    # inside the pane; run.sh finds the binary, one place only.
    tmux new-session -d -s "$session" -c "$repo" 'sleep 86400'
    # A window option needs a window target; a session name alone gets
    # "no such window". Resolve the window's own id (@N) and use that.
    local win; win="$(tmux list-windows -t "$target" -F '#{window_id}' | head -1)"
    local pane; pane="$(pane_id)"
    # "failed" needs tmux >= 3.2; fall back to "on" rather than leaving it unset,
    # where a failed start would vanish without a trace.
    tmux set-option -w -t "$win" remain-on-exit failed >/dev/null 2>&1 \
        || tmux set-option -w -t "$win" remain-on-exit on >/dev/null
    # Spell the way out along the bottom of the screen. Without it there is no
    # clue on screen that Ctrl-b d exists — and if the program dies, the pane
    # freezes with no process behind it, so keys and Ctrl-C do nothing and it
    # reads as a hung ssh session. tmux's status line stays alive either way.
    # These are *session* options, and set-option rejects the "=name" exact-match
    # form that every other subcommand here takes ("no such session: =fgc"), so
    # target the pane and let tmux resolve the session from it.
    tmux set-option -t "$pane" status on >/dev/null
    tmux set-option -t "$pane" status-style 'bg=colour24,fg=colour255' >/dev/null
    tmux set-option -t "$pane" status-left ' fire-gimbal-control ' >/dev/null
    tmux set-option -t "$pane" status-left-length 30 >/dev/null
    tmux set-option -t "$pane" status-right-length 80 >/dev/null
    tmux set-option -t "$pane" status-right \
        ' Ctrl-b d = detach, keeps running   |   q = quit the program ' >/dev/null

    tmux respawn-pane -k -t "$win" -c "$repo" "$launch" >/dev/null

    # Give it a moment to fail fast (bad config, lock already held, no camera).
    sleep 2
    local dead; dead="$(pane_dead)"
    if [[ -z "$dead" ]]; then
        # No session at all: the program exited 0 within the 2s window, so
        # remain-on-exit=failed tore it down and took the output with it.
        die "session '$session' is gone — the program exited immediately (status 0).
     Reproduce it in the foreground to see why:  scripts/run.sh $*"
    fi
    if [[ "$dead" == "1" ]]; then
        local status
        status="$(tmux list-panes -t "$target" -F '#{pane_dead_status}' | head -1)"
        echo "fgc: failed to start (exit ${status:-?}):" >&2
        tmux capture-pane -p -t "$(pane_id)" | sed '/^$/d' | tail -20 >&2
        tmux kill-session -t "$target" 2>/dev/null || true
        exit "${status:-1}"
    fi

    echo "fgc: started detached as PID $(pane_pid) (session '$session')"
    echo "     attach: scripts/fgc attach     (Ctrl-b d to detach; 'q' quits the program)"
}

# Hidden: the command tmux runs inside the pane. Not for direct use.
#
# The app is run as a CHILD and waited on, rather than exec'd. That looks like a
# pointless extra process, and it is load-bearing: when the app is itself the
# pane's process, tmux never records an exit status for it (#{pane_dead_status}
# comes back empty and the process lingers as a zombie), so remain-on-exit=failed
# cannot tell success from failure and keeps *every* finished pane — which is
# what left an operator staring at a frozen dead pane after pressing q. With a
# shell in between to reap the child and exit with its status, tmux gets a real
# status and tears the session down on a clean quit. Measured: 3/3 kept when
# exec'd, 3/3 clean when parented.
cmd__run() {
    resolve_vimba_env

    # <&0 is required: bash redirects a background command's stdin from
    # /dev/null unless it is given one explicitly, which would leave the
    # dashboard unable to read a single keystroke.
    "$here/run.sh" "$@" <&0 &
    local child=$!

    # Forward the signals `fgc stop` and a pane kill use, so the app still runs
    # its own clean teardown (stop capture, close camera, save exposure store)
    # instead of being killed under the wrapper.
    trap 'kill -TERM "$child" 2>/dev/null' TERM
    trap 'kill -INT  "$child" 2>/dev/null' INT
    trap 'kill -HUP  "$child" 2>/dev/null' HUP

    # A trapped signal makes `wait` return 128+n while the child is still
    # shutting down, so keep waiting until it is actually gone before reporting.
    local rc=0
    wait "$child"; rc=$?
    while kill -0 "$child" 2>/dev/null; do
        wait "$child"; rc=$?
    done
    exit "$rc"
}

cmd_attach() {
    need_tmux
    running || die "not running — start it with 'fgc start'"
    if [[ -n "${TMUX:-}" ]]; then
        die "already inside tmux; use 'tmux switch-client -t $session'"
    fi
    echo "fgc: attaching — Ctrl-b d detaches and leaves it running ('q' would quit it)" >&2
    exec tmux attach-session -t "$target"
}

cmd_status() {
    need_tmux
    local lock; lock="$(lock_path)"
    if ! running; then
        if session_exists; then
            echo "fgc: not running — session '$session' holds a finished pane:"
            tmux capture-pane -p -t "$(pane_id)" | sed '/^$/d' | tail -10 | sed 's/^/     /'
            echo "     clear it with 'fgc stop' (or just 'fgc start')"
        else
            echo "fgc: not running (no tmux session '$session')"
        fi
        # The lock outliving the session means an instance is running that fgc
        # did not start — systemd, or someone's bare ./fire_gimbal_control.
        if lock_held "$lock"; then
            local pid; read -r pid <"$lock" || pid="?"
            echo "     but $lock is held by PID $pid — an instance is running outside tmux"
            return 0
        fi
        return 1
    fi
    local app; app="$(app_pid || echo "$(pane_pid)")"
    echo "fgc: running (session '$session', PID $app)"
    ps -o lstart=,etime=,args= -p "$app" 2>/dev/null | sed 's/^/     /'
    echo "     lockfile: $lock"
}

cmd_stop() {
    need_tmux
    if ! running; then
        if session_exists; then
            tmux kill-session -t "$target" 2>/dev/null || true
            echo "fgc: cleared the finished session '$session'"
        else
            echo "fgc: not running"
        fi
        return 0
    fi
    local pid; pid="$(pane_pid)"

    # The pane should be our `_run` wrapper, which forwards the signal to the
    # app. Signalling something else would look like a clean stop while the
    # program kept running, so check rather than assume.
    if ! ps -o args= -p "$pid" 2>/dev/null | grep -qE 'fgc _run|fire_gimbal'; then
        echo "fgc: pane process $pid is not the fgc launcher:" >&2
        ps -o args= -p "$pid" 2>/dev/null | sed 's/^/     /' >&2
        die "refusing to signal it — attach and shut it down by hand"
    fi

    # SIGTERM, not a keystroke: Application installs a SIGTERM handler that runs
    # the same clean teardown as the `exit` command (stop capture, close the
    # camera, save the exposure store, park the motor), and unlike sending keys
    # it works the same in TUI and headless mode. Never SIGKILL — an unclean
    # exit leaves the camera wedged for the next run.
    echo "fgc: stopping fire_gimbal_control (PID $(app_pid || echo '?')) via SIGTERM…"
    kill -TERM "$pid" 2>/dev/null || true

    # Wait on tmux's view of the pane, not on `kill -0`: while a pane is kept
    # after exit, tmux holds the finished process as a zombie until the pane is
    # destroyed, so `kill -0` would report it alive forever and every stop would
    # "time out". `running` covers both endings — a clean exit destroys the
    # session outright, a failed one leaves the pane behind with dead=1.
    #
    # 120s, because shutdown is not instant: the control loop only notices the
    # flag between ticks, and a capture attempt that is timing out on the camera
    # (3 tries x 2s) holds it for several seconds before that.
    local i
    for i in $(seq 1 120); do
        running || break
        sleep 1
    done
    if running; then
        echo "fgc: PID $pid still running 120s after SIGTERM — attach and look at why;" >&2
        echo "     do NOT kill -9, that wedges the camera." >&2
        return 1
    fi
    tmux kill-session -t "$target" 2>/dev/null || true
    echo "fgc: stopped"
}

cmd_peek() {
    need_tmux
    running || die "not running"
    # In TUI mode FTXUI draws on the alternate screen, so this is a snapshot of
    # the live dashboard; in headless mode it is the log scrollback.
    tmux capture-pane -p -S - -t "$(pane_id)"
}

cmd_restart() {
    cmd_stop
    cmd_start "$@"
}

sub="${1:-}"
[[ $# -gt 0 ]] && shift
case "$sub" in
    start)    cmd_start "$@" ;;
    attach)   cmd_attach ;;
    status)   cmd_status ;;
    stop)     cmd_stop ;;
    peek|logs) cmd_peek ;;
    restart)  cmd_restart "$@" ;;
    _run)     cmd__run "$@" ;;
    ""|-h|--help|help)
        # The header block above, minus the shebang: one source of truth.
        awk 'NR>2 && /^#/ {sub(/^# ?/, ""); print; next} NR>2 {exit}' "${BASH_SOURCE[0]}"
        [[ -z "$sub" ]] && exit 1 || exit 0 ;;
    *) die "unknown command: $sub (try 'fgc --help')" ;;
esac