#!/usr/bin/env bash # # fgc — run fire_gimbal_control detached from the terminal, in a tmux session. # # The tower is operated over ssh. Launched directly, the program is a child of # the ssh session's shell and dies with it — mid-scan, and with the camera torn # down uncleanly (which wedges it for the next run). Here tmux owns the pty, so # disconnecting is invisible to the program. # # scripts/fgc start [args...] start it detached (args go to the binary) # scripts/fgc attach open the dashboard; Ctrl-b d to leave it running # scripts/fgc status is it running, since when, as which PID # scripts/fgc stop clean shutdown (camera + motor teardown) # scripts/fgc peek print what the pane shows right now # scripts/fgc restart [args...] stop, then start # # Detaching is Ctrl-b d. NOTE: the dashboard's own `q` key *exits the program*, # it does not detach — that is the one thing to get right when attached. # # Only one instance may run. The binary enforces that itself with a lockfile # (see src/core/SingleInstance.cpp), so a bare ./build/fire_gimbal_control is # refused too, not just a second `fgc start`. # # Env: FGC_SESSION overrides the tmux session name (default "fgc"). set -euo pipefail here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" repo="$(cd "$here/.." && pwd)" session="${FGC_SESSION:-fgc}" target="=$session" # "=" makes tmux match the name exactly, not as a prefix die() { echo "fgc: $*" >&2; exit 1; } need_tmux() { command -v tmux >/dev/null 2>&1 || die "tmux is not installed (apt-get install tmux)" } session_exists() { tmux has-session -t "$target" 2>/dev/null; } # "1" if the pane's program has exited, "0" if alive, "" if there is no session. # remain-on-exit keeps a finished pane around, so a session existing is not on # its own proof that the program is running. pane_dead() { tmux list-panes -t "$target" -F '#{pane_dead}' 2>/dev/null | head -1; } running() { [[ "$(pane_dead)" == "0" ]]; } # PID of the pane's process: the `_run` wrapper shell, which forwards signals to # the app and reports its exit status to tmux. For the app's own PID use # app_pid() — the lockfile is written by the program itself. pane_pid() { tmux list-panes -t "$target" -F '#{pane_pid}' 2>/dev/null | head -1; } # PID of fire_gimbal_control itself, from the lockfile it writes at startup. app_pid() { local f; f="$(lock_path)" [[ -e "$f" ]] || return 1 local p; read -r p <"$f" 2>/dev/null || return 1 [[ -n "$p" ]] && echo "$p" } # capture-pane needs a pane target; a session name alone gets "can't find pane". # %N pane ids are unambiguous, so resolve one rather than guessing at "fgc:0.0". pane_id() { tmux list-panes -t "$target" -F '#{pane_id}' 2>/dev/null | head -1; } # Mirrors SingleInstance::defaultLockPath(). Kept env-independent for the same # reason it is there: $XDG_RUNTIME_DIR is absent in a non-interactive ssh shell. lock_path() { local uid; uid="$(id -u)" if [[ -d "/run/user/$uid" ]]; then echo "/run/user/$uid/fire_gimbal_control.lock" else echo "/tmp/fire_gimbal_control-$uid.lock" fi } # Is the lock actually held right now? Asked by trying to take it, not by # checking whether the recorded PID exists: with remain-on-exit set, tmux does # not reap the pane's process until the pane is destroyed, so a finished run # leaves a zombie that `kill -0` happily reports as alive. The lock itself is # the only thing that answers the question, and the kernel drops it on death. lock_held() { local f="$1" [[ -e "$f" ]] || return 1 ! flock -n "$f" true 2>/dev/null } # Vimba X needs the GenTL transport-layer dir on GENICAM_GENTL64_PATH or # VmbStartup() fails with "Could not start Vimba X API". The SDK installs that # via /etc/profile.d, which non-interactive ssh shells don't source. Resolving it # *here* — i.e. inside the pane, at launch — rather than in the caller's shell # matters: a tmux server left over from an earlier, differently-configured ssh # session would otherwise hand the new session its stale environment. resolve_vimba_env() { local s d # The SDK's profile.d script reads GENICAM_GENTL64_PATH before setting it, so # sourcing it under `set -u` aborts with "unbound variable" and the app never # launches. Relax nounset for the duration of the source only. set +u for s in /etc/profile.d/*Vimba*GenTL*.sh; do [[ -f "$s" ]] && . "$s" done set -u if [[ -z "${GENICAM_GENTL64_PATH:-}" ]]; then for d in "${VIMBA_CTI_PATH:-}" /opt/VimbaX/cti /opt/VimbaX_*/cti; do if [[ -n "$d" && -d "$d" ]]; then export GENICAM_GENTL64_PATH="$d"; break; fi done fi } cmd_start() { need_tmux if running; then die "already running (session '$session') — use 'fgc attach', or 'fgc restart'" fi if session_exists; then # A dead pane left over by remain-on-exit from a previous run. Clear it, # but say so — it is the only trace of how that run ended. echo "fgc: clearing the finished session from a previous run" >&2 tmux kill-session -t "$target" 2>/dev/null || true fi # tmux hands the command to `sh -c`, so quote each forwarded argument rather # than relying on the split surviving the round trip. No `exec` here, and no # exec in `_run` either — see the comment there for why the wrapper shell has # to stay in place. `fgc stop` signals the wrapper, which forwards. local launch launch="$(printf '%q ' "$here/fgc" _run "$@")" # remain-on-exit=failed, not =on: a *failed* start must leave its output on # screen rather than destroying the session and looking like nothing # happened, but a clean exit must take the session with it. With =on, quitting # the dashboard left an operator staring at a frozen dead pane with no process # behind it — keys and Ctrl-C did nothing, and it read as a hung ssh session. # # It is a *window* option, and a pane that dies immediately takes the window # with it before the option can be set — so create the session on a # placeholder, set the option, then respawn the pane with the real command. # -c: the program resolves ./config.ini and the output dir from the cwd. The # pane runs this script's hidden `_run`, so the Vimba environment is resolved # inside the pane; run.sh finds the binary, one place only. tmux new-session -d -s "$session" -c "$repo" 'sleep 86400' # A window option needs a window target; a session name alone gets # "no such window". Resolve the window's own id (@N) and use that. local win; win="$(tmux list-windows -t "$target" -F '#{window_id}' | head -1)" local pane; pane="$(pane_id)" # "failed" needs tmux >= 3.2; fall back to "on" rather than leaving it unset, # where a failed start would vanish without a trace. tmux set-option -w -t "$win" remain-on-exit failed >/dev/null 2>&1 \ || tmux set-option -w -t "$win" remain-on-exit on >/dev/null # Spell the way out along the bottom of the screen. Without it there is no # clue on screen that Ctrl-b d exists — and if the program dies, the pane # freezes with no process behind it, so keys and Ctrl-C do nothing and it # reads as a hung ssh session. tmux's status line stays alive either way. # These are *session* options, and set-option rejects the "=name" exact-match # form that every other subcommand here takes ("no such session: =fgc"), so # target the pane and let tmux resolve the session from it. tmux set-option -t "$pane" status on >/dev/null tmux set-option -t "$pane" status-style 'bg=colour24,fg=colour255' >/dev/null tmux set-option -t "$pane" status-left ' fire-gimbal-control ' >/dev/null tmux set-option -t "$pane" status-left-length 30 >/dev/null tmux set-option -t "$pane" status-right-length 80 >/dev/null tmux set-option -t "$pane" status-right \ ' Ctrl-b d = detach, keeps running | q = quit the program ' >/dev/null tmux respawn-pane -k -t "$win" -c "$repo" "$launch" >/dev/null # Give it a moment to fail fast (bad config, lock already held, no camera). sleep 2 local dead; dead="$(pane_dead)" if [[ -z "$dead" ]]; then # No session at all: the program exited 0 within the 2s window, so # remain-on-exit=failed tore it down and took the output with it. die "session '$session' is gone — the program exited immediately (status 0). Reproduce it in the foreground to see why: scripts/run.sh $*" fi if [[ "$dead" == "1" ]]; then local status status="$(tmux list-panes -t "$target" -F '#{pane_dead_status}' | head -1)" echo "fgc: failed to start (exit ${status:-?}):" >&2 tmux capture-pane -p -t "$(pane_id)" | sed '/^$/d' | tail -20 >&2 tmux kill-session -t "$target" 2>/dev/null || true exit "${status:-1}" fi echo "fgc: started detached as PID $(pane_pid) (session '$session')" echo " attach: scripts/fgc attach (Ctrl-b d to detach; 'q' quits the program)" } # Hidden: the command tmux runs inside the pane. Not for direct use. # # The app is run as a CHILD and waited on, rather than exec'd. That looks like a # pointless extra process, and it is load-bearing: when the app is itself the # pane's process, tmux never records an exit status for it (#{pane_dead_status} # comes back empty and the process lingers as a zombie), so remain-on-exit=failed # cannot tell success from failure and keeps *every* finished pane — which is # what left an operator staring at a frozen dead pane after pressing q. With a # shell in between to reap the child and exit with its status, tmux gets a real # status and tears the session down on a clean quit. Measured: 3/3 kept when # exec'd, 3/3 clean when parented. cmd__run() { resolve_vimba_env # <&0 is required: bash redirects a background command's stdin from # /dev/null unless it is given one explicitly, which would leave the # dashboard unable to read a single keystroke. "$here/run.sh" "$@" <&0 & local child=$! # Forward the signals `fgc stop` and a pane kill use, so the app still runs # its own clean teardown (stop capture, close camera, save exposure store) # instead of being killed under the wrapper. trap 'kill -TERM "$child" 2>/dev/null' TERM trap 'kill -INT "$child" 2>/dev/null' INT trap 'kill -HUP "$child" 2>/dev/null' HUP # A trapped signal makes `wait` return 128+n while the child is still # shutting down, so keep waiting until it is actually gone before reporting. local rc=0 wait "$child"; rc=$? while kill -0 "$child" 2>/dev/null; do wait "$child"; rc=$? done exit "$rc" } cmd_attach() { need_tmux running || die "not running — start it with 'fgc start'" if [[ -n "${TMUX:-}" ]]; then die "already inside tmux; use 'tmux switch-client -t $session'" fi echo "fgc: attaching — Ctrl-b d detaches and leaves it running ('q' would quit it)" >&2 exec tmux attach-session -t "$target" } cmd_status() { need_tmux local lock; lock="$(lock_path)" if ! running; then if session_exists; then echo "fgc: not running — session '$session' holds a finished pane:" tmux capture-pane -p -t "$(pane_id)" | sed '/^$/d' | tail -10 | sed 's/^/ /' echo " clear it with 'fgc stop' (or just 'fgc start')" else echo "fgc: not running (no tmux session '$session')" fi # The lock outliving the session means an instance is running that fgc # did not start — systemd, or someone's bare ./fire_gimbal_control. if lock_held "$lock"; then local pid; read -r pid <"$lock" || pid="?" echo " but $lock is held by PID $pid — an instance is running outside tmux" return 0 fi return 1 fi local app; app="$(app_pid || echo "$(pane_pid)")" echo "fgc: running (session '$session', PID $app)" ps -o lstart=,etime=,args= -p "$app" 2>/dev/null | sed 's/^/ /' echo " lockfile: $lock" } cmd_stop() { need_tmux if ! running; then if session_exists; then tmux kill-session -t "$target" 2>/dev/null || true echo "fgc: cleared the finished session '$session'" else echo "fgc: not running" fi return 0 fi local pid; pid="$(pane_pid)" # The pane should be our `_run` wrapper, which forwards the signal to the # app. Signalling something else would look like a clean stop while the # program kept running, so check rather than assume. if ! ps -o args= -p "$pid" 2>/dev/null | grep -qE 'fgc _run|fire_gimbal'; then echo "fgc: pane process $pid is not the fgc launcher:" >&2 ps -o args= -p "$pid" 2>/dev/null | sed 's/^/ /' >&2 die "refusing to signal it — attach and shut it down by hand" fi # SIGTERM, not a keystroke: Application installs a SIGTERM handler that runs # the same clean teardown as the `exit` command (stop capture, close the # camera, save the exposure store, park the motor), and unlike sending keys # it works the same in TUI and headless mode. Never SIGKILL — an unclean # exit leaves the camera wedged for the next run. echo "fgc: stopping fire_gimbal_control (PID $(app_pid || echo '?')) via SIGTERM…" kill -TERM "$pid" 2>/dev/null || true # Wait on tmux's view of the pane, not on `kill -0`: while a pane is kept # after exit, tmux holds the finished process as a zombie until the pane is # destroyed, so `kill -0` would report it alive forever and every stop would # "time out". `running` covers both endings — a clean exit destroys the # session outright, a failed one leaves the pane behind with dead=1. # # 120s, because shutdown is not instant: the control loop only notices the # flag between ticks, and a capture attempt that is timing out on the camera # (3 tries x 2s) holds it for several seconds before that. local i for i in $(seq 1 120); do running || break sleep 1 done if running; then echo "fgc: PID $pid still running 120s after SIGTERM — attach and look at why;" >&2 echo " do NOT kill -9, that wedges the camera." >&2 return 1 fi tmux kill-session -t "$target" 2>/dev/null || true echo "fgc: stopped" } cmd_peek() { need_tmux running || die "not running" # In TUI mode FTXUI draws on the alternate screen, so this is a snapshot of # the live dashboard; in headless mode it is the log scrollback. tmux capture-pane -p -S - -t "$(pane_id)" } cmd_restart() { cmd_stop cmd_start "$@" } sub="${1:-}" [[ $# -gt 0 ]] && shift case "$sub" in start) cmd_start "$@" ;; attach) cmd_attach ;; status) cmd_status ;; stop) cmd_stop ;; peek|logs) cmd_peek ;; restart) cmd_restart "$@" ;; _run) cmd__run "$@" ;; ""|-h|--help|help) # The header block above, minus the shebang: one source of truth. awk 'NR>2 && /^#/ {sub(/^# ?/, ""); print; next} NR>2 {exit}' "${BASH_SOURCE[0]}" [[ -z "$sub" ]] && exit 1 || exit 0 ;; *) die "unknown command: $sub (try 'fgc --help')" ;; esac