Add a one-file tower bootstrap and a legacy-tower survey, and use an SDK already installed

docs/tower-installation.md

@@ -25,7 +25,7 @@ design, not yet exercised.
 |---|---|
 | The tower host | Powered, on the network, an OS installed, `ssh` running, reachable by key (section 1) |
 | An account | `ggs`, in the `sudo` and `dialout` groups |
-| Both repositories on it | `~/projects/fwt_2a/software` and `~/projects/fwt_2a/firmware` - cloned (`git.ggs-solutions.eu:GGS/fwt_software.git` and `fwt_firmware.git`, which needs a key on the tower), or synced by each repo's `deploy.sh` |
+| Both repositories on it | `~/projects/fwt_2a/software` and `~/projects/fwt_2a/firmware`. `scripts/bootstrap-tower` sets that up from one copied file (below), or each repo's `deploy.sh` syncs them from a dev box |
 | The Vimba X SDK, staged on it | Login-walled, so it cannot be fetched by a script: copy it on (USB stick, `scp`) and give its directory when asked. `none` for a tower with no camera yet |
 | The facts of the build | Which axes, which IMU, which ambient sensor, how the driver boards are wired, and - per axis - the drivetrain: sense resistor, currents, travel, `steps_per_rev`, `sw_mode`, `enc_deviation`, encoder or not. Anything not yet measured can be answered `unknown` |
 
@@ -36,6 +36,50 @@ The installer takes the board from DMI and picks its kit:
 | LattePanda 3 Delta | ATmega32U4 (Leonardo), 5 V | [`hardware/lattepanda-3-delta/`](../hardware/lattepanda-3-delta/) | `leonardo` |
 | LattePanda IOTA | RP2040, 3.3 V | [`hardware/lattepanda-iota/`](../hardware/lattepanda-iota/) | `iota_rp2040` |
 
+### Getting the code onto a tower: `scripts/bootstrap-tower`
+
+One file to copy; it fetches the rest. It needs the Gitea deploy key (tower
+111's `~/.ssh/id_ed25519_deploy_gitea`) in the tower's `~/.ssh/`:
+
+```bash
+scp scripts/bootstrap-tower ggs@<tower>:
+scp -3 ggs@10.11.12.111:.ssh/id_ed25519_deploy_gitea ggs@<tower>:.ssh/   # -3: via this machine
+ssh -t ggs@<tower> bash bootstrap-tower          # any install-tower options pass through
+```
+
+What it does:
+- adds `Host gitea` (port 2222) to `~/.ssh/config` and records the server's
+  host key;
+- clones both repositories side by side, or brings existing clones up to date
+  (fast-forward only);
+- starts `install-tower`.
+
+It stops, rather than overwrite, when it finds a checkout with local changes or
+a directory that isn't a clone.
+
+### A legacy tower: read it first, with `scripts/survey-legacy-tower`
+
+The legacy towers run an older host program and a TMC-API Arduino sketch.
+Before migrating one, record what it runs, so its `hardware.ini` comes from its
+own values:
+
+```bash
+bash survey-legacy-tower --firmware      # pauses the old service ~20 s; changes nothing on disk
+```
+
+**What it records:**
+- the machine: board, OS, groups, disk, whether Gitea, GitHub, PyPI and
+  PlatformIO are reachable;
+- the camera and the Vimba install;
+- the old service and its `config.ini`;
+- with `--firmware`, from the old firmware:
+  - the endstop travel (`?`);
+  - its stored north calibration (`y`);
+  - its TMC registers (`i`), with the run current worked out for the usual
+    sense resistors.
+
+The report is saved as `~/legacy-survey-<date>.txt`.
+
 ### Running it
 
 On the tower, as `ggs`:

scripts/bootstrap-tower

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+#
+# bootstrap-tower - the one file to copy onto a tower: it fetches everything
+# else. Run it ON the tower, as the tower's user (ggs):
+#
+#   scp scripts/bootstrap-tower ggs@<tower>:        # from a machine with this repo
+#   scp -3 ggs@10.11.12.111:.ssh/id_ed25519_deploy_gitea ggs@<tower>:.ssh/   # the key, via this machine
+#   ssh ggs@<tower> bash bootstrap-tower [install-tower options]
+#
+# It needs the tower's deploy key for the Gitea (~/.ssh/id_ed25519_deploy_gitea,
+# the same key tower 111 uses). Then it:
+#   1. sets up ~/.ssh/config "Host gitea" (git.ggs-solutions.eu, port 2222) and
+#      records the server's host key;
+#   2. clones fwt_software and fwt_firmware side by side under ~/projects/fwt_2a/,
+#      or brings existing clones up to date (fast-forward only);
+#   3. runs software/scripts/install-tower with whatever options it was given
+#      (e.g. --dry-run, --answers FILE).
+#
+# Nothing is overwritten: a checkout with local changes, or a directory that is
+# not a clone, stops it with the reason.
+#
+set -euo pipefail
+
+BASE="${FGC_BASE:-$HOME/projects/fwt_2a}"   # FGC_BASE: rehearse somewhere else
+KEY="$HOME/.ssh/id_ed25519_deploy_gitea"
+HOST=git.ggs-solutions.eu
+PORT=2222
+
+say() { printf '>> %s\n' "$*"; }
+die() { printf 'bootstrap-tower: %s\n' "$*" >&2; exit 1; }
+
+[[ $EUID -ne 0 ]] || die "run as the tower's user (ggs), not root"
+
+for tool in git ssh ssh-keyscan; do
+    command -v "$tool" >/dev/null || {
+        say "installing git and the ssh client"
+        sudo apt-get update -qq && sudo apt-get install -y -qq git openssh-client
+        break
+    }
+done
+
+# ---- the deploy key and the gitea alias ------------------------------------
+[[ -r $KEY ]] || die "no deploy key at $KEY. Copy it from tower 111:
+    scp -3 ggs@10.11.12.111:.ssh/id_ed25519_deploy_gitea ggs@<this tower>:.ssh/   (from your laptop)
+(or put this tower's own key there, registered read-only on both repositories)"
+chmod 600 "$KEY"
+mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh"
+touch "$HOME/.ssh/config" && chmod 600 "$HOME/.ssh/config"
+if ! grep -qE '^Host[[:space:]]+gitea([[:space:]]|$)' "$HOME/.ssh/config"; then
+    say "adding 'Host gitea' to ~/.ssh/config"
+    printf '\nHost gitea\n    Hostname %s\n    User git\n    Port %s\n    IdentityFile %s\n' \
+        "$HOST" "$PORT" "$KEY" >> "$HOME/.ssh/config"
+fi
+if ! ssh-keygen -F "[$HOST]:$PORT" >/dev/null 2>&1; then
+    say "recording the host key of $HOST:$PORT"
+    ssh-keyscan -p "$PORT" "$HOST" 2>/dev/null >> "$HOME/.ssh/known_hosts"
+    ssh-keygen -F "[$HOST]:$PORT" -l | sed 's/^/   /'
+fi
+git ls-remote gitea:GGS/fwt_software.git >/dev/null 2>&1 ||
+    die "the deploy key cannot read gitea:GGS/fwt_software.git - is port $PORT reachable from here, and is the key registered on the repository?"
+
+# ---- both repositories, side by side ----------------------------------------
+fetch() {   # repo-name directory
+    local repo="$1" dir="$BASE/$2"
+    if [[ -d $dir/.git ]]; then
+        [[ -z $(git -C "$dir" status --porcelain) ]] ||
+            die "$dir has local changes; commit or set them aside first"
+        say "updating $dir"
+        git -C "$dir" pull --ff-only -q
+    elif [[ -e $dir ]]; then
+        die "$dir exists but is not a git clone (a tree copied by deploy.sh?). Move it aside, then run this again."
+    else
+        say "cloning $repo into $dir"
+        git clone -q "gitea:GGS/$repo.git" "$dir"
+    fi
+    say "  $2 at $(git -C "$dir" log --oneline -1)"
+}
+mkdir -p "$BASE"
+fetch fwt_software software
+fetch fwt_firmware firmware
+
+# ---- the installer ---------------------------------------------------------
+say "starting the installer"
+cd "$BASE/software"
+exec scripts/install-tower "$@"

scripts/install-tower

@@ -80,6 +80,10 @@ die()  { printf 'install-tower: %s\n' "$*" >&2; exit 1; }
 hr()   { printf '\n== %s ==\n' "$*"; }
 
 [[ $EUID -ne 0 ]] || die "run as the tower's user (ggs), not root: builds and ~/.platformio must belong to it. It uses sudo where it has to."
+# The motor MCU is a serial device: without dialout the program cannot open it
+# and the flash cannot reach it - found here, not as a failure an hour in.
+id -nG | tr ' ' '\n' | grep -qx dialout ||
+    die "$USER is not in the dialout group, so it cannot use the motor MCU's serial port. Run: sudo usermod -aG dialout $USER - then log out and back in (the group applies to new logins) and run this again."
 
 # ------------------------------------------------------------- questions ---
 # Answers come from --answer / --answers first; otherwise from the terminal. With
@@ -463,11 +467,21 @@ VIMBA_SRC=""
 if [[ -d /opt/VimbaX/cti ]]; then
     plan vimba skip "$(readlink -f /opt/VimbaX)"
 else
+    # An SDK already installed under a versioned name - the legacy towers' own
+    # software put it at /opt/VimbaX_2024-1, with no /opt/VimbaX - is used where it
+    # is: installing it registers it and links /opt/VimbaX to it, copying nothing.
+    existing="$(ls -d /opt/VimbaX_*/ 2>/dev/null | sed 's:/$::' | sort | tail -1 || true)"
+    [[ -x ${existing:-/nonexistent}/cti/Install_GenTL_Path.sh ]] || existing=""
     staged="$(ls -d "$HOME"/VimbaX_* 2>/dev/null | head -1 || true)"
+    if [[ -n $existing ]]; then
+        echo "A Vimba X SDK is already on this machine at $existing, but not as /opt/VimbaX."
+        echo "Take it (it is registered and linked, not copied), or give another directory."
+    else
         echo "The Vimba X SDK is not installed. It cannot be downloaded by a script: copy it"
         echo "onto this machine (USB stick or scp) and give its directory, or 'none' for a"
         echo "tower with no camera yet."
-    ask vimba_src "Staged Vimba X directory" "${staged:-none}"
+    fi
+    ask vimba_src "Vimba X directory" "${existing:-${staged:-none}}"
     if [[ $REPLY_VAL == none ]]; then plan vimba skip "no camera SDK (answered none)"
     else
         [[ -x $REPLY_VAL/cti/Install_GenTL_Path.sh ]] || die "$REPLY_VAL does not look like a Vimba X SDK"

scripts/survey-legacy-tower

@@ -0,0 +1,147 @@
+#!/usr/bin/env bash
+#
+# survey-legacy-tower - read what a legacy tower (the old host program and the
+# TMC-API Arduino sketch) is running, BEFORE it is migrated, so its
+# hardware.ini is written from its own values rather than another tower's.
+# Run it ON the legacy tower. Changes nothing on disk.
+#
+#   bash survey-legacy-tower               the machine, the old install, the camera
+#   bash survey-legacy-tower --firmware    also ask the old firmware for its
+#                                          registers, travel and north calibration:
+#                                          pauses the old service for ~20 s
+#
+# The report is printed and saved to ~/legacy-survey-<date>.txt.
+#
+# The firmware part talks the old sketch's own serial protocol:
+#   ?        endstop positions and the travel between them (set by its last home)
+#   y        its north calibration from EEPROM (the same `y` the old host sends
+#            after every home - it reloads the stored value, it stores nothing)
+#   id<n>    read TMC5160 register n (write-only ones answer from the library's copy)
+#
+set -uo pipefail
+
+FIRMWARE=0
+[[ ${1:-} == --firmware ]] && FIRMWARE=1
+OUT="$HOME/legacy-survey-$(date +%Y%m%d-%H%M%S).txt"
+exec > >(tee "$OUT") 2>&1
+
+hr() { printf '\n== %s ==\n' "$*"; }
+reach() { timeout 6 bash -c "</dev/tcp/$1/$2" 2>/dev/null && echo "ok" || echo "NO"; }
+
+hr "machine"
+echo "board     : $(tr -d '\0' < /sys/class/dmi/id/product_name 2>/dev/null || echo '?')"
+echo "os        : $(. /etc/os-release 2>/dev/null && echo "$PRETTY_NAME") ($(uname -r))"
+echo "user      : $USER  groups: $(id -nG)"
+echo "disk /    : $(df -h / | awk 'NR==2 {print $4 " free of " $2}')"
+echo "memory    : $(free -h | awk '/^Mem/ {print $2}')"
+echo "internet  : gitea:2222 $(reach git.ggs-solutions.eu 2222)  github $(reach github.com 443)  pypi $(reach pypi.org 443)  platformio $(reach dl.registry.platformio.org 443)"
+
+hr "USB"
+lsusb 2>/dev/null | grep -Ei '1ab2|2341|arduino|allied' || echo "(no Alvium or Arduino on lsusb)"
+ls /dev/serial/by-id/ 2>/dev/null | sed 's/^/by-id: /'
+
+hr "Vimba X and the camera"
+ls -d /opt/VimbaX* 2>/dev/null || echo "no /opt/VimbaX*"
+for tool in /opt/VimbaX*/bin/ListCameras_VmbC /opt/VimbaX*/bin/ListCameras; do
+    [[ -x $tool ]] || continue
+    cti="$(dirname "$(dirname "$tool")")/cti"
+    GENICAM_GENTL64_PATH="$cti" "$tool" 2>/dev/null | grep -E 'Camera (ID|Name)|Model|Transport Layer Path' | sed 's/^/  /'
+    break
+done
+
+hr "the old install"
+systemctl is-enabled fire-gimbal-control 2>/dev/null | sed 's/^/service enabled: /'
+systemctl is-active fire-gimbal-control 2>/dev/null | sed 's/^/service active : /'
+systemctl cat fire-gimbal-control 2>/dev/null | grep -E '^(ExecStart|WorkingDirectory|User|Environment)=' | sed 's/^/  /'
+for c in /opt/fire_gimbal_control/config.ini "$HOME"/*/config.ini; do
+    [[ -r $c ]] || continue
+    echo "-- $c (passwords masked)"
+    grep -vE '^[[:space:]]*(;|#|$)' "$c" | sed -E 's/^(mqtt_pw[[:space:]]*=).*/\1 *****/' | sed 's/^/  /'
+done
+
+[[ $FIRMWARE -eq 1 ]] || { hr "firmware"; echo "not asked (run with --firmware)"; echo; echo "saved: $OUT"; exit 0; }
+
+hr "the old firmware"
+PORT="$(ls /dev/serial/by-id/*Arduino* 2>/dev/null | head -1)"
+[[ -n $PORT ]] || { echo "no Arduino on /dev/serial/by-id"; exit 1; }
+WAS_ACTIVE="$(systemctl is-active fire-gimbal-control 2>/dev/null || true)"
+if [[ $WAS_ACTIVE == active ]]; then
+    echo "pausing the old service (it holds the port)"
+    sudo systemctl stop fire-gimbal-control
+    trap 'echo "restarting the old service"; sudo systemctl start fire-gimbal-control' EXIT
+    sleep 2
+fi
+
+python3 - "$PORT" <<'PYEOF'
+import os, sys, termios, time, math
+
+port = sys.argv[1]
+fd = os.open(port, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK)
+a = termios.tcgetattr(fd)
+a[0] = 0; a[1] = 0; a[2] = termios.CS8 | termios.CREAD | termios.CLOCAL; a[3] = 0
+a[4] = a[5] = termios.B115200
+termios.tcsetattr(fd, termios.TCSANOW, a)
+buf = b""
+
+def lines_for(seconds):
+    global buf
+    out, end = [], time.time() + seconds
+    while time.time() < end:
+        try:
+            buf += os.read(fd, 4096)
+        except BlockingIOError:
+            time.sleep(0.02)
+        while b"\n" in buf:
+            line, buf = buf.split(b"\n", 1)
+            out.append(line.decode(errors="replace").strip())
+    return out
+
+def ask(cmd, seconds=0.8):
+    lines_for(0.2)   # drop what was already streaming
+    os.write(fd, cmd.encode() + b"\n")
+    got = lines_for(seconds)
+    return [l for l in got if l and not l.startswith("$")], [l for l in got if l.startswith("$")]
+
+_, status = ask("", 1.0)
+print("status line  :", status[-1] if status else "(none - is this the legacy firmware?)")
+
+reply, _ = ask("?")
+print("? (endstops) :", " | ".join(reply) or "(no answer)")
+nums = [int(x) for x in reply if x.lstrip("-").isdigit()]
+if len(nums) >= 3:
+    r, l = nums[0], nums[1]
+    print(f"  right {r}  left {l}  travel {abs(l - r)} counts = {abs(l - r) / 528.253966:.1f} deg at 528.25 counts/deg")
+
+reply, _ = ask("y")
+print("y (north)    :", " | ".join(reply) or "(no answer)")
+
+REGS = [("GCONF", 0x00), ("GLOBALSCALER", 0x0B), ("IHOLD_IRUN", 0x10), ("TPOWERDOWN", 0x11),
+        ("XACTUAL", 0x21), ("AMAX", 0x26), ("VMAX", 0x27), ("DMAX", 0x28), ("SW_MODE", 0x34),
+        ("ENCMODE", 0x38), ("X_ENC", 0x39), ("ENC_CONST", 0x3A), ("ENC_DEVIATION", 0x3D),
+        ("CHOPCONF", 0x6C), ("COOLCONF", 0x6D), ("PWMCONF", 0x70)]
+vals = {}
+print("registers    :")
+for name, n in REGS:
+    reply, _ = ask(f"id{n}")
+    v = None
+    for x in reply:
+        if x.startswith("READ"):
+            try: v = int(x.split(",")[-1])
+            except ValueError: pass
+    vals[name] = v
+    shown = "(no answer)" if v is None else f"{v} = 0x{v & 0xFFFFFFFF:08X}"
+    print(f"  {name:14s} 0x{n:02X}  {shown}")
+
+gs, ii = vals.get("GLOBALSCALER"), vals.get("IHOLD_IRUN")
+if ii is not None:
+    irun, ihold = (ii >> 8) & 0x1F, ii & 0x1F
+    scale = (gs or 256) / 256.0
+    print(f"current      : IRUN {irun}, IHOLD {ihold}, GLOBALSCALER {gs if gs else '0 (=256, full)'}")
+    for rs in (0.075, 0.05):
+        i_run = scale * (irun + 1) / 32 * 0.325 / rs / math.sqrt(2) * 1000
+        print(f"  with a {rs * 1000:.0f} mOhm sense resistor: run {i_run:.0f} mA RMS"
+              "  (read the resistor off the board before trusting either)")
+os.close(fd)
+PYEOF
+echo
+echo "saved: $OUT"

src/core/Config.cpp

@@ -826,16 +826,50 @@ std::string updateIniSectionKeys(const std::string& contents, const std::string&
         }
         out << line << "\n";
     }
-    // Any keys not present in the section get appended in a fresh [section] block
+    // Keys the section does not have yet go INTO it, after its last key line, with
-    // (inih merges duplicate sections on read).
+    // a note of when and why. A second [section] block at the end reads the same
+    // (inih merges duplicates) but is not where a person reading the file looks -
+    // on tower 111 it left `cooling` a hundred lines away from `env`. Only a
+    // section the file lacks altogether is appended.
     bool missing = false;
     for (bool w : written) missing = missing || !w;
-    if (missing) {
+    if (!missing) return out.str();
-        out << "\n[" << section << "]\n";
+
+    std::vector<std::string> block;
+    if (keep_history) block.push_back("; added   ; " + provenance);
     for (size_t i = 0; i < kv.size(); ++i)
-            if (!written[i]) out << kv[i].first << " = " << kv[i].second << "\n";
+        if (!written[i]) block.push_back(kv[i].first + " = " + kv[i].second);
+
+    std::vector<std::string> lines;
+    {
+        std::istringstream split(out.str());
+        for (std::string l; std::getline(split, l);) lines.push_back(l);
     }
-    return out.str();
+    long header = -1, last_key = -1;
+    for (size_t i = 0; i < lines.size(); ++i) {
+        const std::string t = trimmed(lines[i]);
+        if (t.size() >= 2 && t.front() == '[' && t.back() == ']') {
+            if (header >= 0) break;   // the end of the first block of this section
+            if (trimmed(t.substr(1, t.size() - 2)) == section) header = static_cast<long>(i);
+            continue;
+        }
+        if (header >= 0 && !t.empty() && t[0] != ';' && t[0] != '#' &&
+            t.find('=') != std::string::npos)
+            last_key = static_cast<long>(i);
+    }
+    std::ostringstream res;
+    if (header < 0) {
+        res << out.str() << "\n[" << section << "]\n";
+        for (const auto& b : block) res << b << "\n";
+        return res.str();
+    }
+    const long after = last_key >= 0 ? last_key : header;
+    for (size_t i = 0; i < lines.size(); ++i) {
+        res << lines[i] << "\n";
+        if (static_cast<long>(i) == after)
+            for (const auto& b : block) res << b << "\n";
+    }
+    return res.str();
 }
 
 bool saveMotorGeometry(const std::string& path, const Geometry& geo, long yaw_margin,

tests/test_config.cpp

@@ -1,6 +1,7 @@
 #include <doctest/doctest.h>
 
 #include "fgc/Config.h"
+#include "fgc/IniEdit.h"
 
 #include <cmath>
 #include <cstdio>
@@ -274,6 +275,22 @@ TEST_CASE("updateIniSectionKeys appends keys missing from the section") {
     CHECK(out.find("pitch_center_count = 7") != std::string::npos);
 }
 
+TEST_CASE("a key the section lacks goes into that section, not a second block") {
+    const std::string in = "[Hardware]\naxes = yaw\nenv = dht11\n\n; about pins\n[Pins]\n"
+                           "yaw_cs = 8\n\n[Camera.rgb]\nid = X\n";
+    auto out = updateIniSectionKeys(in, "Hardware", {{"cooling", "fan"}}, "t1, fan test");
+    CHECK(out.find("[Hardware]") == out.rfind("[Hardware]"));          // still one block
+    CHECK(out.find("env = dht11\n; added   ; t1, fan test\ncooling = fan\n") !=
+          std::string::npos);                                            // right after its keys
+    CHECK(out.find("cooling = fan") < out.find("; about pins"));       // before the next one's comment
+    auto kv = parseIniString(out);
+    CHECK(kv["Hardware.cooling"] == "fan");
+
+    // A section the file does not have is still appended whole.
+    auto out2 = updateIniSectionKeys(in, "Env", {{"i2c_device", "/dev/i2c-2"}});
+    CHECK(out2.find("\n[Env]\ni2c_device = /dev/i2c-2\n") != std::string::npos);
+}
+
 TEST_CASE("updateIniSectionKeys keeps the superseded value as a ';prev' comment") {
     const std::string in =
         "[Motor]\n"