diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a6f643..eaeb46b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,6 +60,7 @@ endif() add_library(fgc_core STATIC src/core/Config.cpp src/core/IniEdit.cpp + src/core/CameraStall.cpp src/core/Paths.cpp src/core/SingleInstance.cpp src/core/Logger.cpp diff --git a/config/config.example.ini b/config/config.example.ini index 4ebe8b7..ca660b7 100644 --- a/config/config.example.ini +++ b/config/config.example.ini @@ -284,11 +284,18 @@ imu_drift_window_ms = 60000 ; long static window for the yaw-drift fit imu_yaw_noise_max_deg = 0.5 imu_yaw_drift_max_deg_min = 1.0 imu_expected_hz = 100 -; test camera: frames to grab, frames to let pass after forcing a setting before -; judging one, and the share of pixels that must clip at each forced extreme. +; test camera. frames: how many to grab, and how long the first may take after +; the stream starts. dark: the share of bytes that must be at or below +; camera_black_max at minimum exposure and gain (a U-1240c's floor is 2..17). +; bright: at maximum gain and this exposure, the mean must rise by at least +; camera_bright_min_delta over dark. settle: frames to let pass after a change. camera_frames = 5 +camera_warmup_ms = 10000 camera_settle_frames = 3 camera_clip_min_pct = 99 +camera_black_max = 20 +camera_bright_exposure_us = 500000 +camera_bright_min_delta = 2 ; imu_expected_profile = VRU_general ; assert the active XKF profile, if set backlash_step_counts = 200,500,1000 ; small steps probing reversal dead-band balance_steps = 10 ; pitch up/down current-profile resolution diff --git a/docs/deployment.md b/docs/deployment.md index d6f29a9..f751743 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -69,9 +69,11 @@ ssh ggs@ 'cd ~/projects/fwt_2a/software && scripts/install-tower --verify It changes nothing; it runs `test hardware` and `test camera` inside the running program (pausing and then resuming capture) and checks the files, the firmware's -pins and the boot service. `test camera` takes about 15 s: it grabs frames, then -drives the camera's own settings to force every pixel black and then white - -which holds whatever the camera is pointed at - and puts the profile back. +pins and the boot service. `test camera` takes 15-30 s: it grabs frames, then +drives the camera's own settings - minimum exposure and gain must take every +pixel black, maximum gain and a long exposure must make the frame measurably +different - which holds whatever the camera is pointed at, and puts the profile +back. `--push-config` also overwrites the tower's `config.ini` with the dev box's, backing the tower's copy up first. It exists for the development tower; a @@ -286,6 +288,7 @@ list below and says which. | `Config error` naming a key | `build/fgc-config check --config config.ini --hardware /etc/fire_gimbal_control/hardware.ini` | | `Failed to open serial port` | MCU plugged in, `ggs` in `dialout`, `[Serial] device` in `hardware.ini` - a `/dev/serial/by-id/` path, which **changes when the MCU is flashed** | | An axis will not move, `ERR need config` | `hardware.ini` is missing a drivetrain value for it; the log names the key | +| Camera opened, but no images and `no frame yet (stream warming up)` for minutes | the stream is stuck; the program restarts it and then resets the camera by itself (known-issues #15), or type `camera reset` | | `No camera found` / Vimba startup error | SDK installed, GenTL path set (`/etc/profile.d/*Vimba*GenTL*.sh` or `VIMBA_CTI_PATH`), `[Camera.rgb] id` in `hardware.ini` | | `MQTT connect failed` (degraded mode) | broker reachable, `credentials.env` readable by `ggs`, firewall | | Runs but no images | capture started (`--start`, or the `start` command) and the gimbal homed; `output_dir` writable | diff --git a/docs/known-issues.md b/docs/known-issues.md index e9dc482..cb8bff3 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -29,7 +29,7 @@ doctest unit-test suite (`ctest`). |---|-------|--------| | 13 | `[Motor]` degrees↔counts calibration | The `config.example.ini` values are **placeholders**. Home, then run `gimbal heading setup` (yaw, needs no IMU) and `gimbal pitch setup` (pitch, needs the IMU) on the rig. | | 14 | Capture sweep on hardware | **Mechanically verified on the rig (2026-08-07).** A full `--init --start` run homed both axes and completed `MOVE → settle → trigger` for 15/15 waypoints with 0 dropped frames, 0 stalls and 0 acquisition failures. `kSettleTolCounts` (600) and the per-interval timing in [CaptureScheduler.cpp](../src/core/CaptureScheduler.cpp) have **not** been tuned against observed `ST` behaviour yet, and the settle tolerance has not been checked against a sharp image (see #16). | -| 15 | **Camera profiles unverified on a lit scene** | The `standard`/`night` profiles must be exported from the VimbaX Viewer (see `config/camera_profiles/rgb/README.md`) and have not been measured against real light. Confirm on the rig that `LoadSettings` succeeds, that `standard` sustains 10 fps with `frames_dropped` at 0, and that the auto-exposure lands on a sensible `ExposureTime` outdoors. The only previous rig test ran with the lens covered. **2026-09-24:** tower 121 has still never imaged a scene - a frame grabbed directly and the program's own saved images (2026-09-21) are a smooth grey gradient with no edges, i.e. the lens is covered or grossly out of focus. Tower 111's images from the same afternoon show nothing sharp anywhere either (soft blobs, no edges) - defocus, a cover or a fogged window; unexplained. `test camera` (below) proves the camera and its data path work in both cases; it cannot, by design, judge the picture. | +| 15 | **Camera profiles unverified on a lit scene** | The `standard`/`night` profiles must be exported from the VimbaX Viewer (see `config/camera_profiles/rgb/README.md`) and have not been measured against real light. Confirm on the rig that `LoadSettings` succeeds, that `standard` sustains 10 fps with `frames_dropped` at 0, and that the auto-exposure lands on a sensible `ExposureTime` outdoors. The only previous rig test ran with the lens covered. **2026-09-24:** tower 121 has still never imaged a scene - a frame grabbed directly and the program's own saved images (2026-09-21) are a smooth grey gradient with no edges, i.e. the lens is covered or grossly out of focus. Tower 111's images from the same afternoon show nothing sharp anywhere either (soft blobs, no edges) - defocus, a cover or a fogged window; unexplained. `test camera` (below) proves the camera and its data path work in both cases; it cannot, by design, judge the picture. **Stream stuck, 2026-09-24:** from 16:10 to 16:19 tower 111's camera delivered no frames at all - not to the program, not to a fresh Vimba session, not at a fixed 10 ms exposure - and it survived program restarts and a USB re-authorisation. The camera's own `DeviceReset` command cleared it. Production captured nothing for those 9 minutes and logged only `no frame yet (stream warming up)`: a session that has NEVER delivered is not restarted by `trigger()`, by design, so a stream stuck from its start stays stuck. It began just after a `test camera` run by an early version that stopped and restarted the stream several times in a few seconds (to write `BlackLevel`, which the camera refuses while streaming); whether that caused it or met 111's known USB stall is not established. `test camera` itself reported it correctly. **Answered the same day:** a stream that never delivers is now restarted once the wait exceeds what its exposure can explain (at least 30 s), and after two fruitless restarts the camera is reset (DeviceReset) and reopened on its profile - both by the program itself (`CameraStall.h`), and on demand with the console command `camera reset` (verified on tower 121: back in ~14 s, frames straight after). `test camera` no longer stops the stream at all: it changes only exposure and gain, which are writable live, and puts them back live. With that, 111 ran it inside the capturing program, passed, and kept capturing. | | 16 | **`settle_wait_ms` untuned** | `standard` waits 1 s and `night` 2.5 s after standstill before keeping a frame. Both are reasoned (≫ one frame period; long enough for the auto-exposure to see the new scene) but not measured. Check on a lit scene that the kept frame shows no slew blur and that its exposure has actually converged for that heading, and shorten the wait if it has. **Seen in real data 2026-09-24:** of eight consecutive images tower 111 kept at 8 deg/s, one was 98.6 % clipped white, one 57 % black and two about half clipped - frames kept before the continuous auto-exposure had caught up with the new view. | | 19 | **USB preconditions do not survive a reboot** | `usbcore.usbfs_memory_mb` reverts to 16 (needs 1000) and USB3 LPM re-enables on the camera's port. **Both were wrong on the LattePanda when tested on 2026-08-07**, and frame delivery stalls without them regardless of application code. **Solved on tower 121 by the setup kit** (`hardware/lattepanda-iota/preconditions/`): a systemd-tmpfiles drop-in for usbfs and two udev rules, all installed by `install.sh --preconditions`. Verified 2026-09-23 — `usbfs_memory_mb` reads 1000 after **6 days of uptime**, so it survived the boot, and both rules are in `/etc/udev/rules.d/`; and again across a deliberate reboot the same day, when `install-tower --verify` read `usbfs_memory_mb` 1000 and the camera port's `usb3_lpm_permit` 0 (a check `test hardware camera` now makes too, once `[Test] usb3_lpm_path` is set - the installer sets it). Note it is **tmpfiles, not the kernel cmdline** the original entry proposed. **On the Delta fleet**: tower 111 read 16 and `u1_u2` on 2026-09-23, and the Delta kit (`hardware/lattepanda-3-delta/`, the same two files) was applied the same day - 1000 and `0` afterwards, confirmed by `test hardware camera`. Confirmed across a reboot on 2026-09-24 (`install-tower --verify`). **Solved on both boards**; open only for Delta towers the kit has not reached. | diff --git a/docs/tower-installation.md b/docs/tower-installation.md index a0db08c..ce39a29 100644 --- a/docs/tower-installation.md +++ b/docs/tower-installation.md @@ -1071,7 +1071,7 @@ freshly flashed board is not failed for values the host had not sent yet. | IMU / ambient sensor | `test hardware imu` / `env` | none declared | ✅ MTi streaming, SHT41 CRC-valid on a real I2C bus | | Camera enumerates | `ListCameras_VmbC` reports the `DEV_` id in `hardware.ini` | ✅ `DEV_1AB22C0B7D4E` | ✅ `DEV_1AB22C049CD1` | | Camera profile applies | startup logs `profile 'standard' applied ... (N settings)` | ✅ 17206 settings | ✅ 3743 settings | -| Camera works, whatever it sees | `test camera`: whole frames, then every pixel forced to black (minimum exposure and gain) and to white (black level at maximum), then the profile restored | ✅ 2026-09-24, with its lens covered: 100.00 % black, 100.00 % white | not run yet (needs the current host) | +| Camera works, whatever it sees | `test camera`: whole frames; every pixel forced to black (minimum exposure and gain); the sensor responding at maximum gain and a long exposure; the profile restored | ✅ 2026-09-24, lens covered: 100 % ≤ 20 dark, bright mean 0 → 255 | ✅ 2026-09-24, inside the running program: 5 frames of 4024x3036 in 9.2 s (a dark scene, ~2 s exposures), 100 % ≤ 20 dark (mean 5.6), bright mean 5.6 → 255; capture resumed. An earlier run correctly reported the camera delivering nothing at all - known-issues #15 | | Firmware identity | `DUMP BEGIN build=` is the revision flashed | ✅ | ✅ `a0b1e4a` | | Driver answers over SPI | `IOIN` reads version `0x30` for every axis | ✅ | ✅ both axes | | Firmware holds `hardware.ini` | `test hardware motor`: values, current, pins | ✅ | ✅ every leaf, 2026-09-24 | diff --git a/include/fgc/CameraStall.h b/include/fgc/CameraStall.h new file mode 100644 index 0000000..0ca4e64 --- /dev/null +++ b/include/fgc/CameraStall.h @@ -0,0 +1,39 @@ +#pragma once + +// When a camera stream delivers nothing: keep waiting, restart the stream, or +// reset the camera. Pure policy, so it is testable without a camera; the Vimba +// source measures and acts. +// +// Why a reset exists at all: on 2026-09-24 tower 111's camera stopped delivering +// frames entirely - to the program, to a fresh Vimba session, at a fixed 10 ms +// exposure - and survived program restarts and a USB re-authorisation. Only the +// camera's own DeviceReset brought it back, and until then production captured +// nothing while logging "stream warming up", because a session that had never +// delivered was never restarted at all. + +namespace fgc { + +enum class StallAction { + Wait, // within what this exposure can legitimately take + Restart, // stop and start the stream + Reset, // DeviceReset: reboot the camera, reopen it, re-apply its profile +}; + +// How long a fresh stream may go without its first frame. Long exposures are +// legitimate - tower 111 auto-exposes up to 9 s in the dark - so the limit +// grows with the exposure, and never drops below 30 s. +long long firstFrameLimitMs(double exposure_us); + +// A stream that has delivered nothing since it started `waited_ms` ago. +// `failed_restarts` counts restarts since the last frame that arrived. +StallAction onNoFirstFrame(long long waited_ms, double exposure_us, int failed_restarts); + +// A stream that delivered before but has gone stale, and a restart has just +// failed to bring a fresh frame. Restart again, or reset once restarts have +// stopped working. +StallAction onRestartFailed(int failed_restarts); + +// Restarts without a frame, after which the camera is reset instead. +constexpr int kRestartsBeforeReset = 2; + +} // namespace fgc diff --git a/include/fgc/HardwareChecks.h b/include/fgc/HardwareChecks.h index 7228c73..9bd6b56 100644 --- a/include/fgc/HardwareChecks.h +++ b/include/fgc/HardwareChecks.h @@ -141,10 +141,12 @@ CheckResult checkTravelFitsPositive(const std::string& axis, long center, long h // What fraction of a frame's bytes sit at each end of the range. Every channel // byte counts, so an RGB8 frame's 12 M bytes are 12 M samples. struct PixelStats { - size_t samples = 0; - double mean = 0.0; - double pct_low = 0.0; // bytes <= low - double pct_high = 0.0; // bytes >= high + size_t samples = 0; + double mean = 0.0; + double pct_low = 0.0; // bytes <= low + double pct_high = 0.0; // bytes >= high + uint8_t low = 5; // the bounds they were counted against + uint8_t high = 250; }; PixelStats pixelStats(const std::vector& data, uint8_t low = 5, uint8_t high = 250); @@ -165,11 +167,21 @@ struct FrameRun { CheckResult checkFramesDelivered(const std::string& camera, const FrameRun& run); // Did a frame taken at forced settings clip to the end they force? `white` -// false = black. The camera is driven there by its own settings (minimum -// exposure and gain for black; black level at maximum for white), so this holds -// in fog, at night and under a lens cap alike - and proves the settings reach the -// sensor, the sensor's output reaches the host, and the full range survives. +// false = black. For black the camera is driven there by its own settings - +// minimum exposure and gain - so it holds in daylight, at night and under a lens +// cap alike. "Black" is `stats.low`, not 0: sensors differ, and an Alvium 1800 +// U-1240c's floor sits at 2..17 of 255 where a U-1620c's is 0. CheckResult checkClipped(const std::string& camera, bool white, const PixelStats& stats, double min_pct); +// Is the sensor alive? A frame at maximum gain and a long exposure must differ +// from the forced-dark one by at least `min_delta` in mean level. Nothing about +// the picture is interpreted, and nothing needs to clip: in daylight the bright +// frame saturates, at night or under a cap amplified dark current and noise still +// lift it - either way it is different, which only a sensor that took the new +// settings can produce. A frame that does not move is a camera that ignored +// them, or a sensor that is not in the loop. +CheckResult checkSensorResponds(const std::string& camera, const PixelStats& dark, + const PixelStats& bright, double min_delta); + } // namespace fgc diff --git a/include/fgc/ICameraSource.h b/include/fgc/ICameraSource.h index a544a43..8b7fbf2 100644 --- a/include/fgc/ICameraSource.h +++ b/include/fgc/ICameraSource.h @@ -114,6 +114,11 @@ public: // setFeature() since. False if that failed - which leaves the camera on // settings nobody chose, so callers must say so loudly. virtual bool restoreProfile(int /*camera_index*/) { return false; } + + // Reboot one camera (GenICam DeviceReset), reopen it and put it back on its + // profile. The last resort for a stream that nothing else will restart - and + // the operator's `camera reset`. Blocks while the camera comes back (seconds). + virtual bool resetCamera(int /*camera_index*/) { return false; } }; } // namespace fgc diff --git a/include/fgc/TestRunner.h b/include/fgc/TestRunner.h index a82493a..a6a1c77 100644 --- a/include/fgc/TestRunner.h +++ b/include/fgc/TestRunner.h @@ -42,19 +42,20 @@ enum TestLeaf : uint32_t { T_HW_IMU = 1u << 14, T_HW_ENV = 1u << 15, T_HW_CAMERA = 1u << 16, - // `test camera`: does the camera deliver whole frames, and can its own - // settings drive every pixel to black and to white? Forced settings, so the - // verdict does not depend on the scene, the weather or the time of day. + // `test camera`: does the camera deliver whole frames, do its own settings + // drive every pixel to black, and does the sensor respond when they are + // turned up? Forced settings, and nothing about the picture is interpreted, + // so the verdict does not depend on the scene, the weather or the time of day. T_CAM_FRAMES = 1u << 17, T_CAM_DARK = 1u << 18, - T_CAM_WHITE = 1u << 19, + T_CAM_BRIGHT = 1u << 19, }; constexpr uint32_t T_GIMBAL_ALL = T_GIMBAL_HOMING | T_GIMBAL_ENCODER | T_GIMBAL_FRICTION | T_GIMBAL_BACKLASH | T_GIMBAL_BALANCE; constexpr uint32_t T_IMU_ALL = T_IMU_CONFIG | T_IMU_HEALTH | T_IMU_DRIFT; constexpr uint32_t T_HOST_ALL = T_HOST_THERMAL | T_HOST_DISK | T_HOST_MEMORY | T_HOST_LOAD; constexpr uint32_t T_HW_ALL = T_HW_AXES | T_HW_MOTOR | T_HW_IMU | T_HW_ENV | T_HW_CAMERA; -constexpr uint32_t T_CAM_ALL = T_CAM_FRAMES | T_CAM_DARK | T_CAM_WHITE; +constexpr uint32_t T_CAM_ALL = T_CAM_FRAMES | T_CAM_DARK | T_CAM_BRIGHT; constexpr uint32_t T_ALL = T_GIMBAL_ALL | T_IMU_ALL | T_HOST_ALL | T_HW_ALL | T_CAM_ALL; // Which leaf masks need real motor / IMU hardware (used by the caller to guard). diff --git a/include/fgc/VimbaCameraSource.h b/include/fgc/VimbaCameraSource.h index 672d75d..4828c3d 100644 --- a/include/fgc/VimbaCameraSource.h +++ b/include/fgc/VimbaCameraSource.h @@ -40,6 +40,7 @@ public: bool setFeature(int camera_index, const std::string& name, const std::string& value) override; bool featureRange(int camera_index, const std::string& name, double& min, double& max) override; bool restoreProfile(int camera_index) override; + bool resetCamera(int camera_index) override; private: struct Impl; diff --git a/include/fgc/mock/MockCameraSource.h b/include/fgc/mock/MockCameraSource.h index 849044c..e2be91b 100644 --- a/include/fgc/mock/MockCameraSource.h +++ b/include/fgc/mock/MockCameraSource.h @@ -46,10 +46,9 @@ public: return true; } - // `test camera`. The mock behaves like the real one: its own settings drive - // it to black (minimum exposure and gain) and to white (black level at its - // maximum), so the self-test passes against it - and a fault can be switched - // on to prove the test fails when it should. + // `test camera`. The mock behaves like the real one: minimum exposure and gain + // drive it to black, maximum gain brightens it, so the self-test passes against + // it - and a fault can be switched on to prove the test fails when it should. enum class Fault { None, NoFrames, IgnoresSettings }; void setFault(Fault f) { fault_ = f; } @@ -82,6 +81,13 @@ public: features_.clear(); return true; } + bool resetCamera(int camera_index) override { + if (camera_index < 0 || camera_index >= count_) return false; + LOG_INFO << "[mock] camera " << camera_index << " reset"; + features_.clear(); + fault_ = Fault::None; // a reset is what clears a stuck camera + return true; + } // Synthetic device info so the expanded camera view renders (and is testable) // without hardware. Non-hardware fields (temperature, exposure...) stay at 0. @@ -123,11 +129,11 @@ private: auto it = features_.find(k); return it == features_.end() ? dflt : std::stod(it->second); }; - if (fault_ != Fault::IgnoresSettings) { - if (num("BlackLevel", 0.0) >= 4095.0) level = 400.0; // clips white - else if (features_.count("ExposureTime") && num("ExposureTime", 1e9) < 100.0 && - num("Gain", 99.0) <= 0.0) - level = -100.0; // clips black + if (fault_ != Fault::IgnoresSettings && features_.count("ExposureTime")) { + if (num("ExposureTime", 1e9) < 100.0 && num("Gain", 99.0) <= 0.0) + level = -100.0; // minimum exposure and gain: black + else if (num("Gain", 0.0) >= 48.0) + level = 180.0; // maximum gain: visibly brighter } const uint8_t phase = static_cast(f.timestamp_ms); diff --git a/src/camera/VimbaCameraSource.cpp b/src/camera/VimbaCameraSource.cpp index 8e611b6..f738e42 100644 --- a/src/camera/VimbaCameraSource.cpp +++ b/src/camera/VimbaCameraSource.cpp @@ -11,6 +11,7 @@ #undef LOG_ERROR #undef LOG_FREE_TEXT +#include "fgc/CameraStall.h" #include "fgc/Logger.h" #include @@ -21,6 +22,8 @@ #include #include #include +#include +#include using namespace VmbCPP; @@ -189,6 +192,12 @@ struct VimbaCameraSource::Impl { bool started = false; long long last_captured_ts = 0; // freshness watermark long long last_no_frame_warn_ms = 0; + long long session_start_ms = 0; // this acquisition session + int failed_restarts = 0; // since the last frame arrived + // Per camera: the value each feature held before `test camera` first changed + // it, in the order changed - so it can be put back live, without stopping + // the stream (which is what a profile reload has to do). + std::vector>> touched; // Begin/end a free-run acquisition session (fresh observers each time). Split out so // trigger() can restart the stream when it stalls. Stats persist across restarts. @@ -202,6 +211,7 @@ struct VimbaCameraSource::Impl { // A fresh session has delivered nothing, so a watermark from the previous // one would make every genuinely-new frame look stale to trigger(). last_captured_ts = 0; + session_start_ms = nowMs(); int cam_id = 0; for (auto& cam : cameras) { IFrameObserverPtr observer(new FrameObserver(cam, cam_id, stats[cam_id].get())); @@ -215,6 +225,57 @@ struct VimbaCameraSource::Impl { for (auto& cam : cameras) cam->StopContinuousImageAcquisition(); observers.clear(); } + // Longest exposure any camera is set to, for how long a frame may take. + double longestExposureUs() { + double e = 0; + for (auto& cam : cameras) e = std::max(e, getFloat(cam, "ExposureTime")); + return e; + } + + // Reboot one camera, reopen it by id and re-apply its profile. See + // CameraStall.h for why this exists. + bool resetOne(size_t idx) { + const std::string& id = specs[idx].id; + LOG_WARN << "camera '" << specs[idx].name << "': resetting it (DeviceReset)"; + const bool was_started = started; + stopAcquisition(); + FeaturePtr f; + if (SP_ACCESS(cameras[idx])->GetFeatureByName("DeviceReset", f) != VmbErrorSuccess || + f->RunCommand() != VmbErrorSuccess) { + LOG_ERROR << "camera '" << specs[idx].name << "': DeviceReset is not available"; + if (was_started) startAcquisition(); + return false; + } + cameras[idx]->Close(); + // It drops off the bus and re-enumerates: ~10 s for an Alvium 1800 U on + // tower 111. Reopen it by id as soon as it answers. + std::this_thread::sleep_for(std::chrono::seconds(3)); + CameraPtr fresh; + const long long deadline = nowMs() + 45000; + bool opened = false; + while (!opened && nowMs() < deadline) { + CameraPtr c; + if (sys.GetCameraByID(id.c_str(), c) == VmbErrorSuccess && + c->Open(VmbAccessModeFull) == VmbErrorSuccess) { + fresh = c; + opened = true; + } else { + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + } + if (!opened) { + LOG_ERROR << "camera '" << specs[idx].name << "' did not come back within 45 s of " + "a DeviceReset - it needs a power cycle"; + return false; + } + cameras[idx] = fresh; + if (idx < touched.size()) touched[idx].clear(); + applyProfile(fresh, profiles[idx]); + if (was_started) startAcquisition(); + LOG_INFO << "camera '" << specs[idx].name << "' is back after a reset"; + return true; + } + // Newest frame timestamp across all cameras (0 if none delivered yet). long long newestTs() { long long t = 0; @@ -259,6 +320,7 @@ void VimbaCameraSource::open() { // Per-camera stats (persist across restart) + identity (read once here). impl_->stats.push_back(std::make_unique()); impl_->telemetry.emplace_back(); + impl_->touched.emplace_back(); impl_->identity.push_back({id, getString(cam, "DeviceModelName"), getString(cam, "DeviceSerialNumber")}); LOG_INFO << "Opened camera '" << spec.name << "' " << id << " (" @@ -300,6 +362,8 @@ bool VimbaCameraSource::setProfile(int camera_index, const CameraProfile& profil } impl_->profiles[idx] = profile; + // A profile sets every feature, so nothing is left to put back by hand. + if (idx < impl_->touched.size()) impl_->touched[idx].clear(); if (!impl_->started) { // Idle: every feature is writable, so apply directly. start() no longer // applies profiles, so skipping this would leave the camera on the old one. @@ -344,11 +408,41 @@ bool VimbaCameraSource::trigger() { // thread for 3 s, so a cold stream would be repeatedly reset just as it was // about to produce its first frame. Return empty-handed instead; the caller // retries every tick. + // + // But not forever. A session that never delivers is not warming up, and it + // used never to be restarted at all: tower 111's camera sat like that for 9 + // minutes on 2026-09-24, capturing nothing and saying only "warming up". + // CameraStall.h decides when waiting has become a stall, allowing for long + // exposures, and when restarting has stopped helping. if (newest == 0) { - long long now = nowMs(); - if (now - impl_->last_no_frame_warn_ms > 5000) { - LOG_WARN << "camera: no frame yet (stream warming up)"; - impl_->last_no_frame_warn_ms = now; + const long long now = nowMs(); + const long long waited = now - impl_->session_start_ms; + const double exposure = impl_->longestExposureUs(); + switch (onNoFirstFrame(waited, exposure, impl_->failed_restarts)) { + case StallAction::Wait: + if (now - impl_->last_no_frame_warn_ms > 5000) { + LOG_WARN << "camera: no frame yet (stream warming up)"; + impl_->last_no_frame_warn_ms = now; + } + break; + case StallAction::Restart: + LOG_WARN << "camera: no frame at all " << waited / 1000 + << " s into the stream; restarting it"; + ++impl_->failed_restarts; + for (auto& st : impl_->stats) st->restarts.fetch_add(1); + try { + impl_->stopAcquisition(); + impl_->startAcquisition(); + } catch (const std::exception& e) { + LOG_WARN << "camera: acquisition restart failed: " << e.what(); + } + break; + case StallAction::Reset: + LOG_ERROR << "camera: still no frame after " << impl_->failed_restarts + << " stream restarts; resetting the camera"; + for (size_t i = 0; i < impl_->cameras.size(); ++i) impl_->resetOne(i); + impl_->failed_restarts = 0; + break; } return false; } @@ -365,6 +459,13 @@ bool VimbaCameraSource::trigger() { const long long deadline = nowMs() + 3000; while (impl_->newestTs() <= impl_->last_captured_ts && nowMs() < deadline) std::this_thread::sleep_for(std::chrono::milliseconds(50)); + if (impl_->newestTs() <= impl_->last_captured_ts && + onRestartFailed(++impl_->failed_restarts) == StallAction::Reset) { + LOG_ERROR << "camera: the stream stays stale after " << impl_->failed_restarts + << " restarts; resetting the camera"; + for (size_t i = 0; i < impl_->cameras.size(); ++i) impl_->resetOne(i); + impl_->failed_restarts = 0; + } } bool any = false; @@ -375,6 +476,7 @@ bool VimbaCameraSource::trigger() { impl_->last_captured_ts = std::max(impl_->last_captured_ts, f.timestamp_ms); if (impl_->callback) impl_->callback(f); any = true; + impl_->failed_restarts = 0; } } if (!any) { @@ -462,6 +564,39 @@ bool VimbaCameraSource::setFeature(int camera_index, const std::string& name, if (SP_ACCESS(impl_->cameras[idx])->GetFeatureByName(name.c_str(), f) != VmbErrorSuccess || f->GetDataType(type) != VmbErrorSuccess) return false; + // The value before the first change, so restoreProfile() can put it back + // live. Only the first: a second change must not record the test's own value. + auto& mine = impl_->touched[idx]; + const bool seen = std::any_of(mine.begin(), mine.end(), + [&](const std::pair& t) { + return t.first == name; + }); + if (!seen) { + std::string before; + switch (type) { + case VmbFeatureDataFloat: { + double v = 0; + if (f->GetValue(v) == VmbErrorSuccess) { + char b[40]; std::snprintf(b, sizeof(b), "%.17g", v); before = b; + } + break; + } + case VmbFeatureDataInt: { + VmbInt64_t v = 0; + if (f->GetValue(v) == VmbErrorSuccess) before = std::to_string(v); + break; + } + case VmbFeatureDataBool: { + bool v = false; + if (f->GetValue(v) == VmbErrorSuccess) before = v ? "true" : "false"; + break; + } + case VmbFeatureDataEnum: + case VmbFeatureDataString: f->GetValue(before); break; + default: break; + } + if (!before.empty()) mine.emplace_back(name, before); + } auto write = [&]() -> VmbErrorType { try { switch (type) { @@ -521,9 +656,48 @@ bool VimbaCameraSource::featureRange(int camera_index, const std::string& name, bool VimbaCameraSource::restoreProfile(int camera_index) { const size_t idx = static_cast(camera_index); if (camera_index < 0 || idx >= impl_->profiles.size()) return false; - // setProfile() rather than applyProfile(): it cycles the stream when one is - // running, since a profile may carry features that are not writable live. + + // Put back what setFeature() changed, live and in reverse order - values + // first, then the auto modes that take over from them. No stream restart: + // stopping and starting it repeatedly is exactly what preceded tower 111's + // camera getting stuck (known-issues #15). + auto undo = std::move(impl_->touched[idx]); + impl_->touched[idx].clear(); + bool ok = !undo.empty(); + for (auto it = undo.rbegin(); ok && it != undo.rend(); ++it) { + FeaturePtr f; + VmbFeatureDataType type = VmbFeatureDataUnknown; + if (SP_ACCESS(impl_->cameras[idx])->GetFeatureByName(it->first.c_str(), f) != VmbErrorSuccess || + f->GetDataType(type) != VmbErrorSuccess) { ok = false; break; } + VmbErrorType rc = VmbErrorWrongType; + try { + switch (type) { + case VmbFeatureDataFloat: rc = f->SetValue(std::stod(it->second)); break; + case VmbFeatureDataInt: rc = f->SetValue(static_cast(std::stoll(it->second))); break; + case VmbFeatureDataBool: rc = f->SetValue(it->second == "true"); break; + case VmbFeatureDataEnum: + case VmbFeatureDataString: rc = f->SetValue(it->second.c_str()); break; + default: break; + } + } catch (const std::exception&) { rc = VmbErrorBadParameter; } + ok = rc == VmbErrorSuccess; + } + if (ok) { + LOG_INFO << "camera: " << undo.size() << " setting(s) put back live"; + return true; + } + // Nothing recorded, or a write refused: the whole profile, the slow way. + // setProfile() cycles the stream when one is running, since a profile may + // carry features that are not writable live. return setProfile(camera_index, impl_->profiles[idx]); } +bool VimbaCameraSource::resetCamera(int camera_index) { + const size_t idx = static_cast(camera_index); + if (camera_index < 0 || idx >= impl_->cameras.size()) return false; + const bool ok = impl_->resetOne(idx); + impl_->failed_restarts = 0; + return ok; +} + } // namespace fgc diff --git a/src/core/Application.cpp b/src/core/Application.cpp index 56cbd0c..32337e2 100644 --- a/src/core/Application.cpp +++ b/src/core/Application.cpp @@ -1634,6 +1634,18 @@ struct Application::Impl { << wrap360(cfg.geometry.yaw.max_deg) << " true) and will be skipped"; } + // `camera reset`: reboot every camera (DeviceReset) and put it back on its + // profile. For a stream that has stopped delivering and will not restart - + // the state tower 111's camera was in on 2026-09-24, which only this cleared. + void resetCameras() { + const bool was = scheduler->captureActive(); + stopCapture(); + for (int i = 0; i < camera->cameraCount(); ++i) + LOG_INFO << "camera " << i << (camera->resetCamera(i) ? ": reset, back on its profile" + : ": reset FAILED (see above)"); + if (was) startCapture(); + } + void startCapture() { // Heading setup owns the gimbal by hand. Starting a scan under it would // move the axis between a sighting and the bearing being typed for it, @@ -2399,7 +2411,7 @@ struct Application::Impl { // long op, or a DUMP over the same serial link). Ignored while busy. static bool interferingVerb(const std::string& verb) { return verb == "gimbal" || verb == "start" || verb == "test" || - verb == "refresh" || verb == "snapshot"; + verb == "refresh" || verb == "snapshot" || verb == "camera"; } void handleCommand(const std::string& line) { @@ -2437,6 +2449,9 @@ struct Application::Impl { } } else if (c.verb == "exit") { running = false; + } else if (c.verb == "camera") { + if (c.device == "reset") resetCameras(); + else LOG_WARN << "usage: camera reset"; } else if (c.verb == "snapshot") { takeSnapshot(); } else if (c.verb == "start") { diff --git a/src/core/CameraStall.cpp b/src/core/CameraStall.cpp new file mode 100644 index 0000000..275b349 --- /dev/null +++ b/src/core/CameraStall.cpp @@ -0,0 +1,21 @@ +#include "fgc/CameraStall.h" + +#include + +namespace fgc { + +long long firstFrameLimitMs(double exposure_us) { + const long long exposure_ms = exposure_us > 0 ? static_cast(exposure_us / 1000) : 0; + return std::max(30000, 3 * exposure_ms + 5000); +} + +StallAction onNoFirstFrame(long long waited_ms, double exposure_us, int failed_restarts) { + if (waited_ms < firstFrameLimitMs(exposure_us)) return StallAction::Wait; + return failed_restarts >= kRestartsBeforeReset ? StallAction::Reset : StallAction::Restart; +} + +StallAction onRestartFailed(int failed_restarts) { + return failed_restarts >= kRestartsBeforeReset ? StallAction::Reset : StallAction::Restart; +} + +} // namespace fgc diff --git a/src/core/HardwareChecks.cpp b/src/core/HardwareChecks.cpp index f31aca0..e7322c8 100644 --- a/src/core/HardwareChecks.cpp +++ b/src/core/HardwareChecks.cpp @@ -270,6 +270,8 @@ CheckResult checkTravelFitsPositive(const std::string& axis, long center, long h PixelStats pixelStats(const std::vector& data, uint8_t low, uint8_t high) { PixelStats st; + st.low = low; + st.high = high; st.samples = data.size(); if (data.empty()) return st; // A histogram, not per-byte arithmetic: a frame is 12 M bytes, and this runs @@ -337,8 +339,8 @@ CheckResult checkClipped(const std::string& camera, bool white, const PixelStats } const double got = white ? st.pct_high : st.pct_low; char buf[160]; - std::snprintf(buf, sizeof(buf), "%.2f%% of pixels %s (want >= %.1f%%), mean %.1f", got, - white ? "clipped white" : "clipped black", min_pct, st.mean); + std::snprintf(buf, sizeof(buf), "%.2f%% of pixels %s %d (want >= %.1f%%), mean %.1f", got, + white ? ">=" : "<=", white ? st.high : st.low, min_pct, st.mean); r.pass = got >= min_pct; r.detail = camera + ": " + buf; if (!r.pass) @@ -350,4 +352,24 @@ CheckResult checkClipped(const std::string& camera, bool white, const PixelStats return r; } +CheckResult checkSensorResponds(const std::string& camera, const PixelStats& dark, + const PixelStats& bright, double min_delta) { + CheckResult r; + if (dark.samples == 0 || bright.samples == 0) { + r.pass = false; + r.detail = camera + ": no frame to compare"; + return r; + } + const double delta = bright.mean - dark.mean; + char buf[160]; + std::snprintf(buf, sizeof(buf), "mean %.1f at maximum gain and long exposure against %.1f " + "forced dark (+%.1f, want >= +%.1f)", bright.mean, dark.mean, delta, min_delta); + r.pass = delta >= min_delta; + r.detail = camera + ": " + buf; + if (!r.pass) + r.detail += " -- the sensor did not respond: the settings were not applied, or the " + "frames are not coming from a live sensor"; + return r; +} + } // namespace fgc diff --git a/src/core/HelpText.cpp b/src/core/HelpText.cpp index d22820a..6827edd 100644 --- a/src/core/HelpText.cpp +++ b/src/core/HelpText.cpp @@ -127,13 +127,19 @@ const std::vector& helpCatalog() { "a declared IMU streams; a declared ambient sensor sits on a REAL I2C bus (not", "SMBus-only) and its CRC checks out; the cameras opened and the USB", "preconditions hold. Nothing declared reports 'not checked', never PASS."}}, - {"test camera [frames|dark|white]", + {"test camera [frames|dark|bright]", "Check the camera works, whatever it is pointed at.", { - "frames: whole frames arrive through the program's own path, within the", - "profile's frame_timeout_ms. dark / white: the camera's own settings force", - "every pixel to black (minimum exposure and gain) and to white (black level", - "at its maximum) - so fog, night or a lens cap cannot decide the verdict.", - "The camera is put back on its profile afterwards; capture pauses meanwhile."}}, + "frames: whole frames arrive in time. dark: minimum exposure and gain take", + "every pixel to black. bright: maximum gain and a long exposure make the", + "frame measurably different - the sensor is alive and took the settings.", + "The picture itself is never judged, so fog, night or a lens cap cannot", + "decide it. The profile is put back after; capture pauses meanwhile."}}, + {"camera reset", + "Reboot the camera and put it back on its profile.", { + "Sends the camera its own DeviceReset, waits for it to come back on USB, and", + "re-applies the profile; capture pauses meanwhile. For a stream that has", + "stopped delivering frames and will not restart. The program also does this", + "by itself when restarting the stream stops helping."}}, {"gimbal pitch setup", "Calibrate PITCH steps<->degrees against gravity, using the IMU.", { "Auto-homes if needed, sweeps pitch dwelling at each step, fits, then", diff --git a/src/core/TestRunner.cpp b/src/core/TestRunner.cpp index 7b6b2cd..667702a 100644 --- a/src/core/TestRunner.cpp +++ b/src/core/TestRunner.cpp @@ -132,7 +132,7 @@ bool resolveTestSelection(const std::string& subsystem, const std::string& test, if (test.empty()) m = T_CAM_ALL; else if (test == "frames") m = T_CAM_FRAMES; else if (test == "dark") m = T_CAM_DARK; - else if (test == "white") m = T_CAM_WHITE; + else if (test == "bright") m = T_CAM_BRIGHT; else { err = "unknown camera test: " + test; return false; } } else if (subsystem == "host") { sub_ok = true; @@ -1153,15 +1153,19 @@ void TestRunner::testHwCamera(TestReport& r) { r.sections.push_back(std::move(s)); } -// `test camera` - frames, and the two forced extremes. +// `test camera` - frames, forced dark, and whether the sensor responds. // // The image test is deliberately not "is the picture good": fog, night and a -// lens cap all make a bad picture out of a healthy camera. Instead the camera is -// driven by its OWN settings to each end of its range - minimum exposure and gain -// for black, black level at maximum for white - and every pixel must follow. That -// holds whatever it is pointed at, and still proves the settings reach the -// sensor, its output reaches the host whole, and the full range survives. -// Measured on tower 121 with its lens covered: 100.00 % black, 100.00 % white. +// lens cap all make a bad picture out of a healthy camera. Instead the camera's +// OWN settings drive it: minimum exposure and gain must take every pixel to +// black, and maximum gain with a long exposure must then produce a frame that is +// measurably different - which only a live sensor that took the settings can do, +// in daylight, at night or under a cap. Nothing is read into what the picture +// shows. +// +// A white end, forced by black level, was tried first and dropped: on tower 121's +// Alvium U-1620c it clips 100 % white, but tower 111's U-1240c has a black level +// range of only 0..255 of 4095 and cannot reach white by any setting at all. void TestRunner::testCamera(TestReport& r) { ICameraSource* cam = hw_.camera; const int n = cam ? cam->cameraCount() : 0; @@ -1174,10 +1178,10 @@ void TestRunner::testCamera(TestReport& r) { }; const bool want_frames = selection_ & T_CAM_FRAMES; const bool want_dark = selection_ & T_CAM_DARK; - const bool want_white = selection_ & T_CAM_WHITE; + const bool want_bright = selection_ & T_CAM_BRIGHT; const char* kFramesTitle = "the camera delivers whole frames"; const char* kDarkTitle = "forced dark: every pixel follows to black"; - const char* kWhiteTitle = "forced bright: every pixel follows to white"; + const char* kBrightTitle = "forced bright: the sensor responds"; double lo = 0, hi = 0; if (n == 0 || !cam->featureRange(0, "ExposureTime", lo, hi)) { @@ -1185,13 +1189,17 @@ void TestRunner::testCamera(TestReport& r) { : "this camera source cannot be driven by the self-test"; if (want_frames) notRun("cam_frames", kFramesTitle, why); if (want_dark) notRun("cam_dark", kDarkTitle, why); - if (want_white) notRun("cam_white", kWhiteTitle, why); + if (want_bright) notRun("cam_bright", kBrightTitle, why); return; } - const int frames = static_cast(profile_.get("camera_frames", 5)); - const int discard = static_cast(profile_.get("camera_settle_frames", 3)); - const double min_pct = profile_.get("camera_clip_min_pct", 99.0); + const int frames = static_cast(profile_.get("camera_frames", 5)); + const int discard = static_cast(profile_.get("camera_settle_frames", 3)); + const int warmup_ms = static_cast(profile_.get("camera_warmup_ms", 10000)); + const double min_pct = profile_.get("camera_clip_min_pct", 99.0); + const uint8_t black_max = static_cast(profile_.get("camera_black_max", 20)); + const double bright_us = profile_.get("camera_bright_exposure_us", 500000); + const double min_delta = profile_.get("camera_bright_min_delta", 2.0); auto name = [&](int i) { return i < static_cast(hw_.configured_cameras.size()) ? hw_.configured_cameras[i] @@ -1205,26 +1213,42 @@ void TestRunner::testCamera(TestReport& r) { const auto info = cam->deviceInfo(); return i < static_cast(info.size()) ? info[i].frames_dropped : 0; }; + // What the camera is exposing for right now. A frame cannot arrive faster + // than that: in a dark scene tower 111 auto-exposes for seconds, and a + // timeout that ignores it fails a healthy camera. + auto exposureUs = [&](int i) -> double { + const auto info = cam->deviceInfo(); + return i < static_cast(info.size()) ? info[i].exposure_us : 0.0; + }; + auto num = [](double v) { + char b[32]; std::snprintf(b, sizeof(b), "%.6g", v); return std::string(b); + }; // The capture scheduler stopped the stream for the test; it needs to run. cam->start(); - TestSection sf, sd, sw; + TestSection sf, sd, sb; sf.id = "cam_frames"; sf.title = kFramesTitle; sd.id = "cam_dark"; sd.title = kDarkTitle; - sw.id = "cam_white"; sw.title = kWhiteTitle; + sb.id = "cam_bright"; sb.title = kBrightTitle; for (int i = 0; i < n && !cancelled(); ++i) { const std::string cn = name(i); + // A freshly started stream is slow to deliver its first frame - 2 s on + // tower 111 at 1 fps and full resolution - so the first frame gets a + // warm-up allowance, and the rest are held to the profile's own timeout. + Frame first; + const bool warm = cam->grabFrame(i, first, warmup_ms); + if (want_frames) { setProgress(sf.id, i + 1, n, "grabbing frames"); FrameRun run; run.wanted = frames; - run.timeout_ms = timeout(i); + run.timeout_ms = timeout(i) + static_cast(2 * exposureUs(i) / 1000); const long long dropped0 = dropped(i); const auto t0 = std::chrono::steady_clock::now(); - for (int k = 0; k < frames && !cancelled(); ++k) { + for (int k = 0; k < frames && !cancelled() && warm; ++k) { Frame f; if (!cam->grabFrame(i, f, run.timeout_ms)) break; if (run.got == 0) { @@ -1240,62 +1264,97 @@ void TestRunner::testCamera(TestReport& r) { const double secs = std::chrono::duration( std::chrono::steady_clock::now() - t0).count(); run.dropped = dropped(i) - dropped0; - verdict(sf, cn + "_frames", checkFramesDelivered(cn, run)); + CheckResult c = checkFramesDelivered(cn, run); + if (!warm) c.detail = cn + ": no first frame within " + std::to_string(warmup_ms) + + " ms of starting the stream"; + verdict(sf, cn + "_frames", c); if (run.got > 1 && secs > 0) sf.notes.push_back(cn + ": " + std::to_string(run.got) + " frames in " + std::to_string(static_cast(secs * 1000)) + " ms"); } - if (!want_dark && !want_white) continue; + if (!want_dark && !want_bright) continue; // From here the camera is off its profile. It is put back below on every - // path out of these two phases, and a failure to put it back is a failure - // of the test. - auto force = [&](bool white, TestSection& s) { - double emin = 0, emax = 0, gmin = 0, gmax = 0, bmin = 0, bmax = 0; - CheckResult c; - const bool ranges = cam->featureRange(i, "ExposureTime", emin, emax) && - cam->featureRange(i, "Gain", gmin, gmax) && - cam->featureRange(i, "BlackLevel", bmin, bmax); - auto num = [](double v) { - char b[32]; std::snprintf(b, sizeof(b), "%.6g", v); return std::string(b); - }; - const bool set = ranges && - cam->setFeature(i, "ExposureAuto", "Off") && - cam->setFeature(i, "GainAuto", "Off") && - cam->setFeature(i, "ExposureTime", num(emin)) && - cam->setFeature(i, "Gain", num(gmin)) && - cam->setFeature(i, "BlackLevel", num(white ? bmax : bmin)); - if (!set) { - c.pass = false; - c.detail = cn + ": the camera refused the forced settings (ExposureAuto, " - "GainAuto, ExposureTime, Gain, BlackLevel)"; - verdict(s, cn + (white ? "_white" : "_dark"), c); - return; - } - // Frames already exposed under the old settings are still arriving; - // judge one taken well after the change. + // path out of these phases, and a failure to put it back is a failure of + // the test. + // Exposure and gain only. Both are writable while the camera streams; + // BlackLevel is not (on either Alvium here), and writing it meant + // restarting the stream mid-test - which on tower 111, slow to deliver a + // first frame and prone to stalling after a restart, left the test judging + // frames it could not vouch for. Black level stays at the profile's. + double emin = 0, emax = 0, gmin = 0, gmax = 0; + const bool ranges = cam->featureRange(i, "ExposureTime", emin, emax) && + cam->featureRange(i, "Gain", gmin, gmax); + + // Set, then judge a frame exposed well after the change: frames already + // in flight carry the old settings. Every grab is logged, so an odd + // verdict can be read back from the log rather than reconstructed. + auto forced = [&](const char* phase, double exposure, double gain, + PixelStats& out) -> std::string { + // A frame still being exposed under the OLD settings arrives after the + // change: allow for whichever exposure is longer, and count only + // frames that actually arrived before judging one. Measured on tower + // 111: with a 2+ s auto-exposure, the first frame after switching to + // the dark settings arrived 7 s later - and was nearly white. + const double before_us = exposureUs(i); + if (!ranges || + !cam->setFeature(i, "ExposureAuto", "Off") || + !cam->setFeature(i, "GainAuto", "Off") || + !cam->setFeature(i, "ExposureTime", num(exposure)) || + !cam->setFeature(i, "Gain", num(gain))) + return "the camera refused the forced settings (ExposureAuto, GainAuto, " + "ExposureTime, Gain)"; + const int t = timeout(i) + + static_cast(2 * std::max(before_us, exposure) / 1000); + const auto give_up = std::chrono::steady_clock::now() + + std::chrono::milliseconds(static_cast(t) * (discard + 2)); Frame f; - bool got = false; - for (int k = 0; k <= discard && !cancelled(); ++k) - got = cam->grabFrame(i, f, timeout(i)) || got; - if (!got) { - c.pass = false; - c.detail = cn + ": no frame arrived at the forced settings"; - verdict(s, cn + (white ? "_white" : "_dark"), c); - return; + int got = 0; + for (int k = 0; got <= discard && !cancelled() && + std::chrono::steady_clock::now() < give_up; ++k) { + Frame g; + if (cam->grabFrame(i, g, t)) { + LOG_DEBUG << "test camera: " << cn << ' ' << phase << " frame " << got + << " ts " << g.timestamp_ms << " mean " + << pixelStats(g.data, black_max, 250).mean; + f = std::move(g); + ++got; + } else { + LOG_DEBUG << "test camera: " << cn << ' ' << phase << " wait " << k + << " timed out after " << t << " ms"; + } } - verdict(s, cn + (white ? "_white" : "_dark"), - checkClipped(cn, white, pixelStats(f.data), min_pct)); + if (got == 0) return "no frame arrived at the forced settings"; + if (got <= discard) + return "only " + std::to_string(got) + " frame(s) arrived at the forced " + "settings - too few to be sure one was exposed after the change"; + out = pixelStats(f.data, black_max, 250); + return ""; }; - if (want_dark && !cancelled()) { - setProgress(sd.id, i + 1, n, "forcing black"); - force(false, sd); + PixelStats dark; + std::string dark_err = "not taken"; + if (!cancelled()) { + setProgress(sd.id, i + 1, n, "forcing dark"); + dark_err = forced("dark", emin, gmin, dark); } - if (want_white && !cancelled()) { - setProgress(sw.id, i + 1, n, "forcing white"); - force(true, sw); + if (want_dark) { + CheckResult c; + if (!dark_err.empty()) { c.pass = false; c.detail = cn + ": " + dark_err; } + else c = checkClipped(cn, false, dark, min_pct); + verdict(sd, cn + "_dark", c); + } + if (want_bright && !cancelled()) { + setProgress(sb.id, i + 1, n, "forcing bright"); + PixelStats bright; + const std::string err = !dark_err.empty() + ? "no dark frame to compare against (" + dark_err + ")" + : forced("bright", std::min(emax, bright_us), gmax, bright); + CheckResult c; + if (!err.empty()) { c.pass = false; c.detail = cn + ": " + err; } + else c = checkSensorResponds(cn, dark, bright, min_delta); + verdict(sb, cn + "_bright", c); } CheckResult back; @@ -1307,12 +1366,12 @@ void TestRunner::testCamera(TestReport& r) { "forced settings; restart the program"; LOG_ERROR << "test camera: " << back.detail; } - verdict(want_white ? sw : sd, cn + "_restored", back); + verdict(want_bright ? sb : sd, cn + "_restored", back); } cam->stop(); - for (TestSection* s : {&sf, &sd, &sw}) { - if ((s == &sf && !want_frames) || (s == &sd && !want_dark) || (s == &sw && !want_white)) + for (TestSection* s : {&sf, &sd, &sb}) { + if ((s == &sf && !want_frames) || (s == &sd && !want_dark) || (s == &sb && !want_bright)) continue; rollup(*s); r.sections.push_back(std::move(*s)); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 67c66ec..658a843 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -23,6 +23,7 @@ add_executable(fgc_tests test_singleinstance.cpp test_config.cpp test_iniedit.cpp + test_camerastall.cpp test_tmccurrent.cpp test_hardwarepresence.cpp test_hardwarechecks.cpp diff --git a/tests/test_camerastall.cpp b/tests/test_camerastall.cpp new file mode 100644 index 0000000..561f299 --- /dev/null +++ b/tests/test_camerastall.cpp @@ -0,0 +1,27 @@ +#include + +#include "fgc/CameraStall.h" + +using namespace fgc; + +TEST_CASE("the first-frame limit never drops below 30 s and grows with exposure") { + CHECK(firstFrameLimitMs(0) == 30000); + CHECK(firstFrameLimitMs(5000) == 30000); // 5 ms: a daylight frame + CHECK(firstFrameLimitMs(9e6) == 3 * 9000 + 5000); // tower 111's 9 s auto-exposure ceiling +} + +TEST_CASE("a stream that has never delivered waits, then restarts, then resets") { + // Within the limit: a slow first frame is not a stall. + CHECK(onNoFirstFrame(10000, 2.26e6, 0) == StallAction::Wait); + // Past it: restart - which production used never to do, so a stream stuck + // from its start stayed stuck (tower 111, 2026-09-24). + CHECK(onNoFirstFrame(31000, 5000, 0) == StallAction::Restart); + CHECK(onNoFirstFrame(31000, 5000, 1) == StallAction::Restart); + // Restarts are not helping: reboot the camera. + CHECK(onNoFirstFrame(31000, 5000, kRestartsBeforeReset) == StallAction::Reset); +} + +TEST_CASE("a stale stream escalates to a reset once restarts stop working") { + CHECK(onRestartFailed(1) == StallAction::Restart); + CHECK(onRestartFailed(kRestartsBeforeReset) == StallAction::Reset); +} diff --git a/tests/test_hardwarechecks.cpp b/tests/test_hardwarechecks.cpp index 647cd68..464513b 100644 --- a/tests/test_hardwarechecks.cpp +++ b/tests/test_hardwarechecks.cpp @@ -347,3 +347,27 @@ TEST_CASE("clipping: the forced end must be reached by the threshold share of pi CHECK(r.detail.find("did not reach white") != std::string::npos); CHECK_FALSE(checkClipped("rgb", false, PixelStats{}, 99.0).pass); } + +TEST_CASE("sensor responds: any measurable lift over forced dark passes, none fails") { + PixelStats dark; dark.samples = 100; dark.mean = 4.3; + PixelStats night; night.samples = 100; night.mean = 9.0; // amplified dark noise only + PixelStats day; day.samples = 100; day.mean = 254.9; // everything clipped + PixelStats dead; dead.samples = 100; dead.mean = 4.6; // did not move + CHECK(checkSensorResponds("rgb", dark, night, 2.0).pass); + CHECK(checkSensorResponds("rgb", dark, day, 2.0).pass); + auto r = checkSensorResponds("rgb", dark, dead, 2.0); + CHECK_FALSE(r.pass); + CHECK(r.detail.find("did not respond") != std::string::npos); + CHECK_FALSE(checkSensorResponds("rgb", PixelStats{}, day, 2.0).pass); +} + +TEST_CASE("the black bound is the camera's floor, not zero") { + // Tower 111's U-1240c at minimum exposure and gain: p99.9 at 17 of 255. + std::vector d(1000, 12); + for (int i = 0; i < 3; ++i) d[i] = 30; + auto st = pixelStats(d, 20, 250); + CHECK(st.pct_low == doctest::Approx(99.7)); + auto r = checkClipped("rgb", false, st, 99.0); + CHECK(r.pass); + CHECK(r.detail.find("<= 20") != std::string::npos); +} diff --git a/tests/test_testrunner.cpp b/tests/test_testrunner.cpp index 16bc908..ea9160d 100644 --- a/tests/test_testrunner.cpp +++ b/tests/test_testrunner.cpp @@ -176,17 +176,18 @@ TEST_CASE("the camera subsystem resolves, and is part of a bare 'test'") { uint32_t m = 0; std::string err; REQUIRE(resolveTestSelection("camera", "", m, err)); CHECK(m == T_CAM_ALL); - REQUIRE(resolveTestSelection("camera", "white", m, err)); - CHECK(m == T_CAM_WHITE); + REQUIRE(resolveTestSelection("camera", "bright", m, err)); + CHECK(m == T_CAM_BRIGHT); + CHECK_FALSE(resolveTestSelection("camera", "white", m, err)); // dropped: see testCamera CHECK_FALSE(resolveTestSelection("camera", "focus", m, err)); CHECK((T_ALL & T_CAM_ALL) == T_CAM_ALL); } -TEST_CASE("a healthy camera passes frames, forced black and forced white, and is put back") { +TEST_CASE("a healthy camera passes frames, forced dark and forced bright, and is put back") { MockCameraSource cam; auto rep = runCamera(cam); REQUIRE(rep); - for (const char* id : {"cam_frames", "cam_dark", "cam_white"}) { + for (const char* id : {"cam_frames", "cam_dark", "cam_bright"}) { const TestSection* s = section(*rep, id); REQUIRE(s); CHECK(s->ran); @@ -216,10 +217,10 @@ TEST_CASE("a camera that delivers nothing fails the frames leaf") { TEST_CASE("a camera that ignores its settings fails both extremes - the scene does not rescue it") { MockCameraSource cam; cam.setFault(MockCameraSource::Fault::IgnoresSettings); - auto rep = runCamera(cam, T_CAM_DARK | T_CAM_WHITE); + auto rep = runCamera(cam, T_CAM_DARK | T_CAM_BRIGHT); REQUIRE(rep); CHECK_FALSE(section(*rep, "cam_dark")->pass); - CHECK_FALSE(section(*rep, "cam_white")->pass); + CHECK_FALSE(section(*rep, "cam_bright")->pass); CHECK_FALSE(rep->all_pass); } @@ -229,9 +230,19 @@ TEST_CASE("with no camera, the camera leaves are not checked rather than passed" TestRunner tr(motor, nullptr, unitGeometry(), T_CAM_ALL, fastProfile(), "."); auto rep = runToCompletion(tr); REQUIRE(rep); - for (const char* id : {"cam_frames", "cam_dark", "cam_white"}) { + for (const char* id : {"cam_frames", "cam_dark", "cam_bright"}) { const TestSection* s = section(*rep, id); REQUIRE(s); CHECK_FALSE(s->ran); } } + +TEST_CASE("bright on its own still compares against a dark frame it takes itself") { + MockCameraSource cam; + auto rep = runCamera(cam, T_CAM_BRIGHT); + REQUIRE(rep); + const TestSection* s = section(*rep, "cam_bright"); + REQUIRE(s); + CHECK(s->pass); + CHECK(section(*rep, "cam_dark") == nullptr); // taken, but not reported +}