From cfd8ba9978137784e1a4cbcd5dfab81bf607e329 Mon Sep 17 00:00:00 2001 From: pgdalmeida Date: Mon, 5 Oct 2026 10:14:53 +0200 Subject: [PATCH] Save raw 12-bit camera frames as 12-bit colour images, keep a waypoint's picture once the image has settled, and hold the scan when saving falls behind --- CMakeLists.txt | 17 +- config/camera_profiles/rgb/README.md | 39 +- config/config.example.ini | 21 +- config/hardware.example.ini | 9 + docs/architecture.md | 9 +- docs/configuration.md | 42 +- docs/known-issues.md | 4 +- docs/raw12-pipeline-implementation-plan.md | 27 +- hardware/common/kit.sh | 6 + hardware/lattepanda-3-delta/install.sh | 2 + hardware/lattepanda-iota/install.sh | 2 + include/fgc/CaptureScheduler.h | 26 + include/fgc/Config.h | 15 + include/fgc/ICameraSource.h | 33 +- include/fgc/ImagePipeline.h | 34 +- include/fgc/JpegXlEncoder.h | 34 +- include/fgc/RawImage.h | 56 + include/fgc/SettleRule.h | 76 + include/fgc/VimbaCameraSource.h | 3 + include/fgc/mock/MockCameraSource.h | 72 +- scripts/export_camera_profile.py | 80 +- src/camera/ImagePipeline.cpp | 92 +- src/camera/JpegXlEncoder.cpp | 59 +- src/camera/VimbaCameraSource.cpp | 104 +- src/core/Application.cpp | 63 +- src/core/CaptureScheduler.cpp | 58 +- src/core/Config.cpp | 25 +- src/core/RawImage.cpp | 94 + src/core/SettleRule.cpp | 43 + tests/CMakeLists.txt | 12 + tests/data/ae_steps.csv | 3574 ++++++++++++++++++++ tests/jxl_test_decode.h | 57 + tests/test_config.cpp | 52 + tests/test_imagepipeline.cpp | 222 ++ tests/test_jpegxl.cpp | 140 + tests/test_rawimage.cpp | 133 + tests/test_scheduler.cpp | 139 + tests/test_settlerule.cpp | 138 + 38 files changed, 5473 insertions(+), 139 deletions(-) create mode 100644 include/fgc/RawImage.h create mode 100644 include/fgc/SettleRule.h create mode 100644 src/core/RawImage.cpp create mode 100644 src/core/SettleRule.cpp create mode 100644 tests/data/ae_steps.csv create mode 100644 tests/jxl_test_decode.h create mode 100644 tests/test_imagepipeline.cpp create mode 100644 tests/test_jpegxl.cpp create mode 100644 tests/test_rawimage.cpp create mode 100644 tests/test_settlerule.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index fd70ee0..5e1c8d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -93,6 +93,8 @@ add_library(fgc_core STATIC src/core/Hash.cpp src/core/ChannelHistory.cpp src/core/ChannelTopics.cpp + src/core/RawImage.cpp + src/core/SettleRule.cpp src/sensors/I2cBus.cpp src/sensors/Sht41Protocol.cpp src/sensors/McuEnvSensor.cpp @@ -109,6 +111,18 @@ target_link_libraries(fgc_core PUBLIC Threads::Threads) # PUBLIC so the application and the tests both see the same build identity. target_compile_definitions(fgc_core PUBLIC FGC_GIT_REV="${FGC_GIT_REV}") +# --------------------------------------------------------------------------- +# fgc_imaging: turning camera frames into saved images (de-Bayer, JPEG XL). Its +# own library so the image tests can link OpenCV and libjxl without the rest of +# the application; fgc_core stays free of both. +# --------------------------------------------------------------------------- +add_library(fgc_imaging STATIC + src/camera/JpegXlEncoder.cpp + src/camera/ImagePipeline.cpp +) +target_include_directories(fgc_imaging PUBLIC ${OpenCV_INCLUDE_DIRS}) +target_link_libraries(fgc_imaging PUBLIC fgc_core ${OpenCV_LIBS} PkgConfig::JXL) + # --------------------------------------------------------------------------- # Application executable. Camera image pipeline + serial controller always # build (OpenCV/libjxl/Boost.Asio are available everywhere); MQTT and Vimba @@ -117,8 +131,6 @@ target_compile_definitions(fgc_core PUBLIC FGC_GIT_REV="${FGC_GIT_REV}") set(FGC_SOURCES main.cpp src/core/Application.cpp - src/camera/JpegXlEncoder.cpp - src/camera/ImagePipeline.cpp src/serial/SerialMotorController.cpp src/serial/MtiImuSource.cpp src/sensors/Sht41EnvSensor.cpp @@ -135,6 +147,7 @@ target_include_directories(fire_gimbal_control PRIVATE ${CMAKE_SOURCE_DIR}) target_link_libraries(fire_gimbal_control PRIVATE fgc_core + fgc_imaging ${OpenCV_LIBS} Boost::headers Boost::program_options diff --git a/config/camera_profiles/rgb/README.md b/config/camera_profiles/rgb/README.md index 42a48cb..85ff890 100644 --- a/config/camera_profiles/rgb/README.md +++ b/config/camera_profiles/rgb/README.md @@ -40,28 +40,33 @@ overwritten whenever the XML changes. ## Profiles this rig expects -`config.ini` ships referencing `standard` and `night`. They were previously -hand-written; these are the values they used, as a starting point when -recreating them in the Viewer. Everything not listed was identical. +`config.ini` ships referencing `standard` and `night`. Both are **raw capture** +(docs/raw12-pipeline-measurements.md): full resolution, binning 1, packed 12-bit +Bayer, converted to colour on the tower and saved as 12-bit JPEG XL. +`scripts/export_camera_profile.py` builds them on the tower; these are its recipe. | Setting | `standard` | `night` | |---|---|---| -| `AcquisitionFrameRate` | 10 | 1 | -| `SensorBitDepth` | Bpp8 | Bpp12 | -| `ExposureAutoMin` | 200 | 1000 | -| `ExposureAutoMax` | 20000 | 500000 | +| `BinningHorizontal` / `Vertical` | 1 | 1 | +| `Width` / `Height` | sensor maximum | sensor maximum | +| `PixelFormat` | `BayerRG12p` | `BayerRG12p` | +| `DeviceLinkThroughputLimit` | 250000000 | 250000000 | +| `AcquisitionFrameRate` | camera maximum (9.86 on the 1800 U-1620c) | camera maximum (exposure-limited at night) | +| `ChunkModeActive` + `ChunkEnable` | ExposureTime, Gain, BalanceRatioRed/Blue, Timestamp | same | +| `BalanceWhiteAuto` | Off (fixed ratios) | Off | +| `ExposureAutoMin` / `Max` | 200 / 20000 µs | 1000 / 500000 µs | | `GainAutoMax` | 24 | 48 | -| `IntensityAutoPrecedence` | Minimizeexposuretime | Minimizenoise | -| `DeviceIndicatorMode` | Active | Off | -| `DeviceIndicatorLuminance` | 10 | 0 | +| `IntensityAutoPrecedence` | MinimizeNoise | MinimizeNoise | +| `IntensityControllerRate` | 100 | 100 | +| `DeviceIndicatorMode` / `Luminance` | Active / 10 | Inactive / 0 | -Both used 2×2 binning at `RGB8` (2256×2256, 15.3 MB/frame) and a 250 MB/s -`DeviceLinkThroughputLimit`. **That binning is not optional at 10 Hz**: a -full-resolution RGB8 frame is 61 MB, which caps out near 4 fps under that limit. +The chunk data is what `settle_rule = brightness` reads (each item has to be +enabled before `ChunkModeActive`, which then locks them). A profile left on +`RGB8` still works: the program saves it as before. -The full previous definitions are in git history at -the hand-written `config/camera_modes/rgb/{standard,night}.ini`, removed in the -same change that made the XML the only thing applied. +Earlier profiles used 2×2 binning at RGB8. On the 1620c that gave **no** extra +frame rate (the camera budgets the link against the unbinned frame), which is one +reason they were replaced. Their values are in git history. -`settle_wait_ms` / `frame_timeout_ms` are *not* camera settings and are not in +`settle_wait_ms` / `frame_timeout_ms` / `settle_rule` are *not* camera settings and are not in the XML — they live in `config.ini` under `[Camera.rgb.]`. diff --git a/config/config.example.ini b/config/config.example.ini index 55887bc..8c2855b 100644 --- a/config/config.example.ini +++ b/config/config.example.ini @@ -50,6 +50,10 @@ mqtt_pw = CHANGE_ME ; JPEG XL encoding. distance: 0 = lossless, ~0.8 = near-lossless; effort 1..9. jxl_distance = 0.8 jxl_effort = 4 +; Raw profiles (PixelFormat BayerRG8 / BayerRG12p in the profile XML): the tower +; converts the sensor mosaic to colour itself. edge_aware (sharper; ~30 ms for +; 16 MP) or bilinear. A 12-bit raw frame is saved as a 12-bit file. +demosaic = edge_aware ; --- One [Camera.] block per camera ----------------------------------- ; Nothing is implied by position: the block names the camera, `label` names its @@ -71,13 +75,20 @@ profile = standard ; /standard.xml rotation_deg = 0 ; Per-profile timing. Not camera settings - how long the program waits around -; them. After the gimbal reports standstill it holds still this long before -; keeping a frame, so the image is exposed at rest with the auto-exposure -; settled; frame_timeout_ms then abandons the waypoint rather than parking on it. +; them. After the gimbal reports standstill it holds still settle_wait_ms, then +; keeps a frame according to settle_rule: +; fixed the newest frame, straight away +; brightness the first frame once brightness is in the camera's target band +; and steady for 3 frames (right 21 of 24 times in the tower tests, +; docs/raw12-pipeline-measurements.md); settle_wait_ms is then just +; the mechanical minimum +; frame_timeout_ms bounds the wait after settle_wait_ms; past it the newest frame +; is kept anyway (or the waypoint skipped if the camera delivers nothing). ; With several cameras the scan uses the LONGEST wait across them. [Camera.rgb.standard] -settle_wait_ms = 1000 -frame_timeout_ms = 2000 +settle_wait_ms = 500 +frame_timeout_ms = 7000 +settle_rule = brightness [Camera.rgb.night] settle_wait_ms = 2500 diff --git a/config/hardware.example.ini b/config/hardware.example.ini index b7984bb..2dd931e 100644 --- a/config/hardware.example.ini +++ b/config/hardware.example.ini @@ -58,6 +58,15 @@ cameras = rgb ; How much bearing one picture covers, in degrees. Pan mode ('mode pan') cuts ; its span into as many pictures as this takes. 6 until the lens is measured. pan_step_deg = 6 +; Image saving: how this computer shares its 4 cores between saved frames. +; Measured with a 16 MP 12-bit frame (docs/raw12-pipeline-measurements.md): the +; IOTA (N150) keeps up with one waypoint every 0.7 s with 1 worker; the 3 Delta +; (N5105) needs 2 workers for that. jxl_threads is libjxl's threads per frame; +; 0 or absent = 4 / save_workers. save_queue_max is how many frames may wait +; or be saving before the scan holds at its waypoint (each raw frame ~24 MB). +save_workers = 1 +;jxl_threads = 0 +save_queue_max = 4 ; ; What happens when this does not match the rig: ; declared but absent -> warned about by name, and with [Features] diff --git a/docs/architecture.md b/docs/architecture.md index 55e2b0a..5340219 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -132,7 +132,14 @@ At each waypoint the scheduler therefore: 1. moves, and waits for both axes to report standstill at the target; 2. waits a further `settle_wait_ms` (1 s in `standard`, 2.5 s in `night`); -3. keeps the newest complete frame. +3. keeps the newest complete frame - or, with `settle_rule = brightness`, the first + frame once brightness is in the camera's target band and steady for 3 frames + ([configuration.md](configuration.md#which-frame-a-waypoint-keeps-settle_rule)). + +Saving runs in background workers and never blocks this loop; if more than +`save_queue_max` frames are waiting, the scan holds at the waypoint until there is +room. Raw profiles (`BayerRG8`, `BayerRG12p`) are converted to colour on the tower +and saved at their own bit depth ([raw12-pipeline-measurements.md](raw12-pipeline-measurements.md)). The wait is the whole mechanism. It guarantees the kept frame was exposed with the gimbal **at rest** (no slew blur) and after the auto-exposure has seen this waypoint's light for several frames. It also sets the diff --git a/docs/configuration.md b/docs/configuration.md index 756b7f5..c5fb821 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -44,9 +44,14 @@ Parsed and validated by `ConfigLoader` ([src/core/Config.cpp](../src/core/Config | `Camera.` | `enabled` | bool | `true` | A disabled camera is not opened and does not affect the scan timing | | `Camera.` | `rotation_deg` | `0`/`90`/`180`/`270` | `0` | Clockwise rotation applied to that camera's saved images | | `Camera..` | `settle_wait_ms` | int ≥ 0 | `1000` | Hold after standstill before keeping a frame | -| `Camera..` | `frame_timeout_ms` | int ≥ 0 | `2000` | Abandon the waypoint after this | +| `Camera..` | `frame_timeout_ms` | int ≥ 0 | `2000` | Longest wait after `settle_wait_ms`; then keep the newest frame, or skip the waypoint if none | +| `Camera..` | `settle_rule` | `fixed` \| `brightness` | `fixed` | `fixed`: keep the newest frame once `settle_wait_ms` has passed. `brightness`: keep the first frame whose brightness is in the camera's target band and steady for 3 frames ([keep rule](#which-frame-a-waypoint-keeps-settle_rule)) | | `Camera` | `jxl_distance` / `jxl_effort` | double ≥ 0 / int 1..9 | `0.8` / `4` | Encoder defaults inherited by every camera block | -| `Camera.` | `jxl_distance` / `jxl_effort` | double ≥ 0 / int 1..9 | the `[Camera]` value | Per-camera override of the encoder settings | +| `Camera.` | `jxl_distance` / `jxl_effort` | double ≥ 0 / int 1..9 | the `[Camera]` value | Per-camera override of the encoder settings. Distance 0 is truly lossless | +| `Camera` / `Camera.` | `demosaic` | `edge_aware` \| `bilinear` | `edge_aware` | How raw profiles (`BayerRG8`, `BayerRG12p`) are converted to colour on the tower | +| `Hardware` (hardware.ini) | `save_workers` | int 1..4 | `1` | Frames saved at once. 1 on the IOTA, 2 on the 3 Delta ([measurements](raw12-pipeline-measurements.md)) | +| `Hardware` (hardware.ini) | `jxl_threads` | int 0..8 | `0` = 4 / `save_workers` | libjxl threads per saved frame | +| `Hardware` (hardware.ini) | `save_queue_max` | int 1..32 | `4` | Frames waiting or being saved before the scan holds at its waypoint (a raw 16 MP frame is ~24 MB) | | `Paths` | `output_dir` | string | `$XDG_DATA_HOME/fire_gimbal_control/images` | Image output dir; supports `~`/`$ENV` | | `Features` | `enable_mqtt` | bool | `true` | Use MQTT (vs null channel) | | `Features` | `enable_camera` | bool | `true` | (reserved) | @@ -432,6 +437,39 @@ Every `*.xml` in `profile_dir` is loaded at start-up, so a profile nobody is using today still fails at launch rather than the first time someone switches to it. +#### Raw profiles + +A profile whose XML sets `PixelFormat = BayerRG12p` (or `BayerRG8`) sends the +camera's raw sensor mosaic. The tower converts it to colour (`[Camera] demosaic`) +and saves it at the frame's own bit depth: a 12-bit raw frame becomes a 12-bit +JPEG XL file. `RGB8` profiles are saved exactly as before. At the 250 MB/s link +cap the 1800 U-1620c delivers 9.86 fps in `BayerRG12p` against 4.95 in RGB8, so +auto-exposure settles in about half the time; the tower-side cost is the same as +saving RGB8 ([measurements](raw12-pipeline-measurements.md)). +`scripts/export_camera_profile.py` builds such profiles; it also turns on the +per-frame data (chunk mode) the keep rule reads. + +#### Which frame a waypoint keeps: `settle_rule` + +With `settle_rule = brightness`, after `settle_wait_ms` (now the mechanical +minimum) the scan watches every frame the camera delivers and keeps the first one +once the last three are all inside the camera's own auto-exposure target band +(`IntensityControllerTarget ± IntensityControllerTolerance`, read from the camera) +and their brightness changed by at most 2% from frame to frame. A steady frame +outside the band also counts when exposure and gain sit at the limits that would +move it (waiting cannot help). Past `frame_timeout_ms` the newest frame is kept +anyway, with a warning. On the 24 recorded exposure steps this picked a good frame +21 times; "exposure and gain unchanged" never did, because auto-exposure pauses +mid-way. A camera that reports no frame data falls back to the fixed wait. + +#### When saving falls behind + +Saving runs in background workers (`[Hardware] save_workers`, `jxl_threads`). If +more than `save_queue_max` frames are waiting or being saved, the scan **holds at +its waypoint** until there is room (logged once), so no picture is lost and memory +stays bounded. A requested snapshot is never held. Measured: the IOTA keeps up +with a frame every 0.7 s on one worker; the 3 Delta needs two. + ### `[Env]` — SHT41 ambient temperature/humidity sensor Enable with `[Features] enable_env = true` and point `[Env] i2c_device` at the LattePanda's **own diff --git a/docs/known-issues.md b/docs/known-issues.md index e3feaa0..4368964 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -30,9 +30,9 @@ doctest unit-test suite (`ctest`). | 13 | `[Motor]` degrees↔counts calibration | The `config.example.ini` values are **placeholders**. Home, then run `gimbal heading setup` (yaw, needs no IMU) and `gimbal pitch setup` (pitch, needs the IMU) on the rig. | | 14 | Capture sweep on hardware | **Mechanically verified on the rig (2026-08-07).** A full `--init --start` run homed both axes and completed `MOVE → settle → trigger` for 15/15 waypoints with 0 dropped frames, 0 stalls and 0 acquisition failures. `kSettleTolCounts` (600) and the per-interval timing in [CaptureScheduler.cpp](../src/core/CaptureScheduler.cpp) have **not** been tuned against observed `ST` behaviour yet, and the settle tolerance has not been checked against a sharp image (see #16). | | 15 | **Camera profiles unverified on a lit scene** | The `standard`/`night` profiles must be exported from the VimbaX Viewer (see `config/camera_profiles/rgb/README.md`) and have not been measured against real light. Confirm on the rig that `LoadSettings` succeeds, that `standard` sustains 10 fps with `frames_dropped` at 0, and that the auto-exposure lands on a sensible `ExposureTime` outdoors. The only previous rig test ran with the lens covered. **2026-09-24:** tower 121 has still never imaged a scene - a frame grabbed directly and the program's own saved images (2026-09-21) are a smooth grey gradient with no edges, i.e. the lens is covered or grossly out of focus. Tower 111's images from the same afternoon show nothing sharp anywhere either (soft blobs, no edges) - defocus, a cover or a fogged window; unexplained. `test camera` (below) proves the camera and its data path work in both cases; it cannot, by design, judge the picture. **Stream stuck, 2026-09-24:** from 16:10 to 16:19 tower 111's camera delivered no frames at all - not to the program, not to a fresh Vimba session, not at a fixed 10 ms exposure - and it survived program restarts and a USB re-authorisation. The camera's own `DeviceReset` command cleared it. Production captured nothing for those 9 minutes and logged only `no frame yet (stream warming up)`: a session that has NEVER delivered is not restarted by `trigger()`, by design, so a stream stuck from its start stays stuck. It began just after a `test camera` run by an early version that stopped and restarted the stream several times in a few seconds (to write `BlackLevel`, which the camera refuses while streaming); whether that caused it or met 111's known USB stall is not established. `test camera` itself reported it correctly. **Answered the same day:** a stream that never delivers is now restarted once the wait exceeds what its exposure can explain (at least 30 s), and after two fruitless restarts the camera is reset (DeviceReset) and reopened on its profile - both by the program itself (`CameraStall.h`), and on demand with the console command `camera reset` (verified on tower 121: back in ~14 s, frames straight after). `test camera` no longer stops the stream at all: it changes only exposure and gain, which are writable live, and puts them back live. With that, 111 ran it inside the capturing program, passed, and kept capturing. **2026-09-30:** tower 121's camera is indoors, facing a window, still badly out of focus, gain at its 24 dB maximum ([camera-capture-measurements.md](camera-capture-measurements.md)). | -| 16 | **`settle_wait_ms` untuned** | `standard` waits 1 s and `night` 2.5 s after standstill before keeping a frame. Both are reasoned (≫ one frame period; long enough for the auto-exposure to see the new scene) but not measured. Check on a lit scene that the kept frame shows no slew blur and that its exposure has actually converged for that heading, and shorten the wait if it has. **Seen in real data 2026-09-24:** of eight consecutive images tower 111 kept at 8 deg/s, one was 98.6 % clipped white, one 57 % black and two about half clipped - frames kept before the continuous auto-exposure had caught up with the new view. **Measured 2026-09-30 (tower 121, still camera, forced ×16 brightness steps):** auto-exposure took 1.4–1.8 s to come within ±10% at 4.95 fps, so 1 s is too short for a large step; from a dark step it first overshoots to its maximum exposure, 3.4–4.1× too bright. Exposure and gain holding still for a few frames does not mean it has finished ([camera-capture-measurements.md](camera-capture-measurements.md)). | +| 16 | **`settle_wait_ms` untuned** | `standard` waits 1 s and `night` 2.5 s after standstill before keeping a frame. Both are reasoned (≫ one frame period; long enough for the auto-exposure to see the new scene) but not measured. Check on a lit scene that the kept frame shows no slew blur and that its exposure has actually converged for that heading, and shorten the wait if it has. **Seen in real data 2026-09-24:** of eight consecutive images tower 111 kept at 8 deg/s, one was 98.6 % clipped white, one 57 % black and two about half clipped - frames kept before the continuous auto-exposure had caught up with the new view. **Measured 2026-09-30 (tower 121, still camera, forced ×16 brightness steps):** auto-exposure took 1.4–1.8 s to come within ±10% at 4.95 fps, so 1 s is too short for a large step; from a dark step it first overshoots to its maximum exposure, 3.4–4.1× too bright. Exposure and gain holding still for a few frames does not mean it has finished ([camera-capture-measurements.md](camera-capture-measurements.md)). **2026-10-05:** `settle_rule = brightness` keeps the frame once brightness is in the camera's band and steady (21 of 24 recorded steps right), and raw profiles double the frame rate it works with; to be confirmed with real slews ([raw12-pipeline-measurements.md](raw12-pipeline-measurements.md)). | | 19 | **USB preconditions do not survive a reboot** | `usbcore.usbfs_memory_mb` reverts to 16 (needs 1000) and USB3 LPM re-enables on the camera's port. **Both were wrong on the LattePanda when tested on 2026-08-07**, and frame delivery stalls without them regardless of application code. **Solved on tower 121 by the setup kit** (`hardware/lattepanda-iota/preconditions/`): a systemd-tmpfiles drop-in for usbfs and two udev rules, all installed by `install.sh --preconditions`. Verified 2026-09-23 — `usbfs_memory_mb` reads 1000 after **6 days of uptime**, so it survived the boot, and both rules are in `/etc/udev/rules.d/`; and again across a deliberate reboot the same day, when `install-tower --verify` read `usbfs_memory_mb` 1000 and the camera port's `usb3_lpm_permit` 0 (a check `test hardware camera` now makes too, once `[Test] usb3_lpm_path` is set - the installer sets it). Note it is **tmpfiles, not the kernel cmdline** the original entry proposed. **On the Delta fleet**: tower 111 read 16 and `u1_u2` on 2026-09-23, and the Delta kit (`hardware/lattepanda-3-delta/`, the same two files) was applied the same day - 1000 and `0` afterwards, confirmed by `test hardware camera`. Confirmed across a reboot on 2026-09-24 (`install-tower --verify`). **Solved on both boards**; open only for Delta towers the kit has not reached. **2026-09-30:** with the kit applied, tower 121 streamed full resolution in every format at 250 and 450 MB/s for 60 s with nothing lost; 2 incomplete frames in 1,574 at 450 MB/s while encoding. **2026-10-01:** tower 111 (Delta kit applied) likewise, at 200, 250 and 450 MB/s, also while encoding, with nothing lost ([camera-capture-measurements.md](camera-capture-measurements.md)). | -| 20 | **`jxl_distance = 0` is not lossless** | `JpegXlEncoder::encodeToFile` always sets `uses_original_profile = false`; for distance 0 libjxl prints `Set uses_original_profile=true for lossless encoding`, still reports success, and encodes through its lossy colour space. Measured 2026-09-30: decoded pixels differ by up to 1 (raw mosaic) or 2 (RGB8) levels in 1.5% / 7.3% of values ([camera-capture-measurements.md](camera-capture-measurements.md)). Fix: set `uses_original_profile = true` when distance is 0. | +| 20 | **`jxl_distance = 0` is not lossless** | `JpegXlEncoder::encodeToFile` always sets `uses_original_profile = false`; for distance 0 libjxl prints `Set uses_original_profile=true for lossless encoding`, still reports success, and encodes through its lossy colour space. Measured 2026-09-30: decoded pixels differ by up to 1 (raw mosaic) or 2 (RGB8) levels in 1.5% / 7.3% of values ([camera-capture-measurements.md](camera-capture-measurements.md)). **Fixed 2026-10-05:** the encoder sets `uses_original_profile` for distance 0, and `tests/test_jpegxl.cpp` checks 8-bit and 12-bit lossless decode exactly; 8-bit lossy output is byte-identical to before (72 of 72 compared). | ## Camera acquisition (USB3) — root cause + approach diff --git a/docs/raw12-pipeline-implementation-plan.md b/docs/raw12-pipeline-implementation-plan.md index cb9d40a..4da8e8d 100644 --- a/docs/raw12-pipeline-implementation-plan.md +++ b/docs/raw12-pipeline-implementation-plan.md @@ -1,7 +1,32 @@ # Implementation plan — raw capture, colour on the tower, 12-bit JPEG XL Based on [raw12-pipeline-measurements.md](raw12-pipeline-measurements.md) (towers 121 and 111, -2026-10-02/04). Not started; decisions marked **[decide]** are the user's. +2026-10-02/04). + +## Status (branch `raw12-capture`, 2026-10-05) + +Decisions taken: `BayerRG12p` for both profiles; colour JPEG XL only (no raw-planes save); white +balance stays fixed; a full save queue **holds the scan** (no lost waypoints). + +| Phase | State | +|---|---| +| 1 Encoder | **done**: 12-bit input, true lossless (#20), threads from config; 8-bit lossy byte-identical to before (72 of 72) | +| 2 Frames | **done**: pixel layout from the camera, chunk data (exposure, gain, white balance), brightness sample per frame. Not done: keeping frames in the SDK buffer instead of copying (optional) | +| 3 Raw core | **done**: `RawImage` (layouts, `unpack12p`, brightness sample) | +| 4 Pipeline | **done**: de-Bayer (edge-aware/bilinear), save at the frame's bit depth, N workers, `canAccept()`; scan holds when full | +| 5 Keep rule | **done** for scan waypoints (`SettleRule`, `settle_rule = brightness`); snapshots and continuous capture keep the fixed wait | +| 6 Config | **done**: `[Camera] demosaic`; `[Hardware] save_workers`, `jxl_threads`, `save_queue_max` (kit writes 1 / 2 per board); per-profile `settle_rule` | +| 7 Profiles | **recipe done** (`scripts/export_camera_profile.py`, dry-run checked on the 1620c); profiles still to be exported on each tower | +| 8 Docs | **done** (configuration, architecture, known issues, profile README). Downstream 12-bit decode still to check | + +Tests: 457, of which 36 new, covering the encoder, the pipeline, raw handling, the keep rule (including +a replay of the 24 recorded steps), the scheduler and config. Builds with the +camera SDK on tower 121. + +Still to do before switching a tower: export the new profiles on the tower; check the ground +station and detection read 12-bit JPEG XL; a daylight run in auto mode on each tower +(verification below); tune `settle_wait_ms` (500 ms is a guess) and `frame_timeout_ms` (7 s from +the worst measured step). ## Goal diff --git a/hardware/common/kit.sh b/hardware/common/kit.sh index 6604673..8974487 100644 --- a/hardware/common/kit.sh +++ b/hardware/common/kit.sh @@ -5,6 +5,7 @@ # # The facts a kit must set before sourcing: # KIT_BOARD lattepanda-iota | lattepanda-3-delta (matches probe_board) +# KIT_SAVE_WORKERS image-saving workers for this board (1 IOTA, 2 3 Delta) # KIT_FW_ENV the PlatformIO env for this board's MCU # KIT_MCU_VID USB vendor of the motor MCU (2e8a RP2040, 2341 Arduino) # KIT_DEFAULT_AXES what this board's towers are usually built with @@ -392,6 +393,11 @@ imu = $IMU env = $ENV_SENSOR cooling = $COOLING cameras = $CAMERAS +; Image saving on this computer: frames encoded at once (libjxl threads per +; frame default to 4 / save_workers), and frames that may wait before the +; scan holds. Measured for this board in docs/raw12-pipeline-measurements.md. +save_workers = ${KIT_SAVE_WORKERS:-1} +save_queue_max = 4 [Serial] ; The motor controller MCU, by stable id - /dev/ttyACM* moves when the modem diff --git a/hardware/lattepanda-3-delta/install.sh b/hardware/lattepanda-3-delta/install.sh index 5d43f80..63e2340 100755 --- a/hardware/lattepanda-3-delta/install.sh +++ b/hardware/lattepanda-3-delta/install.sh @@ -16,6 +16,8 @@ KIT_BOARD=lattepanda-3-delta KIT_FW_ENV=leonardo KIT_MCU_VID=2341 KIT_DEFAULT_AXES=yaw,pitch +# Image saving: the N5105 needs two workers x 2 threads to keep up (docs/raw12-pipeline-measurements.md). +KIT_SAVE_WORKERS=2 KIT_IMU_DEVICE=/dev/ttyS0 KIT_IMU_NOTE="On the Delta the MTi is on the RS-232 header: a hardware UART, not a USB node." KIT_ENV_DEVICE=/dev/i2c-2 diff --git a/hardware/lattepanda-iota/install.sh b/hardware/lattepanda-iota/install.sh index 98bfb58..ee41f60 100755 --- a/hardware/lattepanda-iota/install.sh +++ b/hardware/lattepanda-iota/install.sh @@ -18,6 +18,8 @@ KIT_BOARD=lattepanda-iota KIT_FW_ENV=iota_rp2040 KIT_MCU_VID=2e8a KIT_DEFAULT_AXES=yaw +# Image saving: the N150 keeps up with one worker x 4 threads (docs/raw12-pipeline-measurements.md). +KIT_SAVE_WORKERS=1 KIT_IMU_DEVICE=/dev/ttyS0 KIT_IMU_NOTE="On the IOTA the only PCH serial on the header is COM1; confirm it enumerates first." # No PCH-side I2C reaches the header, so an ambient sensor needs a USB-I2C bridge diff --git a/include/fgc/CaptureScheduler.h b/include/fgc/CaptureScheduler.h index 69400c5..db031e9 100644 --- a/include/fgc/CaptureScheduler.h +++ b/include/fgc/CaptureScheduler.h @@ -5,6 +5,7 @@ #include "fgc/IMotorController.h" #include "fgc/OperatingMode.h" #include "fgc/ScanGrid.h" +#include "fgc/SettleRule.h" #include #include @@ -67,12 +68,29 @@ public: double imageRate() const { return image_rate_; } // images per second + // Is there room to save another frame (ImagePipeline::canAccept)? Asked + // before moving to the next waypoint and before keeping a continuous-capture + // frame. While it says no, the scan holds where it is, so a scan that + // outruns saving slows down instead of losing pictures or filling memory. + // A requested snapshot is never held. Unset: always room. + void setSaveGate(std::function can_save) { can_save_ = std::move(can_save); } + // True while the scan is held for saving to catch up. + bool holdingForSave() const { return holding_; } + // Per-camera-profile timing (see CameraProfile): how long to wait after standstill // before keeping a frame, and how long to keep retrying before abandoning the // waypoint. Called on the control thread, same as tick(). void setSettleWait(int settle_ms, int timeout_ms); int settleWaitMs() const { return settle_wait_ms_; } + // The keep rule for scan waypoints (profile setting settle_rule). Off: keep + // the newest frame once settle_wait_ms has passed, as always. On: after + // settle_wait_ms (now the mechanical minimum), wait until the camera's + // frames say the image has settled (SettleRule), within frame_timeout_ms; a + // camera that reports no frame data falls back to the fixed wait. + void setSettleRule(bool on, const SettleRule::Params& params = {}); + bool settleRuleOn() const { return use_rule_; } + // Replace the degrees<->counts maps (e.g. after `gimbal heading fit`) so subsequent // scan waypoints target with the calibrated conversion. Called on the control // thread, same as tick(), so no locking is needed. @@ -99,6 +117,8 @@ private: // budget runs out. `settled_ms` is that request's own settle clock. enum class Shot { Waiting, Captured, TimedOut }; Shot shootAtRest(const MotorTelemetry& t, long long& settled_ms); + // The save gate, with one log line when a hold starts and one when it ends. + bool roomToSave(); IMotorController& motor_; ICameraSource& camera_; @@ -106,6 +126,8 @@ private: ScanGrid& grid_; std::function now_ms_; + std::function can_save_; + bool holding_ = false; double image_rate_; long long timer_start_ = 0; @@ -115,6 +137,10 @@ private: std::vector pan_headings_; size_t pan_index_ = 0; + bool use_rule_ = false; // keep by SettleRule, not after a fixed wait + SettleRule rule_; + bool rule_started_ = false; // this waypoint's rule has its first frame number + uint64_t rule_seq_ = 0; // last frame fed to the rule bool moving_ = false; // a MOVE has been issued, awaiting settle long long settled_at_ms_ = 0; // when standstill was first seen (0 = not yet) int settle_wait_ms_ = 1000; diff --git a/include/fgc/Config.h b/include/fgc/Config.h index 2aed1ae..308f0e4 100644 --- a/include/fgc/Config.h +++ b/include/fgc/Config.h @@ -49,6 +49,13 @@ struct HardwareConfig { // a span is cut into as many pictures as it takes (OperatingMode.h). double pan_step_deg = 6.0; + // Saving images - a property of the computer, not of what the tower does. + // Measured (docs/raw12-pipeline-measurements.md): the IOTA keeps up with one + // worker x 4 threads, the 3 Delta needs two x 2. + int save_workers = 1; // frames encoded at once + int jxl_threads = 0; // libjxl threads per frame; 0 = 4 / save_workers + int save_queue_max = 4; // frames waiting or saving before the scan holds + // Was a hardware.ini actually read? False on every tower that has not been // migrated yet, and the switch that keeps this change invisible on rollout. bool declared = false; @@ -179,6 +186,11 @@ struct SerialConfig { struct CameraProfileTiming { int settle_wait_ms = 1000; // hold after standstill before keeping a frame int frame_timeout_ms = 2000; // give up on the waypoint after this + // How to tell the frame is good to keep, after settle_wait_ms: + // fixed keep the newest frame as soon as settle_wait_ms has passed + // brightness wait until brightness is in the camera's target band and + // steady for 3 frames (SettleRule.h), up to frame_timeout_ms + std::string settle_rule = "fixed"; }; // One camera, from a [Camera.] block. Nothing is implied by position: the @@ -196,6 +208,8 @@ struct CameraSpec { // Encoding, defaulted from [Camera] and overridable per camera. double jxl_distance = 0.8; int jxl_effort = 4; + // De-Bayering for raw profiles (BayerRG8/12p): "edge_aware" or "bilinear". + std::string demosaic = "edge_aware"; std::map timing; // by profile name @@ -208,6 +222,7 @@ struct CameraConfig { // [Camera] defaults, inherited by each block unless it overrides them. double jxl_distance = 0.8; int jxl_effort = 4; + std::string demosaic = "edge_aware"; // Enabled cameras only. std::vector active() const; diff --git a/include/fgc/ICameraSource.h b/include/fgc/ICameraSource.h index 8b7fbf2..04cc4f5 100644 --- a/include/fgc/ICameraSource.h +++ b/include/fgc/ICameraSource.h @@ -1,6 +1,8 @@ #pragma once #include "fgc/CameraProfile.h" +#include "fgc/RawImage.h" +#include "fgc/SettleRule.h" #include #include @@ -11,12 +13,25 @@ namespace fgc { // One captured frame, owning a copy of its pixel buffer (was image_store_8bit). struct Frame { - std::vector data; + std::vector data; // as delivered: BayerRG12p stays packed uint32_t width = 0; uint32_t height = 0; - int channels = 0; // 1 (mono) or 3 (RGB) + int channels = 0; // values per pixel as delivered: 3 (RGB) or 1 (mono, Bayer mosaic) + // How `data` is laid out. Unknown only for a format the program does not + // handle; `channels` then says what it holds, as before layouts existed. + PixelLayout layout = PixelLayout::Unknown; long long timestamp_ms = 0; // Unix epoch ms int cam_id = 0; // index into the configured camera list + + // What the camera says it exposed this frame with (its per-frame chunk + // data). Negative when the profile does not turn chunk data on. + double exposure_us = -1.0; + double gain_db = -1.0; + double wb_red = -1.0; // BalanceRatio, red against green + double wb_blue = -1.0; + // Sampled from `data` on arrival (see RawImage.h); invalid when the layout + // is Unknown. + BrightnessSample brightness; }; // Live per-camera identity, sensor telemetry, and acquisition stats, for the @@ -77,6 +92,20 @@ public: virtual void setFrameCallback(FrameCallback cb) = 0; + // ---- For the keep rule (SettleRule.h) --------------------------------------- + // The most recent frames camera 0 delivered, with their brightness, exposure + // and gain. Sequence numbers count up for the session; framesSince(n) returns + // the frames after n, oldest first (only the last few dozen are kept). + // frameSeq() is the newest sequence number. Default: no frames, and the scan + // keeps its fixed settle wait. + virtual uint64_t frameSeq() { return 0; } + virtual std::vector framesSince(uint64_t /*after*/) { return {}; } + // A numeric camera feature (IntensityControllerTarget, ExposureAutoMax...). + // False if the camera has no such feature. + virtual bool readFeature(int /*camera_index*/, const std::string& /*name*/, double& /*value*/) { + return false; + } + // Number of cameras this source manages. virtual int cameraCount() const = 0; diff --git a/include/fgc/ImagePipeline.h b/include/fgc/ImagePipeline.h index e8c8c1b..c9d00fa 100644 --- a/include/fgc/ImagePipeline.h +++ b/include/fgc/ImagePipeline.h @@ -23,9 +23,11 @@ struct Orientation { // Consumes captured frames and turns them into stored artifacts: rotate by // Camera.rotation_deg, encode to JPEG XL, write to -// /