diff --git a/CMakeLists.txt b/CMakeLists.txt index fd70ee0..5e1c8d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -93,6 +93,8 @@ add_library(fgc_core STATIC src/core/Hash.cpp src/core/ChannelHistory.cpp src/core/ChannelTopics.cpp + src/core/RawImage.cpp + src/core/SettleRule.cpp src/sensors/I2cBus.cpp src/sensors/Sht41Protocol.cpp src/sensors/McuEnvSensor.cpp @@ -109,6 +111,18 @@ target_link_libraries(fgc_core PUBLIC Threads::Threads) # PUBLIC so the application and the tests both see the same build identity. target_compile_definitions(fgc_core PUBLIC FGC_GIT_REV="${FGC_GIT_REV}") +# --------------------------------------------------------------------------- +# fgc_imaging: turning camera frames into saved images (de-Bayer, JPEG XL). Its +# own library so the image tests can link OpenCV and libjxl without the rest of +# the application; fgc_core stays free of both. +# --------------------------------------------------------------------------- +add_library(fgc_imaging STATIC + src/camera/JpegXlEncoder.cpp + src/camera/ImagePipeline.cpp +) +target_include_directories(fgc_imaging PUBLIC ${OpenCV_INCLUDE_DIRS}) +target_link_libraries(fgc_imaging PUBLIC fgc_core ${OpenCV_LIBS} PkgConfig::JXL) + # --------------------------------------------------------------------------- # Application executable. Camera image pipeline + serial controller always # build (OpenCV/libjxl/Boost.Asio are available everywhere); MQTT and Vimba @@ -117,8 +131,6 @@ target_compile_definitions(fgc_core PUBLIC FGC_GIT_REV="${FGC_GIT_REV}") set(FGC_SOURCES main.cpp src/core/Application.cpp - src/camera/JpegXlEncoder.cpp - src/camera/ImagePipeline.cpp src/serial/SerialMotorController.cpp src/serial/MtiImuSource.cpp src/sensors/Sht41EnvSensor.cpp @@ -135,6 +147,7 @@ target_include_directories(fire_gimbal_control PRIVATE ${CMAKE_SOURCE_DIR}) target_link_libraries(fire_gimbal_control PRIVATE fgc_core + fgc_imaging ${OpenCV_LIBS} Boost::headers Boost::program_options diff --git a/README.md b/README.md index 830fb05..49a4c9f 100644 --- a/README.md +++ b/README.md @@ -130,8 +130,9 @@ flowchart TD A([next waypoint]) --> B["move yaw and pitch
to the waypoint"] B --> C{"both axes
stopped?"} C -- no --> C - C -- yes --> D["hold still for settle_wait_ms
(1 s by day, 2.5 s at night)"] - D --> E["keep the newest frame
the camera delivered"] + C -- yes --> D["hold still for settle_wait_ms
(and, with settle_rule = brightness,
until the brightness has settled)"] + D --> W["wait out what is left of
image_interval since the last picture"] + W --> E["keep the newest frame
the camera delivered"] E --> F["encode to JPEG XL
save to the output folder"] F --> G["publish a CamEvent
(camera, heading, pitch, time)"] G --> A diff --git a/config/camera_profiles/rgb/README.md b/config/camera_profiles/rgb/README.md index 42a48cb..85ff890 100644 --- a/config/camera_profiles/rgb/README.md +++ b/config/camera_profiles/rgb/README.md @@ -40,28 +40,33 @@ overwritten whenever the XML changes. ## Profiles this rig expects -`config.ini` ships referencing `standard` and `night`. They were previously -hand-written; these are the values they used, as a starting point when -recreating them in the Viewer. Everything not listed was identical. +`config.ini` ships referencing `standard` and `night`. Both are **raw capture** +(docs/raw12-pipeline-measurements.md): full resolution, binning 1, packed 12-bit +Bayer, converted to colour on the tower and saved as 12-bit JPEG XL. +`scripts/export_camera_profile.py` builds them on the tower; these are its recipe. | Setting | `standard` | `night` | |---|---|---| -| `AcquisitionFrameRate` | 10 | 1 | -| `SensorBitDepth` | Bpp8 | Bpp12 | -| `ExposureAutoMin` | 200 | 1000 | -| `ExposureAutoMax` | 20000 | 500000 | +| `BinningHorizontal` / `Vertical` | 1 | 1 | +| `Width` / `Height` | sensor maximum | sensor maximum | +| `PixelFormat` | `BayerRG12p` | `BayerRG12p` | +| `DeviceLinkThroughputLimit` | 250000000 | 250000000 | +| `AcquisitionFrameRate` | camera maximum (9.86 on the 1800 U-1620c) | camera maximum (exposure-limited at night) | +| `ChunkModeActive` + `ChunkEnable` | ExposureTime, Gain, BalanceRatioRed/Blue, Timestamp | same | +| `BalanceWhiteAuto` | Off (fixed ratios) | Off | +| `ExposureAutoMin` / `Max` | 200 / 20000 µs | 1000 / 500000 µs | | `GainAutoMax` | 24 | 48 | -| `IntensityAutoPrecedence` | Minimizeexposuretime | Minimizenoise | -| `DeviceIndicatorMode` | Active | Off | -| `DeviceIndicatorLuminance` | 10 | 0 | +| `IntensityAutoPrecedence` | MinimizeNoise | MinimizeNoise | +| `IntensityControllerRate` | 100 | 100 | +| `DeviceIndicatorMode` / `Luminance` | Active / 10 | Inactive / 0 | -Both used 2×2 binning at `RGB8` (2256×2256, 15.3 MB/frame) and a 250 MB/s -`DeviceLinkThroughputLimit`. **That binning is not optional at 10 Hz**: a -full-resolution RGB8 frame is 61 MB, which caps out near 4 fps under that limit. +The chunk data is what `settle_rule = brightness` reads (each item has to be +enabled before `ChunkModeActive`, which then locks them). A profile left on +`RGB8` still works: the program saves it as before. -The full previous definitions are in git history at -the hand-written `config/camera_modes/rgb/{standard,night}.ini`, removed in the -same change that made the XML the only thing applied. +Earlier profiles used 2×2 binning at RGB8. On the 1620c that gave **no** extra +frame rate (the camera budgets the link against the unbinned frame), which is one +reason they were replaced. Their values are in git history. -`settle_wait_ms` / `frame_timeout_ms` are *not* camera settings and are not in +`settle_wait_ms` / `frame_timeout_ms` / `settle_rule` are *not* camera settings and are not in the XML — they live in `config.ini` under `[Camera.rgb.]`. diff --git a/config/config.example.ini b/config/config.example.ini index 55887bc..0aaf9ce 100644 --- a/config/config.example.ini +++ b/config/config.example.ini @@ -14,7 +14,7 @@ [General] ; Tower identity, substituted into all MQTT topics and CamEvent payloads. tower_name = ExampleTower -; Seconds between captures. +; Desired seconds from one picture to the next (decimals allowed, e.g. 2.5). image_interval = 5 ; 1 = print motor telemetry each loop tick, 0 = quiet. debug = 0 @@ -50,6 +50,10 @@ mqtt_pw = CHANGE_ME ; JPEG XL encoding. distance: 0 = lossless, ~0.8 = near-lossless; effort 1..9. jxl_distance = 0.8 jxl_effort = 4 +; Raw profiles (PixelFormat BayerRG8 / BayerRG12p in the profile XML): the tower +; converts the sensor mosaic to colour itself. edge_aware (sharper; ~30 ms for +; 16 MP) or bilinear. A 12-bit raw frame is saved as a 12-bit file. +demosaic = edge_aware ; --- One [Camera.] block per camera ----------------------------------- ; Nothing is implied by position: the block names the camera, `label` names its @@ -71,13 +75,20 @@ profile = standard ; /standard.xml rotation_deg = 0 ; Per-profile timing. Not camera settings - how long the program waits around -; them. After the gimbal reports standstill it holds still this long before -; keeping a frame, so the image is exposed at rest with the auto-exposure -; settled; frame_timeout_ms then abandons the waypoint rather than parking on it. +; them. After the gimbal reports standstill it holds still settle_wait_ms, then +; keeps a frame according to settle_rule: +; fixed the newest frame, straight away +; brightness the first frame once brightness is in the camera's target band +; and steady for 3 frames (right 21 of 24 times in the tower tests, +; docs/raw12-pipeline-measurements.md); settle_wait_ms is then just +; the mechanical minimum +; frame_timeout_ms bounds the wait after settle_wait_ms; past it the newest frame +; is kept anyway (or the waypoint skipped if the camera delivers nothing). ; With several cameras the scan uses the LONGEST wait across them. [Camera.rgb.standard] -settle_wait_ms = 1000 -frame_timeout_ms = 2000 +settle_wait_ms = 500 +frame_timeout_ms = 7000 +settle_rule = brightness [Camera.rgb.night] settle_wait_ms = 2500 diff --git a/config/hardware.example.ini b/config/hardware.example.ini index b7984bb..2dd931e 100644 --- a/config/hardware.example.ini +++ b/config/hardware.example.ini @@ -58,6 +58,15 @@ cameras = rgb ; How much bearing one picture covers, in degrees. Pan mode ('mode pan') cuts ; its span into as many pictures as this takes. 6 until the lens is measured. pan_step_deg = 6 +; Image saving: how this computer shares its 4 cores between saved frames. +; Measured with a 16 MP 12-bit frame (docs/raw12-pipeline-measurements.md): the +; IOTA (N150) keeps up with one waypoint every 0.7 s with 1 worker; the 3 Delta +; (N5105) needs 2 workers for that. jxl_threads is libjxl's threads per frame; +; 0 or absent = 4 / save_workers. save_queue_max is how many frames may wait +; or be saving before the scan holds at its waypoint (each raw frame ~24 MB). +save_workers = 1 +;jxl_threads = 0 +save_queue_max = 4 ; ; What happens when this does not match the rig: ; declared but absent -> warned about by name, and with [Features] diff --git a/docs/architecture.md b/docs/architecture.md index 55e2b0a..4079304 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -79,8 +79,10 @@ corrupted. pan's pitch, converted to counts via `Geometry`. - manual: nothing, bar a requested snapshot or continuous capture (`capture on`), which skip the move and wait only for standstill. - - Once **both axes report standstill at the target**, wait `settle_wait_ms`, then keep the newest - frame and advance the grid or pan position. + - Once **both axes report standstill at the target**, wait `settle_wait_ms` (and, with + `settle_rule = brightness`, until the brightness has settled), then whatever is left of + `image_interval` since the last frame kept; keep the newest frame and move straight on to the next + grid or pan position - unless saving is behind, when the scan holds until it catches up. 5. **Acquisition** — the camera free-runs at the active profile's `AcquisitionFrameRate` with its own auto-exposure tracking the light; `trigger()` hands over the freshest complete frame. See *Image capture* below. @@ -94,20 +96,29 @@ corrupted. ## Capture state machine ``` - interval elapsed AND mode auto or pan AND not already moving + mode auto or pan AND not already moving (straight after the last frame) + AND room to save (else hold here until saving catches up) │ ▼ - ┌──────────────────────┐ auto → next ScanGrid (yaw,pitch) - │ MOVE , │ pan → next pan heading, at the pan's pitch - │ (yaw-only: MOVE Y) │ deg→counts via Geometry; set moving, reset timer + ┌──────────────────────┐ auto → next ScanGrid (yaw,pitch); pan → next pan heading + │ MOVE , │ unreachable targets skipped in the same tick + │ (yaw-only: MOVE Y) │ deg→counts via Geometry; set moving └──────────┬───────────┘ │ (both axes standstill AND |xenc − target| ≤ tol) ▼ ┌──────────────────────┐ the camera keeps streaming throughout; this pause is - │ wait settle_wait_ms │ what makes the kept frame one exposed at rest + │ wait settle_wait_ms, │ what makes the kept frame one exposed at rest. + │ then the keep rule │ settle_rule = brightness: until brightness is in band + │ if on │ and steady, at most frame_timeout_ms (then keep, warn) + └──────────┬───────────┘ move + wait > interval: warn (not on the return to the + │ first waypoint, nor the first move after a mode starts) + ▼ + ┌──────────────────────┐ + │ wait out the rest of │ interval counted from the last frame kept + │ image_interval │ └──────────┬───────────┘ ▼ - ┌──────────────────────┐ on success: clear moving, advance the grid / pan position + ┌──────────────────────┐ on success: clear moving, advance, restart the interval │ camera.trigger() │ no fresh frame: retry next tick └──────────────────────┘ still none after frame_timeout_ms: warn, skip, advance ``` @@ -132,15 +143,27 @@ At each waypoint the scheduler therefore: 1. moves, and waits for both axes to report standstill at the target; 2. waits a further `settle_wait_ms` (1 s in `standard`, 2.5 s in `night`); -3. keeps the newest complete frame. +3. with `settle_rule = brightness`, waits on until brightness is in the camera's + target band and steady for 3 frames + ([configuration.md](configuration.md#which-frame-a-waypoint-keeps-settle_rule)); +4. waits out what is left of `image_interval`, counted from the last frame kept, and + keeps the newest complete frame. + +Saving runs in background workers and never blocks this loop; if more than +`save_queue_max` frames are waiting, the scan holds at the waypoint until there is +room. Raw profiles (`BayerRG8`, `BayerRG12p`) are converted to colour on the tower +and saved at their own bit depth ([raw12-pipeline-measurements.md](raw12-pipeline-measurements.md)). The wait is the whole mechanism. It guarantees the kept frame was exposed with the gimbal **at rest** (no slew blur) and after the auto-exposure has seen this waypoint's light for several frames. It also sets the floor on a profile's frame rate: a profile must produce comfortably more than one frame inside its own settle wait, which is why `night` at 1 Hz waits 2.5 s where `standard` at 10 Hz waits 1 s. -If no fresh frame has arrived by `frame_timeout_ms` past the wait, the waypoint is logged and skipped. A -stalled camera costs single images, never the whole sweep. +If no fresh frame arrives within `frame_timeout_ms` of the frame falling due, the waypoint is logged and +skipped. A stalled camera costs single images, never the whole sweep. A merely slow camera is not stalled: +with multi-second exposures the next frame may not be due yet, so the stream is only restarted once nothing +newer has come for twice the longer of the exposure and the frame period, plus a second (at least 2 s; +`CameraStall.h`). Restarting earlier throws the exposure in progress away. ### Camera profiles diff --git a/docs/configuration.md b/docs/configuration.md index 756b7f5..a4ad64e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -27,7 +27,7 @@ Parsed and validated by `ConfigLoader` ([src/core/Config.cpp](../src/core/Config | Section | Key | Type | Default | Meaning | |---------|-----|------|---------|---------| | `General` | `tower_name` | string | `Unnamed` | Tower identity; used in all MQTT topics/payloads | -| `General` | `image_interval` | int > 0 | `5` | Seconds between captures (→ `image_rate = 1/interval`) | +| `General` | `image_interval` | number > 0 | `5` | Desired seconds from one picture to the next, decimals allowed (→ `image_rate = 1/interval`). Counted from the last frame kept; the move and the exposure wait come out of it. A waypoint whose move + wait outlasts it warns (not the return to the first waypoint, nor the first move after a mode starts) | | `General` | `debug` | bool | `false` | Start with debug-level logging | | `Startup` | `home` | bool | `false` | Home the gimbal at every start (see [Startup](#startup--homing-and-the-start-mode)). A tower in service needs `true`, and then homes **unattended** at every boot | | `Startup` | `mode` | `manual`/`auto` | `manual` | Mode the tower starts in. `auto` is refused, leaving it in manual, if the gimbal is not homed by then. `pan` is by command only | @@ -44,9 +44,14 @@ Parsed and validated by `ConfigLoader` ([src/core/Config.cpp](../src/core/Config | `Camera.` | `enabled` | bool | `true` | A disabled camera is not opened and does not affect the scan timing | | `Camera.` | `rotation_deg` | `0`/`90`/`180`/`270` | `0` | Clockwise rotation applied to that camera's saved images | | `Camera..` | `settle_wait_ms` | int ≥ 0 | `1000` | Hold after standstill before keeping a frame | -| `Camera..` | `frame_timeout_ms` | int ≥ 0 | `2000` | Abandon the waypoint after this | +| `Camera..` | `frame_timeout_ms` | int ≥ 0 | `2000` | With `settle_rule = brightness`, the longest wait for the brightness after `settle_wait_ms`; then keep the newest frame anyway. Also how long to wait for a frame once one is due before skipping the waypoint | +| `Camera..` | `settle_rule` | `fixed` \| `brightness` | `fixed` | `fixed`: keep the newest frame once `settle_wait_ms` has passed. `brightness`: keep the first frame whose brightness is in the camera's target band and steady for 3 frames ([keep rule](#which-frame-a-waypoint-keeps-settle_rule)) | | `Camera` | `jxl_distance` / `jxl_effort` | double ≥ 0 / int 1..9 | `0.8` / `4` | Encoder defaults inherited by every camera block | -| `Camera.` | `jxl_distance` / `jxl_effort` | double ≥ 0 / int 1..9 | the `[Camera]` value | Per-camera override of the encoder settings | +| `Camera.` | `jxl_distance` / `jxl_effort` | double ≥ 0 / int 1..9 | the `[Camera]` value | Per-camera override of the encoder settings. Distance 0 is truly lossless | +| `Camera` / `Camera.` | `demosaic` | `edge_aware` \| `bilinear` | `edge_aware` | How raw profiles (`BayerRG8`, `BayerRG12p`) are converted to colour on the tower | +| `Hardware` (hardware.ini) | `save_workers` | int 1..4 | `1` | Frames saved at once. 1 on the IOTA, 2 on the 3 Delta ([measurements](raw12-pipeline-measurements.md)) | +| `Hardware` (hardware.ini) | `jxl_threads` | int 0..8 | `0` = 4 / `save_workers` | libjxl threads per saved frame | +| `Hardware` (hardware.ini) | `save_queue_max` | int 1..32 | `4` | Frames waiting or being saved before the scan holds at its waypoint (a raw 16 MP frame is ~24 MB) | | `Paths` | `output_dir` | string | `$XDG_DATA_HOME/fire_gimbal_control/images` | Image output dir; supports `~`/`$ENV` | | `Features` | `enable_mqtt` | bool | `true` | Use MQTT (vs null channel) | | `Features` | `enable_camera` | bool | `true` | (reserved) | @@ -432,6 +437,43 @@ Every `*.xml` in `profile_dir` is loaded at start-up, so a profile nobody is using today still fails at launch rather than the first time someone switches to it. +#### Raw profiles + +A profile whose XML sets `PixelFormat = BayerRG12p` (or `BayerRG8`) sends the +camera's raw sensor mosaic. The tower converts it to colour (`[Camera] demosaic`) +and saves it at the frame's own bit depth: a 12-bit raw frame becomes a 12-bit +JPEG XL file. `RGB8` profiles are saved exactly as before. At the 250 MB/s link +cap the 1800 U-1620c delivers 9.86 fps in `BayerRG12p` against 4.95 in RGB8, so +auto-exposure settles in about half the time; the tower-side cost is the same as +saving RGB8 ([measurements](raw12-pipeline-measurements.md)). +`scripts/export_camera_profile.py` builds such profiles; it also turns on the +per-frame data (chunk mode) the keep rule reads. + +#### Which frame a waypoint keeps: `settle_rule` + +With `settle_rule = brightness`, after `settle_wait_ms` (now the mechanical +minimum) the scan watches every frame the camera delivers and keeps the first one +once the last three are all inside the camera's own auto-exposure target band +(`IntensityControllerTarget ± IntensityControllerTolerance`, read from the camera) +and their brightness changed by at most 2% from frame to frame. A steady frame +outside the band also counts when exposure and gain sit at the limits that would +move it (waiting cannot help). Past `frame_timeout_ms` the newest frame is kept +anyway, with a warning. On the 24 recorded exposure steps this picked a good frame +21 times; "exposure and gain unchanged" never did, because auto-exposure pauses +mid-way. A camera that reports no frame data falls back to the fixed wait. The +rule applies to scan and pan waypoints; a snapshot or continuous capture keeps +the fixed wait. Its time counts towards `image_interval`, like the move's. + +#### When saving falls behind + +Saving runs in background workers (`[Hardware] save_workers`, `jxl_threads`). If +more than `save_queue_max` frames are waiting or being saved, the scan **holds at +its waypoint** until there is room (logged once), so no picture is lost and memory +stays bounded. That waypoint does not also warn about running past +`image_interval`. Continuous capture holds the same way; a requested snapshot is +never held. Measured: the IOTA keeps up +with a frame every 0.7 s on one worker; the 3 Delta needs two. + ### `[Env]` — SHT41 ambient temperature/humidity sensor Enable with `[Features] enable_env = true` and point `[Env] i2c_device` at the LattePanda's **own @@ -654,7 +696,7 @@ Handled in `Application::Impl::handleCommand`. | `trace all` / `trace off` | Enable every category / silence all | | `set camera profile [] ` | Switch a camera to another profile from its `profile_dir` (the camera may be omitted when only one is configured) | | `snapshot` | One image now, through the scan's settle-and-trigger path (manual mode, homed) | -| `capture [on\|off]` | Continuous capture where the gimbal is aimed: one image per `image_interval` + standstill + `settle_wait_ms`, kept only at rest. Manual mode, homed; ends on any mode change, homing, heading setup or test. Bare: report | +| `capture [on\|off]` | Continuous capture where the gimbal is aimed: one image per `image_interval` since the last, each once the gimbal has stood still for `settle_wait_ms`. Manual mode, homed; ends on any mode change, homing, heading setup or test. Bare: report | | `camera reset` | Reboot the cameras and put them back on their profiles | | `test camera [frames\|dark\|bright]` | Check the camera delivers frames and responds to exposure/gain | | `test [ []] []` | Run the hardware self-test suite (see below) | diff --git a/docs/known-issues.md b/docs/known-issues.md index e3feaa0..efff870 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -29,10 +29,10 @@ doctest unit-test suite (`ctest`). |---|-------|--------| | 13 | `[Motor]` degrees↔counts calibration | The `config.example.ini` values are **placeholders**. Home, then run `gimbal heading setup` (yaw, needs no IMU) and `gimbal pitch setup` (pitch, needs the IMU) on the rig. | | 14 | Capture sweep on hardware | **Mechanically verified on the rig (2026-08-07).** A full `--init --start` run homed both axes and completed `MOVE → settle → trigger` for 15/15 waypoints with 0 dropped frames, 0 stalls and 0 acquisition failures. `kSettleTolCounts` (600) and the per-interval timing in [CaptureScheduler.cpp](../src/core/CaptureScheduler.cpp) have **not** been tuned against observed `ST` behaviour yet, and the settle tolerance has not been checked against a sharp image (see #16). | -| 15 | **Camera profiles unverified on a lit scene** | The `standard`/`night` profiles must be exported from the VimbaX Viewer (see `config/camera_profiles/rgb/README.md`) and have not been measured against real light. Confirm on the rig that `LoadSettings` succeeds, that `standard` sustains 10 fps with `frames_dropped` at 0, and that the auto-exposure lands on a sensible `ExposureTime` outdoors. The only previous rig test ran with the lens covered. **2026-09-24:** tower 121 has still never imaged a scene - a frame grabbed directly and the program's own saved images (2026-09-21) are a smooth grey gradient with no edges, i.e. the lens is covered or grossly out of focus. Tower 111's images from the same afternoon show nothing sharp anywhere either (soft blobs, no edges) - defocus, a cover or a fogged window; unexplained. `test camera` (below) proves the camera and its data path work in both cases; it cannot, by design, judge the picture. **Stream stuck, 2026-09-24:** from 16:10 to 16:19 tower 111's camera delivered no frames at all - not to the program, not to a fresh Vimba session, not at a fixed 10 ms exposure - and it survived program restarts and a USB re-authorisation. The camera's own `DeviceReset` command cleared it. Production captured nothing for those 9 minutes and logged only `no frame yet (stream warming up)`: a session that has NEVER delivered is not restarted by `trigger()`, by design, so a stream stuck from its start stays stuck. It began just after a `test camera` run by an early version that stopped and restarted the stream several times in a few seconds (to write `BlackLevel`, which the camera refuses while streaming); whether that caused it or met 111's known USB stall is not established. `test camera` itself reported it correctly. **Answered the same day:** a stream that never delivers is now restarted once the wait exceeds what its exposure can explain (at least 30 s), and after two fruitless restarts the camera is reset (DeviceReset) and reopened on its profile - both by the program itself (`CameraStall.h`), and on demand with the console command `camera reset` (verified on tower 121: back in ~14 s, frames straight after). `test camera` no longer stops the stream at all: it changes only exposure and gain, which are writable live, and puts them back live. With that, 111 ran it inside the capturing program, passed, and kept capturing. **2026-09-30:** tower 121's camera is indoors, facing a window, still badly out of focus, gain at its 24 dB maximum ([camera-capture-measurements.md](camera-capture-measurements.md)). | -| 16 | **`settle_wait_ms` untuned** | `standard` waits 1 s and `night` 2.5 s after standstill before keeping a frame. Both are reasoned (≫ one frame period; long enough for the auto-exposure to see the new scene) but not measured. Check on a lit scene that the kept frame shows no slew blur and that its exposure has actually converged for that heading, and shorten the wait if it has. **Seen in real data 2026-09-24:** of eight consecutive images tower 111 kept at 8 deg/s, one was 98.6 % clipped white, one 57 % black and two about half clipped - frames kept before the continuous auto-exposure had caught up with the new view. **Measured 2026-09-30 (tower 121, still camera, forced ×16 brightness steps):** auto-exposure took 1.4–1.8 s to come within ±10% at 4.95 fps, so 1 s is too short for a large step; from a dark step it first overshoots to its maximum exposure, 3.4–4.1× too bright. Exposure and gain holding still for a few frames does not mean it has finished ([camera-capture-measurements.md](camera-capture-measurements.md)). | +| 15 | **Camera profiles unverified on a lit scene** | The `standard`/`night` profiles must be exported from the VimbaX Viewer (see `config/camera_profiles/rgb/README.md`) and have not been measured against real light. Confirm on the rig that `LoadSettings` succeeds, that `standard` sustains 10 fps with `frames_dropped` at 0, and that the auto-exposure lands on a sensible `ExposureTime` outdoors. The only previous rig test ran with the lens covered. **2026-09-24:** tower 121 has still never imaged a scene - a frame grabbed directly and the program's own saved images (2026-09-21) are a smooth grey gradient with no edges, i.e. the lens is covered or grossly out of focus. Tower 111's images from the same afternoon show nothing sharp anywhere either (soft blobs, no edges) - defocus, a cover or a fogged window; unexplained. `test camera` (below) proves the camera and its data path work in both cases; it cannot, by design, judge the picture. **Stream stuck, 2026-09-24:** from 16:10 to 16:19 tower 111's camera delivered no frames at all - not to the program, not to a fresh Vimba session, not at a fixed 10 ms exposure - and it survived program restarts and a USB re-authorisation. The camera's own `DeviceReset` command cleared it. Production captured nothing for those 9 minutes and logged only `no frame yet (stream warming up)`: a session that has NEVER delivered is not restarted by `trigger()`, by design, so a stream stuck from its start stays stuck. It began just after a `test camera` run by an early version that stopped and restarted the stream several times in a few seconds (to write `BlackLevel`, which the camera refuses while streaming); whether that caused it or met 111's known USB stall is not established. `test camera` itself reported it correctly. **Answered the same day:** a stream that never delivers is now restarted once the wait exceeds what its exposure can explain (at least 30 s), and after two fruitless restarts the camera is reset (DeviceReset) and reopened on its profile - both by the program itself (`CameraStall.h`), and on demand with the console command `camera reset` (verified on tower 121: back in ~14 s, frames straight after). `test camera` no longer stops the stream at all: it changes only exposure and gain, which are writable live, and puts them back live. With that, 111 ran it inside the capturing program, passed, and kept capturing. **2026-09-30:** tower 121's camera is indoors, facing a window, still badly out of focus, gain at its 24 dB maximum ([camera-capture-measurements.md](camera-capture-measurements.md)). **Slow, not stuck, 2026-10-04:** at dusk tower 111 ran multi-second exposures, one frame every 7-9 s, and the program restarted the stream whenever a picture fell due before the next frame had come - throwing that exposure away, so the next frame came later still. A stream that has delivered is now only restarted once nothing newer has come for twice the longer of the exposure and the frame period, plus a second (`CameraStall.h`); not yet checked on the tower. | +| 16 | **`settle_wait_ms` untuned** | `standard` waits 1 s and `night` 2.5 s after standstill before keeping a frame. Both are reasoned (≫ one frame period; long enough for the auto-exposure to see the new scene) but not measured. Check on a lit scene that the kept frame shows no slew blur and that its exposure has actually converged for that heading, and shorten the wait if it has. **Seen in real data 2026-09-24:** of eight consecutive images tower 111 kept at 8 deg/s, one was 98.6 % clipped white, one 57 % black and two about half clipped - frames kept before the continuous auto-exposure had caught up with the new view. **Measured 2026-09-30 (tower 121, still camera, forced ×16 brightness steps):** auto-exposure took 1.4–1.8 s to come within ±10% at 4.95 fps, so 1 s is too short for a large step; from a dark step it first overshoots to its maximum exposure, 3.4–4.1× too bright. Exposure and gain holding still for a few frames does not mean it has finished ([camera-capture-measurements.md](camera-capture-measurements.md)). **2026-10-05:** `settle_rule = brightness` keeps the frame once brightness is in the camera's band and steady (21 of 24 recorded steps right), and raw profiles double the frame rate it works with; to be confirmed with real slews ([raw12-pipeline-measurements.md](raw12-pipeline-measurements.md)). | | 19 | **USB preconditions do not survive a reboot** | `usbcore.usbfs_memory_mb` reverts to 16 (needs 1000) and USB3 LPM re-enables on the camera's port. **Both were wrong on the LattePanda when tested on 2026-08-07**, and frame delivery stalls without them regardless of application code. **Solved on tower 121 by the setup kit** (`hardware/lattepanda-iota/preconditions/`): a systemd-tmpfiles drop-in for usbfs and two udev rules, all installed by `install.sh --preconditions`. Verified 2026-09-23 — `usbfs_memory_mb` reads 1000 after **6 days of uptime**, so it survived the boot, and both rules are in `/etc/udev/rules.d/`; and again across a deliberate reboot the same day, when `install-tower --verify` read `usbfs_memory_mb` 1000 and the camera port's `usb3_lpm_permit` 0 (a check `test hardware camera` now makes too, once `[Test] usb3_lpm_path` is set - the installer sets it). Note it is **tmpfiles, not the kernel cmdline** the original entry proposed. **On the Delta fleet**: tower 111 read 16 and `u1_u2` on 2026-09-23, and the Delta kit (`hardware/lattepanda-3-delta/`, the same two files) was applied the same day - 1000 and `0` afterwards, confirmed by `test hardware camera`. Confirmed across a reboot on 2026-09-24 (`install-tower --verify`). **Solved on both boards**; open only for Delta towers the kit has not reached. **2026-09-30:** with the kit applied, tower 121 streamed full resolution in every format at 250 and 450 MB/s for 60 s with nothing lost; 2 incomplete frames in 1,574 at 450 MB/s while encoding. **2026-10-01:** tower 111 (Delta kit applied) likewise, at 200, 250 and 450 MB/s, also while encoding, with nothing lost ([camera-capture-measurements.md](camera-capture-measurements.md)). | -| 20 | **`jxl_distance = 0` is not lossless** | `JpegXlEncoder::encodeToFile` always sets `uses_original_profile = false`; for distance 0 libjxl prints `Set uses_original_profile=true for lossless encoding`, still reports success, and encodes through its lossy colour space. Measured 2026-09-30: decoded pixels differ by up to 1 (raw mosaic) or 2 (RGB8) levels in 1.5% / 7.3% of values ([camera-capture-measurements.md](camera-capture-measurements.md)). Fix: set `uses_original_profile = true` when distance is 0. | +| 20 | **`jxl_distance = 0` is not lossless** | `JpegXlEncoder::encodeToFile` always sets `uses_original_profile = false`; for distance 0 libjxl prints `Set uses_original_profile=true for lossless encoding`, still reports success, and encodes through its lossy colour space. Measured 2026-09-30: decoded pixels differ by up to 1 (raw mosaic) or 2 (RGB8) levels in 1.5% / 7.3% of values ([camera-capture-measurements.md](camera-capture-measurements.md)). **Fixed 2026-10-05:** the encoder sets `uses_original_profile` for distance 0, and `tests/test_jpegxl.cpp` checks 8-bit and 12-bit lossless decode exactly; 8-bit lossy output is byte-identical to before (72 of 72 compared). | ## Camera acquisition (USB3) — root cause + approach diff --git a/docs/modules-reference.md b/docs/modules-reference.md index 8944dcb..9b03f5c 100644 --- a/docs/modules-reference.md +++ b/docs/modules-reference.md @@ -8,6 +8,8 @@ Per-file reference for the refactored tree, plus the shared data structures. |------|----------| | [include/fgc/Config.h](../include/fgc/Config.h), [src/core/Config.cpp](../src/core/Config.cpp) | Typed `AppConfig` (General/Network/Serial/Camera/Paths/Features/Logging/UI/Motor/Scan/**IMU**) + `ConfigLoader` (INI parse, env overrides, validation) | | [include/fgc/Paths.h](../include/fgc/Paths.h), [src/core/Paths.cpp](../src/core/Paths.cpp) | `~`/`$ENV` expansion, executable dir, config search order, default output dir | +| [include/fgc/IniEdit.h](../include/fgc/IniEdit.h), [src/core/IniEdit.cpp](../src/core/IniEdit.cpp) | Editing `config.ini` and `hardware.ini` without breaking them, as pure functions on file contents: set keys keeping comments, order and one `;prev` generation, comment out keys `hardware.ini` owns (without their value, so `deploy.sh`'s placeholder guard stays quiet), and refuse lines too long for inih to read whole. `tools/fgc_config.cpp` does the file IO | +| [include/fgc/SingleInstance.h](../include/fgc/SingleInstance.h), [src/core/SingleInstance.cpp](../src/core/SingleInstance.cpp) | Refuses a second running copy of the program, which would fight the first over the motor's serial port and the camera: an advisory `flock` on a lockfile holding the owner's PID, taken in the binary so every way of starting it is covered | | [include/fgc/Logger.h](../include/fgc/Logger.h), [src/core/Logger.cpp](../src/core/Logger.cpp) | Leveled, thread-safe logger + per-category wire trace; `LOG_TRACE..LOG_ERROR`, `LOG_TRACE_CAT` | | [include/fgc/FileLogSink.h](../include/fgc/FileLogSink.h), [src/core/FileLogSink.cpp](../src/core/FileLogSink.cpp) | The persistent log file: append, size rotation, count-based pruning, a self-describing header per file. Runs inside `Logger`'s non-recursive lock, so it never logs - failures latch into `failed()`/`lastError()` for the caller to report | | [include/fgc/Geometry.h](../include/fgc/Geometry.h), [src/core/Geometry.cpp](../src/core/Geometry.cpp) | Per-axis degrees↔encoder-counts affine map (`[Motor]` calibration); `encoderPitchDeg` (the CamEvent pitch when there is no IMU — the fixed `deg_at_center` on a yaw-only tower) | @@ -22,6 +24,9 @@ Per-file reference for the refactored tree, plus the shared data structures. | [include/fgc/FirmwareConfig.h](../include/fgc/FirmwareConfig.h), [src/core/FirmwareConfig.cpp](../src/core/FirmwareConfig.cpp) | `parseNeedLine` (firmware `NEED` line -> axis + keys) + `planFirmwareConfig` (-> the `SET` lines that answer it). Reports a key it cannot answer separately from one it does not recognise: incomplete `hardware.ini` vs. a firmware newer than the host | | [include/fgc/DumpParser.h](../include/fgc/DumpParser.h), [src/core/DumpParser.cpp](../src/core/DumpParser.cpp) | `parseDump` firmware `DUMP` block → `DumpData` (per-axis state + decoded TMC registers/flags); `formatDump`. Prefers the firmware's own `reset=` name over any host-side decode - `mcusr` is an AVR bitmask on one board and a plain enum on the other, so a single table cannot be right for both | | [include/fgc/DiagParser.h](../include/fgc/DiagParser.h), [src/core/DiagParser.cpp](../src/core/DiagParser.cpp) | `parseDiag` firmware `DG` self-test stream → `DiagResult` (per-axis tests, PASS/FAIL); `formatDiag` | +| [include/fgc/TestRunner.h](../include/fgc/TestRunner.h), [src/core/TestRunner.cpp](../src/core/TestRunner.cpp) | The `test` command: resolves `test [ []]` to a set of leaf tests, knows which need a homed gimbal (the `test hardware` leaves deliberately do not), and runs them on a worker thread with live progress, handing back a `TestReport` | +| [include/fgc/TestReport.h](../include/fgc/TestReport.h), [src/core/TestReport.cpp](../src/core/TestReport.cpp) | A self-test run's result as plain data: formats a human-readable text report that `parseTestReport` reads back, so a run can be compared with the baseline or the previous one | +| [include/fgc/HostMetrics.h](../include/fgc/HostMetrics.h), [src/core/HostMetrics.cpp](../src/core/HostMetrics.cpp) | Tower computer health for `test host`: CPU temperature, disk, memory and load. Pure parsers split from the live `/sys` and `/proc` readers so they unit-test anywhere; readers report `ok = false` rather than throw | | [include/fgc/Calibration.h](../include/fgc/Calibration.h), [src/core/Calibration.cpp](../src/core/Calibration.cpp) | `linearFit` (least-squares, R²) + `circularMeanDeg` for the IMU-referenced calibration | | [include/fgc/PitchSetup.h](../include/fgc/PitchSetup.h), [src/core/PitchSetup.cpp](../src/core/PitchSetup.cpp) | `gimbal pitch setup` worker thread: home-if-needed → pitch sweep dwelling on the gravity-referenced IMU pitch → least-squares fit → park level; tunable via `CalibParams`; exposes `progress()`/`report()`/`takeResult()`. Yaw is not touched | | [include/fgc/HeadingFit.h](../include/fgc/HeadingFit.h), [src/core/HeadingFit.cpp](../src/core/HeadingFit.cpp) | `gimbal heading fit`: least-squares fit of the yaw map from operator sightings of landmarks with known true bearings. Sorts by encoder position, unwraps bearings so a set spanning north fits one line, and refuses a baseline under 10° (two points always fit exactly, so a short baseline looks perfect and is wrong). No IMU involved | @@ -29,14 +34,20 @@ Per-file reference for the refactored tree, plus the shared data structures. | [include/fgc/ImuMount.h](../include/fgc/ImuMount.h), [src/core/ImuMount.cpp](../src/core/ImuMount.cpp) | `ImuMount` (the `[IMU] x/y/z` mounting rotation) and the pure math around it: `mountRotationZXY` (intrinsic Z-X-Y), the `rotationFromEulerZYX`/`eulerZYXFromRotation` pair for the MTi's own yaw-pitch-roll output, `applyFrames`, which moves a whole `ImuSample` into the gimbal's NED frame by re-referencing the earth frame (`ImuReference`/`nedFromReference`, left-multiplied) and then applying the mount (right-multiplied) — the two are not interchangeable, which is why a mount alone cannot correct a Z-up device; `applyMount` is the NED-already shorthand. Body-frame vectors are rotated by the mount only. Returns the sample untouched when there is nothing to do | | [include/fgc/MountedImuSource.h](../include/fgc/MountedImuSource.h) | Decorator over any `IImuSource` applying that rotation in `sample()`. Wrapped around the factory's output in `Application::makeImu()`, so every consumer reads gimbal-frame values without knowing a mount exists; skipped entirely when the mount is the identity | | [include/fgc/sensors/Sht41Protocol.h](../include/fgc/sensors/Sht41Protocol.h), [src/sensors/Sht41Protocol.cpp](../src/sensors/Sht41Protocol.cpp) | Sensirion SHT4x wire protocol, pure: command bytes, `sht41Crc8` (poly 0x31/init 0xFF), `decodeSht41Measurement` (6-byte response → `Sht41Reading`, CRC-checked, raw→°C/%RH with RH clamped 0..100) | +| [include/fgc/sensors/McuEnvSensor.h](../include/fgc/sensors/McuEnvSensor.h), [src/sensors/McuEnvSensor.cpp](../src/sensors/McuEnvSensor.cpp) | `IEnvSensor` for the older towers' DHT11, wired to the motor board (`[Hardware] env = dht11`): serves the firmware's latest `ENV` line, and no reading after a failed checksum or 2.5 read periods of silence | | [include/fgc/CameraProfile.h](../include/fgc/CameraProfile.h), [src/core/CameraProfile.cpp](../src/core/CameraProfile.cpp) | `loadCameraProfile` / `loadCameraProfiles`: one VimbaX settings XML per profile. Hashes the XML, regenerates the `.txt` record when it changes, and reports which settings moved. A missing XML throws — the record is never a fallback | | [include/fgc/CameraSettingsXml.h](../include/fgc/CameraSettingsXml.h), [src/core/CameraSettingsXml.cpp](../src/core/CameraSettingsXml.cpp) | Parses the Viewer export's `` features (ordered, selector-aware), renders the human-readable `.txt` record, reads one back, and diffs two settings lists. Used for display and change detection only — never to configure a camera | | [include/fgc/ChannelTopics.h](../include/fgc/ChannelTopics.h), [src/core/ChannelTopics.cpp](../src/core/ChannelTopics.cpp) | The MQTT wire format in one place: topic builders for the `GGS/FWT//…` tree and the `CamEvent`/status payload builders. Pure, so the documented format is unit-tested rather than assembled inline at each publish site | | [include/fgc/ChannelHistory.h](../include/fgc/ChannelHistory.h), [src/core/ChannelHistory.cpp](../src/core/ChannelHistory.cpp) | Ring buffer of recently published messages (time, topic, payload, outcome) plus the delivery counters — sent, failed, connects, losses, last status. Feeds the expanded Connectivity view so a publish can be confirmed without a broker-side subscriber | +| [include/fgc/RemoteCommands.h](../include/fgc/RemoteCommands.h), [src/core/RemoteCommands.cpp](../src/core/RemoteCommands.cpp) | Console commands that arrive over MQTT: run through the same handler as typed ones, on the control thread, and answered on `CommandReply` with every line they logged. Retained messages are never run, a repeated id is answered again rather than re-run, each is audited in the log, and a reply that depends on the firmware (`gimbal dump`) waits for its answer | | [include/fgc/Hash.h](../include/fgc/Hash.h), [src/core/Hash.cpp](../src/core/Hash.cpp) | FNV-1a 64 over a string or a file, with an algorithm-prefixed label. Change detection for the profile records, deliberately not a crypto hash | -| [include/fgc/CaptureScheduler.h](../include/fgc/CaptureScheduler.h), [src/core/CaptureScheduler.cpp](../src/core/CaptureScheduler.cpp) | Capture state machine over the interfaces; injectable clock; `setGeometry` adopts a recalibration | +| [include/fgc/CaptureScheduler.h](../include/fgc/CaptureScheduler.h), [src/core/CaptureScheduler.cpp](../src/core/CaptureScheduler.cpp) | Capture state machine over the interfaces: move, standstill, settle wait (+ `SettleRule`), the rest of the interval since the last frame, keep; holds while saving is behind; injectable clock; `setGeometry` adopts a recalibration | +| [include/fgc/SettleRule.h](../include/fgc/SettleRule.h), [src/core/SettleRule.cpp](../src/core/SettleRule.cpp) | The `settle_rule = brightness` keep rule, pure: fed one frame's brightness, exposure and gain at a time, says when the image has settled (in the camera's target band and steady for 3 frames) or is pinned at the auto-exposure limits | +| [include/fgc/RawImage.h](../include/fgc/RawImage.h), [src/core/RawImage.cpp](../src/core/RawImage.cpp) | Pixel layouts the program accepts (`Mono8`, `RGB8`, `BayerRG8`, `BayerRG12p`), their sizes and bit depths, `unpack12p`, and `sampleBrightness` - the cheap per-frame sample the keep rule reads | +| [include/fgc/CameraStall.h](../include/fgc/CameraStall.h), [src/core/CameraStall.cpp](../src/core/CameraStall.cpp) | When a camera stream delivers nothing new: wait, restart the stream, or reset the camera. Pure policy, scaled by the exposure and frame period so a slow frame is not taken for a stall | | [include/fgc/Application.h](../include/fgc/Application.h), [src/core/Application.cpp](../src/core/Application.cpp) | Factory (real vs mock, headless vs TUI), wiring, control loop, `gimbal …` commands, background-result polling, `buildSnapshot()` | | [include/fgc/ui/UiSnapshot.h](../include/fgc/ui/UiSnapshot.h), [src/ui/UiSnapshot.cpp](../src/ui/UiSnapshot.cpp) | Plain-data view model (incl. `ImuView`, `EnvView`, `ActivityView`, `CalibResultView`, `DumpView`) + pure formatting helpers (state label/colour, degrees, time-ago, pending-sensors) | +| [include/fgc/ui/Scroll.h](../include/fgc/ui/Scroll.h), [src/ui/Scroll.cpp](../src/ui/Scroll.cpp) | Viewport arithmetic for the scrollable TUI panes, line-anchored. Pure integer maths with no FTXUI types, so it is in `fgc_core` and unit-tested without a terminal | | [include/fgc/ui/HeadlessUi.h](../include/fgc/ui/HeadlessUi.h), [src/ui/HeadlessUi.cpp](../src/ui/HeadlessUi.cpp) | Default line console: stdin → command sink; logs via the default stdout/stderr writer | | [ini.c](../ini.c), [ini.h](../ini.h) | Bundled third-party inih INI parser | @@ -62,8 +73,8 @@ Per-file reference for the refactored tree, plus the shared data structures. | [src/mqtt/MqttControlChannel.cpp](../src/mqtt/MqttControlChannel.cpp) | `IControlChannel` over Eclipse Paho | `WITH_MQTT` | | [src/ui/TuiUi.cpp](../src/ui/TuiUi.cpp) | `IUserInterface` over FTXUI: panels, key bar, log pane | `WITH_TUI` | | [src/camera/VimbaCameraSource.cpp](../src/camera/VimbaCameraSource.cpp) | `ICameraSource` over Vimba X (pImpl): free-running stream + keep-latest, restart-on-stall, and per-camera profiles applied with the SDK's `LoadSettings` on the Viewer's settings XML | `WITH_VIMBA` | -| [src/camera/JpegXlEncoder.cpp](../src/camera/JpegXlEncoder.cpp) | libjxl encode-to-file | always | -| [src/camera/ImagePipeline.cpp](../src/camera/ImagePipeline.cpp) | frame → rotate → encode → write → CamEvent (worker thread). The pose is sampled in `submit()` and carried with the frame, so it belongs to the pixels rather than to whenever the encode finished | always | +| [src/camera/JpegXlEncoder.cpp](../src/camera/JpegXlEncoder.cpp) | libjxl encode-to-file, 8-bit or up to 16-bit values (a 12-bit frame keeps its depth); distance 0 is lossless | always | +| [src/camera/ImagePipeline.cpp](../src/camera/ImagePipeline.cpp) | frame → de-Bayer (raw profiles) → rotate → encode → write → CamEvent (`save_workers` background workers; `canAccept()` is the scan's save gate). The pose is sampled in `submit()` and carried with the frame, so it belongs to the pixels rather than to whenever the encode finished | always | ## Mock / null implementations @@ -80,6 +91,8 @@ Per-file reference for the refactored tree, plus the shared data structures. | File | Role | |------|------| | [main.cpp](../main.cpp) | CLI parsing → `AppConfig` + `RuntimeOptions` → `Application::run()` | +| [tools/fgc_config.cpp](../tools/fgc_config.cpp) | `fgc-config`: writes and edits a tower's `config.ini` and `hardware.ini` (`seed`, `set`, `get`, `check`) so what lands on disk is always a file the program accepts. Used by `scripts/install-tower`; the rules are in `IniEdit` | +| [tools/camera-bench/](../tools/camera-bench/README.md) | Camera and CPU measurements behind the capture design (frame rates, losses, settle time, encode cost). Not part of the build; results in [camera-capture-measurements.md](camera-capture-measurements.md) | | [scripts/run.sh](../scripts/run.sh) | Path-independent launcher | | [scripts/fire-gimbal-control.service](../scripts/fire-gimbal-control.service) | systemd unit template | diff --git a/docs/mqtt-api.md b/docs/mqtt-api.md index 029423d..a47f35d 100644 --- a/docs/mqtt-api.md +++ b/docs/mqtt-api.md @@ -46,7 +46,7 @@ edges included: 75, 81, 87, 93, 99, 105 - six pictures. A span that does not divide evenly gets one more picture, never a gap (32° → seven, 5.33° apart). A span under one picture is the one heading, photographed once per loop. Pitch is optional and defaults to 0 (level); a yaw-only tower ignores it, with a warning. Pictures are taken at the same pace as auto -(`image_interval`), and each publishes a CamEvent exactly as a sweep picture +(one per `image_interval`, counted from the last picture), and each publishes a CamEvent exactly as a sweep picture does. **Headings are true bearings and can be out of reach.** Yaw travel is under a diff --git a/docs/raw12-pipeline-implementation-plan.md b/docs/raw12-pipeline-implementation-plan.md index cb9d40a..ad3df23 100644 --- a/docs/raw12-pipeline-implementation-plan.md +++ b/docs/raw12-pipeline-implementation-plan.md @@ -1,7 +1,32 @@ # Implementation plan — raw capture, colour on the tower, 12-bit JPEG XL Based on [raw12-pipeline-measurements.md](raw12-pipeline-measurements.md) (towers 121 and 111, -2026-10-02/04). Not started; decisions marked **[decide]** are the user's. +2026-10-02/04). + +## Status (branch `raw12-capture`, 2026-10-05) + +Decisions taken: `BayerRG12p` for both profiles; colour JPEG XL only (no raw-planes save); white +balance stays fixed; a full save queue **holds the scan** (no lost waypoints). + +| Phase | State | +|---|---| +| 1 Encoder | **done**: 12-bit input, true lossless (#20), threads from config; 8-bit lossy byte-identical to before (72 of 72) | +| 2 Frames | **done**: pixel layout from the camera, chunk data (exposure, gain, white balance), brightness sample per frame. Not done: keeping frames in the SDK buffer instead of copying (optional) | +| 3 Raw core | **done**: `RawImage` (layouts, `unpack12p`, brightness sample) | +| 4 Pipeline | **done**: de-Bayer (edge-aware/bilinear), save at the frame's bit depth, N workers, `canAccept()`; scan holds when full | +| 5 Keep rule | **done** for scan waypoints (`SettleRule`, `settle_rule = brightness`); snapshots and continuous capture keep the fixed wait | +| 6 Config | **done**: `[Camera] demosaic`; `[Hardware] save_workers`, `jxl_threads`, `save_queue_max` (kit writes 1 / 2 per board); per-profile `settle_rule` | +| 7 Profiles | **recipe done** (`scripts/export_camera_profile.py`, dry-run checked on the 1620c); profiles still to be exported on each tower | +| 8 Docs | **done** (configuration, architecture, known issues, profile README). Downstream 12-bit decode still to check | + +Tests: 457, of which 36 new, covering the encoder, the pipeline, raw handling, the keep rule (including +a replay of the 24 recorded steps), the scheduler and config. Builds with the +camera SDK on tower 121. + +Still to do before switching a tower: export the new profiles on the tower; check the ground +station and detection read 12-bit JPEG XL; a daylight run in auto mode on each tower +(verification below); tune `settle_wait_ms` (500 ms is a guess) and `frame_timeout_ms` (7 s from +the worst measured step). ## Goal @@ -26,7 +51,7 @@ exactly as today. ## Processing budget the design must fit Measured on the **identical frame** on both computers (5328×3040, real scene; 4 threads; see the -report's [Where the time goes](raw12-pipeline-measurements.md#where-the-time-goes)). +report's [Section 3.5](raw12-pipeline-measurements.md#35-processing-time-per-step)). ### Time per saved frame, step by step diff --git a/docs/raw12-pipeline-measurements.md b/docs/raw12-pipeline-measurements.md index dc99313..05e4965 100644 --- a/docs/raw12-pipeline-measurements.md +++ b/docs/raw12-pipeline-measurements.md @@ -1,561 +1,917 @@ -# 12-bit raw capture pipeline — final report +# Raw 12-bit capture with on-tower colour conversion: measurements on the LattePanda IOTA and 3 Delta -**Question:** should the towers stop asking the camera for 8-bit colour (RGB8) and take raw -12-bit Bayer (`BayerRG12p`) instead, with the LattePanda converting it to colour and saving 12-bit -JPEG XL? The aim is a higher frame rate, so auto-exposure and auto white balance settle faster, -and more tonal detail for the smoke-detection algorithms. Can the IOTA (tower 121) and the -3 Delta (tower 111) carry the work, how long does each step take, and how much space do the -images need? +*Fire-watch tower camera system — technical report, October 2026* -**Campaign:** 2–5 October 2026. Camera tests on tower 121 (2 Oct) and tower 111 (4 Oct) **with the -same camera** (Alvium 1800 U-1620c, 5328×3040), moved from 121 to 111 and looking at the same -scene, so the two towers differ only in their computer. Processing tests repeated on **one -identical frame** on both towers (4 Oct). Earlier blurred-scene measurements: -[camera-capture-measurements.md](camera-capture-measurements.md). What to build from this: +## Abstract + +The towers' camera currently delivers 8-bit colour images (RGB8), converted to colour inside the +camera. This study tested whether the camera should instead deliver its raw 12-bit sensor data +(`BayerRG12p`), with the tower's computer converting it to colour and saving 12-bit JPEG XL. The +same camera (Allied Vision Alvium 1800 U-1620c, 16.2 MP) was measured on both tower computers, the +LattePanda IOTA (tower 121) and the LattePanda 3 Delta (tower 111). Frame rate, auto-exposure and +auto-white-balance settling, the duration of every processing step, file sizes, CPU temperature and +power were recorded, and the processing was repeated on one identical image on both computers. + +Raw 12-bit doubles the camera's frame rate over the same USB link (9.86 fps against 4.95 fps), so +auto-exposure settles in about 60% of the time, with no lost frames on either tower. Converting to +colour on the tower costs 38–50 ms; the JPEG XL encode dominates, and the complete 12-bit save takes +slightly less time than today's RGB8 save (0.52 s on the IOTA, 0.82 s on the Delta, which is 1.6× +slower). At a realistic scan pace both computers keep up and stay cool (53 °C and 66 °C); the Delta +needs two save workers to match the IOTA's single one. A 12-bit colour file at quality setting +(distance) 1.0 is 2.7 MB, smaller than today's 3.3–4.0 MB. A rule for when to keep a frame +(brightness in the camera's target band and steady for 3 frames) selected a good frame in 21 of 24 +tests; waiting for exposure and gain to stop changing never did. + +The extra bits themselves add less than expected. In the daylight scene measured, the sensor's own +noise was 6–17 times larger than the error from rounding to 8 bits, which raised the total noise by +at most 1.5%; and JPEG XL compression at distance 1.0 adds about 13 times more error than that +rounding. 12 bits are expected to make a meaningful difference only where part of the image is dark +while the gain is low, as in high-contrast scenes (a bright sky over shaded forest or a backlit ridge +at dawn or dusk), where faint smoke over dark terrain is a small change in a dark value; and only if +those values are saved with little or no loss. These conditions were not measured. The 12-bit format +costs little against 8-bit raw (same file size, about 10% more processing time), but the camera +runs at 9.86 instead of 14.75 fps. + +## Units and abbreviations + +*Units, symbols and abbreviations used in this report.* + +| Symbol | Meaning | +|---|---| +| MB, GB, TB | 10⁶, 10⁹, 10¹² bytes | +| MB/s | 10⁶ bytes per second | +| fps | frames per second | +| ms, s | milliseconds, seconds | +| dB | decibel (camera gain; image quality PSNR) | +| DN | digital number: a pixel value, on a 0–255 scale (8-bit) or 0–4095 (12-bit) as stated | +| °C, W, GHz | CPU package temperature, package power, average core clock | +| MP | megapixels (10⁶ pixels) | +| d, e | JPEG XL distance (quality: 0 = lossless, higher = smaller and lossier) and effort (1–9) | +| PSNR | peak signal-to-noise ratio, in dB | +| SSIM | structural similarity index, 0–1 (1 = identical) [6] | +| stop | a factor of 2 in light or brightness: brightened by 2 stops = pixel values ×4; ×16 = 4 stops | +| IOTA, Delta | LattePanda IOTA (tower 121) and LattePanda 3 Delta (tower 111) | + +--- + +## 1. Introduction + +### 1.1 Background and question + +Each tower scans a set of waypoints and keeps one picture per waypoint. After the gimbal stops, the +camera's auto-exposure needs a number of frames to adapt to the new view, so the scan waits; and +each kept picture is compressed to JPEG XL [2] on the tower. Today the camera converts its sensor +data to 8-bit colour (RGB8) itself. That format is three times larger than the raw sensor data, which +limits the frame rate over the camera's USB link and thereby lengthens the exposure settling. + +This report answers: should the towers take raw 12-bit data from the camera instead, and convert it +to colour on the tower? Specifically: (1) how much faster is the camera, and does exposure settle +faster; (2) how long does each processing step take on each tower computer; (3) how large are the +files; (4) can both computers keep up without piling up images or overheating; and (5) what do the +extra bits add. + +### 1.2 The camera's pixel formats + +The sensor sees through a colour-filter mosaic: each pixel measures only red, green or blue, in the +pattern of Figure 1a ("BayerRG": a red pixel first). Producing a colour image means estimating the +two missing colours at every pixel from its neighbours, called **de-Bayering** (demosaicing). The +camera can either send the raw mosaic or de-Bayer it itself. + +``` +R G R G R G ... +G B G B G B ... +R G R G R G ... +``` + +***Figure 1a.** The BayerRG colour-filter pattern of the sensor: each pixel records one colour.* + +***Table 1.** Pixel formats considered, at the 1620c's full resolution (5328 × 3040 pixels) and a +USB link cap of 250 MB/s [4]. Bits per value: the precision of each stored measurement (8 bits = 256 +levels, 10 bits = 1024, 12 bits = 4096). Packed: values stored back to back without padding bits. +Maximum rate: the highest frame rate the camera accepts at this link cap.* + +| Format | What the camera sends | Bits per value | Bytes per pixel | Frame size (MB) | Maximum rate (fps) | Colour made by | +|---|---|---|---|---|---|---| +| `RGB8` (today) | three values per pixel, two of them estimated | 8 | 3 | 48.6 | 4.95 | the camera | +| `BayerRG8` | the raw mosaic, top 8 of the sensor's 12 bits | 8 | 1 | 16.2 | 14.75 | the tower | +| `BayerRG10p` | the raw mosaic, 10 bits, packed (4 pixels in 5 bytes) | 10 | 1.25 | 20.2 | 11.81 | the tower | +| **`BayerRG12p`** (proposed) | the raw mosaic, all 12 bits, packed (2 pixels in 3 bytes) | 12 | 1.5 | 24.3 | **9.86** | the tower | +| `BayerRG12` | the same 12 bits, each in a 16-bit word (4 bits of padding) | 12 | 2 | 32.4 | 7.41 | the tower | + +RGB8 is the largest because two of its three values per pixel are estimates; it carries no more +measured information than `BayerRG8`. `BayerRG12p` and `BayerRG12` hold identical data; packing saves +25% of the bytes and so allows 33% more frames. The camera applies its white balance to the raw +formats too, so the tower does not need to. The camera also offers grey (`Mono8/10/12`, which +discards colour), `BGR8` (RGB8 in reverse order) and video formats (`YCbCr…`, which reduce colour +detail); none of these was considered. + +**Saving raw data: mosaic and planes.** Besides converting to colour, the tower can save the raw +12-bit data as received, each pixel keeping the single value measured through its filter. Two +layouts were tested; they hold identical values and differ only in their arrangement in the file: + +- **Raw mosaic:** the sensor data as the camera sends it, a single-channel (grey) image of + 5328 × 3040 values still in the R G / G B pattern. Viewed directly it looks like a grey image with + a fine checkerboard texture. Because colours alternate from pixel to pixel, the encoder sees the + pattern as extremely fine detail: it compresses poorly (20.9 MB lossless), and lossy compression + would mix values of different colours. +- **Raw planes:** the same values reordered into four quarter-size images (2664 × 1520 each), one per + filter position (red, first green, second green, blue), tiled 2 × 2 into one grey image of the + same total size (Figure 1b). Within each tile neighbouring values belong to the same colour and are + similar, so the image is smooth to the encoder: 17.6 MB lossless, and lossy compression works + (distance 0.5: 6.0 MB at a PSNR of 47.2 dB, against 38.8 dB for 12-bit colour of similar size; + Section 3.7). The reordering is exactly reversible, so the mosaic can always be rebuilt. + +``` +Mosaic (5328 × 3040) Planes (2 × 2 tiles of 2664 × 1520) +R G R G R G ... R R R ... | G G G ... +G B G B G B ... → R R R ... | G G G ... +R G R G R G ... ----------+---------- +G B G B G B ... G G G ... | B B B ... + G G G ... | B B B ... +``` + +***Figure 1b.** Rearranging the raw mosaic into four single-colour planes.* + +Neither layout is a viewable colour picture: the reader must de-Bayer it, as the tower does before +saving colour. Their value is that they preserve the measured sensor values, with nothing estimated +and no colour processing applied, which matters only if the detection algorithms need exact sensor +data. Lossless planes take about 6 times the storage of 12-bit colour at distance 1.0 (17.6 MB +against 2.65 MB per frame), so they are an option (recommendation 6, Section 5) and are not part of +the current implementation. + +### 1.3 Earlier work + +An earlier campaign ([camera-capture-measurements.md](camera-capture-measurements.md)) used blurred, +dim indoor scenes and found that the camera link is stable at full resolution once the USB settings +of known issue #19 are applied; its compression figures were not representative of real scenes. +The implementation built on this report is described in [raw12-pipeline-implementation-plan.md](raw12-pipeline-implementation-plan.md). -## Conclusions +--- -| Topic | Finding | Evidence | -|---|---|---| -| Frame rate | `BayerRG12p` runs at **9.86 fps**, 2× RGB8's 4.95 fps, at the 250 MB/s cap; `BayerRG8` 14.75 fps. Unpacked `BayerRG12` only 7.4 fps: use the packed format. | [streaming](#frame-rate-and-streaming) | -| Link | No lost or incomplete frames in any format on either tower, also while saving 12-bit flat out. | [streaming](#frame-rate-and-streaming) | -| Auto-exposure | From **under-exposure** (1/16, brightness ~11 of 255) or **over-exposure** (×16, 97–100% blown out), settling takes **~25–35 frames**, so time scales with 1/fps: raw 8-bit 1.8–2.7 s, **12-bit 2.2–6.6 s (typically ~3 s)**, RGB8 4.6–6.6 s. | [method](#method), [auto-exposure](#auto-exposure) | -| Which frame to keep | **"Brightness in the camera's target band and steady for 3 frames"**: right in 21 of 24 steps. "Exposure and gain unchanged" was never right. | [settle rules](#which-frame-to-keep) | -| Auto white balance | Recovers in **2–4 frames** (0.2–0.8 s) at every rate; stable within ±1–3% on a still scene. Exposure, not white balance, sets the wait. | [white balance](#auto-white-balance) | -| Processing cost | Same frame on both computers: unpack + colour conversion **38 ms (IOTA) / 50 ms (Delta)**; JPEG XL encode 480 / 765 ms. **A 12-bit save (518 / 820 ms) costs a little less than today's RGB8 save (567 / 872 ms).** | [stages](#where-the-time-goes) | -| IOTA vs 3 Delta | **The Delta takes 1.6× as long** on the identical frame (820 vs 518 ms for a 12-bit save); it saves 1.19 frames/s with one worker, 1.53 with two (IOTA 1.79 / 2.15). | [stages](#where-the-time-goes) | -| Pace | The camera side sets the pace: a waypoint needs ~1.5–5 s (move + exposure), a 12-bit save 0.56 s (IOTA) or 0.65 s (Delta, two workers). On the shared frame the IOTA keeps up with one frame every **0.7 s** with one worker; the Delta needs **two workers** for that. | [pace](#overall-pace-how-fast-can-a-tower-capture), [pile-up](#keeping-up-with-the-scan) | -| Heat | One frame every 2 s: IOTA **53 °C / 5.1 W**, Delta **66 °C / 5.7 W**. Flat out: ~80 °C on both; the IOTA slowed 16% over 10 min, the Delta held steady at its 8 W power limit. | [heat](#heat-and-power) | -| Storage | 12-bit colour at distance 1.0: **2.7 MB/frame**, less than today's 3.7–4.0 MB. Lossless 12-bit: 44–47 MB as colour, **17–18 MB as raw planes**. | [quality](#quality-setting-size-and-quality) | -| **Value of 12 bits** | Sensor noise in this sunlit scene is **2–6 steps of 8-bit even at 0 dB gain**, so 8-bit already holds what the sensor delivers in mid and bright tones. 12-bit adds information only below about **300 of 4095** (deep shadow), which daylight scenes barely have and dusk or night scenes do. | [bit depth](#what-12-bit-adds) | +## 2. Materials and methods -### Recommendation +### 2.1 Hardware and software -1. **Take raw from the camera** (`BayerRG12p`, 250 MB/s) and convert to colour on the tower, - edge-aware. It doubles the frame rate and roughly halves the settle time, at no extra CPU, - heat or storage cost compared with today, on either computer. -2. **Save 12-bit colour JPEG XL at distance 1.0, effort 3, 4 threads.** 2.7 MB per frame. -3. **Workers per computer:** one worker on the IOTA; two workers × 2 threads on the Delta, which - then keeps up with a frame every 1.0 s. With the measured settle times (~2–3 s per waypoint), - the scan pace stays below either limit. The save queue still needs a bound. -4. **Keep the frame by "target band and steady for 3 frames",** with a time limit. -5. **12-bit vs raw 8-bit is a choice, not a cost.** `BayerRG8` (14.75 fps) settles about 1 s faster - in daylight and loses nothing measurable there; `BayerRG12p` keeps the deep shadows at dusk and - night. The pixel format lives in the camera profile, so the code should accept both, and - `standard` and `night` can each use the right one. -6. **Store raw 12-bit planes losslessly (17–18 MB) only if the detection pipeline wants exact sensor - values**; it is 6× the storage. - -## Setup +***Table 2.** Test systems. The same camera was moved from tower 121 to tower 111, so only the +computer differs between the towers. Storage write speed: sequential writing to the built-in eMMC +flash storage.* | | Tower 121 | Tower 111 | |---|---|---| -| Computer | LattePanda IOTA, Intel N150, 4 cores, up to 3.6 GHz, 7 GB | LattePanda 3 Delta, Celeron N5105, 4 cores, up to 2.9 GHz, 7.6 GB | -| Storage write | eMMC, 218 MB/s | eMMC, 76 MB/s | -| Libraries | libjxl 0.11.1, OpenCV 4.10 | libjxl 0.11.0, OpenCV 4.5.4 | -| Date, light | 2 Oct, 15:01–16:50, sun | 4 Oct, 16:19–18:19, sun then evening | -| Camera | Alvium 1800 U-1620c `DEV_1AB22C0B7D4E`, 5328×3040, link cap 250 MB/s | same camera | -| Exposure | capped at 20 ms (the `standard` profile), so ~15 dB gain in this scene | same | +| Computer | LattePanda IOTA: Intel N150, 4 cores, ≤ 3.6 GHz, 7 GB RAM | LattePanda 3 Delta: Intel Celeron N5105, 4 cores, ≤ 2.9 GHz, 7.6 GB RAM | +| Storage write speed (MB/s) | 218 (eMMC) | 76 (eMMC) | +| Libraries | libjxl 0.11.1 [2], OpenCV 4.10 [3] | libjxl 0.11.0, OpenCV 4.5.4 | +| Camera | Alvium 1800 U-1620c `DEV_1AB22C0B7D4E` [5], 5328 × 3040 pixels | same camera | +| Link cap (MB/s) | 250 | 250 | +| Test dates and light | 2 Oct, 15:01–16:50, sun | 4 Oct, 16:19–18:19, sun, then evening | -The scene, through the camera: spruce, birch trunks, leaves, a building, ~13% blown-out sky. +The tower control program was stopped during every test (the camera accepts one program at a time, +and the CPU figures should have a single cause). Afterwards both cameras were returned to their +`standard` profile. + +### 2.2 Scene ![The scene](images/raw12/examples/121/thumbs/c12_ea_lossless.jpg) -The tower program was stopped during every test, and both cameras went back to their `standard` -profiles afterwards. +***Figure 2.** The test scene from tower 121's camera (2 Oct, 15:07; exposure 20 ms, gain ≈ 15 dB): +spruce, birch trunks, leaves and a building. About 13% of the pixels are clipped sky: saturated at +the maximum value, so they show pure white without detail.* -## Method +### 2.3 Camera control -Everything ran on the towers themselves, with the tower program stopped (the camera accepts one -program at a time, and the CPU figures should have a single cause). The test programs are in -[tools/camera-bench](../tools/camera-bench/README.md); the raw results, one JSON line per -measurement, are linked at the end. +A Python program (`camera_bench.py`, Allied Vision vmbpy) drives the camera through the same GenICam +interface [4] the tower program uses. Each setting change is a GenICam feature write sent over the +USB control channel **while the camera keeps streaming**. `ExposureAuto`, `GainAuto` and +`BalanceWhiteAuto` take the values `Off`, `Once` or `Continuous`; setting one to `Off` freezes the +value the automatic control had reached, after which `ExposureTime`, `Gain` and `BalanceRatio` can be +written directly and take effect from the next frame exposed. No profile, user set or file on the +camera is changed. -### How the camera is driven +***Table 3.** Camera settings common to all camera tests. Names in code font are the camera's +GenICam features [4]. Chunk data: values the camera attaches to each frame, describing the settings +it was taken with.* -`camera_bench.py` talks to the camera through Allied Vision's Python SDK (vmbpy), the same -GenICam interface the tower program uses through C++. Every setting change is a **GenICam feature -write** (`feature.set(value)`), sent over the USB control channel **while the camera keeps -streaming**. `ExposureAuto`, `GainAuto` and `BalanceWhiteAuto` are switches with the values `Off`, -`Once` and `Continuous`. Setting one to `Off` freezes the value the automatic control had reached; -`ExposureTime`, `Gain` and `BalanceRatio` can then be written directly and take effect from the next -frame the camera exposes. No profile, user set or file on the camera is changed; afterwards the -camera is put back by reloading its `standard.xml` profile, exactly as the tower program does. - -Common settings for every camera test: - -| Setting | Value | Why | +| Setting | Value | Reason | |---|---|---| -| Resolution | 5328×3040, binning 1 | full resolution, as decided | -| `DeviceLinkThroughputLimit` | 250 MB/s | the agreed conservative link cap | -| `AcquisitionFrameRate` | the camera's maximum for the format | 9.86 fps 12-bit, 14.75 fps 8-bit raw, 4.95 fps RGB8 | -| `ExposureAutoMax` | 20 ms | as in the `standard` profile; keeps long exposures from limiting the frame rate | -| `IntensityAutoPrecedence` | `MinimizeNoise` | as the camera was left (favours exposure over gain; with exposure capped at 20 ms, the camera then uses gain) | +| Resolution | 5328 × 3040 pixels, binning 1 | full resolution, as decided | +| `DeviceLinkThroughputLimit` | 250 MB/s | agreed conservative link cap | +| `AcquisitionFrameRate` | the camera's maximum for the format (fps) | 9.86 (12-bit), 14.75 (8-bit raw), 4.95 (RGB8) | +| `ExposureAutoMax` | 20 ms | as in the `standard` profile | +| `IntensityAutoPrecedence` | `MinimizeNoise` | favours exposure over gain; with exposure at its 20 ms cap, the camera then uses gain | | `IntensityControllerTarget` / `Tolerance` / `Rate` | 50% / 5% / 100 | camera defaults; rate 100 is the fastest | -| `GainAutoMax` | as left in the camera (≥ 43.5 dB) | the bright steps reached 43.5 dB | -| Chunk data | on: `ExposureTime`, `Gain`, `BalanceRatioRed`, `BalanceRatioBlue`, `Timestamp` | each frame carries the exact settings it was exposed with | -| Buffers | 5 | as production | +| `GainAutoMax` | ≥ 43.5 dB (as left in the camera) | the bright steps reached 43.5 dB | +| Chunk data (per-frame metadata) | `ExposureTime`, `Gain`, `BalanceRatioRed`, `BalanceRatioBlue`, `Timestamp` | each frame carries the settings it was exposed with | +| Buffers | 5 | as in production | -**Per-frame record.** For every frame the script keeps the host arrival time, the camera's frame -number, complete or not, the chunk data above, and the frame's **brightness and blown-out share**, -sampled from the image buffer itself: every 997th byte for RGB8 and 8-bit raw (mixed colours), the -high byte of every 331st pixel pair for `BayerRG12p` (green and blue sites), each scaled to 0–255. -"Blown out" means a sample ≥ 250 of 255. +For every frame the program records the host arrival time, the camera's frame number, completeness, +the chunk data, and the frame's **brightness** (DN, 0–255) and **clipped share** (% of samples +≥ 250 DN), sampled from the image buffer: every 997th byte for RGB8 and 8-bit raw, and the high byte +of every 331st pixel pair for `BayerRG12p`, scaled to 0–255 DN. -### Frame rate and streaming +### 2.4 Frame rate and streaming -For each format: set it, read the camera's maximum frame rate (the camera computes it from the -link cap and frame size), stream 60 s at that rate, and count frames received, incomplete frames -(the SDK's frame status) and lost frames (gaps in the camera's frame counter). Once more for -`BayerRG12p` while the tower saved 12-bit images flat out in parallel. +For each format, the camera's maximum frame rate was read, then the camera streamed for 60 s at that +rate. Counted: frames received, incomplete frames (SDK frame status) and lost frames (gaps in the +camera's frame counter). `BayerRG12p` was streamed once more while the tower saved 12-bit images +continuously in parallel. -### Auto-exposure step test +### 2.5 Auto-exposure step test -One run, per format and direction (`camera_bench.py ae`): +***Table 4.** The three phases of one auto-exposure step run. Exposure is the time the sensor +collects light (ms); gain amplifies the sensor signal (dB; +6 dB doubles the brightness, +20 dB +multiplies it by 10). A step of ×16 makes the image 16 times brighter (×1/16: 16 times darker), i.e. +4 stops, where one stop is a factor of 2.* -| Phase | Duration | Exactly what is done | +| Phase | Duration (s) | Procedure | |---|---|---| -| 1. Settle | 8 s | Stream with `ExposureAuto = Continuous`, `GainAuto = Continuous`. "Before" = median exposure and gain of the last 5 frames (chunk data). The brightness the camera steers is exposure × linear gain: E₀ = exposure × 10^(gain/20). | -| 2. Force | 2 s | `ExposureAuto = Off`, `GainAuto = Off` (freezes them). Then write a new exposure and gain. **Dark:** `ExposureTime` = max(`ExposureAutoMin`, E₀/16), `Gain` = 0 dB. **Bright:** `ExposureTime` = min(20 ms, 16·E₀), and the rest as gain: `Gain` = 20·log10(16·E₀ / exposure), up to `GainAutoMax`. Streaming continues; the camera delivers 10–30 frames at the forced settings. | -| 3. Release | 15 s | Time zero is taken, then `ExposureAuto = Continuous` and `GainAuto = Continuous` are written. Every frame for 15 s is recorded. | +| 1. Settle | 8 | `ExposureAuto` and `GainAuto` = `Continuous`. "Before" = median exposure and gain of the last 5 frames. The brightness the camera steers is E₀ = exposure × 10^(gain/20). | +| 2. Force | 2 | `ExposureAuto` = `GainAuto` = `Off`. **Dark:** `ExposureTime` = max(`ExposureAutoMin`, E₀/16), `Gain` = 0 dB. **Bright:** `ExposureTime` = min(20 ms, 16·E₀); the remainder as `Gain` = 20·log₁₀(16·E₀ / exposure) dB, up to `GainAutoMax`. The camera delivers 10–30 frames at the forced settings. | +| 3. Release | 15 | Time zero is taken, then `ExposureAuto` = `GainAuto` = `Continuous`; every frame is recorded. | -With exposure at its 20 ms cap before the step, the forced values came out as: +***Table 5.** Starting conditions of the auto-exposure steps: the first frame after release, ranges +over all 24 runs. Before: the settled state before forcing. Dark / bright step: the settings forced +16 times darker / brighter. Brightness: mean pixel value on the 0–255 scale (DN). Clipped share: +percentage of sampled pixels at or above 250 DN, i.e. saturated white without detail.* -| | Before (settled) | Forced "dark" (start) | Forced "bright" (start) | +| | Before (settled) | Start of "dark" step | Start of "bright" step | |---|---|---|---| -| Exposure | 20 ms (at its cap) | 7.4–11.5 ms | 20 ms | -| Gain | 15.4–19.5 dB | 0 dB | 39.6–43.5 dB (before + 24.1 dB) | -| Brightness (0–255), first frame after release | 117–122 | **9–13: under-exposed** | **254–255: over-exposed** | -| Blown-out share | 10–15% (sky) | 0% | **97–100%** | +| Exposure (ms) | 20 (at its cap) | 7.4–11.5 | 20 | +| Gain (dB) | 15.4–19.5 | 0 | 39.6–43.5 (before + 24.1) | +| Brightness (DN, 0–255) | 117–122 | **9–13 (under-exposed)** | **254–255 (over-exposed)** | +| Clipped share (%) | 10–15 (sky) | 0 | **97–100** | -So settling was measured from **both sides, under-exposure and over-exposure**. The bright start is -fully saturated: the camera cannot see how far over it is and can only step down until detail -reappears. +Settling was therefore measured from both under- and over-exposure. The bright start is fully +saturated, so the camera cannot measure how far over it is and can only step down until detail +reappears. Each format and direction was run twice: 12 runs per tower, 24 in total. -Each format × direction was run twice: 12 runs per tower, 24 in all. **"Settled"** is judged on the -image: the time from release after which the frame's brightness stays within ±10% of where it -finally settles (median brightness of the run's last 2 s), with the number of frames that took. -The first 1–5 frames after release still carry the forced values (frames already in the -camera-to-host pipeline); they count towards the settle time. +**Settling time** is judged on the image: the time from release after which brightness stays within +±10% of its final level (median over the run's last 2 s), and the number of frames this took. The +first 1–5 frames after release still carry the forced values (frames in transit); they count +towards the settling time. -**Keep rules** were then scored offline on the same 24 per-frame logs: for each rule, the first frame -at which it would have kept a picture, and whether that frame's brightness was within 10% of the -final level ("correct") or not ("early"). The camera's target band is 50% ± 5% of full scale, i.e. -114.8–140.3 on the 0–255 scale used here. +### 2.6 Evaluation of keep rules -### Auto white balance step test +A keep rule decides which frame a waypoint keeps. Each candidate rule was applied offline to the 24 +per-frame logs; for each, the first frame at which it would keep a picture was found, and counted as +**correct** if that frame's brightness was within 10% of the final level, otherwise **early**. The +camera's target band is 50 ± 5% of full scale, i.e. 114.8–140.3 DN on the 0–255 scale. -One run (`camera_bench.py awb`), per format and direction: +### 2.7 Auto white balance step test -| Phase | Duration | Exactly what is done | +***Table 6.** The four phases of one auto-white-balance step run. White-balance ratios are the gains +the camera applies to the red and blue values relative to green (no unit; 1 = unchanged); ×0.5 +halves them and ×2 doubles them.* + +| Phase | Duration (s) | Procedure | |---|---|---| -| 1. Exposure | 6 s | Auto-exposure on, settle; then `ExposureAuto = Off`, `GainAuto = Off`, and `ExposureTime`/`Gain` written to the median of the last 5 frames, so only white balance moves from here on. `BalanceWhiteAutoRate = 100` (fastest). | -| 2. Reference | 8 s | `BalanceWhiteAuto = Continuous`. The reference = median red and blue ratios of the frames in the last 3 s (the camera's own answer for this scene: red 2.34–2.46, blue 1.86–1.97). | -| 3. Force | 2 s | `BalanceWhiteAuto = Off`; `BalanceRatioSelector = Red`, `BalanceRatio` = reference × factor; the same for Blue (green is the camera's fixed reference). **×0.5**: red ≈ 1.2, blue ≈ 0.95 (image too green); **×2**: red ≈ 4.8, blue ≈ 3.8 (too magenta). | -| 4. Release | 15 s | Time zero, then `BalanceWhiteAuto = Continuous`; every frame's ratios recorded (chunk data). | +| 1. Exposure | 6 | Auto-exposure settles; then `ExposureAuto` = `GainAuto` = `Off` with `ExposureTime` and `Gain` fixed at the median of the last 5 frames, so only white balance can change. `BalanceWhiteAutoRate` = 100 (fastest). | +| 2. Reference | 8 | `BalanceWhiteAuto` = `Continuous`. Reference = median red and blue ratios over the last 3 s (red 2.34–2.46, blue 1.86–1.97). | +| 3. Force | 2 | `BalanceWhiteAuto` = `Off`; red and blue `BalanceRatio` = reference × factor (green is the fixed reference). ×0.5: red ≈ 1.2, blue ≈ 0.95 (image too green); ×2: red ≈ 4.8, blue ≈ 3.8 (too magenta). | +| 4. Release | 15 | Time zero, then `BalanceWhiteAuto` = `Continuous`; every frame's ratios recorded. | -"Settled" = both ratios within ±5% of the reference and staying there. Each case twice; also the -spread of the reference over phase 2 on the still scene. +Settled = both ratios within ±5% of the reference and remaining there. Each case was run twice. -### Processing tests +### 2.8 Processing tests -`raw12_bench.cpp`, built on each tower against its own libjxl and OpenCV, runs the steps the tower -program would run for one saved frame and **times each with a monotonic clock**: +A C++ program (`raw12_bench.cpp`), built on each tower against its own libraries, performs the steps +the tower program would perform for one saved frame and times each with a monotonic clock. -| Step | What it does | +***Table 7.** Processing steps timed for one saved frame. Bilinear conversion estimates each missing +colour as the average of the nearest pixels of that colour; edge-aware conversion interpolates along +edges rather than across them, giving fewer false colour fringes at a slightly higher cost. Distance +and effort are the JPEG XL quality and speed settings (Units and abbreviations).* + +| Step | Operation | |---|---| | Unpack | `BayerRG12p` (two 12-bit pixels in three bytes) → one 16-bit word per pixel | -| Colour conversion | OpenCV `cvtColor`, `COLOR_BayerBG2RGB` (bilinear) or `…_EA` (edge-aware), 8- or 16-bit; the camera's `BayerRG` is OpenCV's `BayerBG` (checked against the camera's own RGB8) | -| JPEG XL encode | libjxl into memory; threads via libjxl's thread pool; effort and distance as stated; 12-bit input as 16-bit words with the bit depth taken from the file header; lossless with `uses_original_profile` set | +| Colour conversion | OpenCV `cvtColor`, bilinear (`COLOR_BayerBG2RGB`) or edge-aware (`…_EA`), 8- or 16-bit [3]; the camera's `BayerRG` corresponds to OpenCV's `BayerBG` (verified against the camera's RGB8) | +| JPEG XL encode | libjxl [2], into memory; effort and distance as stated; 12-bit input as 16-bit words with the bit depth taken from the file header; lossless with the original colour profile | | File write | `fopen`/`fwrite`/`fflush`/`fclose` to the tower's eMMC | -Each figure is the median of 3 runs (2 in the quality sweep). Before every run on every tower the -program checks that its 8-bit encoder produces **byte for byte** the file production's encoder -produces, that 12-bit lossless decodes to **exactly** the input, and that its colour order matches -the camera's. +Each time is the median of 3 runs (2 in the quality sweep). Before every run, the program verified +that its 8-bit encoder produces byte-for-byte the file of the production encoder, that 12-bit +lossless decodes exactly to the input, and that its colour order matches the camera's. **Input frames.** -- **Shared frame, identical on both towers (the tower comparison uses this):** tower 121's frame of - 2 Oct (15:07, 20 ms, ~15 dB), rebuilt bit-exactly from its lossless 12-bit file and repacked to - `BayerRG12p`; its 8-bit raw is the 12-bit value shifted down 4 bits; its "RGB8" is the edge-aware - colour conversion of the 12-bit frame shifted to 8 bits (the camera's own RGB8 of that instant - does not exist). Copied to both towers; MD5 checksums identical. -- **Synthetic frame, identical on both towers:** a "dead leaves" texture (the camera texture chart - of ISO 19567-2) with sensor noise, same generator and seed: sharper than any real scene, the worst - case for the encoder. +- **Shared frame** (identical on both towers; used for the tower comparison): tower 121's frame of + 2 Oct (15:07, 20 ms, ≈ 15 dB), rebuilt bit-exactly from its lossless 12-bit file. Its 8-bit raw + version is the 12-bit value shifted down by 4 bits; its "RGB8" version is the edge-aware colour + conversion of the 12-bit frame, scaled to 8 bits (the camera's own RGB8 of that instant does not + exist). Identical MD5 checksums on both towers. +- **Synthetic frame** (identical on both towers): a "dead leaves" texture [1] with added sensor + noise; sharper than any real scene and therefore the worst case for the encoder. - Each tower's own camera capture from its test day. -**Quality.** Every 12-bit result is decoded and compared with the exact lossless 12-bit colour image -of the same frame (mosaic results are converted to colour the same way first): PSNR on the 12-bit -scale, and SSIM on luma, which follows structure and is less fooled by noise. +### 2.9 Image quality metrics -**Load.** *Back to back:* workers save the same frame from memory continuously for 60 s. -*Paced:* a producer queues one frame every 2.0, 1.5, 1.0, 0.7 or 0.5 s for 2 minutes, like a scan -keeping one frame per waypoint; each job runs every step above; recorded: time from queueing to file -written, frames still queued at the end, peak memory. *Heat:* 10 minutes flat out and 10 minutes at -one frame every 2 s, each after a 5-minute cool-down, with once a second: CPU package temperature -(`x86_pkg_temp`), average core clock, CPU busy share (`/proc/stat`) and package power (Intel RAPL -energy counter, read through a root helper). +Every 12-bit result was decoded and compared with the exact lossless 12-bit colour image of the same +frame (mosaic results first converted to colour in the same way): PSNR (dB) over all three channels on +the 12-bit scale, and SSIM [6] on luma. -### Noise test +### 2.10 Throughput, pile-up and heat -To see whether the 12th bit carries information, sensor noise was measured on the raw 12-bit -values: the green sites split into 24×24-pixel patches; in each, noise = the robust spread (median -absolute deviation) of differences between horizontal neighbours ÷ √2; patches binned by brightness; -the quietest 10% in each bin taken as the noise floor (texture only adds to it). Once for the -~15 dB frame and once for a frame taken at **0 dB gain** (124 ms exposure, 4 Oct 16:19, exposure -cap temporarily raised to 200 ms). +*Back to back:* save workers process the same frame from memory continuously for 60 s. +*Paced:* one frame is queued every 2.0, 1.5, 1.0, 0.7 or 0.5 s for 120 s, as a scan keeps one frame +per waypoint; recorded are the time from queueing to file written, the frames still queued at the end +("behind") and the peak memory. *Heat:* 10 minutes back to back and 10 minutes at one frame every 2 s, +each after a 5-minute cool-down; recorded once per second: CPU package temperature, average core +clock, CPU busy share and package power (Intel RAPL energy counter [7]). -### Campaign log +### 2.11 Noise measurement -| When | Tower | What | Duration | Notes | +Sensor noise was measured on the raw 12-bit values: the green sites were divided into 24 × 24-pixel +patches; in each, noise σ = median absolute deviation of differences between horizontal neighbours, +÷ √2; patches were grouped by brightness, and the quietest 10% of each group taken as the noise floor +(scene texture can only add to it). This was done for the ≈ 15 dB frame and for a frame taken at +**0 dB gain** (exposure 124 ms, 4 Oct 16:19, exposure cap temporarily raised to 200 ms). + +### 2.12 Campaign log + +***Table 8.** Campaign log: date and local time, tower, activity and duration (min) of each test +session.* + +| Date and time | Tower | Activity | Duration (min) | Notes | |---|---|---|---|---| -| 2 Oct 15:01–15:22 | 121 | camera tests | 21 min | sunny | -| 2 Oct 15:22–16:50 | 121 | processing, pile-up, heat on its own frame and the synthetic frame | 88 min | | -| 2 Oct 19:01 | 121 | attempt at a 0 dB frame | – | too dark (24 dB needed); discarded | -| 4 Oct 16:19 | 111 | 0 dB frame (camera moved from 121) | 2 min | | -| 4 Oct 16:20–16:41 | 111 | camera tests | 21 min | 111's own 1240c camera connected but idle | -| 4 Oct 16:41–18:19 | 111 | processing, pile-up, heat | 98 min | | -| 4 Oct 23:03–23:32 | 121 + 111 | processing on the shared frame, both towers at once | 29 min | no camera needed | +| 2 Oct 15:01–15:22 | 121 | camera tests | 21 | sunny | +| 2 Oct 15:22–16:50 | 121 | processing, pile-up, heat (own and synthetic frames) | 88 | | +| 2 Oct 19:01 | 121 | attempted 0 dB frame | – | too dark (24 dB needed); discarded | +| 4 Oct 16:19 | 111 | 0 dB frame (camera moved from 121) | 2 | | +| 4 Oct 16:20–16:41 | 111 | camera tests | 21 | tower 111's own camera connected but idle | +| 4 Oct 16:41–18:19 | 111 | processing, pile-up, heat | 98 | | +| 4 Oct 23:03–23:32 | 121 and 111 | processing on the shared frame, both towers simultaneously | 29 | no camera needed | -## Frame rate and streaming +--- + +## 3. Results + +### 3.1 Frame rate and streaming ![Frame rate per format](images/raw12/fps_per_format.png) -Same camera on both towers, so identical maximum rates. 60 s at the maximum: +***Figure 3.** Maximum full-resolution frame rate (fps) per pixel format at the 250 MB/s link cap. +The same camera was used on both towers.* -| Format | Frame | Max fps | 121: complete / incomplete / lost | 111: complete / incomplete / lost | +***Table 9.** Results of 60 s streams at each format's maximum rate. Complete: frames received +intact. Incomplete: frames whose data arrived only partly (SDK frame status). Lost: frames missing +from the camera's frame counter. Last row: the same stream while the tower saved 12-bit images +continuously in parallel.* + +| Format | Frame size (MB) | Maximum rate (fps) | 121: complete / incomplete / lost (frames) | 111: complete / incomplete / lost (frames) | |---|---|---|---|---| -| BayerRG12p | 24.3 MB | 9.86 | 593 / 0 / 0 | 592 / 0 / 0 | -| BayerRG12 (unpacked) | 32.4 MB | 7.41 | 445 / 0 / 0 | 445 / 0 / 0 | -| BayerRG10p | 20.2 MB | 11.81 | 711 / 0 / 0 | 709 / 0 / 0 | -| BayerRG8 | 16.2 MB | 14.75 | 887 / 0 / 0 | 887 / 0 / 0 | -| RGB8 (today) | 48.6 MB | 4.95 | 298 / 0 / 0 | 297 / 0 / 0 | -| **BayerRG12p while saving 12-bit flat out** | 24.3 MB | 9.86 | 591 / 0 / 0 | 592 / 0 / 0 | +| BayerRG12p | 24.3 | 9.86 | 593 / 0 / 0 | 592 / 0 / 0 | +| BayerRG12 (unpacked) | 32.4 | 7.41 | 445 / 0 / 0 | 445 / 0 / 0 | +| BayerRG10p | 20.2 | 11.81 | 711 / 0 / 0 | 709 / 0 / 0 | +| BayerRG8 | 16.2 | 14.75 | 887 / 0 / 0 | 887 / 0 / 0 | +| RGB8 (today) | 48.6 | 4.95 | 298 / 0 / 0 | 297 / 0 / 0 | +| BayerRG12p while saving 12-bit continuously | 24.3 | 9.86 | 591 / 0 / 0 | 592 / 0 / 0 | -On tower 111 the camera shares the USB 3 controller with the 5G modem, a USB network adapter and -the tower's second camera (idle during the test). +No frame was lost or incomplete in any format on either tower. On tower 111 the camera shares the +USB 3 controller with a 5G modem, a USB network adapter and a second (idle) camera. -## Auto-exposure +### 3.2 Auto-exposure settling ![Brightness after a step, 121](images/raw12/ae_brightness_121.png) ![Brightness after a step, 111](images/raw12/ae_brightness_111.png) -With exposure at its 20 ms cap, auto-exposure works through gain, 1–2 dB per frame, and sometimes -pauses mid-way (the flat stretches, up to 2.8 s while still far too bright). +***Figure 4.** Frame brightness (DN, 0–255) against time (s) after auto-exposure is switched back on +at time 0, for each format: (a, top) tower 121; (b, bottom) tower 111. Left: starting from +under-exposure, 16 times (4 stops) too dark; right: from over-exposure, 16 times too bright. Dashed +line: settled level, the median brightness over the run's last 2 s.* -Time until brightness stays within ±10% of where it settles (two runs each): +With exposure at its 20 ms cap, auto-exposure adjusts through gain, by 1–2 dB per frame, and +sometimes pauses mid-way (flat segments in Figure 4, up to 2.8 s while still far too bright). -| Format | fps | Step | Tower 121 | Tower 111 | +***Table 10.** Settling time after a forced exposure step: time (s) from release, and in parentheses +the number of frames, until brightness stays within ±10% of its final level (median of the run's +last 2 s). Dark ×1/16: started 16 times (4 stops) too dark; bright ×16: 16 times too bright. Two +runs per cell.* + +| Format | Rate (fps) | Step | Tower 121: time (s) (frames) | Tower 111: time (s) (frames) | |---|---|---|---|---| -| BayerRG8 | 14.75 | dark ×1/16 | 1.94 s (28 frames), 2.21 s (32) | 1.79 s (26), 1.87 s (27) | -| BayerRG8 | 14.75 | bright ×16 | 2.69 s (39), 2.55 s (37) | 2.00 s (29), 1.86 s (27) | -| **BayerRG12p** | 9.86 | dark ×1/16 | 2.90 s (28), 3.00 s (29) | 6.64 s (65), 4.51 s (44) | -| **BayerRG12p** | 9.86 | bright ×16 | 6.45 s (63), 2.69 s (26) | 2.18 s (21), 5.73 s (56) | -| RGB8 (today) | 4.95 | dark ×1/16 | 4.56 s (22), 5.17 s (25) | 5.97 s (29), 5.77 s (28) | -| RGB8 (today) | 4.95 | bright ×16 | 5.17 s (25), 5.17 s (25) | 6.57 s (32), 4.96 s (24) | +| BayerRG8 | 14.75 | dark ×1/16 | 1.94 (28), 2.21 (32) | 1.79 (26), 1.87 (27) | +| BayerRG8 | 14.75 | bright ×16 | 2.69 (39), 2.55 (37) | 2.00 (29), 1.86 (27) | +| **BayerRG12p** | 9.86 | dark ×1/16 | 2.90 (28), 3.00 (29) | 6.64 (65), 4.51 (44) | +| **BayerRG12p** | 9.86 | bright ×16 | 6.45 (63), 2.69 (26) | 2.18 (21), 5.73 (56) | +| RGB8 (today) | 4.95 | dark ×1/16 | 4.56 (22), 5.17 (25) | 5.97 (29), 5.77 (28) | +| RGB8 (today) | 4.95 | bright ×16 | 5.17 (25), 5.17 (25) | 6.57 (32), 4.96 (24) | -Typically ~25–35 frames, so **settling time scales with 1/fps**. The longer 12-bit runs (44–65 -frames) are the mid-way pauses; they happened at every rate but hit the 12-bit runs more often here. +Settling typically takes 25–35 frames, so **settling time scales with 1/frame rate**. The longer +12-bit runs (44–65 frames) contain mid-way pauses, which occurred at every rate but more often in the +12-bit runs here. -### Which frame to keep +### 3.3 Keep rules -Each rule applied to the per-frame log of all 24 steps. "Correct" means the frame it picks is within -10% of the brightness the camera finally settles at. +When the gimbal stops at a new waypoint the view changes, often to a much brighter or darker one, +and auto-exposure needs a few seconds to adapt; frames taken in that time are too dark or too bright. +The program must recognise when the image has settled and keep that frame. Each candidate rule was +replayed on the recordings of the 24 forced steps (Section 2.6) to find the frame it would have kept. -| Rule | 121 correct / early | 111 correct / early | Median time when correct (121 / 111) | +***Table 11.** Candidate keep rules scored on the 24 recorded steps (12 per tower). A keep rule +decides which frame after a gimbal stop is saved. Correct: the kept frame's brightness is within +±10% of the final level; early: the rule kept a frame before that. Target band: brightness between +45% and 55% of full scale (114.8–140.3 DN), the range the camera's auto-exposure aims for. Steady: +brightness changes by at most 2% from one frame to the next. Median time (s) from release to the +kept frame, over the correct runs.* + +| Rule | 121 correct / early (runs) | 111 correct / early (runs) | Median time when correct, 121 / 111 (s) | |---|---|---|---| | Exposure and gain unchanged for 3 frames | 0 / 12 | 0 / 12 | – | -| Brightness in the camera's target band (target 50% ± 5%), 2 frames | 6 / 6 | 6 / 6 | 3.30 / 6.10 s | -| Target band 2 frames and exposure/gain unchanged 2 frames | 6 / 6 | 7 / 5 | 3.35 / 5.02 s | -| Target band 3 frames | 8 / 4 | 8 / 4 | 3.40 / 4.84 s | -| **Target band and brightness steady (≤2% change) over 3 frames** | **11 / 1** | **10 / 2** | 3.61 / 5.45 s | +| Brightness in the target band (50 ± 5%), 2 frames | 6 / 6 | 6 / 6 | 3.30 / 6.10 | +| Target band 2 frames and exposure/gain unchanged 2 frames | 6 / 6 | 7 / 5 | 3.35 / 5.02 | +| Target band 3 frames | 8 / 4 | 8 / 4 | 3.40 / 4.84 | +| **Target band and brightness steady (≤ 2% change) over 3 frames** | **11 / 1** | **10 / 2** | 3.61 / 5.45 | -Coming down from too bright, brightness enters the band about 15% above where it settles, so "in -band" alone fires early; requiring it to have stopped changing fixes most of that. +**Exposure and gain unchanged.** This rule assumes that once the camera stops adjusting, the image +is right. It kept a frame early in every run, for two reasons: -## Auto white balance +- In 16 of 24 runs it fired on the first frames after release (0.2–0.5 s). These frames were already + on their way from the camera and still carried the forced settings, so the settings looked + unchanged while the image was black (brightness 9–13 DN) or white (253–255 DN), against a settled + level of about 120 DN. At a real waypoint the same happens: the settings have not changed yet + because the camera has not reacted to the new view. +- In the other 8 runs it fired during a pause: auto-exposure adjusts in steps and sometimes holds + still for several frames mid-way, here at brightness 33–227 DN. -The profiles fix white balance today (red 2.87, blue 2.0). With `BalanceWhiteAuto = Continuous`, the -camera settled this scene at red ≈ 2.36–2.46, blue ≈ 1.87–1.97. +Settings that hold still therefore do not show that the image is right. + +**Brightness in the target band and steady.** This rule judges the image itself: it keeps a frame +when, for 3 frames in a row, the brightness is within the band the camera's auto-exposure aims for +and changes by no more than 2% between frames. Both conditions are needed. Descending from +over-exposure, brightness passes through the band about 15% above its final level, so "in band" +alone fired early in half the runs; "steady" alone would fire during the pauses. Combined, the rule +kept a good frame in 21 of 24 runs; the 3 failures were slightly early on a descent. The program +implements this rule as `settle_rule = brightness`, with a time limit. + +These were ×16 steps forced on a still camera. Changes between neighbouring waypoints are usually +smaller, so the rule remains to be confirmed on a real scan. + +### 3.4 Auto white balance + +The profiles currently fix white balance (red 2.87, blue 2.0). With `BalanceWhiteAuto` = +`Continuous` the camera settled this scene at red ≈ 2.36–2.46, blue ≈ 1.87–1.97. ![Auto white balance, 121](images/raw12/awb_121.png) ![Auto white balance, 111](images/raw12/awb_111.png) -| Format | fps | Within ±5% and stays, 121 | Within ±5% and stays, 111 | +***Figure 5.** White-balance ratios (the red and blue gains relative to green) divided by their +settled values, against time (s) after automatic white balance is switched back on at time 0, for +each format: (a, top) tower 121; (b, bottom) tower 111. A value of 1.0 means settled. Left: ratios +forced to half (×0.5); right: to double (×2). Shaded: the ±5% band counted as settled.* + +***Table 12.** Settling time after a forced white-balance step: time (s), and in parentheses frames, +until both the red and the blue ratio are within ±5% of the reference and remain there. Ranges over +the ×0.5 and ×2 steps, two runs each.* + +| Format | Rate (fps) | Tower 121: time (s) (frames) | Tower 111: time (s) (frames) | |---|---|---|---| -| BayerRG12p | 9.86 | 0.21–0.32 s (2–3 frames); once 1.33 s | 0.22–0.41 s (2–4) | -| RGB8 | 4.95 | 0.57–0.77 s (2–3) | 0.55–0.76 s (2–3) | -| BayerRG8 | 14.75 | 0.26–0.27 s (3) | 0.19–0.26 s (2–3) | +| BayerRG12p | 9.86 | 0.21–0.32 (2–3); once 1.33 | 0.22–0.41 (2–4) | +| RGB8 | 4.95 | 0.57–0.77 (2–3) | 0.55–0.76 (2–3) | +| BayerRG8 | 14.75 | 0.26–0.27 (3) | 0.19–0.26 (2–3) | -On a still scene the auto ratios held within ±0.8–3.1%. +On a still scene the automatic ratios held within ±0.8–3.1%. White balance settles much faster than +exposure. -## Where the time goes - -### Same frame on both computers - -The fairest comparison: the **identical shared frame** (checksums verified on both towers), every -step timed on its own, 4 threads. Colour paths at distance 1.0 and effort 3; today's RGB8 path at -its production settings (distance 0.8, effort 4). +### 3.5 Processing time per step ![Same frame, time per step](images/raw12/stages_common.png) -| Step | IOTA (121) | 3 Delta (111) | Delta / IOTA | +***Figure 6.** Time per saved frame (ms), split by step, on the shared frame (tower 121's frame +copied bit-exactly to both towers, so only the computer differs). 4 encoder threads. Colour paths at +d1.0 e3 (JPEG XL distance 1.0, effort 3); today's RGB8 path at its current setting d0.8 e4.* + +***Table 13.** Duration (ms) of each processing step for one frame on the shared frame, 4 encoder +threads, median of 3 runs. Delta / IOTA: ratio of the two times (above 1 = the Delta is slower). +d1.0 e3: JPEG XL distance 1.0, effort 3. The last two rows differ mainly through the towers' storage +write speeds.* + +| Step | IOTA (ms) | Delta (ms) | Delta / IOTA | |---|---|---|---| -| Unpack 12-bit packed → 16-bit | 10 ms | 17 ms | 1.7× | -| Colour conversion, 8-bit raw, bilinear / edge-aware | 8 / 9 ms | 11 / 12 ms | 1.3× | -| Colour conversion, 12-bit raw, bilinear / edge-aware | 17 / 28 ms | 22 / 33 ms | 1.2× | -| JPEG XL encode, 12-bit colour, d1.0 e3 | 480 ms | 765 ms | 1.6× | -| JPEG XL encode, today's RGB8, d0.8 e4 | 566 ms | 867 ms | 1.5× | -| File write, 2.7 MB colour file | 1 ms | 4 ms | | -| File write, 17–21 MB lossless raw file | 7–8 ms | 162–173 ms | (storage: 218 vs 76 MB/s) | +| Unpack 12-bit packed → 16-bit | 10 | 17 | 1.7 | +| Colour conversion, 8-bit raw, bilinear / edge-aware | 8 / 9 | 11 / 12 | 1.3 | +| Colour conversion, 12-bit raw, bilinear / edge-aware | 17 / 28 | 22 / 33 | 1.2 | +| JPEG XL encode, 12-bit colour, d1.0 e3 | 480 | 765 | 1.6 | +| JPEG XL encode, today's RGB8, d0.8 e4 | 566 | 867 | 1.5 | +| File write, 2.7 MB colour file | 1 | 4 | | +| File write, 17–21 MB lossless raw file | 7–8 | 162–173 | (storage 218 vs 76 MB/s) | -Per saved frame, all steps together, and the file each produces (sizes are identical on both -computers, as they must be for the same input): +***Table 14.** Total time per saved frame (ms: unpack, colour conversion, encode and file write) and +file size (MB), on the shared frame, 4 encoder threads. File sizes are identical on both computers. +Raw planes and raw mosaic: raw data saved without colour conversion (Section 1.2).* -| Path | IOTA total | Delta total | File | +| Path | IOTA (ms) | Delta (ms) | File size (MB) | |---|---|---|---| -| Today: RGB8 → JPEG XL, d0.8 e4 | 567 ms | 872 ms | 3.27 MB | -| Raw 8 → colour → JPEG XL, d1.0 e3 | 468 ms | 741 ms | 2.66 MB | -| Raw 12 → colour (bilinear) → JPEG XL, d1.0 e3 | 476 ms | 778 ms | 2.53 MB | -| **Raw 12 → colour (edge-aware) → JPEG XL, d1.0 e3** | **518 ms** | **820 ms** | **2.65 MB** | -| Raw 12 planes, lossless (e1) | 91 ms | 297 ms | 17.65 MB | -| Raw 12 mosaic, lossless (e1) | 87 ms | 303 ms | 20.94 MB | +| Today: RGB8 → JPEG XL, d0.8 e4 | 567 | 872 | 3.27 | +| Raw 8-bit → colour → JPEG XL, d1.0 e3 | 468 | 741 | 2.66 | +| Raw 12-bit → colour (bilinear) → JPEG XL, d1.0 e3 | 476 | 778 | 2.53 | +| **Raw 12-bit → colour (edge-aware) → JPEG XL, d1.0 e3** | **518** | **820** | **2.65** | +| Raw 12-bit planes, lossless, e1 | 91 | 297 | 17.65 | +| Raw 12-bit mosaic, lossless, e1 | 87 | 303 | 20.94 | -Unpacking and colour conversion together cost **38 ms on the IOTA and 50 ms on the Delta**, 7% of -the save. The JPEG XL encode is 90% of it. +Unpacking and colour conversion together cost 38 ms on the IOTA and 50 ms on the Delta (7% of the +save); the JPEG XL encode accounts for 90%. Here the "RGB8" input is the tower's own colour +conversion of the frame, so its file equals the raw 8-bit path's; the camera's real RGB8 files were +somewhat larger (3.7–4.0 MB, Appendix B). -The "RGB8" input here is the edge-aware colour conversion of the same frame (the camera's own RGB8 -of that instant does not exist), so its file matches the raw 8-bit path's. The camera's real RGB8 -files were a little larger in the per-tower runs below (3.7–4.0 MB). - -### Size and time per quality setting, same frame - -![Same frame, size and time by quality](images/raw12/sizes_common.png) - -| Distance | File (12-bit colour) | IOTA | Delta | File (raw 12 planes) | -|---|---|---|---|---| -| lossless | 44.8 MB | 1,225 ms | 2,906 ms | 16.8 MB (455 / 860 ms) | -| 0.3 | 7.19 MB | 742 ms | 1,197 ms | 7.31 MB | -| 0.5 | 5.15 MB | 678 ms | 1,033 ms | 5.99 MB | -| 0.8 | 3.16 MB | 570 ms | 848 ms | 4.32 MB | -| **1.0** | **2.65 MB** | **558 ms** | **847 ms** | 3.78 MB | -| 1.5 | 1.63 MB | 515 ms | 751 ms | 2.44 MB | -| 2.0 | 1.32 MB | 558 ms | 738 ms | 1.96 MB | - -(Effort 3, 4 threads, file write included. A separate pass of 2 runs per point, so it differs from -the stage table above by run-to-run variation, up to ~40 ms.) - -### Each tower's own capture - -Same camera and scene, different day and light; consistent with the shared-frame result. - -![Stage times, real frame](images/raw12/stages_real.png) -![Stage times, synthetic frame](images/raw12/stages_synth.png) - -| Path (effort 3, 4 threads, d1.0) | IOTA, own frame / synthetic | Delta, own frame / synthetic | -|---|---|---| -| Today: camera RGB8 | 463 / 652 ms | 788 / 1,071 ms | -| Raw 12 → colour (edge-aware) | 569 / 727 ms | 849 / 1,088 ms | -| Raw 12 planes, lossless (e1) | 90 / 96 ms | 276 / 298 ms | - -### Effort and threads +### 3.6 Effort and threads ![Effort × threads, 121](images/raw12/effort_threads_121.png) ![Effort × threads, 111](images/raw12/effort_threads_111.png) -- Effort 1–4 are within ~15% of each other; **effort 3 gives the smallest files**. -- On the IOTA, **3 threads is slower than 2** at every effort (e.g. 761 / 582 / 672 / 512 ms for - 1–4 threads at effort 1); the Delta scales normally (1,440 / 985 / 886 / 802 ms). 4 is fastest on - both computers. +***Figure 7.** Time per saved frame (ms) by JPEG XL effort (rows) and thread count (columns), each +tower's own real frame, distance 1.0: (a, top) tower 121; (b, bottom) tower 111. Effort: how hard +the encoder searches for a compact encoding (1 = fastest; higher = slower, usually smaller). +Threads: processor threads one encode may use.* -## Quality setting: size and quality +Effort 1–4 lie within about 15% of each other, and effort 3 gives the smallest files. On the IOTA, +3 threads is slower than 2 at every effort (761 / 582 / 672 / 512 ms for 1–4 threads at effort 1); +the Delta scales normally (1440 / 985 / 886 / 802 ms). Four threads is fastest on both computers. + +### 3.7 Quality setting: size, time and quality + +![Same frame, size and time by quality](images/raw12/sizes_common.png) + +***Figure 8.** File size (MB, left) and time per saved frame (ms, right) against JPEG XL distance on +the shared frame, effort 3, 4 threads, both towers. Distance is the encoder's quality setting: 0 is +lossless, 1.0 is intended to be visually indistinguishable from the original, and higher values give +smaller files with more loss.* + +***Table 15.** File size (MB) and time per saved frame (ms) per JPEG XL distance on the shared frame +(effort 3, 4 threads, file write included, median of 2 runs). Last column: raw planes (Section 1.2) +at the same distance.* + +| Distance | 12-bit colour size (MB) | IOTA time (ms) | Delta time (ms) | Raw 12-bit planes size (MB) | +|---|---|---|---|---| +| 0 (lossless) | 44.8 | 1225 | 2906 | 16.8 (IOTA 455 ms, Delta 860 ms) | +| 0.3 | 7.19 | 742 | 1197 | 7.31 | +| 0.5 | 5.15 | 678 | 1033 | 5.99 | +| 0.8 | 3.16 | 570 | 848 | 4.32 | +| **1.0** | **2.65** | **558** | **847** | 3.78 | +| 1.5 | 1.63 | 515 | 751 | 2.44 | +| 2.0 | 1.32 | 558 | 738 | 1.96 | ![Size and quality, real frame](images/raw12/distance_real.png) -Against the exact lossless 12-bit colour image of the same frame (real plant frame, ~15 dB gain; -the two towers agree within 0.3 dB): +***Figure 9.** File size (MB), PSNR (dB) and fine-detail retention against JPEG XL distance, each +tower's own real frame, each method compared with its own uncompressed image. Fine-detail retention: +the amount of fine detail (Laplacian energy) in the decoded image relative to the uncompressed one; +1 = all detail kept, below 1 = smoothed.* -| Saved as | File 121 / 111 | PSNR | SSIM | +***Table 16.** Image quality against the exact lossless 12-bit colour image of the same frame (real +frame, ≈ 15 dB gain; the towers agree within 0.3 dB). PSNR (dB): peak signal-to-noise ratio, higher +means closer to the reference; +6 dB means half the error. SSIM: structural similarity, comparing +local patterns of brightness and contrast, 0–1 with 1 = identical. Exact: decodes bit for bit to the +original. Colour rows use edge-aware conversion unless stated.* + +| Saved as | File size, 121 / 111 (MB) | PSNR (dB) | SSIM | |---|---|---|---| -| 12-bit colour, distance 0.3 | 7.2 / 7.4 MB | 40.3 dB | 0.994 | -| 12-bit colour, distance 0.5 | 5.1 / 5.3 MB | 38.8 dB | 0.989 | -| 12-bit colour, distance 0.8 | 3.2 / 3.3 MB | 37.1 dB | 0.978 | -| **12-bit colour, distance 1.0** | **2.6 / 2.7 MB** | 36.6 dB | 0.973 | -| 12-bit colour, distance 1.5 | 1.6 / 1.7 MB | 35.4 dB | 0.960 | -| 12-bit colour, distance 2.0 | 1.3 / 1.3 MB | 34.8 dB | 0.953 | -| 12-bit colour, bilinear, distance 1.0 | 2.5 / 2.6 MB | 36.6 dB | 0.968 | -| 12-bit raw planes, lossy 0.5 | 6.0 / 6.0 MB | **47.2 dB** | 0.990 | -| 12-bit raw planes, lossless | 17.6 / 17.7 MB | exact | 1 | -| 12-bit raw mosaic, lossless | 20.9 / 19.0 MB | exact | 1 | -| 12-bit colour, lossless | 47.4 / 46.3 MB | exact | 1 | +| 12-bit colour, d0.3 | 7.2 / 7.4 | 40.3 | 0.994 | +| 12-bit colour, d0.5 | 5.1 / 5.3 | 38.8 | 0.989 | +| 12-bit colour, d0.8 | 3.2 / 3.3 | 37.1 | 0.978 | +| **12-bit colour, d1.0** | **2.6 / 2.7** | 36.6 | 0.973 | +| 12-bit colour, d1.5 | 1.6 / 1.7 | 35.4 | 0.960 | +| 12-bit colour, d2.0 | 1.3 / 1.3 | 34.8 | 0.953 | +| 12-bit colour, bilinear, d1.0 | 2.5 / 2.6 | 36.6 | 0.968 | +| 12-bit raw planes, d0.5 | 6.0 / 6.0 | **47.2** | 0.990 | +| 12-bit raw planes, lossless | 17.6 / 17.7 | exact | 1 | +| 12-bit raw mosaic, lossless | 20.9 / 19.0 | exact | 1 | +| 12-bit colour, lossless | 47.4 / 46.3 | exact | 1 | -- **Colour JPEG XL is tuned to the eye:** it drops sensor noise and fine colour, so PSNR is - moderate while SSIM (structure) stays high. -- **Raw planes keep the sensor values much closer** at the same size (47 vs 39 dB, equal SSIM). - They suit a detector that wants raw values and can convert to colour itself. Lossless, they take - a third of the space of lossless colour. +Colour JPEG XL is tuned to human vision: it discards sensor noise and fine colour, so PSNR is +moderate while SSIM stays high. Raw planes preserve the sensor values much more closely at the same +size (47 vs 39 dB, equal SSIM); lossless, they take a third of the space of lossless colour. -## Example images +### 3.8 Example images -Every compression type, from tower 121's frame: a 100% crop of spruce needles over bright leaves; -its difference from lossless (×8, black = identical); and a darker crop brightened 2 stops. -Full-size `.jxl` files are kept outside git in `tools/camera-bench/examples-2026-10-02-tower121/`, -`examples-2026-10-04-tower111/` and `examples-2026-10-04-lowgain/` (~115 MB each). -Crop positions: ![crops](images/raw12/examples/121/crops.jpg) +![crops](images/raw12/examples/121/crops.jpg) -| Saved as | File, time (121) | Detail (100%) | Difference ×8 | Shadow, +2 stops | +***Figure 10.** Tower 121's frame (downscaled), with the positions of the detail crop (red) and +shadow crop (blue) used in Figure 11.* + +***Figure 11.** One example per compression method, tower 121's frame, as 480 × 320-pixel crops at +100% (one image pixel per screen pixel). Detail: spruce needles over bright leaves. Difference: the +absolute difference from the lossless image, multiplied by 8 so small errors become visible (black = +identical). Shadow: a darker crop brightened by 2 stops, i.e. every pixel value multiplied by 4 +(each stop doubles), so detail in the dark parts becomes visible. File size (MB) and save time (s) +are single runs at 4 threads. The two 8-bit rows are separate captures taken seconds apart (branches +move), so they have no difference map.* + +| Method | File size (MB), time (s) | Detail (100%) | Difference (×8) | Shadow (+2 stops) | |---|---|---|---|---| -| Camera RGB8, d0.8 e4 (today; another capture) | 3.8 MB, 0.62 s | ![](images/raw12/examples/121/detail/rgb8_d0.8_e4_today.png) | – | ![](images/raw12/examples/121/shadow/rgb8_d0.8_e4_today.png) | -| Raw 8 → colour, d1.0 (another capture) | 2.7 MB, 0.45 s | ![](images/raw12/examples/121/detail/c8_ea_d1.0.png) | – | ![](images/raw12/examples/121/shadow/c8_ea_d1.0.png) | -| 12-bit colour, lossless (reference) | 47.4 MB, 0.23 s | ![](images/raw12/examples/121/detail/c12_ea_lossless.png) | ![](images/raw12/examples/121/diff/c12_ea_lossless.png) | ![](images/raw12/examples/121/shadow/c12_ea_lossless.png) | -| 12-bit colour, d0.3 | 7.2 MB, 0.77 s | ![](images/raw12/examples/121/detail/c12_ea_d0.3.png) | ![](images/raw12/examples/121/diff/c12_ea_d0.3.png) | ![](images/raw12/examples/121/shadow/c12_ea_d0.3.png) | -| 12-bit colour, d0.5 | 5.1 MB, 0.69 s | ![](images/raw12/examples/121/detail/c12_ea_d0.5.png) | ![](images/raw12/examples/121/diff/c12_ea_d0.5.png) | ![](images/raw12/examples/121/shadow/c12_ea_d0.5.png) | -| 12-bit colour, d0.8 | 3.2 MB, 0.58 s | ![](images/raw12/examples/121/detail/c12_ea_d0.8.png) | ![](images/raw12/examples/121/diff/c12_ea_d0.8.png) | ![](images/raw12/examples/121/shadow/c12_ea_d0.8.png) | -| **12-bit colour, d1.0** | 2.6 MB, 0.53 s | ![](images/raw12/examples/121/detail/c12_ea_d1.0.png) | ![](images/raw12/examples/121/diff/c12_ea_d1.0.png) | ![](images/raw12/examples/121/shadow/c12_ea_d1.0.png) | -| 12-bit colour, d1.5 | 1.6 MB, 0.46 s | ![](images/raw12/examples/121/detail/c12_ea_d1.5.png) | ![](images/raw12/examples/121/diff/c12_ea_d1.5.png) | ![](images/raw12/examples/121/shadow/c12_ea_d1.5.png) | -| 12-bit colour, d2.0 | 1.3 MB, 0.42 s | ![](images/raw12/examples/121/detail/c12_ea_d2.0.png) | ![](images/raw12/examples/121/diff/c12_ea_d2.0.png) | ![](images/raw12/examples/121/shadow/c12_ea_d2.0.png) | -| 12-bit colour, bilinear, d1.0 | 2.5 MB, 0.48 s | ![](images/raw12/examples/121/detail/c12_bil_d1.0.png) | ![](images/raw12/examples/121/diff/c12_bil_d1.0.png) | ![](images/raw12/examples/121/shadow/c12_bil_d1.0.png) | -| 12-bit raw planes, d0.5 | 6.0 MB, 0.54 s | ![](images/raw12/examples/121/detail/raw12_planes_d0.5.png) | ![](images/raw12/examples/121/diff/raw12_planes_d0.5.png) | ![](images/raw12/examples/121/shadow/raw12_planes_d0.5.png) | -| 12-bit raw planes, lossless | 17.6 MB, 0.09 s | ![](images/raw12/examples/121/detail/raw12_planes_lossless.png) | ![](images/raw12/examples/121/diff/raw12_planes_lossless.png) | ![](images/raw12/examples/121/shadow/raw12_planes_lossless.png) | -| 12-bit raw mosaic, lossless | 20.9 MB, 0.09 s | ![](images/raw12/examples/121/detail/raw12_lossless.png) | ![](images/raw12/examples/121/diff/raw12_lossless.png) | ![](images/raw12/examples/121/shadow/raw12_lossless.png) | +| Camera RGB8, d0.8 e4 (today) | 3.8, 0.62 | ![](images/raw12/examples/121/detail/rgb8_d0.8_e4_today.png) | – | ![](images/raw12/examples/121/shadow/rgb8_d0.8_e4_today.png) | +| Raw 8-bit → colour, d1.0 | 2.7, 0.45 | ![](images/raw12/examples/121/detail/c8_ea_d1.0.png) | – | ![](images/raw12/examples/121/shadow/c8_ea_d1.0.png) | +| 12-bit colour, lossless (reference) | 47.4, 0.23 | ![](images/raw12/examples/121/detail/c12_ea_lossless.png) | ![](images/raw12/examples/121/diff/c12_ea_lossless.png) | ![](images/raw12/examples/121/shadow/c12_ea_lossless.png) | +| 12-bit colour, d0.3 | 7.2, 0.77 | ![](images/raw12/examples/121/detail/c12_ea_d0.3.png) | ![](images/raw12/examples/121/diff/c12_ea_d0.3.png) | ![](images/raw12/examples/121/shadow/c12_ea_d0.3.png) | +| 12-bit colour, d0.5 | 5.1, 0.69 | ![](images/raw12/examples/121/detail/c12_ea_d0.5.png) | ![](images/raw12/examples/121/diff/c12_ea_d0.5.png) | ![](images/raw12/examples/121/shadow/c12_ea_d0.5.png) | +| 12-bit colour, d0.8 | 3.2, 0.58 | ![](images/raw12/examples/121/detail/c12_ea_d0.8.png) | ![](images/raw12/examples/121/diff/c12_ea_d0.8.png) | ![](images/raw12/examples/121/shadow/c12_ea_d0.8.png) | +| **12-bit colour, d1.0** | 2.6, 0.53 | ![](images/raw12/examples/121/detail/c12_ea_d1.0.png) | ![](images/raw12/examples/121/diff/c12_ea_d1.0.png) | ![](images/raw12/examples/121/shadow/c12_ea_d1.0.png) | +| 12-bit colour, d1.5 | 1.6, 0.46 | ![](images/raw12/examples/121/detail/c12_ea_d1.5.png) | ![](images/raw12/examples/121/diff/c12_ea_d1.5.png) | ![](images/raw12/examples/121/shadow/c12_ea_d1.5.png) | +| 12-bit colour, d2.0 | 1.3, 0.42 | ![](images/raw12/examples/121/detail/c12_ea_d2.0.png) | ![](images/raw12/examples/121/diff/c12_ea_d2.0.png) | ![](images/raw12/examples/121/shadow/c12_ea_d2.0.png) | +| 12-bit colour, bilinear, d1.0 | 2.5, 0.48 | ![](images/raw12/examples/121/detail/c12_bil_d1.0.png) | ![](images/raw12/examples/121/diff/c12_bil_d1.0.png) | ![](images/raw12/examples/121/shadow/c12_bil_d1.0.png) | +| 12-bit raw planes, d0.5 | 6.0, 0.54 | ![](images/raw12/examples/121/detail/raw12_planes_d0.5.png) | ![](images/raw12/examples/121/diff/raw12_planes_d0.5.png) | ![](images/raw12/examples/121/shadow/raw12_planes_d0.5.png) | +| 12-bit raw planes, lossless | 17.6, 0.09 | ![](images/raw12/examples/121/detail/raw12_planes_lossless.png) | ![](images/raw12/examples/121/diff/raw12_planes_lossless.png) | ![](images/raw12/examples/121/shadow/raw12_planes_lossless.png) | +| 12-bit raw mosaic, lossless | 20.9, 0.09 | ![](images/raw12/examples/121/detail/raw12_lossless.png) | ![](images/raw12/examples/121/diff/raw12_lossless.png) | ![](images/raw12/examples/121/shadow/raw12_lossless.png) | -The two 8-bit rows are separate captures seconds apart (branches move), so they have no difference -map. Times are single runs at 4 threads. Tower 111's examples (same camera and scene) and the -0 dB examples are kept as full-size `.jxl` only; `tools/camera-bench/make_figures.py` makes the same -crops from them. +Full-size `.jxl` files are kept outside the repository in `tools/camera-bench/examples-*` +(about 115 MB per set), including tower 111's and the 0 dB examples. -## What 12-bit adds - -To see whether the extra 4 bits carry information or noise, a frame was also taken at **0 dB gain** -(124 ms exposure, 4 Oct). The noise floor, from the quietest patches at each brightness: +### 3.9 Information content of the 12-bit data ![Noise vs brightness](images/raw12/noise_vs_signal.png) -| Signal (12-bit, green) | Noise at 0 dB | Noise at ~15 dB | In 8-bit steps (0 dB / 15 dB) | +***Figure 12.** Sensor noise σ (DN, 12-bit scale) against signal level (DN, 12-bit scale), measured +on the green pixels, at 0 dB gain and at ≈ 15 dB gain. σ is the standard deviation of the random +pixel-to-pixel variation. Dashed: one 8-bit quantisation step, the smallest change an 8-bit value +can record (16 DN on the 12-bit scale). Dotted: the RMS error added by rounding to 8 bits (16/√12 = +4.6 DN). Noise above these lines means rounding to 8 bits loses nothing measurable.* + +***Table 17a.** Sensor noise floor σ (DN, 12-bit scale, 0–4095) at three signal levels, green pixels, +at 0 dB and ≈ 15 dB gain. Last column: the same noise in 8-bit steps (one step = 16 DN on the 12-bit +scale); above 1, the 8-bit values already resolve all the sensor delivers.* + +| Signal (DN, 12-bit) | Noise at 0 dB (DN) | Noise at ≈ 15 dB (DN) | Noise in 8-bit steps, 0 dB / 15 dB | |---|---|---|---| -| ~700 (darkest in this scene) | 26 | 42 | 1.6 / 2.6 | -| ~1,500 | 35 | 58 | 2.2 / 3.6 | -| ~3,000 | 37 | 80 | 2.3 / 5.0 | +| ≈ 700 (darkest in this scene) | 26 | 42 | 1.6 / 2.6 | +| ≈ 1500 | 35 | 58 | 2.2 / 3.6 | +| ≈ 3000 | 37 | 80 | 2.3 / 5.0 | -Rounding to 8 bits adds an error of about 4.6 (12-bit units): small next to this noise. **In -daylight, 8-bit already holds what the sensor delivers.** By extrapolation, 12-bit starts to carry -information below about 300 (0 dB) or 100 (15 dB) of 4095, i.e. in deep shadow: dusk, night, the -dark side of a ridge. That is where a 12-bit night profile would pay off. Not measured at night. +**Rounding against noise.** Rounding to 8 bits adds an RMS error of 4.6 DN (12-bit scale). +Independent errors add as the square root of the sum of their squares, so in the worst case measured +(26 DN of noise at 0 dB) the total grows from 26 to 26.4 DN, i.e. by 1.5%; at ≈ 15 dB gain by 0.6%, +and in bright areas by less than 0.2%. Because the noise is larger than one 8-bit step, it also acts +as dither: averaging over an area recovers differences finer than one step even from 8-bit values. +For a single daylight frame, rounding to 8 bits therefore loses almost nothing measurable. -Same frame, a darker crop brightened 2 stops, 8-bit (left, the 12-bit value >> 4) vs 12-bit (right), -at ~15 dB and at 0 dB. 3,138 vs 227 distinct levels at 15 dB, yet no visible difference: +**Averaging over many pixels.** Averaging an area of N pixels reduces the noise by √N, which might +seem to favour the 12-bit image. But while the noise exceeds one 8-bit step, the rounding error is +also random and independent from pixel to pixel, so it averages down at the same rate: an average of +8-bit values resolves differences much finer than one step, as precisely as the 12-bit average. A +12-bit image always holds at least as much information as its 8-bit version, but at a noise of +26 DN (≈ 2^4.7) the four bits removed by rounding mostly record the noise itself; the information +they carry about the scene is about 0.02 bits per pixel. This changes when the noise falls below one +8-bit step: the rounding error is then the same for neighbouring pixels, averaging cannot remove it, +and the dropped bits hold real scene detail. + +**Rounding against compression.** The saved file is lossy. At distance 1.0 the 12-bit colour file +has a PSNR of 36.6 dB (Table 16), an RMS error of about 61 DN, which is 13 times the rounding error; +even at distance 0.3 (40.3 dB) it is about 40 DN, and raw planes at distance 0.5 (47.2 dB) about +18 DN. Much of this error is noise the encoder discards, and JPEG XL spends relatively more +precision on dark tones, so the error in shadows is smaller than these averages; it was not measured +separately. Still, with lossy files the compression, not the bit depth, decides how much fine tonal +detail survives. Correspondingly, 12-bit and 8-bit raw give the same file size at distance 1.0 +(2.65 and 2.66 MB, Table 14). + +**Where 12 bits are expected to matter.** The noise falls with the signal, while the 8-bit step +stays at 16 DN. Extrapolating Figure 12, the noise drops below one 8-bit step at about 300 DN at +0 dB gain, or 100 DN at ≈ 15 dB. The 8-bit data is linear (no gamma curve), so this region, the +darkest 2–7% of the range, holds only 6–19 of the 256 8-bit levels. Gain multiplies the noise, so at +high gain the noise exceeds one 8-bit step again almost everywhere. Table 17b combines these +observations. + +***Table 17b.** Situations in which 12-bit data is expected to differ from 8-bit data. Signal in DN +on the 12-bit scale (0–4095). "Measured" refers to this campaign.* + +| Situation | Signal (DN), gain (dB) | Noise against one 8-bit step | Expected effect of 12 bits | Measured | +|---|---|---|---|---| +| Daylight, mid-tones and highlights (lit terrain, sky) | 700–4095, 0–15 | 1.6–5 steps | none measurable: total noise +0.2–1.5% | yes | +| Shadows in a high-contrast scene at low gain: shaded forest under a bright sky, backlit ridges, valleys at dawn and dusk | below ≈ 300 at 0 dB, ≈ 100 at 15 dB | below 1 step | 8 bits merge neighbouring levels; 12 bits keep small changes in dark areas, such as thin smoke over dark terrain | no (extrapolated) | +| Uniformly dark scene at high gain: night, gain up to 48 dB | any, high (up to 48) | above 1 step (gain amplifies noise) | little, except in the very darkest values | no | +| Strong processing of dark areas: brightening by 3 stops (×8) or more, contrast stretching | dark areas, any | – | 12 bits delay visible banding | 2 stops (×4) only: no difference (Figure 13) | +| Saving lossy at distance 1.0 | any | compression error ≈ 61 DN ≫ rounding 4.6 DN | largely lost to compression; kept in lossless files | yes, overall PSNR only | + +The case most relevant to fire detection is the second: in a scene exposed for a bright sky, smoke +rising in front of shaded forest is a small brightness increase in an already dark area, at low gain. +Whether 12 bits help there, and whether that benefit survives distance 1.0, requires a test on such a +scene (a dawn or dusk frame with dark terrain under a bright sky, compared as in Figures 12 and 13). ![8 vs 12 bit at 15 dB](images/raw12/examples/121/shadow_8_vs_12.png) ![8 vs 12 bit at 0 dB](images/raw12/examples/lowgain/shadow_8_vs_12.png) -At 0 dB the same settings give **25–30% smaller files and ~3 dB higher PSNR** (less noise to -encode): 12-bit colour at distance 1.0 is 1.9 MB, PSNR 40.0 dB, SSIM 0.987. +***Figure 13.** The same darker crop brightened by 2 stops (pixel values multiplied by 4), saved as +8-bit (left; the 12-bit value shifted down by 4 bits, i.e. divided by 16) and as 12-bit (right): (a, +top) ≈ 15 dB gain, 227 distinct green levels in 8-bit against 3138 in 12-bit; (b, bottom) 0 dB gain. +No visible difference: the noise is larger than the extra levels.* -## Overall pace: how fast can a tower capture? +At 0 dB gain the same settings produce 25–30% smaller files and about 3 dB higher PSNR (less noise +to encode): 12-bit colour at d1.0 is 1.9 MB, PSNR 40.0 dB, SSIM 0.987. -A waypoint goes through these steps; saving runs in the background, overlapping the next move: - -| Step | 12-bit raw (proposed) | Raw 8-bit | RGB8 (today) | Source | -|---|---|---|---|---| -| Camera delivers a frame every | 0.10 s (9.86 fps) | 0.07 s (14.75 fps) | 0.20 s (4.95 fps) | measured | -| Slew to the next waypoint | ~1–1.5 s (6° step at 5–8°/s) | same | same | estimate from the scan speed; not measured here | -| Mechanical settle after standstill | ~0.2 s | same | same | to be measured in the field | -| Exposure settle after a **large** change (×16) | ~3 s (2.2–6.6) | ~2 s (1.8–2.7) | ~5 s (4.6–6.6) | measured, 24 steps per tower | -| Exposure settle after a small change (neighbouring waypoints) | well under the above | | | not measured | -| **Save, one worker × 4 threads (IOTA / Delta)** | **0.56 / 0.84 s** | 0.52 / 0.85 s | 0.59 / 0.88 s | measured, shared frame, back to back | -| **Save, two workers × 2 threads (IOTA / Delta)** | **0.47 / 0.65 s** | 0.42 / 0.60 s | 0.43 / 0.64 s | measured, shared frame, back to back | +### 3.10 Saving throughput and pile-up ![Saves per second, same frame](images/raw12/pace_common.png) -- **The camera side sets the pace.** A waypoint needs roughly 1.5–5 s (move, then exposure), - while a 12-bit save takes 0.56 s on the IOTA and 0.65 s on the Delta with two workers. Saving - uses a fraction of the time between waypoints on either computer. -- **Saving becomes the limit only below about one waypoint per 0.6–0.85 s**, which a real scan does - not reach. On the shared frame, the IOTA kept up with a frame every 0.7 s with one worker; the - Delta needed two workers for 0.7 s (with one it fell 27 frames behind in 2 minutes). -- Settling after small changes, the common case between neighbouring waypoints, still needs a field - measurement; it is what decides the real scan pace. - -## Keeping up with the scan +***Figure 14.** Frames saved per second, back to back (the next save starts as soon as one +finishes), on the shared frame, for one save worker with 4 threads and two workers with 2 threads +each. A worker saves one frame at a time; two workers save two frames in parallel.* ![Pile-up](images/raw12/pileup.png) -Synthetic detailed frame (worst case). *ok* = no backlog after 2 minutes; otherwise frames behind -and the worst capture-to-file time: +***Figure 15.** Worst time (s, logarithmic axis) from queueing a frame to its file being written, +against the interval (s) between queued frames, synthetic frame. A flat line means saving keeps up; +a rising line means a backlog builds.* -| Path | Workers × threads | Tower | Saves/s back to back | One frame every 2.0 / 1.5 / 1.0 / 0.7 / 0.5 s | Peak memory | +***Table 18.** Throughput and pile-up on the synthetic frame, the worst case for the encoder. +Workers × threads: frames saved in parallel × encoder threads each. Back to back: frames saved per +second when saving continuously. Paced columns: one frame queued every given interval for 120 s; +"ok" = no frames waiting at the end, "N behind" = N frames still waiting, "–" = not tested. Peak +memory: the most memory (GB) used by the queue and encoders.* + +| Path | Workers × threads | Tower | Back to back (frames/s) | One frame every 2.0 / 1.5 / 1.0 / 0.7 / 0.5 s | Peak memory (GB) | |---|---|---|---|---|---| -| Today: camera RGB8, d0.8 e4 | 1 × 4 | 121 | 1.33 | ok / – / ok / **3 behind** / – | 0.5 GB | -| Today: camera RGB8, d0.8 e4 | 1 × 4 | 111 | 0.88 | ok / – / **12 behind** / **65 behind** / – | 0.5 GB | -| **12-bit colour, d1.0 e3** | 1 × 4 | 121 | 1.44 | ok / ok / ok / **ok** / – | 0.7 GB | -| **12-bit colour, d1.0 e3** | 1 × 4 | 111 | 0.93 | ok / **ok** / **8 behind** / 60 behind / – | 0.7 GB | -| **12-bit colour, d1.0 e3** | 2 × 2 | 121 | 1.74 | – / – / ok / ok / **34 behind** | 1.1 GB | -| **12-bit colour, d1.0 e3** | 2 × 2 | 111 | 1.18 | – / – / **ok** / **28 behind** / 97 behind | 1.1 GB | -| 12-bit colour, d1.0 e3 | 4 × 1 | 121 / 111 | 1.99 / 1.37 | (back to back only; each save waits 2–3 s) | 2.0 GB | -| 12-bit colour, d0.5 e3 | 1 × 4 | 121 | 1.19 | – / – / ok / 24 behind / – | 0.8 GB | -| 12-bit colour, d0.5 e3 | 1 × 4 | 111 | 0.77 | – / – / 27 behind / 79 behind / – | 0.7 GB | -| 12-bit raw mosaic, lossless | 1 × 4 | 121 / 111 | 10.2 / 1.8 | ok at 1.0 and 0.7 on both | 0.3 GB | +| Today: camera RGB8, d0.8 e4 | 1 × 4 | 121 | 1.33 | ok / – / ok / **3 behind** / – | 0.5 | +| Today: camera RGB8, d0.8 e4 | 1 × 4 | 111 | 0.88 | ok / – / **12 behind** / **65 behind** / – | 0.5 | +| **12-bit colour, d1.0 e3** | 1 × 4 | 121 | 1.44 | ok / ok / ok / **ok** / – | 0.7 | +| **12-bit colour, d1.0 e3** | 1 × 4 | 111 | 0.93 | ok / **ok** / **8 behind** / 60 behind / – | 0.7 | +| **12-bit colour, d1.0 e3** | 2 × 2 | 121 | 1.74 | – / – / ok / ok / **34 behind** | 1.1 | +| **12-bit colour, d1.0 e3** | 2 × 2 | 111 | 1.18 | – / – / **ok** / **28 behind** / 97 behind | 1.1 | +| 12-bit colour, d1.0 e3 | 4 × 1 | 121 / 111 | 1.99 / 1.37 | (back to back only; each save waits 2–3 s) | 2.0 | +| 12-bit colour, d0.5 e3 | 1 × 4 | 121 | 1.19 | – / – / ok / 24 behind / – | 0.8 | +| 12-bit colour, d0.5 e3 | 1 × 4 | 111 | 0.77 | – / – / 27 behind / 79 behind / – | 0.7 | +| 12-bit raw mosaic, lossless | 1 × 4 | 121 / 111 | 10.2 / 1.8 | ok at 1.0 and 0.7 on both | 0.3 | -On the real plant frame (lighter), one worker kept up with a frame every 1.0 s on both towers -(saved within 0.63 s on 121, 0.96 s on 111). +On the shared real frame, back to back, a 12-bit save took 0.56 s (IOTA, 1 × 4) and 0.65 s (Delta, +2 × 2); the IOTA kept up with a frame every 0.7 s with one worker, the Delta with two (with one it +fell 27 frames behind in 120 s). -## Heat and power +### 3.11 Heat and power ![Flat out](images/raw12/heat_max.png) ![Paced](images/raw12/heat_paced.png) -10 minutes each: +***Figure 16.** CPU package temperature (°C) and package power (W) over 10 minutes of 12-bit saving; +the package is the whole processor chip. (a, top) back to back with 2 workers × 2 threads; (b, +bottom) one frame every 2 s with 1 worker × 4 threads. Solid: tower 121; dashed: tower 111.* -| Load | Tower | Temp mean / max | Clock | CPU busy | Power | Saves per minute, first → last | +***Table 19.** Processor temperature (mean and maximum, °C), average core clock (GHz), CPU busy (% +of time the cores were working), and package power (W, read from the Intel RAPL energy counter [7]), +10 minutes per condition after a 5-minute cool-down. Saves per minute in the first and last minute +show any slowdown as the processor heats up and lowers its clock.* + +| Load | Tower | Temperature mean / max (°C) | Clock (GHz) | CPU busy (%) | Power (W) | Saves per minute, first → last minute | |---|---|---|---|---|---|---| -| Idle | 121 | 38 / 40 °C | 0.79 GHz | 2% | 2.2 W | – | -| Idle | 111 | 52 / 54 °C | 0.96 GHz | 4% | 2.9 W | – | -| 12-bit, one frame every 2 s (1 × 4) | 121 | **53 / 84 °C** | 1.71 GHz | 21% | **5.1 W** | 30 → 29 | -| 12-bit, one frame every 2 s (1 × 4) | 111 | **66 / 86 °C** | 1.84 GHz | 33% | **5.7 W** | 30 → 30 | -| 12-bit, flat out (2 × 2) | 121 | 80 / 87 °C | 2.88 GHz | 81% | 12.5 W | 122 → 103 (−16%) | -| 12-bit, flat out (2 × 2) | 111 | 78 / 83 °C | 2.44 GHz | 81% | 8.1 W | 74 → 73 | +| Idle | 121 | 38 / 40 | 0.79 | 2 | 2.2 | – | +| Idle | 111 | 52 / 54 | 0.96 | 4 | 2.9 | – | +| 12-bit, one frame every 2 s (1 × 4) | 121 | **53 / 84** | 1.71 | 21 | **5.1** | 30 → 29 | +| 12-bit, one frame every 2 s (1 × 4) | 111 | **66 / 86** | 1.84 | 33 | **5.7** | 30 → 30 | +| 12-bit, back to back (2 × 2) | 121 | 80 / 87 | 2.88 | 81 | 12.5 | 122 → 103 (−16%) | +| 12-bit, back to back (2 × 2) | 111 | 78 / 83 | 2.44 | 81 | 8.1 | 74 → 73 | -At scan pace both computers stay far from their limits. The Delta runs ~13 °C warmer; flat out it -sits at an 8 W power limit, so it does not slow further, while the IOTA starts fast (12.5 W) and -settles 16% lower as it heats. +At scan pace both computers stay far from their limits; the Delta runs about 13 °C warmer. Back to +back, the Delta sits at an 8 W power limit and does not slow further, whereas the IOTA starts at +12.5 W and loses 16% of its throughput as it heats. -## Storage +### 3.12 Storage ![Storage per day](images/raw12/storage.png) -Real plant frame sizes, 24 hours of scanning: +***Figure 17.** Storage per day (GB) at one saved frame every 2 s, per saving method, file sizes +from each tower's own real frame.* -| Saved as | Per frame (121 / 111) | One frame / 2 s | / 5 s | / 10 s | +***Table 20.** Storage (GB/day) for 24 hours of scanning at one saved frame every 2, 5 or 10 s. +Per-frame sizes (MB) are from each tower's own real frame; ranges span the two towers.* + +| Saved as | Per frame, 121 / 111 (MB) | Every 2 s (GB/day) | Every 5 s (GB/day) | Every 10 s (GB/day) | |---|---|---|---|---| -| Camera RGB8, d0.8 (today's quality) | 3.7 / 4.0 MB | 160–172 GB | 64–69 GB | 32–34 GB | -| **12-bit colour, d1.0** | **2.65 / 2.75 MB** | **114–119 GB** | **46–48 GB** | **23–24 GB** | -| 12-bit colour, d0.5 | 5.2 / 5.3 MB | 222–230 GB | 89–92 GB | 44–46 GB | -| 12-bit raw planes, lossless | 16.8 / 17.0 MB | 727–736 GB | 291–294 GB | 145–147 GB | -| 12-bit colour, lossless | 44.8 / 44.1 MB | ~1.9 TB | ~770 GB | ~385 GB | +| Camera RGB8, d0.8 (today's quality) | 3.7 / 4.0 | 160–172 | 64–69 | 32–34 | +| **12-bit colour, d1.0** | **2.65 / 2.75** | **114–119** | **46–48** | **23–24** | +| 12-bit colour, d0.5 | 5.2 / 5.3 | 222–230 | 89–92 | 44–46 | +| 12-bit raw planes, lossless | 16.8 / 17.0 | 727–736 | 291–294 | 145–147 | +| 12-bit colour, lossless | 44.8 / 44.1 | ≈ 1900 | ≈ 770 | ≈ 385 | -The towers have 11–22 GB free, so only a short buffer can stay on the tower. +The towers have 11–22 GB of free storage, so images must leave the tower continuously. -## Scope and caveats +--- -- **One scene, two afternoons,** ~15 dB gain in the main runs; one 0 dB frame. Night is not measured. -- **Auto-exposure steps on a still camera,** not after real slews. Real waypoint-to-waypoint changes - are usually much smaller than ×16, so real settling should be shorter. -- **The tower program was stopped,** so its own load (camera stream, control loop) is not included. - The stream-while-saving test shows the camera link is unaffected by saving. -- **Detection quality is not measured.** PSNR and SSIM stand in for "what the algorithms receive". -- **Tower 111's own camera (1240c, 12 MP) was not tested here.** Its frames are 25% smaller, so the +## 4. Discussion + +### 4.1 Summary of findings + +***Table 21.** Summary of findings, with the section where each is shown.* + +| Topic | Finding | Section | +|---|---|---| +| Frame rate | `BayerRG12p`: 9.86 fps, twice RGB8's 4.95 fps at 250 MB/s; `BayerRG8`: 14.75 fps. Unpacked `BayerRG12`: 7.41 fps. | 3.1 | +| Link | No lost or incomplete frames in any format on either tower, also while saving continuously. | 3.1 | +| Auto-exposure | From under- or over-exposure, settling takes about 25–35 frames: raw 8-bit 1.8–2.7 s, 12-bit 2.2–6.6 s (typically ≈ 3 s), RGB8 4.6–6.6 s. | 3.2 | +| Keep rule | Target band and steady for 3 frames: correct in 21 of 24 steps; "exposure and gain unchanged": 0 of 24. | 3.3 | +| White balance | Recovers in 2–4 frames (0.2–0.8 s), stable within ±1–3%. Exposure, not white balance, determines the wait. | 3.4 | +| Processing | Unpack + colour conversion 38 ms (IOTA) / 50 ms (Delta); a 12-bit save 518 / 820 ms, slightly faster than today's RGB8 save (567 / 872 ms). | 3.5 | +| Computers | The Delta takes 1.6 times as long on the identical frame. | 3.5 | +| Quality | d1.0: 2.65 MB, SSIM 0.973. Raw planes keep sensor values much closer (47 vs 39 dB at 6 MB). | 3.7 | +| 12 bits | In daylight, rounding to 8 bits adds at most 1.5% to the noise, and compression at d1.0 adds 13 times more error than rounding. 12 bits are expected to matter only in dark areas at low gain (high-contrast scenes), and only in lightly compressed files; not measured. | 3.9 | +| Throughput | IOTA keeps up with a frame every 0.7 s with one worker; the Delta needs two. | 3.10 | +| Heat | One frame every 2 s: IOTA 53 °C / 5.1 W, Delta 66 °C / 5.7 W. | 3.11 | +| Storage | 12-bit colour d1.0: 2.7 MB per frame, below today's 3.3–4.0 MB. | 3.12 | + +### 4.2 Overall pace + +Saving runs in the background and overlaps the next movement, so the scan pace is set by the +slowest of the camera-side steps and the save rate. + +***Table 22.** Time budget per waypoint (s) for each pixel format. Slew: the gimbal turning to the +next waypoint. Mechanical settling: vibration dying down after it stops. Exposure settling: +auto-exposure adapting to the new view. Save: runs in the background, overlapping the next waypoint. +Source: whether the value was measured or estimated.* + +| Step | 12-bit raw (proposed) | Raw 8-bit | RGB8 (today) | Source | +|---|---|---|---|---| +| Camera frame interval (s) | 0.10 (9.86 fps) | 0.07 (14.75 fps) | 0.20 (4.95 fps) | measured | +| Slew to the next waypoint (s) | ≈ 1–1.5 (6° at 5–8 °/s) | same | same | estimate, not measured | +| Mechanical settling after standstill (s) | ≈ 0.2 | same | same | estimate, to be measured | +| Exposure settling after a large (×16) change (s) | ≈ 3 (2.2–6.6) | ≈ 2 (1.8–2.7) | ≈ 5 (4.6–6.6) | measured, 24 steps per tower | +| Exposure settling after a small change (s) | below the above | | | not measured | +| Save, 1 worker × 4 threads, IOTA / Delta (s) | **0.56 / 0.84** | 0.52 / 0.85 | 0.59 / 0.88 | measured, shared frame | +| Save, 2 workers × 2 threads, IOTA / Delta (s) | **0.47 / 0.65** | 0.42 / 0.60 | 0.43 / 0.64 | measured, shared frame | + +A waypoint takes roughly 1.5–5 s, whereas a 12-bit save takes 0.56 s on the IOTA and 0.65 s on the +Delta with two workers. Saving only becomes the limit below about one waypoint per 0.6–0.85 s, which +a real scan does not reach. The camera side therefore sets the pace; settling after small changes +between neighbouring waypoints, which decides the real pace, remains to be measured. + +### 4.3 Limitations + +- One scene on two afternoons, at ≈ 15 dB gain in the main runs, plus one 0 dB frame. Night + conditions were not measured. +- Auto-exposure steps were forced on a still camera, not caused by real slews. Real waypoint-to- + waypoint changes are usually much smaller than ×16, so real settling should be shorter. +- The tower program was stopped, so its own load (camera stream, control loop) is not included; the + stream-while-saving test shows the camera link is unaffected by saving. +- Detection quality was not measured; PSNR and SSIM stand in for what the algorithms receive. +- Tower 111's own camera (1800 U-1240c, 12 MP) was not tested; its frames are 25% smaller, so the Delta's limits above are conservative for it. -- **Slew time and mechanical settle** in the pace table are estimates, not measurements. +- Slew time and mechanical settling in Table 22 are estimates. -## Raw data and tools +--- -- Programs and how to rerun every test: [tools/camera-bench](../tools/camera-bench/README.md). -- Camera tests, processing on own frames, heat: [tower 121](../tools/camera-bench/results-2026-10-02-tower121/), +## 5. Conclusions and recommendations + +1. **Take raw data from the camera** (`BayerRG12p`, 250 MB/s) and convert to colour on the tower, + edge-aware. This doubles the frame rate and roughly halves the settling time, at no additional + CPU, heat or storage cost compared with today, on either computer. +2. **Save 12-bit colour JPEG XL at distance 1.0, effort 3, 4 threads**: 2.7 MB per frame. +3. **Use one save worker on the IOTA and two workers × 2 threads on the Delta.** With the measured + settling times (≈ 2–3 s per waypoint) the scan stays below either limit; the save queue still + needs a bound. +4. **Keep a waypoint's frame when brightness is in the camera's target band and steady for 3 + frames**, with a time limit. +5. **12-bit versus raw 8-bit is a choice, not a cost.** `BayerRG8` (14.75 fps) settles about 1 s + faster in daylight and loses nothing measurable there; `BayerRG12p` is expected to help in dark areas + of high-contrast scenes at low gain, such as shaded terrain under a bright sky at dawn or dusk + (Section 3.9), which remains to be tested. The pixel format is part of the camera profile, so the program should accept both. +6. **Store raw 12-bit planes losslessly (17–18 MB per frame) only if the detection pipeline needs + exact sensor values**; it requires about 6 times the storage. + +--- + +## References + +1. ISO 19567-2:2019. *Photography — Digital cameras — Texture reproduction measurements — Part 2: + Measurements using dead leaves pattern.* International Organization for Standardization. +2. ISO/IEC 18181. *Information technology — JPEG XL image coding system*; reference implementation + libjxl, versions 0.11.0 and 0.11.1, https://github.com/libjxl/libjxl. +3. OpenCV, versions 4.5.4 and 4.10: `cvtColor` colour conversions, Bayer demosaicing (bilinear and + edge-aware), https://docs.opencv.org. +4. EMVA. *GenICam Pixel Format Naming Convention (PFNC)* and *GenICam Standard Features Naming + Convention (SFNC)*. European Machine Vision Association. +5. Allied Vision. *Alvium 1800 U series technical manual* and *Vimba X SDK* (VmbCPP, vmbpy 1.0.5 and + 1.2.0). +6. Z. Wang, A. C. Bovik, H. R. Sheikh, E. P. Simoncelli (2004). Image quality assessment: from error + visibility to structural similarity. *IEEE Transactions on Image Processing* 13(4), 600–612. +7. Intel Running Average Power Limit (RAPL) energy counters, Linux `powercap` interface. + +--- + +## Appendix A. Raw data and tools + +- Programs and instructions to rerun every test: [tools/camera-bench](../tools/camera-bench/README.md). +- Camera tests, processing on each tower's own frame, heat: + [tower 121](../tools/camera-bench/results-2026-10-02-tower121/), [tower 111](../tools/camera-bench/results-2026-10-04-tower111/). - Shared-frame processing: [tower 121](../tools/camera-bench/results-common-tower121/), [tower 111](../tools/camera-bench/results-common-tower111/) (each with `frame_md5.txt`). -- Full-size example `.jxl` files (git-ignored, ~115 MB per set): `tools/camera-bench/examples-*`. -- Charts: `plot_results.py` and `plot_common.py`; example figures: `make_figures.py`; +- Full-size example `.jxl` files (not in the repository, about 115 MB per set): + `tools/camera-bench/examples-*`. +- Figures: `plot_results.py`, `plot_common.py` (charts), `make_figures.py` (example crops); quality scores: `compare_examples.py`. + +## Appendix B. Processing times on each tower's own frame + +![Stage times, real frame](images/raw12/stages_real.png) +![Stage times, synthetic frame](images/raw12/stages_synth.png) + +***Figure B1.** Time per saved frame (ms) split by step: (a, top) each tower's own real frame; (b, +bottom) the synthetic frame (a computer-generated "dead leaves" texture of overlapping discs with +added noise [1], harder to compress than any real scene). Effort 3 (raw lossless: effort 1), 4 +threads, distance 1.0.* + +***Table B1.** Total time per saved frame (ms) on each tower's own real frame and on the synthetic +frame, effort 3, 4 threads, distance 1.0 (raw lossless: effort 1). The real frames differ between +towers, so Table 14 is the fair comparison.* + +| Path | IOTA, own frame (ms) | IOTA, synthetic (ms) | Delta, own frame (ms) | Delta, synthetic (ms) | +|---|---|---|---|---| +| Today: camera RGB8 | 463 | 652 | 788 | 1071 | +| Raw 12-bit → colour (edge-aware) | 569 | 727 | 849 | 1088 | +| Raw 12-bit planes, lossless | 90 | 96 | 276 | 298 | + +The camera's real RGB8 files on these frames were 3.7–4.0 MB. The results agree with the shared-frame +comparison (Section 3.5). diff --git a/docs/raw12-pipeline-measurements_llm_context.md b/docs/raw12-pipeline-measurements_llm_context.md new file mode 100644 index 0000000..c644f61 --- /dev/null +++ b/docs/raw12-pipeline-measurements_llm_context.md @@ -0,0 +1,193 @@ +# Context for discussing `raw12-pipeline-measurements.md` + +This file gives an AI assistant the background needed to discuss the report +[raw12-pipeline-measurements.md](raw12-pipeline-measurements.md): what was asked, how the tests were +run, what the numbers mean, what was decided, which points have been questioned, and what is still +open. Read the report itself for the full tables and figures; this file explains around them. +State as of 2026-10-05, branch `raw12-capture`. + +## 1. The system in one paragraph + +Fire-watch towers scan waypoints with a gimbal-mounted Allied Vision Alvium camera and keep one +picture per waypoint, saved as JPEG XL on the tower's computer. After the gimbal stops, the camera's +continuous auto-exposure needs frames to adapt to the new view, so the scan waits ("settling"). The +tower program is C++ (`fgc`), using Vimba X (VmbCPP), OpenCV and libjxl. Two computers are in use: +LattePanda IOTA (Intel N150, tower 121) and LattePanda 3 Delta (Celeron N5105, tower 111). + +## 2. The question and how it evolved + +- Origin: speeding up the per-waypoint capture. Today the camera sends RGB8 (colour made in the + camera, 48.6 MB per full frame), which limits the frame rate over USB to 4.95 fps at the agreed + 250 MB/s link cap, so exposure settles slowly. +- Proposal tested: camera sends raw `BayerRG12p` (24.3 MB, 9.86 fps), the tower unpacks, + de-Bayers (converts to colour) and saves 12-bit colour JPEG XL. +- An earlier campaign ([camera-capture-measurements.md](camera-capture-measurements.md)) used + blurred dim scenes; its compression numbers were unrepresentative. This campaign used a focused, + real scene (spruce, birch, leaves, building, ~13% clipped sky). +- The report went through several rounds of requested edits: example images per compression type, + methodology detail (how settings were forced), processing on one identical frame on both towers, + an abstract, a pixel-format description, an academic structure (numbered sections, figure and + table captions, units in every header, references), captions that explain every term, an + explanation of raw mosaic vs raw planes, a more careful treatment of what 12 bits add, and a fuller + explanation of the keep rules. + +## 3. User decisions (fixed constraints, do not re-litigate) + +- Full resolution always; **no binning**; **one quality setting** (no switching between a fast low + quality mode and a slow high quality one); continuous acquisition. +- **No exposure memory** across passes or from neighbouring waypoints. +- USB link cap **250 MB/s** (`DeviceLinkThroughputLimit`), conservative. +- **`BayerRG12p` everywhere**, day and night profiles. +- Save **12-bit colour JPEG XL only**; no raw-planes option in the program for now. +- **Fixed white balance** ratios kept in the profiles (auto white balance was measured but not adopted). +- When saving falls behind, **pause the scan** (save gate) rather than drop frames. +- Writing style for the report: academic structure, captions that explain their terms, units given + wherever possible, plain language. Commit messages: one plain-language line, no trailers. + +## 4. Test setup essentials + +- Same camera for both towers: Alvium 1800 U-1620c `DEV_1AB22C0B7D4E`, 5328 × 3040, firmware 14.3. + Moved from 121 to 111 so only the computer differs. Tower 111's own camera (1800 U-1240c, 12 MP, + 4024 × 3036) was not tested; its frames are 25% smaller, so the Delta results are conservative. +- Dates: tower 121 on 2 Oct 2026 (sunny afternoon); tower 111 on 4 Oct (afternoon into evening); + shared-frame processing on both towers simultaneously on 4 Oct 23:03–23:32. +- The tower program was stopped during all tests. +- Storage write speed: IOTA 218 MB/s, Delta 76 MB/s (eMMC). Libraries: 121 libjxl 0.11.1 / + OpenCV 4.10; 111 libjxl 0.11.0 / OpenCV 4.5.4. +- Tools in `tools/camera-bench/` (camera_bench.py with vmbpy; raw12_bench.cpp; run scripts; + plot_results.py, plot_common.py, make_figures.py, compare_examples.py). Results in + `results-2026-10-02-tower121/`, `results-2026-10-04-tower111/`, `results-common-tower121/111/`. + Full-size example `.jxl` files in `tools/camera-bench/examples-*` (git-ignored, ~115 MB per set). + The recorded auto-exposure steps are also in `tests/data/ae_steps.csv` (used by a unit test). + +## 5. Methods that are easy to misunderstand + +- **Forcing settings:** feature writes over USB while streaming. `ExposureAuto`/`GainAuto` set to + `Off` freezes current values, then `ExposureTime`/`Gain` are written; for the 2 s "force" phase the + camera delivers 10–30 frames at the forced values; then auto is set back to `Continuous` (time 0). + Dark step: exposure E₀/16 with 0 dB gain. Bright step: exposure at the 20 ms cap plus gain up to + +24.1 dB (×16). Bright start is fully clipped (97–100%), so the camera cannot see how far over it is. +- **Brightness** = mean of a sparse sample of the frame buffer on 0–255 (12-bit: high byte). + **Settled** = stays within ±10% of the median brightness of the run's last 2 s. +- **Frames in transit:** the first 1–5 frames after release still carry the forced values. +- **Shared frame:** tower 121's real frame rebuilt bit-exactly from its lossless 12-bit file and + copied to both towers (identical MD5). Its 8-bit raw version is the 12-bit value >> 4; its "RGB8" + version is the tower's own edge-aware conversion scaled to 8 bits, so on the shared frame the RGB8 + file size equals raw 8-bit's (2.66 MB). The camera's real RGB8 files were 3.7–4.0 MB. +- **Synthetic frame:** "dead leaves" texture plus noise (ISO 19567-2 style), sharper than any real + scene; worst case for the encoder; used for pile-up and heat. +- **Quality** is against the exact lossless 12-bit colour image of the same frame: PSNR on the + 12-bit scale over 3 channels; SSIM on luma. JPEG XL is perceptual, so PSNR is moderate while SSIM + is high. +- Camera `BayerRG` = OpenCV `COLOR_BayerBG2RGB` (verified). The camera applies white balance to raw. +- 12-bit JPEG XL from 16-bit words needs bit depth "from codestream", or values get rescaled; + lossless needs the original colour profile. An earlier mistake on both points was found and the + 12-bit numbers corrected before the final report. + +## 6. Key results (numbers to quote) + +Frame rate at 250 MB/s (fps): RGB8 4.95, BayerRG8 14.75, BayerRG10p 11.81, BayerRG12p 9.86, +BayerRG12 7.41. 60 s streams: zero lost or incomplete frames in every format on both towers, also +while saving continuously. On 111 the camera shares the USB 3 controller with a 5G modem, a USB +network adapter and the idle second camera. + +Auto-exposure settling after ×16 steps: ~25–35 frames, so time scales with 1/frame rate. +BayerRG8 1.8–2.7 s; BayerRG12p 2.2–6.6 s (typically ≈ 3 s; long runs contain mid-way pauses); +RGB8 4.6–6.6 s. With exposure at its 20 ms cap the camera adjusts gain by 1–2 dB per frame and +sometimes pauses up to 2.8 s while still far too bright. + +Keep rules (24 runs = 3 formats × 2 directions × 2 repeats × 2 towers; good = within ±10% of +final level): +- "Exposure and gain unchanged 3 frames": 0/24. Replay detail (added 2026-10-05 from + `tests/data/ae_steps.csv`): 16 runs fired on the first frames (0.2–0.5 s, frames in transit, image + black 9–13 or white 253–255 vs ≈ 120); 8 fired during mid-way pauses at brightness 33–227. +- "In band 2 frames": 12/24 (passes through the band ~15% above final level when descending). +- "In band and steady (≤ 2%/frame) 3 frames": 21/24 (3 slightly early on descents); median time + when correct 3.61 s (121) / 5.45 s (111). Implemented as `settle_rule = brightness`. +- Band = `IntensityControllerTarget` ± `Tolerance` = 50 ± 5% → 114.75–140.25 on 0–255. + +Auto white balance: recovers in 2–4 frames (0.2–0.8 s), stable ±0.8–3.1%. Exposure, not white +balance, sets the wait. Auto ratios ≈ red 2.36–2.46, blue 1.87–1.97 vs fixed profile 2.87 / 2.0. + +Processing on the shared frame (4 threads; IOTA / Delta, ms): unpack 10 / 17; de-Bayer 12-bit +bilinear 17 / 22, edge-aware 28 / 33; encode 12-bit d1.0 e3 480 / 765; today's RGB8 encode +d0.8 e4 566 / 867. Totals: 12-bit edge-aware d1.0 e3 **518 / 820 ms, 2.65 MB**; today RGB8 +567 / 872 ms, 3.27 MB; raw 8-bit → colour 468 / 741 ms, 2.66 MB. Delta ≈ 1.6× slower. +Unpack + de-Bayer = 7% of a save; encode ≈ 90%. Effort 1–4 within ~15%; effort 3 smallest; +4 threads fastest; on the IOTA 3 threads is oddly slower than 2. + +Distance sweep (12-bit colour, shared frame, MB): lossless 44.8, d0.3 7.19, d0.5 5.15, d0.8 3.16, +d1.0 2.65, d1.5 1.63, d2.0 1.32. Quality at d1.0: PSNR 36.6 dB, SSIM 0.973. Raw planes d0.5: +6.0 MB, PSNR 47.2 dB. Lossless: raw planes 17.6, raw mosaic 20.9, colour 47.4 MB. + +Throughput (saves/s back to back, shared frame): 1 worker × 4 threads IOTA 1.79, Delta 1.19; +2 × 2: 2.15 / 1.53. On the shared frame the IOTA keeps up with one frame per 0.7 s with one worker; +the Delta needs two. Pile-up details on the synthetic frame are in Table 18. + +Heat (10 min each): one frame every 2 s → IOTA 53 °C mean / 5.1 W, Delta 66 °C / 5.7 W. Back to +back (2 × 2): IOTA 80 °C, 12.5 W, loses 16% throughput as it heats; Delta 78 °C, at its 8 W limit, +steady. + +Storage: 12-bit colour d1.0 ≈ 2.65–2.75 MB/frame → 114–119 GB/day at one frame every 2 s. Towers +have 11–22 GB free, so images must leave the tower continuously. + +## 7. The 12-bit question (discussed at length; current position) + +- Noise floor (12-bit DN, green pixels): at ≈ 700 DN 26 (0 dB) / 42 (15 dB); at ≈ 1500 35 / 58; + at ≈ 3000 37 / 80. One 8-bit step = 16 DN; rounding error RMS = 16/√12 = 4.6 DN. +- Errors add in quadrature: √(26² + 4.6²) = 26.4 → rounding raises noise by at most 1.5% in the + measured daylight scene (0.6% at 15 dB, < 0.2% in bright areas). +- Averaging: while noise > one 8-bit step, rounding error is random (dither) and averages down at + the same √N rate as noise, so 8-bit averages resolve sub-step differences as well as 12-bit. A + simulation (not in the report, by user request) showed a 3 DN difference over 100 × 100 pixels + recovered exactly by both at noise 26, but only 0.2 of 3 by 8-bit at noise 2. +- 12 bits always contain at least as much information; at noise 26 the dropped bits carry about + 0.02 bits/pixel of scene information. +- Compression dominates: PSNR 36.6 dB at d1.0 ≈ 61 DN RMS error (13× the rounding error); d0.3 + ≈ 40 DN; raw planes d0.5 ≈ 18 DN. Much of that is discarded noise, and JPEG XL keeps relatively + more precision in dark tones; shadow-only error was not measured. +- Where 12 bits should matter (Table 17b): dark areas at low gain where noise < one 8-bit step + (below ≈ 300 DN at 0 dB, ≈ 100 DN at 15 dB), i.e. high-contrast scenes such as shaded forest or + backlit ridges under a bright sky at dawn or dusk; thin smoke over dark terrain is the relevant + fire-detection case. At night with high gain, noise is amplified above the step again, so little + benefit. Strong shadow lifting (≥ 3 stops) untested; 2 stops showed no difference. +- None of the dark cases were measured (the darkest area in the scene was ≈ 700 DN; a dusk attempt + on 121 needed 24 dB and was discarded). Proposed test: a dawn/dusk frame with dark terrain under a + bright sky, compared as in Figures 12 and 13. +- 12-bit vs 8-bit raw costs little in saving (same size, ~10% more time) but the camera runs at + 9.86 vs 14.75 fps, ~1 s slower settling in daylight. The main gain over today is frame rate, not + bit depth. + +## 8. Limitations to keep in mind + +One scene, two afternoons, main runs at ≈ 15 dB gain plus one 0 dB frame (124 ms exposure, cap +raised to 200 ms); no night data. Steps forced on a still camera, not real slews; real +waypoint-to-waypoint changes are usually smaller. Tower program stopped, so its own load is not +included. Detection quality not measured (PSNR/SSIM stand in). Slew time (≈ 1–1.5 s) and +mechanical settling (≈ 0.2 s) in the pace table are estimates. + +## 9. Implementation status (branch `raw12-capture`) + +Implemented and unit-tested (457 tests pass locally): 12-bit JPEG XL encoding with the bit-depth fix +and true lossless (known issue #20 fixed); frames carry pixel layout, chunk exposure/gain/white +balance and a brightness sample; `ImagePipeline` unpacks and de-Bayers (edge-aware default), +N workers, bounded queue; scheduler save gate pauses the scan when the queue is full; `SettleRule` +(`settle_rule = brightness`) with a time limit and an "at limit" exit; config keys `save_workers`, +`jxl_threads`, `save_queue_max`, `demosaic`, `settle_rule`; profile export script recipe for raw +`standard` and `night` profiles. Plan: [raw12-pipeline-implementation-plan.md](raw12-pipeline-implementation-plan.md). + +Not done yet: exporting the new camera profiles on a tower (the 1620c is currently on tower 111 +and must be moved back to 121 first); field verification with real slews; checking that downstream +consumers decode 12-bit files; the dawn/dusk 12-bit test. The branch is deployed on tower 111, +whose boot service still passes old `--init/--start` flags and needs `scripts/install-tower`. + +## 10. Report structure (for orientation) + +Abstract; Units and abbreviations; 1 Introduction (1.1 question, 1.2 pixel formats incl. raw +mosaic vs raw planes, Figures 1a/1b, Table 1; 1.3 earlier work); 2 Materials and methods (2.1–2.12, +Tables 2–8, Figure 2); 3 Results (3.1 frame rate, 3.2 auto-exposure, 3.3 keep rules, 3.4 white +balance, 3.5 processing per step, 3.6 effort and threads, 3.7 quality setting, 3.8 example images, +3.9 information content of 12-bit data with Tables 17a/17b, 3.10 throughput, 3.11 heat, +3.12 storage; Figures 3–17, Tables 9–20); 4 Discussion (summary Table 21, pace Table 22, +limitations); 5 Conclusions and six recommendations; References [1]–[7]; Appendix A raw data and +tools; Appendix B per-tower processing times. Figures live in `docs/images/raw12/`. diff --git a/hardware/common/kit.sh b/hardware/common/kit.sh index 6604673..8974487 100644 --- a/hardware/common/kit.sh +++ b/hardware/common/kit.sh @@ -5,6 +5,7 @@ # # The facts a kit must set before sourcing: # KIT_BOARD lattepanda-iota | lattepanda-3-delta (matches probe_board) +# KIT_SAVE_WORKERS image-saving workers for this board (1 IOTA, 2 3 Delta) # KIT_FW_ENV the PlatformIO env for this board's MCU # KIT_MCU_VID USB vendor of the motor MCU (2e8a RP2040, 2341 Arduino) # KIT_DEFAULT_AXES what this board's towers are usually built with @@ -392,6 +393,11 @@ imu = $IMU env = $ENV_SENSOR cooling = $COOLING cameras = $CAMERAS +; Image saving on this computer: frames encoded at once (libjxl threads per +; frame default to 4 / save_workers), and frames that may wait before the +; scan holds. Measured for this board in docs/raw12-pipeline-measurements.md. +save_workers = ${KIT_SAVE_WORKERS:-1} +save_queue_max = 4 [Serial] ; The motor controller MCU, by stable id - /dev/ttyACM* moves when the modem diff --git a/hardware/lattepanda-3-delta/install.sh b/hardware/lattepanda-3-delta/install.sh index 5d43f80..63e2340 100755 --- a/hardware/lattepanda-3-delta/install.sh +++ b/hardware/lattepanda-3-delta/install.sh @@ -16,6 +16,8 @@ KIT_BOARD=lattepanda-3-delta KIT_FW_ENV=leonardo KIT_MCU_VID=2341 KIT_DEFAULT_AXES=yaw,pitch +# Image saving: the N5105 needs two workers x 2 threads to keep up (docs/raw12-pipeline-measurements.md). +KIT_SAVE_WORKERS=2 KIT_IMU_DEVICE=/dev/ttyS0 KIT_IMU_NOTE="On the Delta the MTi is on the RS-232 header: a hardware UART, not a USB node." KIT_ENV_DEVICE=/dev/i2c-2 diff --git a/hardware/lattepanda-iota/install.sh b/hardware/lattepanda-iota/install.sh index 98bfb58..ee41f60 100755 --- a/hardware/lattepanda-iota/install.sh +++ b/hardware/lattepanda-iota/install.sh @@ -18,6 +18,8 @@ KIT_BOARD=lattepanda-iota KIT_FW_ENV=iota_rp2040 KIT_MCU_VID=2e8a KIT_DEFAULT_AXES=yaw +# Image saving: the N150 keeps up with one worker x 4 threads (docs/raw12-pipeline-measurements.md). +KIT_SAVE_WORKERS=1 KIT_IMU_DEVICE=/dev/ttyS0 KIT_IMU_NOTE="On the IOTA the only PCH serial on the header is COM1; confirm it enumerates first." # No PCH-side I2C reaches the header, so an ambient sensor needs a USB-I2C bridge diff --git a/include/fgc/CameraStall.h b/include/fgc/CameraStall.h index 0ca4e64..1baaab8 100644 --- a/include/fgc/CameraStall.h +++ b/include/fgc/CameraStall.h @@ -28,6 +28,18 @@ long long firstFrameLimitMs(double exposure_us); // `failed_restarts` counts restarts since the last frame that arrived. StallAction onNoFirstFrame(long long waited_ms, double exposure_us, int failed_restarts); +// How long a stream that has delivered may go without a newer frame before it +// counts as stalled: twice the longer of the exposure and the frame period, plus +// a second, and never under 2 s. A slow frame is not a stall - tower 111 runs +// multi-second exposures, one frame every 7-9 s, and restarting the stream the +// moment a capture fell due before the next frame threw that exposure away, so +// the next frame came later still (2026-10-04). +long long staleFrameLimitMs(double exposure_us, double frame_period_ms); + +// No frame newer than the last one kept, the newest having arrived +// `since_newest_ms` ago: wait for the next one, or restart the stream. +StallAction onStaleFrame(long long since_newest_ms, double exposure_us, double frame_period_ms); + // A stream that delivered before but has gone stale, and a restart has just // failed to bring a fresh frame. Restart again, or reset once restarts have // stopped working. diff --git a/include/fgc/CaptureScheduler.h b/include/fgc/CaptureScheduler.h index 69400c5..06e251b 100644 --- a/include/fgc/CaptureScheduler.h +++ b/include/fgc/CaptureScheduler.h @@ -5,6 +5,7 @@ #include "fgc/IMotorController.h" #include "fgc/OperatingMode.h" #include "fgc/ScanGrid.h" +#include "fgc/SettleRule.h" #include #include @@ -17,13 +18,22 @@ namespace fgc { // // Each tick(): // 1. reads motor telemetry (per-axis encoder counts + state), -// 2. runs the capture cycle as a move -> settle -> wait -> keep machine: when -// the interval elapses it issues an absolute `MOVE ,` to the next -// target, waits until both axes report standstill at that target, waits a -// further settle_wait_ms so the free-running camera has exposed a frame with -// the gimbal at rest, then keeps the newest frame. If none is ready it +// 2. runs the capture cycle. The interval is the time from one kept frame to +// the next: as soon as a frame is kept it issues an absolute +// `MOVE ,` to the next target, waits until both axes report +// standstill there, waits for the exposure to settle, then waits out what +// is left of the interval and keeps the newest frame. If none is ready it // retries each tick, and gives up on the waypoint after frame_timeout_ms. // +// The exposure wait is settle_wait_ms; with the keep rule on (setSettleRule) +// a scan waypoint then also waits until the camera's frames say the image +// has settled (SettleRule), at most frame_timeout_ms more - after which the +// newest frame is kept anyway, with a warning. +// +// When the move and the exposure wait alone take longer than the interval, +// that waypoint warns - except the return from the last waypoint to the +// first, and the first move after a mode starts, which are long by nature. +// // What it captures is the mode (OperatingMode.h): auto sweeps the ScanGrid front // to back, repeating; pan walks panHeadings() left to right at the pan's pitch, // then jumps back to the left edge; manual captures nothing, bar a snapshot or @@ -59,20 +69,38 @@ public: bool snapshotPending() const { return snapshot_pending_; } // Continuous capture (`capture on|off`): in manual, keep a frame on the scan's - // own cycle - the interval, then standstill + settle wait - wherever the - // operator has aimed. The gimbal stays free to nudge; a frame is only kept at - // rest. Ignored outside manual, and cleared by any setMode(). + // own cycle - one per interval since the last, once standstill + the exposure + // wait are done - wherever the operator has aimed. The gimbal stays free to + // nudge; a frame is only kept at rest. No overrun warnings: the operator may be + // jogging. Ignored outside manual, and cleared by any setMode(). void setContinuous(bool on); bool continuous() const { return continuous_; } double imageRate() const { return image_rate_; } // images per second + // Is there room to save another frame (ImagePipeline::canAccept)? Asked + // before moving to the next waypoint and before keeping a continuous-capture + // frame. While it says no, the scan holds where it is, so a scan that + // outruns saving slows down instead of losing pictures or filling memory. + // A requested snapshot is never held. Unset: always room. + void setSaveGate(std::function can_save) { can_save_ = std::move(can_save); } + // True while the scan is held for saving to catch up. + bool holdingForSave() const { return holding_; } + // Per-camera-profile timing (see CameraProfile): how long to wait after standstill - // before keeping a frame, and how long to keep retrying before abandoning the - // waypoint. Called on the control thread, same as tick(). + // for the exposure to settle, and how long to keep retrying before abandoning + // the waypoint. Called on the control thread, same as tick(). void setSettleWait(int settle_ms, int timeout_ms); int settleWaitMs() const { return settle_wait_ms_; } + // The keep rule for scan waypoints (profile setting settle_rule). Off: keep + // the newest frame once settle_wait_ms has passed, as always. On: after + // settle_wait_ms (now the mechanical minimum), wait until the camera's + // frames say the image has settled (SettleRule), within frame_timeout_ms; a + // camera that reports no frame data falls back to the fixed wait. + void setSettleRule(bool on, const SettleRule::Params& params = {}); + bool settleRuleOn() const { return use_rule_; } + // Replace the degrees<->counts maps (e.g. after `gimbal heading fit`) so subsequent // scan waypoints target with the calibrated conversion. Called on the control // thread, same as tick(), so no locking is needed. @@ -85,20 +113,50 @@ public: long yawTargetCounts() const { return yaw_target_; } long pitchTargetCounts() const { return pitch_target_; } bool moving() const { return moving_; } + // Where the current waypoint is: "moving", "settling" (exposure) or "waiting" + // (for the rest of the interval). For the status line. + const char* phase() const; + // Waypoints that warned for running past the interval since construction. + int overruns() const { return overruns_; } private: + // One frame's way to the camera, from standstill: when the gimbal came to + // rest, when the exposure wait ended, when the trigger fell due (0 = not yet). + struct Hold { + long long settled_ms = 0; + long long ready_ms = 0; + long long due_ms = 0; + }; + enum class Shot { Waiting, Captured, TimedOut }; + long long elapsedMs() const; void resetTimer(); void sendMove(); // emit MOVE to (yaw_target_, pitch_target_) bool settledAt(const MotorTelemetry& t) const; + bool startNextMove(); // pick the next target and MOVE; false if none + void finishWaypoint(); + std::string waypointLabel() const; // "waypoint 3/15" / "pan position 2/6" void serviceSnapshot(const MotorTelemetry& t); void serviceContinuous(const MotorTelemetry& t); + // Standstill seen: begin the exposure wait. + void beginHold(Hold& h); + // Whether the exposure wait that began at h.settled_ms is over (sets + // h.ready_ms): settle_wait_ms, then with `keep_rule` the keep rule too. + bool dwellOver(Hold& h, bool keep_rule); + // The keep rule, fed the frames since it last looked: true once it has + // decided - settled, at the exposure limits, timed out (with a warning), or + // no frame data to judge. + bool ruleDecided(long long since_settled); + // The save gate, with one log line when a hold starts and one when it ends. + bool roomToSave(); + // Trigger until a frame comes or frame_timeout_ms passes since it fell due. + Shot fire(Hold& h); + // The part a snapshot and a continuous-capture frame share: wait for the - // gimbal to stand still, then the settle wait, then trigger until the frame - // budget runs out. `settled_ms` is that request's own settle clock. - enum class Shot { Waiting, Captured, TimedOut }; - Shot shootAtRest(const MotorTelemetry& t, long long& settled_ms); + // gimbal to stand still, then the exposure wait, then (continuous only) the + // rest of the interval, then trigger until the frame budget runs out. + Shot shootAtRest(const MotorTelemetry& t, Hold& h, bool wait_interval); IMotorController& motor_; ICameraSource& camera_; @@ -106,8 +164,11 @@ private: ScanGrid& grid_; std::function now_ms_; + std::function can_save_; + bool holding_ = false; double image_rate_; - long long timer_start_ = 0; + double interval_ms_; // 1000 / image_rate_; 0 = never + long long timer_start_ = 0; // the last frame kept or given up, or the mode start Mode mode_ = Mode::Manual; PanSpec pan_; @@ -115,8 +176,17 @@ private: std::vector pan_headings_; size_t pan_index_ = 0; - bool moving_ = false; // a MOVE has been issued, awaiting settle - long long settled_at_ms_ = 0; // when standstill was first seen (0 = not yet) + bool moving_ = false; // a MOVE has been issued, its frame not yet kept + Hold hold_; // the current waypoint's frame + long long move_sent_ms_ = 0; + bool first_move_ = true; // no move yet since setMode() + bool quiet_overrun_ = false; // this waypoint is a long move by nature + int overruns_ = 0; + bool save_held_ = false; // the next move waited for saving to catch up + bool use_rule_ = false; // keep by SettleRule, not after a fixed wait + SettleRule rule_; + bool rule_started_ = false; // this waypoint's rule has its first frame number + uint64_t rule_seq_ = 0; // last frame fed to the rule int settle_wait_ms_ = 1000; int frame_timeout_ms_ = 2000; bool pitch_present_ = false; // a P: segment was seen in telemetry (2-axis rig) @@ -126,9 +196,9 @@ private: bool pan_unreachable_warned_ = false; bool snapshot_pending_ = false; // a manual snapshot is armed - long long snapshot_settled_ms_ = 0; // when standstill was first seen (0 = not yet) + Hold snapshot_hold_; bool continuous_ = false; // `capture on` - long long continuous_settled_ms_ = 0; + Hold continuous_hold_; long yaw_target_ = 0; // current target, encoder counts long pitch_target_ = 0; }; diff --git a/include/fgc/Config.h b/include/fgc/Config.h index 2aed1ae..759ce34 100644 --- a/include/fgc/Config.h +++ b/include/fgc/Config.h @@ -16,7 +16,7 @@ namespace fgc { struct GeneralConfig { std::string tower_name = "Unnamed"; - int image_interval = 5; // seconds between captures + double image_interval = 5; // desired seconds from one picture to the next bool debug = false; // print motor telemetry each loop tick }; @@ -49,6 +49,13 @@ struct HardwareConfig { // a span is cut into as many pictures as it takes (OperatingMode.h). double pan_step_deg = 6.0; + // Saving images - a property of the computer, not of what the tower does. + // Measured (docs/raw12-pipeline-measurements.md): the IOTA keeps up with one + // worker x 4 threads, the 3 Delta needs two x 2. + int save_workers = 1; // frames encoded at once + int jxl_threads = 0; // libjxl threads per frame; 0 = 4 / save_workers + int save_queue_max = 4; // frames waiting or saving before the scan holds + // Was a hardware.ini actually read? False on every tower that has not been // migrated yet, and the switch that keeps this change invisible on rollout. bool declared = false; @@ -179,6 +186,11 @@ struct SerialConfig { struct CameraProfileTiming { int settle_wait_ms = 1000; // hold after standstill before keeping a frame int frame_timeout_ms = 2000; // give up on the waypoint after this + // How to tell the frame is good to keep, after settle_wait_ms: + // fixed keep the newest frame as soon as settle_wait_ms has passed + // brightness wait until brightness is in the camera's target band and + // steady for 3 frames (SettleRule.h), up to frame_timeout_ms + std::string settle_rule = "fixed"; }; // One camera, from a [Camera.] block. Nothing is implied by position: the @@ -196,6 +208,8 @@ struct CameraSpec { // Encoding, defaulted from [Camera] and overridable per camera. double jxl_distance = 0.8; int jxl_effort = 4; + // De-Bayering for raw profiles (BayerRG8/12p): "edge_aware" or "bilinear". + std::string demosaic = "edge_aware"; std::map timing; // by profile name @@ -208,6 +222,7 @@ struct CameraConfig { // [Camera] defaults, inherited by each block unless it overrides them. double jxl_distance = 0.8; int jxl_effort = 4; + std::string demosaic = "edge_aware"; // Enabled cameras only. std::vector active() const; diff --git a/include/fgc/ICameraSource.h b/include/fgc/ICameraSource.h index 8b7fbf2..04cc4f5 100644 --- a/include/fgc/ICameraSource.h +++ b/include/fgc/ICameraSource.h @@ -1,6 +1,8 @@ #pragma once #include "fgc/CameraProfile.h" +#include "fgc/RawImage.h" +#include "fgc/SettleRule.h" #include #include @@ -11,12 +13,25 @@ namespace fgc { // One captured frame, owning a copy of its pixel buffer (was image_store_8bit). struct Frame { - std::vector data; + std::vector data; // as delivered: BayerRG12p stays packed uint32_t width = 0; uint32_t height = 0; - int channels = 0; // 1 (mono) or 3 (RGB) + int channels = 0; // values per pixel as delivered: 3 (RGB) or 1 (mono, Bayer mosaic) + // How `data` is laid out. Unknown only for a format the program does not + // handle; `channels` then says what it holds, as before layouts existed. + PixelLayout layout = PixelLayout::Unknown; long long timestamp_ms = 0; // Unix epoch ms int cam_id = 0; // index into the configured camera list + + // What the camera says it exposed this frame with (its per-frame chunk + // data). Negative when the profile does not turn chunk data on. + double exposure_us = -1.0; + double gain_db = -1.0; + double wb_red = -1.0; // BalanceRatio, red against green + double wb_blue = -1.0; + // Sampled from `data` on arrival (see RawImage.h); invalid when the layout + // is Unknown. + BrightnessSample brightness; }; // Live per-camera identity, sensor telemetry, and acquisition stats, for the @@ -77,6 +92,20 @@ public: virtual void setFrameCallback(FrameCallback cb) = 0; + // ---- For the keep rule (SettleRule.h) --------------------------------------- + // The most recent frames camera 0 delivered, with their brightness, exposure + // and gain. Sequence numbers count up for the session; framesSince(n) returns + // the frames after n, oldest first (only the last few dozen are kept). + // frameSeq() is the newest sequence number. Default: no frames, and the scan + // keeps its fixed settle wait. + virtual uint64_t frameSeq() { return 0; } + virtual std::vector framesSince(uint64_t /*after*/) { return {}; } + // A numeric camera feature (IntensityControllerTarget, ExposureAutoMax...). + // False if the camera has no such feature. + virtual bool readFeature(int /*camera_index*/, const std::string& /*name*/, double& /*value*/) { + return false; + } + // Number of cameras this source manages. virtual int cameraCount() const = 0; diff --git a/include/fgc/ImagePipeline.h b/include/fgc/ImagePipeline.h index e8c8c1b..c9d00fa 100644 --- a/include/fgc/ImagePipeline.h +++ b/include/fgc/ImagePipeline.h @@ -23,9 +23,11 @@ struct Orientation { // Consumes captured frames and turns them into stored artifacts: rotate by // Camera.rotation_deg, encode to JPEG XL, write to -// /