diff --git a/.gitignore b/.gitignore index 311e1de..6b06c89 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,13 @@ bin/x64/Release/NIR/ config.ini bin/x64/Release/config.ini .deploy.env + +# Full-size example JPEG XL files from tools/camera-bench (1-30 MB each); the +# report shows decoded crops from docs/images/raw12/ instead. +tools/camera-bench/examples-*/ + +# The shared processing test frame (87 MB), rebuilt from an example file. +tools/camera-bench/common-frame/ + +# Claude Code worktrees: separate checkouts of this repo on other branches. +.claude/worktrees/ diff --git a/docs/architecture.md b/docs/architecture.md index 9efa7a8..55e2b0a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -186,6 +186,11 @@ full resolution is not reachable in RGB8 at all. (The datasheet's "21 fps at ful Raising the throughput limit to 450 is not the fix: it produced incomplete frames on this host ([known-issues.md](known-issues.md)). +*Tower 121 (1800 U-1620c, 5328×3040) measured differently on 2026-09-30: its camera budgets the limit +against the unbinned frame, so 2×2 binning gives no frame rate there (RGB8 is 4.95 fps at 250 MB/s +either way); BayerRG8 at full resolution reaches 14.75 fps at 250 MB/s and 26.2 at 450, and 450 MB/s +streamed without loss. See [camera-capture-measurements.md](camera-capture-measurements.md).* + Full parameter reference: `[Camera]` in [configuration.md](configuration.md). ## Why this shape diff --git a/docs/camera-capture-measurements.md b/docs/camera-capture-measurements.md new file mode 100644 index 0000000..0ec95dd --- /dev/null +++ b/docs/camera-capture-measurements.md @@ -0,0 +1,454 @@ +# Camera capture measurements — towers 121 and 111 + +These are measured values, not estimates. They were taken to decide how auto mode can capture faster +at **full resolution, no binning, one quality setting, continuous acquisition**. That constraint is +the design decision these numbers serve. Two runs: + +- **2026-09-30, tower 121:** camera frame rates, streaming, auto-exposure, white balance, first CPU + figures. +- **2026-10-01, towers 121 and 111:** JPEG XL compression in detail (input format, effort, threads, + quality setting), whether saving keeps up with the scan (pile-up), CPU heat, storage speed; and + the camera tests repeated on tower 111. + +Each figure applies to the tower it was measured on (see [Scope](#scope-and-caveats)). + +## Summary + +**Camera and USB** + +- **Full resolution does not cost frame rate on tower 121.** With 2×2 binning, its `standard` + profile runs at the same rate as full-resolution RGB8 (4.95 fps at 250 MB/s). The camera budgets the + link limit against the unbinned frame, so binning saves USB traffic but not time. +- **Raw Bayer is about 3× the frame rate of RGB8 over the same link** on both cameras. Tower 121: + BayerRG8 14.75 fps at 250 MB/s, 26.2 at 450 MB/s (RGB8 4.95 and 8.8). Tower 111: 14.2 and 20.6 + (RGB8 5.9 and 9.6). +- **No format stalled on either tower.** Every format, at every link limit (up to 450 MB/s), + streamed for 60 s with nothing lost, also while the CPU was encoding. The only losses were 2 + incomplete frames out of 1,574 on tower 121 at 450 MB/s under encode load. The stalls in + [known-issues.md](known-issues.md#camera-acquisition-usb3--root-cause--approach) did not + reproduce on either tower since the USB fixes (#19). +- **The camera applies white balance to raw Bayer output** exactly as to RGB8, on both cameras, so + colour conversion on the tower needs no white-balance step. + +**Auto-exposure** (tower 121; tower 111's camera saw near-darkness, see below) + +- **It settles faster at a higher frame rate, but not in proportion.** After a large brightness step + it reached ±10% in 1.4–1.8 s at 4.95 fps, 0.7–1.2 s at 14.75 fps and 0.4–0.7 s at 26.2 fps. + Today's 1000 ms `settle_wait_ms` at 4.95 fps is **shorter than a large step takes**, which fits the + badly exposed frames in known issue #16. +- **"Exposure and gain unchanged for N frames" is not a safe settle test.** Auto-exposure moves in + steps and pauses; the rule fired in 6 of 14 runs while brightness was still 17–300% off. +- **From a dark step it overshoots to its maximum exposure** (3.4–4.1× too bright) before coming + back down. +- **The camera's per-frame data (chunk mode) works** on both cameras: exposure, gain and a camera + timestamp on every frame. + +**JPEG XL compression and saving** (2026-10-01; times include writing the file) + +- **The encoder takes ordinary images only:** one channel (grey) or three (RGB). It knows nothing of + Bayer patterns, so raw data is either converted to colour first or stored as a one-channel mosaic, + which must be lossless. +- **Converting raw to colour on the tower costs almost nothing:** 5–13 ms, then the same encode time + as the camera's own RGB8. The colour *encode* is the cost: on a detailed full frame, 0.74 s on + tower 121 (16 MP) and 0.86 s on tower 111 (12 MP), with 4 threads at today's quality setting. +- **Effort 1–4 cost about the same; effort 5 is 3–5× slower.** Effort 3 gave the smallest files of + effort 1–4. +- **Production's 3 encoder threads are a poor choice.** On tower 121, 3 threads was slower than 2; + 4 threads was fastest on both towers. +- **One save worker cannot keep up with a frame every 0.7 s:** colour images piled up on both towers + (9–29 behind after 2 minutes, saved up to 21 s late). **Two workers with 2 threads each kept up** + on both. One worker handled a frame every 1.0 s on both. +- **Lossless raw saving uses a tenth of the CPU** (0.19–0.25 CPU-s per frame against 1.7–1.9), but + files are 2–3× larger, and on tower 111 the storage is the limit: its eMMC writes at 76 MB/s + (tower 121: 218 MB/s). +- **Heat, in the field enclosure:** flat out, the CPU levels at 75–79 °C with peaks of 88–90 °C. + At one colour frame every 2 s: 53 °C on tower 121 and 63 °C on tower 111. Tower 121's throughput + drifted down 13% over 10 minutes flat out; tower 111's by 4%. + +**Bugs found** + +- **`jxl_distance = 0` is not lossless** in the production encoder (known issue #20): decoded pixels + differ by up to 1–2 levels. +- **12-bit lossless needs an explicit bit-depth setting** in libjxl, or 12-bit values held in 16-bit + words are rescaled. This affected only the test program; 12-bit figures before 2026-10-01 were + wrong and are corrected below. + +## Setup + +| | Tower 121 | Tower 111 | +|---|---|---| +| Computer | LattePanda IOTA: Intel N150, 4 cores / 4 threads, 0.7–3.6 GHz, 7 GB | LattePanda 3 Delta: Intel Celeron N5105, 4 cores / 4 threads, 0.8–2.9 GHz, 7.6 GB | +| OS | Ubuntu 26.04.1, kernel 7.0.0-31 | Ubuntu 22.04.5, kernel 6.8.0-138 | +| Storage | eMMC, 218 MB/s write (measured) | eMMC, 76 MB/s write (measured) | +| Camera | Alvium 1800 U-1620c, `DEV_1AB22C0B7D4E`, firmware 14.3.4646D35D, 5328×3040 (16.2 MP) | Alvium 1800 U-1240c, `DEV_1AB22C049CD1`, firmware 13.1.794391F9, 4024×3036 (12.2 MP) | +| Profile in use (`standard`) | RGB8, 2×2 binning, 4.95 fps, 250 MB/s, exposure ≤ 20 ms, `MinimizeBlur` | RGB8, no binning, 1.09 fps, 200 MB/s, exposure ≤ 9 s, `MinimizeNoise` | +| Libraries | vmbpy 1.2.0 (Python 3.14), libjxl 0.11.1, OpenCV 4.10.0 | vmbpy 1.0.5 (Python 3.10), libjxl 0.11.0, OpenCV 4.5.4 | +| Tower program | `f2c61e0-dirty` | `8d81e03` (same JPEG XL encoder source) | +| Enclosure | field enclosure (confirmed) | field enclosure | +| Scene | Indoors facing a window, badly out of focus (known issue #15). 2026-09-30: grey, noisy, gain at 24 dB. 2026-10-01: nearly black, a blurred blue light, exposure and gain at their limits. | Near-darkness: mean pixel value 7 of 255 at 20 ms and 27 dB. | +| USB preconditions | `usbfs_memory_mb` 1000, LPM off ([#19](known-issues.md)) | same | + +The tower program was **stopped for every test**: the camera accepts one program at a time, and the +CPU figures should have one cause. Nothing here is measured under the program's own load. + +## Method + +The programs are in [tools/camera-bench](../tools/camera-bench/README.md). + +**Camera** (`camera_bench.py`, vmbpy): full sensor, binning 1. Its frame handler only records +metadata and hands the buffer back, so it cannot cause the losses it counts. On tower 111 +auto-exposure was capped at 20 ms (tower 121's profile value), or its 0.7 s exposures would have +limited the frame rate instead of the link. +1. **Maximum frame rate** per pixel format and link limit, as the camera reports it. +2. **Streaming** 60 s at that rate, 5 buffers (as production): frames, incomplete frames, gaps in the + camera's frame IDs, gaps over 1 s. +3. **Streaming under load:** the same while encoding colour JXL in a loop. +4. **Auto-exposure step:** settle 8 s, force brightness (exposure × gain) to 1/16 or up to 16×, hold + 2 s, release, and log per-frame exposure and gain for 15 s; twice per setting. +5. **White balance on raw:** exposure and gain fixed; red and blue `BalanceRatio` at 1.0 and 2.0; + compare the mean at those pixel sites in BayerRG8 and RGB8. + +**Compression** (`encode_bench.cpp`), on full-resolution frames in the camera's formats. Colour +lossy encodes use the **production** `src/camera/JpegXlEncoder.cpp` unchanged. Lossless and 12-bit +encodes use a test-only encoder that sets lossless mode correctly (see #20). Every time includes +writing the file, as production does. Two input frames per tower: +- **the real frame** the camera saw that day (dark and blurred, so it compresses unusually easily); +- **a synthetic frame** at the camera's resolution: a "dead leaves" texture (the camera texture + chart of ISO 19567-2) with mild sensor noise, sharp and detailed at every scale, as the realistic + worst case ([image](../tools/camera-bench/results-2026-10-01-tower121/synthetic.jpg)). + +Input kinds: `rgb8` (the camera's colour), `bilinear` and `ea` (raw converted to colour on the tower, +OpenCV bilinear or edge-aware), `raw8` and `raw12` (the raw mosaic, lossless), `colour12` (12-bit raw +converted to 16-bit colour, lossy). + +6. **Matrix:** every colour input at effort 1–5 × 1–4 threads (quality setting `distance` 0.8, as + production); a distance sweep 0.5–3.0 with PSNR against the input; lossless raw at effort 1–3 × + 1–4 threads. Median of 3 runs. +7. **Throughput:** back-to-back saves for 60 s, 1 worker × 4 threads and 2 workers × 2 threads. +8. **Pile-up:** one frame queued every 2.0, 1.0 or 0.7 s for 2 minutes, as a scan keeps one frame + per waypoint; time from queueing to file written, and the queue left at the end. +9. **Heat:** idle baseline; 10 min of back-to-back colour saves; 5 min cool-down; 10 min at one + frame every 2 s. CPU package temperature, average clock, CPU load and package power (RAPL) once a + second. +10. **Storage:** `dd` of 256 MB with a final sync. + +At the end of each tower's run the camera profile was reloaded, frames were confirmed to flow, and +the tower program was restarted; both came back on their `standard` profile. + +## Results: camera and USB + +### Maximum frame rate and streaming, full resolution + +Received / incomplete / lost frames over 60 s at the maximum rate. + +**Tower 121** (5328×3040): + +| Format | Frame | 250 MB/s: max fps | 60 s at 250 | 450 MB/s: max fps | 60 s at 450 | +|---|---|---|---|---|---| +| BayerRG8 | 16.2 MB | **14.75** | 886 / 0 / 0 | **26.17** | 1570 / 0 / 0 | +| BayerRG10p | 20.2 MB | 11.82 | not streamed | 20.95 | not streamed | +| BayerRG12p | 24.3 MB | **9.86** | 592 / 0 / 0 | **17.52** | 1053 / 0 / 0 | +| RGB8 | 48.6 MB | **4.95** | 298 / 0 / 0 | **8.79** | 527 / 0 / 0 | +| RGB8, 2×2 binning (`standard`) | 12.1 MB | 4.95 | 50 / 0 / 0 in 10 s, 60 MB/s actual | not measured | not measured | + +While encoding: BayerRG8 at 450 MB/s 1574 received, **2 incomplete**; BayerRG8 at 250 and RGB8 at 250 +clean. + +**Tower 111** (4024×3036): + +| Format | Frame | 200 MB/s: max fps | 60 s at 200 | 250 MB/s: max fps | 60 s at 250 | 450 MB/s: max fps | 60 s at 450 | +|---|---|---|---|---|---|---|---| +| BayerRG8 | 12.2 MB | **12.10** | 717 / 0 / 0 | **14.25** | 854 / 0 / 0 | **20.60** | 1235 / 0 / 0 | +| BayerRG10p | 15.3 MB | 10.20 | not streamed | 12.10 | not streamed | 18.06 | not streamed | +| BayerRG12p | 18.3 MB | **8.80** | 527 / 0 / 0 | **10.52** | 632 / 0 / 0 | **16.01** | 962 / 0 / 0 | +| RGB8 | 36.7 MB | **4.84** | 290 / 0 / 0 | **5.90** | 354 / 0 / 0 | **9.61** | 575 / 0 / 0 | + +While encoding: BayerRG8 at 450 and 250, and RGB8 at 250, all clean (1235, 854 and 353 frames). + +In every run the measured rate was within 1.5% of the maximum and the longest gap between frames was +about one frame period. Tower 111's raw rates rise less than in proportion to the link limit +(20.6 fps at 450 MB/s carries 251 MB/s), so above 250 MB/s something other than the link limits +that camera. + +### Auto-exposure step response (tower 121, BayerRG8) + +Times from switching auto-exposure back on. *±10%*: first frame within 10% of the pre-step +brightness. *Stopped changing*: last change of exposure or gain in the 15 s run. *3-unchanged rule*: +first frame after which exposure and gain stayed the same for 3 frames. + +| fps | Reaction rate | Step | ±10% | Stopped changing | 3-unchanged rule fired | +|---|---|---|---|---|---| +| 26.17 | 100 | dark ×1/16 | 0.47 s (12 frames), 0.66 s (17) | 0.89 s (23), 1.27 s (33) | 0.62 s, **0.09 s** | +| 26.17 | 100 | bright ×4.1, ×3.7 | 0.54 s (14), 0.39 s (10) | 0.93 s (24), 0.93 s (24) | **0.09 s**, 0.77 s | +| 14.75 | 100 | dark ×1/16 | 1.04 s (15), 0.91 s (13) | 1.17 s (17), 1.18 s (17) | **0.16 s**, **0.16 s** | +| 14.75 | 100 | bright ×4.0, ×3.6 | 1.18 s (17), 0.70 s (10) | 1.45 s (21), 1.38 s (20) | **0.16 s**, 1.11 s | +| 4.95 | 100 | dark ×1/16 | 1.58 s (7), 1.79 s (8) | 2.19 s (10), 2.59 s (12) | 2.59 s, 3.00 s | +| 4.95 | 100 | bright ×3.8, ×3.5 | 1.38 s (6), 1.38 s (6) | 1.79 s (8), 2.59 s (12) | 2.19 s, 3.00 s | +| 14.75 | **50** | dark ×1/16 | 2.19 s (32) | 3.75 s (55) | **0.16 s** | +| 14.75 | **50** | bright ×3.4 | 1.99 s (29) | 3.48 s (51) | **0.16 s** | + +Bold: the rule fired too early, on frames that still carried the forced values. The bright step +reached only ×3.4–4.1 because gain was already at its maximum. In every run exposure and gain ended +within 2% of the pre-step value. From the per-frame series: +- **Once within ±10%, brightness stayed there,** so that column is a real settle time. +- **Old values persist for 1–5 frames** after auto-exposure resumes. +- **Auto-exposure moves in steps and pauses:** holds of 3+ unchanged frames while still more than 10% + off, 0–3 times per run at rate 100 and 5–6 at rate 50. Even skipping the first 3–4 frames, "unchanged + for 3 or 4 frames" fired while 17–300% off in 6 of 14 runs. +- **Overshoot from a dark step:** auto-exposure first jumps to `ExposureAutoMax` (20 ms) and comes + back down, here 3.4–4.1× too bright. + +**Tower 111** saw near-darkness, so with exposure capped at 20 ms both exposure and gain sat at their +limits. The bright step could not be forced at all, and the dark step only measures recovery back to +the limits: 1.25 s (13 frames) at 20.6 fps, 1.56–1.65 s (10–12) at 14.2 fps, 3.6–3.8 s (12–13) at +5.9 fps, 2.43 s at reaction rate 50. Treat these as a lower bound on its settling, not a measurement +of it. + +### White balance on raw output + +Mean pixel values at those sites, exposure and gain fixed: + +| Setting | 121 raw red | 121 RGB8 red | 121 raw blue | 121 RGB8 blue | 111 raw red | 111 RGB8 red | 111 raw blue | 111 RGB8 blue | +|---|---|---|---|---|---|---|---|---| +| Red ratio 1.0 | 58.3 | 57.8 | 104.3 | 102.8 | 2.55 | 6.03 | 7.76 | 17.65 | +| Red ratio 2.0 | 82.4 | 81.4 | 104.9 | 103.3 | 5.54 | 12.47 | 7.75 | 17.55 | +| Blue ratio 1.0 | 99.2 | 98.1 | 74.0 | 73.3 | 7.07 | 16.46 | 4.10 | 9.60 | +| Blue ratio 2.0 | 99.1 | 98.9 | 104.0 | 103.5 | 7.11 | 16.44 | 8.70 | 19.50 | + +On both cameras the ratios change raw pixels as they change RGB8. On tower 111 doubling a ratio +doubled the values (×2.07–2.17). On tower 121 it gave only ×1.41; that frame had 21% of its pixels +clipped at white, which cannot get brighter, and this is the likely reason. On tower 111 the RGB8 +values are about 2.3× the raw ones at the same settings; not explained. + +The camera's `BayerRG8` is OpenCV's **`COLOR_BayerBG2RGB`**, not `BayerRG2RGB`: de-Bayering with it +gives the same channel order as the camera's own RGB8. + +## Results: JPEG XL compression and saving + +### What the encoder takes + +libjxl encodes ordinary images: one channel (grey) or three (RGB), 8 or 16 bits per value. Raw +camera data reaches it in one of two ways: +- **Converted to colour** (by the camera as RGB8, or on the tower), then encoded **lossy**: libjxl + converts to a perceptual colour space, transforms blocks and discards what the eye is least likely + to miss. `distance` sets how much (1.0 is meant to be visually lossless; production uses 0.8). +- **As the raw mosaic in one grey channel**, which must be **lossless**: lossy compression would blend + neighbouring pixels of different colours and ruin colour after de-Bayering. The file then needs + de-Bayering after decoding. + +Not tested: splitting the mosaic into four quarter-size planes (R, G, G, B) and storing them as +channels, a known technique that compresses raw data better and could allow light lossy compression. + +### Every input at production quality (distance 0.8 or lossless), 4 threads + +Time per frame including the file write, and file size. Synthetic (detailed) frame first, then the +real frame. + +| Input | Tower 121, 16.2 MP | Tower 111, 12.2 MP | +|---|---|---| +| `rgb8`, camera's colour, effort 4 | 0.74 s, 7.9 MB · real 0.47–0.50 s, 1.7 MB | 0.87 s, 6.0 MB · real 0.62 s, 1.8 MB | +| `rgb8`, effort 4, **3 threads (production today)** | 0.92 s · real 0.63 s | 0.93 s · real 0.69 s | +| `bilinear`, tower colour, effort 4 | 0.73 s, 7.0 MB · real 0.46 s, 1.2 MB | 0.84 s, 5.3 MB · real 0.49 s, 0.7 MB | +| `ea`, tower colour, effort 4 | 0.74–0.79 s, 7.2 MB · real 0.47–0.49 s, 1.3 MB | 0.86 s, 5.4 MB · real 0.52 s, 0.8 MB | +| `colour12`, 12-bit tower colour, effort 4 | 0.69 s, 7.0 MB · real 0.49 s, 1.2 MB | 0.94 s, 5.2 MB · real 0.59 s, 0.8 MB | +| `raw8` lossless, effort 1 | 0.10 s, 17.9 MB · real 0.06 s, 11.2 MB | 0.23 s, 13.5 MB · real 0.15 s, 6.8 MB | +| `raw12` lossless, effort 1 | 0.07 s, 26.0 MB · real 0.09 s, 19.3 MB | 0.40 s, 19.6 MB · real 0.19 s, 13.0 MB | + +The conversion to colour on the tower took 5–16 ms (bilinear or edge-aware), and 23–41 ms for 12-bit. +On the synthetic frame, 12-bit lossless is *larger* than the 24.3 MB the camera sends: sensor noise in +the low bits does not compress. + +### Effort and threads (synthetic frame, `ea`, distance 0.8) + +| Effort | 121: 1 / 2 / 3 / 4 threads | 121 file | 111: 1 / 2 / 3 / 4 threads | 111 file | +|---|---|---|---|---| +| 1 | 884 / 685 / 786 / 620 ms | 7.4 MB | 1257 / 849 / 731 / 730 ms | 5.6 MB | +| 2 | 882 / 717 / 792 / 587 ms | 7.4 MB | 1253 / 862 / 785 / 718 ms | 5.6 MB | +| 3 | 1027 / 788 / 890 / 645 ms | 7.0 MB | 1385 / 949 / 803 / 768 ms | 5.3 MB | +| 4 | 1070 / 839 / 907 / 742–788 ms | 7.1 MB | 1494 / 1005 / 847 / 858 ms | 5.4 MB | +| 5 | 5637 / 3500 / 2954 / 2440 ms | 8.3 MB | 7287 / 4242 / 3541 / 3161 ms | 6.3 MB | + +`rgb8` and `bilinear` follow the same pattern within about 10% (full tables: +`summarize_encode.py`, see [Raw data](#raw-data-and-tools)). +- **Effort 1–4 are within about 20% of each other; effort 5 is 3–5× slower** and gave larger files + on the detailed frame (smaller ones on the dark real frames). Effort 3 gave the smallest files of 1–4, at effort 4's speed or better. +- **Tower 121: 3 threads is slower than 2** for every colour input and effort, on both frames (and + for most lossless rows); 4 is fastest. The cause is not known. Tower 111 behaves as expected: each thread helps, though 3 and 4 + are close. +- **Per pixel, tower 111 is about 55% slower** than tower 121. + +### Quality setting (effort 4, 4 threads, synthetic frame) + +PSNR against the input; higher is closer. On this noisy synthetic frame PSNR counts removed noise as +loss, so compare settings with each other, not with other images. + +| distance | 121 `rgb8` | 121 `ea` | 111 `rgb8` | 111 `ea` | +|---|---|---|---|---| +| 0.5 | 0.84 s, 10.8 MB, 29.7 dB | 0.81 s, 9.8 MB, 31.7 dB | 0.92 s, 8.1 MB, 29.7 dB | 0.94 s, 7.4 MB, 31.7 dB | +| **0.8** | 0.79 s, 7.9 MB, 27.6 dB | 0.74 s, 7.2 MB, 29.4 dB | 0.85 s, 6.0 MB, 27.6 dB | 0.83 s, 5.4 MB, 29.4 dB | +| 1.0 | 0.68 s, 7.1 MB, 26.9 dB | 0.73 s, 6.4 MB, 28.7 dB | 0.80 s, 5.3 MB, 26.9 dB | 0.87 s, 4.8 MB, 28.8 dB | +| 1.5 | 0.69 s, 5.0 MB, 25.2 dB | 0.66 s, 4.5 MB, 27.0 dB | 0.68 s, 3.8 MB, 25.2 dB | 0.67 s, 3.4 MB, 27.0 dB | +| 2.0 | 0.64 s, 4.2 MB, 24.6 dB | 0.62 s, 3.8 MB, 26.2 dB | 0.65 s, 3.2 MB, 24.6 dB | 0.64 s, 2.9 MB, 26.3 dB | +| 3.0 | 0.59 s, 3.4 MB, 23.8 dB | 0.58 s, 3.1 MB, 25.4 dB | 0.62 s, 2.5 MB, 23.8 dB | 0.61 s, 2.3 MB, 25.4 dB | + +From 0.5 to 3.0, files shrink about 3× and encoding gets only about 30% faster. The quality setting +is a storage decision more than a speed one. + +### Keeping up with the scan (synthetic frame, distance 0.8, effort 4; raw8 effort 1) + +*Saved after*: time from queueing the frame to the file being written, typical / worst. *Behind*: +frames still queued when the 2 minutes ended. + +| Input | Workers × threads | One frame every | 121: saves/s, saved after, behind | 111: saves/s, saved after, behind | +|---|---|---|---|---| +| `rgb8` | 1 × 4 | back to back | 1.32 /s | 1.19 /s | +| `rgb8` | 2 × 2 | back to back | 1.72 /s | 1.58 /s | +| `ea` | 1 × 4 | back to back | 1.31 /s | 1.23 /s | +| `ea` | 2 × 2 | back to back | 1.77 /s | 1.62 /s | +| `raw8` | 1 × 4 | back to back | 9.90 /s | 3.74 /s | +| `raw8` | 2 × 2 | back to back | 13.1 /s | 5.10 /s | +| `rgb8` | 1 × 4 | 2.0 s / 1.0 s | 0.7 / 0.8 s, none | 0.8 / 1.0 s, none | +| `rgb8` | 1 × 4 | **0.7 s** | **3.5 / 7.4 s, 9 behind** | **11.1 / 21.4 s, 29 behind** | +| `ea` | 1 × 4 | 2.0 s / 1.0 s | 0.7 / 0.8 s, none | 0.8 / 1.1 s, none | +| `ea` | 1 × 4 | **0.7 s** | **1.5 / 4.2 s, 4 behind** | **8.9 / 17.0 s, 23 behind** | +| `ea` | 2 × 2 | 1.0 s | 0.8 / 0.9 s, none | 1.0 / 2.1 s, none | +| `ea` | 2 × 2 | 0.7 s | 0.9 / 1.1 s, none | 1.2 / 1.6 s, none | +| `raw8` | 1 × 4 | 2.0 / 1.0 / 0.7 s | 0.1 / 0.16 s, none | 0.22 / 1.6 s, none | +| `ea`, real frame | 1 × 4 | 1.0 s | 0.48 / 0.57 s, none | 0.48 / 0.59 s, none | + +- **One worker keeps up to about one colour frame per 0.8 s (121) or 0.85 s (111)**; faster than + that, the queue grows without limit (the production queue has none). +- **Two workers × 2 threads raise that by about 30%**, at the cost of each image taking a little + longer (about 1.1–1.2 s instead of 0.8 s), because one encode does not keep 4 cores busy: one + worker × 4 threads kept the CPU 60–64% busy. +- **Raw lossless** keeps up easily; on tower 111 its occasional slow saves (up to 1.6 s) come from + storage (see below). + +### Heat and power (field enclosure) + +| | Tower 121 | Tower 111 | +|---|---|---| +| Idle, program stopped | 37 °C, 0.78 GHz, 2.2 W | 51 °C, 1.01 GHz, 2.9 W | +| Idle, program running (before the tests) | 39 °C | 64 °C | +| One `ea` frame every 2 s, 10 min | **53 °C** mean (peaks 83), 1.79 GHz, 5.2 W, 23% busy | **63 °C** mean (peaks 84), 1.77 GHz, 5.0 W, 26% busy | +| Flat out, 10 min (1 × 4) | **75 °C** mean (peaks 88), 2.82 GHz, 11.2 W | **77 °C** mean (peaks 90), 2.51 GHz, 7.9 W | +| Throughput over the 10 min flat out | 92 → 80 per minute (**−13%**) | 76 → 73 per minute (−4%) | +| Back to idle after flat out | ~38 °C within 3–4 min | not separately measured | +| CPU-seconds per colour frame | 1.7–1.9 | 1.7–1.9 | +| CPU-seconds per raw lossless frame | 0.19–0.21 | 0.23–0.25 | + +Minute by minute, flat out, the temperature levelled after 2 minutes on both towers and the clock +averages stayed flat. Tower 121's throughput still fell 13%, so it slowed in a way the one-second +clock average does not show. At one frame every 2 s nothing slowed on either tower: 30 frames every +minute. Peaks are momentary, during each encode, and stay well below the usual 100–105 °C junction limit +of these CPUs. + +### Storage + +256 MB written with a final sync: **tower 111 76 MB/s, tower 121 218 MB/s** (both eMMC). A +13–18 MB raw file takes 0.17–0.24 s just to write on tower 111, which caps it at about 4 raw saves per +second whatever the CPU could do. Colour files (1–8 MB) are not limited by it. + +### CPU cost per frame, first figures (tower 121, 2026-09-30) + +Real frame of that day (grey, noisy), 3 threads unless stated. + +| Step | Time | File size | +|---|---|---| +| Copy a frame (as `FrameReceived` does for every frame) | 1.6 ms (BayerRG8), 4.2 ms (RGB8) | – | +| De-Bayer, OpenCV bilinear / edge-aware / VNG | 8.0 / 4.1 / 107 ms | – | +| Rotate 180° (RGB8) | 31 ms | – | +| Colour JXL, distance 0.8, effort 1 / 2 / 3 / **4** / 5 | 655 / 619 / 672 / **670** / 2466 ms | 2.82 / 2.82 / 2.58 / **2.65** / 2.23 MB | +| same, 4 threads | 412 / 426 / 464 / **504** / 2049 ms | same | +| same, effort 4, while streaming | 711–742 ms | – | +| Raw BayerRG8, truly lossless, effort 1 / 2 / 3 | 98 / 400 / 520 ms | 9.29 / 8.44 / 8.24 MB | +| Raw BayerRG8 through production `jxl_distance = 0` (**not lossless**), effort 1–4 | 718–789 ms | 14.9–15.1 MB | + +The 12-bit figures first published in this row were wrong (see +[12-bit lossless](#12-bit-lossless-needs-a-bit-depth-setting)); the corrected 12-bit results are in +the tables above. + +### `jxl_distance = 0` is not lossless + +`JpegXlEncoder::encodeToFile` sets `basic_info.uses_original_profile = JXL_FALSE` in every case. For +distance 0, libjxl prints `Set uses_original_profile=true for lossless encoding`, still returns +success, and encodes through its lossy colour space: + +| Input | File | Pixel values differing | Largest difference | +|---|---|---|---| +| BayerRG8 (1 channel) | 15.1 MB | 243,403 of 16.2 M (1.5%) | 1 | +| RGB8 (3 channels) | 31.2 MB | 3,564,870 of 48.6 M (7.3%) | 2 | + +`config.example.ini` and the encoder header both describe distance 0 as lossless. Known issue #20. + +### 12-bit lossless needs a bit-depth setting + +When 12-bit values are passed in 16-bit words, libjxl by default reads them as the full 0–65535 +range and rescales, so a "lossless" encode keeps only the top bits: in a round-trip test every pixel +changed, by up to 3,839. Setting `JxlEncoderSetFrameBitDepth(JXL_BIT_DEPTH_FROM_CODESTREAM)` makes it +exact (0 differences). The production encoder is 8-bit only and not affected; the test program was, +and all 12-bit figures before 2026-10-01 (12-bit lossless 8.4 MB and 0.3 s, smaller than 8-bit) were +wrong. Any future 12-bit save path needs this setting. + +## What this means for the design + +- **Format.** Raw Bayer gives about 3× the frames of RGB8 over the same link on both cameras. + Converting it to colour on the tower costs 5–16 ms and no white-balance work, and the colour encode + afterwards costs the same as for the camera's RGB8. So "raw from the camera, colour on the tower" + gains frame rate (faster auto-exposure) without costing save time. Saving the mosaic losslessly + instead costs a tenth of the CPU and heat, but 2–3× the storage, and is limited by tower 111's + storage speed. +- **Saving.** One save worker handles about one colour frame per 0.8 s; the production queue has no + limit, so a faster scan would fall further behind for as long as it runs. Two workers × 2 threads + add about 30%. Production's 3 encoder threads should be 4 (one worker) or 2 (two workers). Effort + 3 is as fast as 4 with smaller files here; effort 5 is not usable at scan pace. +- **Heat.** At scan paces of a frame every 1–2 s both towers ran far from throttling (53–71 °C). + Flat-out encoding for 10 minutes reached 75–79 °C with momentary peaks of 90 °C and a small + slowdown on tower 121. Tower 111 runs 10–14 °C hotter than tower 121 at rest and at scan pace. +- **Settling.** A higher frame rate settles faster (about 1.6 s at 4.95 fps to 0.5 s at 26 fps for the + same ×16 step). Exposure and gain holding still is not enough to know auto-exposure has finished; + the kept frame's brightness (mean against the camera's target, clipped share) has to be part of the + test, with exposure and gain at a limit marking when waiting cannot help. **Not yet measured.** +- **Link limit.** 450 MB/s was clean on both towers alone and nearly clean under load. If used, the + capture path should count and skip incomplete frames. +- **Profile.** On tower 121, `MinimizeBlur` raises gain to its 24 dB maximum before exposure goes past + about 5 ms. With the gimbal standing still, `MinimizeNoise` may give cleaner images. Not tested. + +## Scope and caveats + +- **Two towers, two cameras, two computers.** Tower 121 (IOTA, 1800 U-1620c) and tower 111 (3 Delta, + 1800 U-1240c). Other Delta towers have the same board as 111 but were not measured. +- **The tower program was stopped throughout.** Nothing was measured under its own load (camera + stream, frame copies, control loop, MQTT), nor in auto mode. +- **Binning was not tested** beyond tower 121's maximum frame rate and a 10 s stream with its binned + profile, by design (full resolution only). +- **Poor scenes on both towers.** Tower 121 indoors and out of focus; tower 111 in near-darkness. + The synthetic frame stands in for a sharp outdoor view in the compression tests; auto-exposure in + daylight, and on tower 111 at all, is not measured. +- **A step on a still camera is not a slew.** Settling after real slews needs the per-waypoint log in + the scan program. +- **60 s streams, 10 min heat runs.** Hours-long stability and summer ambient temperatures are not + covered; the ambient temperature during the runs was not recorded. +- **Not measured:** BayerRG10p streaming; `IntensityControllerTolerance` and `Target`; colour quality + of the tower's de-Bayer against the camera's; the four-plane raw storage option. +- **vmbpy 1.2.0 cannot stream on tower 121's Python 3.14 as installed** (`NameError: name 'Camera' is + not defined` from its runtime type check); the test script bypasses the check in its own process. + The production program (C++) is not affected. +- **Per-frame data** (`ChunkEnable`) can only be switched per item while `ChunkModeActive` is off. + Chunk data adds 48 bytes per frame. + +## Raw data and tools + +The programs and how to rerun them: [tools/camera-bench](../tools/camera-bench/README.md). Raw +results, one JSON line per measurement (including per-frame auto-exposure series and per-second heat +samples): + +- [results-2026-09-30-tower121/](../tools/camera-bench/results-2026-09-30-tower121/) — camera tests + and first CPU figures, with `scene.jpg`. +- [results-2026-10-01-tower121/](../tools/camera-bench/results-2026-10-01-tower121/) — compression, + pile-up, heat; `matrix12_*.jsonl` are the corrected 12-bit rows; `scene.jpg` and `synthetic.jpg`. +- [results-2026-10-01-tower111/](../tools/camera-bench/results-2026-10-01-tower111/) — the same, and + tower 111's camera tests (`camera_*.jsonl`). + +`python3 tools/camera-bench/summarize_encode.py ` prints every compression, pile-up +and heat table in full. diff --git a/docs/images/raw12/ae_brightness_111.png b/docs/images/raw12/ae_brightness_111.png new file mode 100644 index 0000000..198fda7 Binary files /dev/null and b/docs/images/raw12/ae_brightness_111.png differ diff --git a/docs/images/raw12/ae_brightness_121.png b/docs/images/raw12/ae_brightness_121.png new file mode 100644 index 0000000..411df86 Binary files /dev/null and b/docs/images/raw12/ae_brightness_121.png differ diff --git a/docs/images/raw12/awb_111.png b/docs/images/raw12/awb_111.png new file mode 100644 index 0000000..4bfe26c Binary files /dev/null and b/docs/images/raw12/awb_111.png differ diff --git a/docs/images/raw12/awb_121.png b/docs/images/raw12/awb_121.png new file mode 100644 index 0000000..882d42f Binary files /dev/null and b/docs/images/raw12/awb_121.png differ diff --git a/docs/images/raw12/distance_real.png b/docs/images/raw12/distance_real.png new file mode 100644 index 0000000..4281081 Binary files /dev/null and b/docs/images/raw12/distance_real.png differ diff --git a/docs/images/raw12/distance_synth.png b/docs/images/raw12/distance_synth.png new file mode 100644 index 0000000..7d63eec Binary files /dev/null and b/docs/images/raw12/distance_synth.png differ diff --git a/docs/images/raw12/effort_threads_111.png b/docs/images/raw12/effort_threads_111.png new file mode 100644 index 0000000..ad1870e Binary files /dev/null and b/docs/images/raw12/effort_threads_111.png differ diff --git a/docs/images/raw12/effort_threads_121.png b/docs/images/raw12/effort_threads_121.png new file mode 100644 index 0000000..4c60824 Binary files /dev/null and b/docs/images/raw12/effort_threads_121.png differ diff --git a/docs/images/raw12/examples/121/crops.jpg b/docs/images/raw12/examples/121/crops.jpg new file mode 100644 index 0000000..4d4199b Binary files /dev/null and b/docs/images/raw12/examples/121/crops.jpg differ diff --git a/docs/images/raw12/examples/121/detail/c12_bil_d1.0.png b/docs/images/raw12/examples/121/detail/c12_bil_d1.0.png new file mode 100644 index 0000000..92439f1 Binary files /dev/null and b/docs/images/raw12/examples/121/detail/c12_bil_d1.0.png differ diff --git a/docs/images/raw12/examples/121/detail/c12_ea_d0.3.png b/docs/images/raw12/examples/121/detail/c12_ea_d0.3.png new file mode 100644 index 0000000..617b830 Binary files /dev/null and b/docs/images/raw12/examples/121/detail/c12_ea_d0.3.png differ diff --git a/docs/images/raw12/examples/121/detail/c12_ea_d0.5.png b/docs/images/raw12/examples/121/detail/c12_ea_d0.5.png new file mode 100644 index 0000000..3f9b941 Binary files /dev/null and b/docs/images/raw12/examples/121/detail/c12_ea_d0.5.png differ diff --git a/docs/images/raw12/examples/121/detail/c12_ea_d0.8.png b/docs/images/raw12/examples/121/detail/c12_ea_d0.8.png new file mode 100644 index 0000000..770d1a3 Binary files /dev/null and b/docs/images/raw12/examples/121/detail/c12_ea_d0.8.png differ diff --git a/docs/images/raw12/examples/121/detail/c12_ea_d1.0.png b/docs/images/raw12/examples/121/detail/c12_ea_d1.0.png new file mode 100644 index 0000000..97dab01 Binary files /dev/null and b/docs/images/raw12/examples/121/detail/c12_ea_d1.0.png differ diff --git a/docs/images/raw12/examples/121/detail/c12_ea_d1.5.png b/docs/images/raw12/examples/121/detail/c12_ea_d1.5.png new file mode 100644 index 0000000..d918750 Binary files /dev/null and b/docs/images/raw12/examples/121/detail/c12_ea_d1.5.png differ diff --git a/docs/images/raw12/examples/121/detail/c12_ea_d2.0.png b/docs/images/raw12/examples/121/detail/c12_ea_d2.0.png new file mode 100644 index 0000000..516b727 Binary files /dev/null and b/docs/images/raw12/examples/121/detail/c12_ea_d2.0.png differ diff --git a/docs/images/raw12/examples/121/detail/c12_ea_lossless.png b/docs/images/raw12/examples/121/detail/c12_ea_lossless.png new file mode 100644 index 0000000..37a6458 Binary files /dev/null and b/docs/images/raw12/examples/121/detail/c12_ea_lossless.png differ diff --git a/docs/images/raw12/examples/121/detail/c8_ea_d1.0.png b/docs/images/raw12/examples/121/detail/c8_ea_d1.0.png new file mode 100644 index 0000000..608844b Binary files /dev/null and b/docs/images/raw12/examples/121/detail/c8_ea_d1.0.png differ diff --git a/docs/images/raw12/examples/121/detail/raw12_lossless.png b/docs/images/raw12/examples/121/detail/raw12_lossless.png new file mode 100644 index 0000000..37a6458 Binary files /dev/null and b/docs/images/raw12/examples/121/detail/raw12_lossless.png differ diff --git a/docs/images/raw12/examples/121/detail/raw12_planes_d0.5.png b/docs/images/raw12/examples/121/detail/raw12_planes_d0.5.png new file mode 100644 index 0000000..6370337 Binary files /dev/null and b/docs/images/raw12/examples/121/detail/raw12_planes_d0.5.png differ diff --git a/docs/images/raw12/examples/121/detail/raw12_planes_lossless.png b/docs/images/raw12/examples/121/detail/raw12_planes_lossless.png new file mode 100644 index 0000000..37a6458 Binary files /dev/null and b/docs/images/raw12/examples/121/detail/raw12_planes_lossless.png differ diff --git a/docs/images/raw12/examples/121/detail/rgb8_d0.8_e4_today.png b/docs/images/raw12/examples/121/detail/rgb8_d0.8_e4_today.png new file mode 100644 index 0000000..c8d9fd3 Binary files /dev/null and b/docs/images/raw12/examples/121/detail/rgb8_d0.8_e4_today.png differ diff --git a/docs/images/raw12/examples/121/diff/c12_bil_d1.0.png b/docs/images/raw12/examples/121/diff/c12_bil_d1.0.png new file mode 100644 index 0000000..2656108 Binary files /dev/null and b/docs/images/raw12/examples/121/diff/c12_bil_d1.0.png differ diff --git a/docs/images/raw12/examples/121/diff/c12_ea_d0.3.png b/docs/images/raw12/examples/121/diff/c12_ea_d0.3.png new file mode 100644 index 0000000..497622c Binary files /dev/null and b/docs/images/raw12/examples/121/diff/c12_ea_d0.3.png differ diff --git a/docs/images/raw12/examples/121/diff/c12_ea_d0.5.png b/docs/images/raw12/examples/121/diff/c12_ea_d0.5.png new file mode 100644 index 0000000..25b1da1 Binary files /dev/null and b/docs/images/raw12/examples/121/diff/c12_ea_d0.5.png differ diff --git a/docs/images/raw12/examples/121/diff/c12_ea_d0.8.png b/docs/images/raw12/examples/121/diff/c12_ea_d0.8.png new file mode 100644 index 0000000..3c0744a Binary files /dev/null and b/docs/images/raw12/examples/121/diff/c12_ea_d0.8.png differ diff --git a/docs/images/raw12/examples/121/diff/c12_ea_d1.0.png b/docs/images/raw12/examples/121/diff/c12_ea_d1.0.png new file mode 100644 index 0000000..9513387 Binary files /dev/null and b/docs/images/raw12/examples/121/diff/c12_ea_d1.0.png differ diff --git a/docs/images/raw12/examples/121/diff/c12_ea_d1.5.png b/docs/images/raw12/examples/121/diff/c12_ea_d1.5.png new file mode 100644 index 0000000..82864d0 Binary files /dev/null and b/docs/images/raw12/examples/121/diff/c12_ea_d1.5.png differ diff --git a/docs/images/raw12/examples/121/diff/c12_ea_d2.0.png b/docs/images/raw12/examples/121/diff/c12_ea_d2.0.png new file mode 100644 index 0000000..55aaa92 Binary files /dev/null and b/docs/images/raw12/examples/121/diff/c12_ea_d2.0.png differ diff --git a/docs/images/raw12/examples/121/diff/c12_ea_lossless.png b/docs/images/raw12/examples/121/diff/c12_ea_lossless.png new file mode 100644 index 0000000..8762d10 Binary files /dev/null and b/docs/images/raw12/examples/121/diff/c12_ea_lossless.png differ diff --git a/docs/images/raw12/examples/121/diff/raw12_lossless.png b/docs/images/raw12/examples/121/diff/raw12_lossless.png new file mode 100644 index 0000000..8762d10 Binary files /dev/null and b/docs/images/raw12/examples/121/diff/raw12_lossless.png differ diff --git a/docs/images/raw12/examples/121/diff/raw12_planes_d0.5.png b/docs/images/raw12/examples/121/diff/raw12_planes_d0.5.png new file mode 100644 index 0000000..17fbdea Binary files /dev/null and b/docs/images/raw12/examples/121/diff/raw12_planes_d0.5.png differ diff --git a/docs/images/raw12/examples/121/diff/raw12_planes_lossless.png b/docs/images/raw12/examples/121/diff/raw12_planes_lossless.png new file mode 100644 index 0000000..8762d10 Binary files /dev/null and b/docs/images/raw12/examples/121/diff/raw12_planes_lossless.png differ diff --git a/docs/images/raw12/examples/121/levels.json b/docs/images/raw12/examples/121/levels.json new file mode 100644 index 0000000..f85d976 --- /dev/null +++ b/docs/images/raw12/examples/121/levels.json @@ -0,0 +1,21 @@ +{ + "detail_crop": { + "x": 3440, + "y": 720, + "w": 480, + "h": 320 + }, + "shadow_crop": { + "x": 960, + "y": 1760, + "w": 480, + "h": 320 + }, + "shadow_mean_12bit": 811.7, + "shadow_distinct_levels_8bit": 227, + "shadow_distinct_levels_12bit": 3138, + "shadow_green_range_12bit": [ + 473, + 4095 + ] +} \ No newline at end of file diff --git a/docs/images/raw12/examples/121/shadow/c12_bil_d1.0.png b/docs/images/raw12/examples/121/shadow/c12_bil_d1.0.png new file mode 100644 index 0000000..a8994c8 Binary files /dev/null and b/docs/images/raw12/examples/121/shadow/c12_bil_d1.0.png differ diff --git a/docs/images/raw12/examples/121/shadow/c12_ea_d0.3.png b/docs/images/raw12/examples/121/shadow/c12_ea_d0.3.png new file mode 100644 index 0000000..df8e7cb Binary files /dev/null and b/docs/images/raw12/examples/121/shadow/c12_ea_d0.3.png differ diff --git a/docs/images/raw12/examples/121/shadow/c12_ea_d0.5.png b/docs/images/raw12/examples/121/shadow/c12_ea_d0.5.png new file mode 100644 index 0000000..f238f2e Binary files /dev/null and b/docs/images/raw12/examples/121/shadow/c12_ea_d0.5.png differ diff --git a/docs/images/raw12/examples/121/shadow/c12_ea_d0.8.png b/docs/images/raw12/examples/121/shadow/c12_ea_d0.8.png new file mode 100644 index 0000000..a84d590 Binary files /dev/null and b/docs/images/raw12/examples/121/shadow/c12_ea_d0.8.png differ diff --git a/docs/images/raw12/examples/121/shadow/c12_ea_d1.0.png b/docs/images/raw12/examples/121/shadow/c12_ea_d1.0.png new file mode 100644 index 0000000..2d5d3db Binary files /dev/null and b/docs/images/raw12/examples/121/shadow/c12_ea_d1.0.png differ diff --git a/docs/images/raw12/examples/121/shadow/c12_ea_d1.5.png b/docs/images/raw12/examples/121/shadow/c12_ea_d1.5.png new file mode 100644 index 0000000..945bdde Binary files /dev/null and b/docs/images/raw12/examples/121/shadow/c12_ea_d1.5.png differ diff --git a/docs/images/raw12/examples/121/shadow/c12_ea_d2.0.png b/docs/images/raw12/examples/121/shadow/c12_ea_d2.0.png new file mode 100644 index 0000000..f651c7b Binary files /dev/null and b/docs/images/raw12/examples/121/shadow/c12_ea_d2.0.png differ diff --git a/docs/images/raw12/examples/121/shadow/c12_ea_lossless.png b/docs/images/raw12/examples/121/shadow/c12_ea_lossless.png new file mode 100644 index 0000000..0586b66 Binary files /dev/null and b/docs/images/raw12/examples/121/shadow/c12_ea_lossless.png differ diff --git a/docs/images/raw12/examples/121/shadow/c8_ea_d1.0.png b/docs/images/raw12/examples/121/shadow/c8_ea_d1.0.png new file mode 100644 index 0000000..8451e34 Binary files /dev/null and b/docs/images/raw12/examples/121/shadow/c8_ea_d1.0.png differ diff --git a/docs/images/raw12/examples/121/shadow/raw12_lossless.png b/docs/images/raw12/examples/121/shadow/raw12_lossless.png new file mode 100644 index 0000000..0586b66 Binary files /dev/null and b/docs/images/raw12/examples/121/shadow/raw12_lossless.png differ diff --git a/docs/images/raw12/examples/121/shadow/raw12_planes_d0.5.png b/docs/images/raw12/examples/121/shadow/raw12_planes_d0.5.png new file mode 100644 index 0000000..d968797 Binary files /dev/null and b/docs/images/raw12/examples/121/shadow/raw12_planes_d0.5.png differ diff --git a/docs/images/raw12/examples/121/shadow/raw12_planes_lossless.png b/docs/images/raw12/examples/121/shadow/raw12_planes_lossless.png new file mode 100644 index 0000000..0586b66 Binary files /dev/null and b/docs/images/raw12/examples/121/shadow/raw12_planes_lossless.png differ diff --git a/docs/images/raw12/examples/121/shadow/rgb8_d0.8_e4_today.png b/docs/images/raw12/examples/121/shadow/rgb8_d0.8_e4_today.png new file mode 100644 index 0000000..b3d5a45 Binary files /dev/null and b/docs/images/raw12/examples/121/shadow/rgb8_d0.8_e4_today.png differ diff --git a/docs/images/raw12/examples/121/shadow_8_vs_12.png b/docs/images/raw12/examples/121/shadow_8_vs_12.png new file mode 100644 index 0000000..3a2d975 Binary files /dev/null and b/docs/images/raw12/examples/121/shadow_8_vs_12.png differ diff --git a/docs/images/raw12/examples/121/thumbs/c12_bil_d1.0.jpg b/docs/images/raw12/examples/121/thumbs/c12_bil_d1.0.jpg new file mode 100644 index 0000000..f3df1ad Binary files /dev/null and b/docs/images/raw12/examples/121/thumbs/c12_bil_d1.0.jpg differ diff --git a/docs/images/raw12/examples/121/thumbs/c12_ea_d0.3.jpg b/docs/images/raw12/examples/121/thumbs/c12_ea_d0.3.jpg new file mode 100644 index 0000000..36d61fd Binary files /dev/null and b/docs/images/raw12/examples/121/thumbs/c12_ea_d0.3.jpg differ diff --git a/docs/images/raw12/examples/121/thumbs/c12_ea_d0.5.jpg b/docs/images/raw12/examples/121/thumbs/c12_ea_d0.5.jpg new file mode 100644 index 0000000..182e3c8 Binary files /dev/null and b/docs/images/raw12/examples/121/thumbs/c12_ea_d0.5.jpg differ diff --git a/docs/images/raw12/examples/121/thumbs/c12_ea_d0.8.jpg b/docs/images/raw12/examples/121/thumbs/c12_ea_d0.8.jpg new file mode 100644 index 0000000..dc3899e Binary files /dev/null and b/docs/images/raw12/examples/121/thumbs/c12_ea_d0.8.jpg differ diff --git a/docs/images/raw12/examples/121/thumbs/c12_ea_d1.0.jpg b/docs/images/raw12/examples/121/thumbs/c12_ea_d1.0.jpg new file mode 100644 index 0000000..ca4570f Binary files /dev/null and b/docs/images/raw12/examples/121/thumbs/c12_ea_d1.0.jpg differ diff --git a/docs/images/raw12/examples/121/thumbs/c12_ea_d1.5.jpg b/docs/images/raw12/examples/121/thumbs/c12_ea_d1.5.jpg new file mode 100644 index 0000000..e91cbbe Binary files /dev/null and b/docs/images/raw12/examples/121/thumbs/c12_ea_d1.5.jpg differ diff --git a/docs/images/raw12/examples/121/thumbs/c12_ea_d2.0.jpg b/docs/images/raw12/examples/121/thumbs/c12_ea_d2.0.jpg new file mode 100644 index 0000000..d42d9a4 Binary files /dev/null and b/docs/images/raw12/examples/121/thumbs/c12_ea_d2.0.jpg differ diff --git a/docs/images/raw12/examples/121/thumbs/c12_ea_lossless.jpg b/docs/images/raw12/examples/121/thumbs/c12_ea_lossless.jpg new file mode 100644 index 0000000..cd0b96c Binary files /dev/null and b/docs/images/raw12/examples/121/thumbs/c12_ea_lossless.jpg differ diff --git a/docs/images/raw12/examples/121/thumbs/c8_ea_d1.0.jpg b/docs/images/raw12/examples/121/thumbs/c8_ea_d1.0.jpg new file mode 100644 index 0000000..eb5de8d Binary files /dev/null and b/docs/images/raw12/examples/121/thumbs/c8_ea_d1.0.jpg differ diff --git a/docs/images/raw12/examples/121/thumbs/raw12_lossless.jpg b/docs/images/raw12/examples/121/thumbs/raw12_lossless.jpg new file mode 100644 index 0000000..cd0b96c Binary files /dev/null and b/docs/images/raw12/examples/121/thumbs/raw12_lossless.jpg differ diff --git a/docs/images/raw12/examples/121/thumbs/raw12_planes_d0.5.jpg b/docs/images/raw12/examples/121/thumbs/raw12_planes_d0.5.jpg new file mode 100644 index 0000000..b6923eb Binary files /dev/null and b/docs/images/raw12/examples/121/thumbs/raw12_planes_d0.5.jpg differ diff --git a/docs/images/raw12/examples/121/thumbs/raw12_planes_lossless.jpg b/docs/images/raw12/examples/121/thumbs/raw12_planes_lossless.jpg new file mode 100644 index 0000000..cd0b96c Binary files /dev/null and b/docs/images/raw12/examples/121/thumbs/raw12_planes_lossless.jpg differ diff --git a/docs/images/raw12/examples/121/thumbs/rgb8_d0.8_e4_today.jpg b/docs/images/raw12/examples/121/thumbs/rgb8_d0.8_e4_today.jpg new file mode 100644 index 0000000..82669b7 Binary files /dev/null and b/docs/images/raw12/examples/121/thumbs/rgb8_d0.8_e4_today.jpg differ diff --git a/docs/images/raw12/examples/lowgain/shadow_8_vs_12.png b/docs/images/raw12/examples/lowgain/shadow_8_vs_12.png new file mode 100644 index 0000000..ce0be52 Binary files /dev/null and b/docs/images/raw12/examples/lowgain/shadow_8_vs_12.png differ diff --git a/docs/images/raw12/fps_per_format.png b/docs/images/raw12/fps_per_format.png new file mode 100644 index 0000000..0c6fa3e Binary files /dev/null and b/docs/images/raw12/fps_per_format.png differ diff --git a/docs/images/raw12/heat_max.png b/docs/images/raw12/heat_max.png new file mode 100644 index 0000000..284bf14 Binary files /dev/null and b/docs/images/raw12/heat_max.png differ diff --git a/docs/images/raw12/heat_paced.png b/docs/images/raw12/heat_paced.png new file mode 100644 index 0000000..de4979d Binary files /dev/null and b/docs/images/raw12/heat_paced.png differ diff --git a/docs/images/raw12/noise_vs_signal.png b/docs/images/raw12/noise_vs_signal.png new file mode 100644 index 0000000..da3f343 Binary files /dev/null and b/docs/images/raw12/noise_vs_signal.png differ diff --git a/docs/images/raw12/pace_common.png b/docs/images/raw12/pace_common.png new file mode 100644 index 0000000..5a975c8 Binary files /dev/null and b/docs/images/raw12/pace_common.png differ diff --git a/docs/images/raw12/pileup.png b/docs/images/raw12/pileup.png new file mode 100644 index 0000000..90a09f4 Binary files /dev/null and b/docs/images/raw12/pileup.png differ diff --git a/docs/images/raw12/sizes_common.png b/docs/images/raw12/sizes_common.png new file mode 100644 index 0000000..31f5db3 Binary files /dev/null and b/docs/images/raw12/sizes_common.png differ diff --git a/docs/images/raw12/stages_common.png b/docs/images/raw12/stages_common.png new file mode 100644 index 0000000..87982af Binary files /dev/null and b/docs/images/raw12/stages_common.png differ diff --git a/docs/images/raw12/stages_real.png b/docs/images/raw12/stages_real.png new file mode 100644 index 0000000..a32d401 Binary files /dev/null and b/docs/images/raw12/stages_real.png differ diff --git a/docs/images/raw12/stages_synth.png b/docs/images/raw12/stages_synth.png new file mode 100644 index 0000000..ad94f51 Binary files /dev/null and b/docs/images/raw12/stages_synth.png differ diff --git a/docs/images/raw12/storage.png b/docs/images/raw12/storage.png new file mode 100644 index 0000000..1452838 Binary files /dev/null and b/docs/images/raw12/storage.png differ diff --git a/docs/known-issues.md b/docs/known-issues.md index c67ccad..e3feaa0 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -29,9 +29,10 @@ doctest unit-test suite (`ctest`). |---|-------|--------| | 13 | `[Motor]` degrees↔counts calibration | The `config.example.ini` values are **placeholders**. Home, then run `gimbal heading setup` (yaw, needs no IMU) and `gimbal pitch setup` (pitch, needs the IMU) on the rig. | | 14 | Capture sweep on hardware | **Mechanically verified on the rig (2026-08-07).** A full `--init --start` run homed both axes and completed `MOVE → settle → trigger` for 15/15 waypoints with 0 dropped frames, 0 stalls and 0 acquisition failures. `kSettleTolCounts` (600) and the per-interval timing in [CaptureScheduler.cpp](../src/core/CaptureScheduler.cpp) have **not** been tuned against observed `ST` behaviour yet, and the settle tolerance has not been checked against a sharp image (see #16). | -| 15 | **Camera profiles unverified on a lit scene** | The `standard`/`night` profiles must be exported from the VimbaX Viewer (see `config/camera_profiles/rgb/README.md`) and have not been measured against real light. Confirm on the rig that `LoadSettings` succeeds, that `standard` sustains 10 fps with `frames_dropped` at 0, and that the auto-exposure lands on a sensible `ExposureTime` outdoors. The only previous rig test ran with the lens covered. **2026-09-24:** tower 121 has still never imaged a scene - a frame grabbed directly and the program's own saved images (2026-09-21) are a smooth grey gradient with no edges, i.e. the lens is covered or grossly out of focus. Tower 111's images from the same afternoon show nothing sharp anywhere either (soft blobs, no edges) - defocus, a cover or a fogged window; unexplained. `test camera` (below) proves the camera and its data path work in both cases; it cannot, by design, judge the picture. **Stream stuck, 2026-09-24:** from 16:10 to 16:19 tower 111's camera delivered no frames at all - not to the program, not to a fresh Vimba session, not at a fixed 10 ms exposure - and it survived program restarts and a USB re-authorisation. The camera's own `DeviceReset` command cleared it. Production captured nothing for those 9 minutes and logged only `no frame yet (stream warming up)`: a session that has NEVER delivered is not restarted by `trigger()`, by design, so a stream stuck from its start stays stuck. It began just after a `test camera` run by an early version that stopped and restarted the stream several times in a few seconds (to write `BlackLevel`, which the camera refuses while streaming); whether that caused it or met 111's known USB stall is not established. `test camera` itself reported it correctly. **Answered the same day:** a stream that never delivers is now restarted once the wait exceeds what its exposure can explain (at least 30 s), and after two fruitless restarts the camera is reset (DeviceReset) and reopened on its profile - both by the program itself (`CameraStall.h`), and on demand with the console command `camera reset` (verified on tower 121: back in ~14 s, frames straight after). `test camera` no longer stops the stream at all: it changes only exposure and gain, which are writable live, and puts them back live. With that, 111 ran it inside the capturing program, passed, and kept capturing. | -| 16 | **`settle_wait_ms` untuned** | `standard` waits 1 s and `night` 2.5 s after standstill before keeping a frame. Both are reasoned (≫ one frame period; long enough for the auto-exposure to see the new scene) but not measured. Check on a lit scene that the kept frame shows no slew blur and that its exposure has actually converged for that heading, and shorten the wait if it has. **Seen in real data 2026-09-24:** of eight consecutive images tower 111 kept at 8 deg/s, one was 98.6 % clipped white, one 57 % black and two about half clipped - frames kept before the continuous auto-exposure had caught up with the new view. | -| 19 | **USB preconditions do not survive a reboot** | `usbcore.usbfs_memory_mb` reverts to 16 (needs 1000) and USB3 LPM re-enables on the camera's port. **Both were wrong on the LattePanda when tested on 2026-08-07**, and frame delivery stalls without them regardless of application code. **Solved on tower 121 by the setup kit** (`hardware/lattepanda-iota/preconditions/`): a systemd-tmpfiles drop-in for usbfs and two udev rules, all installed by `install.sh --preconditions`. Verified 2026-09-23 — `usbfs_memory_mb` reads 1000 after **6 days of uptime**, so it survived the boot, and both rules are in `/etc/udev/rules.d/`; and again across a deliberate reboot the same day, when `install-tower --verify` read `usbfs_memory_mb` 1000 and the camera port's `usb3_lpm_permit` 0 (a check `test hardware camera` now makes too, once `[Test] usb3_lpm_path` is set - the installer sets it). Note it is **tmpfiles, not the kernel cmdline** the original entry proposed. **On the Delta fleet**: tower 111 read 16 and `u1_u2` on 2026-09-23, and the Delta kit (`hardware/lattepanda-3-delta/`, the same two files) was applied the same day - 1000 and `0` afterwards, confirmed by `test hardware camera`. Confirmed across a reboot on 2026-09-24 (`install-tower --verify`). **Solved on both boards**; open only for Delta towers the kit has not reached. | +| 15 | **Camera profiles unverified on a lit scene** | The `standard`/`night` profiles must be exported from the VimbaX Viewer (see `config/camera_profiles/rgb/README.md`) and have not been measured against real light. Confirm on the rig that `LoadSettings` succeeds, that `standard` sustains 10 fps with `frames_dropped` at 0, and that the auto-exposure lands on a sensible `ExposureTime` outdoors. The only previous rig test ran with the lens covered. **2026-09-24:** tower 121 has still never imaged a scene - a frame grabbed directly and the program's own saved images (2026-09-21) are a smooth grey gradient with no edges, i.e. the lens is covered or grossly out of focus. Tower 111's images from the same afternoon show nothing sharp anywhere either (soft blobs, no edges) - defocus, a cover or a fogged window; unexplained. `test camera` (below) proves the camera and its data path work in both cases; it cannot, by design, judge the picture. **Stream stuck, 2026-09-24:** from 16:10 to 16:19 tower 111's camera delivered no frames at all - not to the program, not to a fresh Vimba session, not at a fixed 10 ms exposure - and it survived program restarts and a USB re-authorisation. The camera's own `DeviceReset` command cleared it. Production captured nothing for those 9 minutes and logged only `no frame yet (stream warming up)`: a session that has NEVER delivered is not restarted by `trigger()`, by design, so a stream stuck from its start stays stuck. It began just after a `test camera` run by an early version that stopped and restarted the stream several times in a few seconds (to write `BlackLevel`, which the camera refuses while streaming); whether that caused it or met 111's known USB stall is not established. `test camera` itself reported it correctly. **Answered the same day:** a stream that never delivers is now restarted once the wait exceeds what its exposure can explain (at least 30 s), and after two fruitless restarts the camera is reset (DeviceReset) and reopened on its profile - both by the program itself (`CameraStall.h`), and on demand with the console command `camera reset` (verified on tower 121: back in ~14 s, frames straight after). `test camera` no longer stops the stream at all: it changes only exposure and gain, which are writable live, and puts them back live. With that, 111 ran it inside the capturing program, passed, and kept capturing. **2026-09-30:** tower 121's camera is indoors, facing a window, still badly out of focus, gain at its 24 dB maximum ([camera-capture-measurements.md](camera-capture-measurements.md)). | +| 16 | **`settle_wait_ms` untuned** | `standard` waits 1 s and `night` 2.5 s after standstill before keeping a frame. Both are reasoned (≫ one frame period; long enough for the auto-exposure to see the new scene) but not measured. Check on a lit scene that the kept frame shows no slew blur and that its exposure has actually converged for that heading, and shorten the wait if it has. **Seen in real data 2026-09-24:** of eight consecutive images tower 111 kept at 8 deg/s, one was 98.6 % clipped white, one 57 % black and two about half clipped - frames kept before the continuous auto-exposure had caught up with the new view. **Measured 2026-09-30 (tower 121, still camera, forced ×16 brightness steps):** auto-exposure took 1.4–1.8 s to come within ±10% at 4.95 fps, so 1 s is too short for a large step; from a dark step it first overshoots to its maximum exposure, 3.4–4.1× too bright. Exposure and gain holding still for a few frames does not mean it has finished ([camera-capture-measurements.md](camera-capture-measurements.md)). | +| 19 | **USB preconditions do not survive a reboot** | `usbcore.usbfs_memory_mb` reverts to 16 (needs 1000) and USB3 LPM re-enables on the camera's port. **Both were wrong on the LattePanda when tested on 2026-08-07**, and frame delivery stalls without them regardless of application code. **Solved on tower 121 by the setup kit** (`hardware/lattepanda-iota/preconditions/`): a systemd-tmpfiles drop-in for usbfs and two udev rules, all installed by `install.sh --preconditions`. Verified 2026-09-23 — `usbfs_memory_mb` reads 1000 after **6 days of uptime**, so it survived the boot, and both rules are in `/etc/udev/rules.d/`; and again across a deliberate reboot the same day, when `install-tower --verify` read `usbfs_memory_mb` 1000 and the camera port's `usb3_lpm_permit` 0 (a check `test hardware camera` now makes too, once `[Test] usb3_lpm_path` is set - the installer sets it). Note it is **tmpfiles, not the kernel cmdline** the original entry proposed. **On the Delta fleet**: tower 111 read 16 and `u1_u2` on 2026-09-23, and the Delta kit (`hardware/lattepanda-3-delta/`, the same two files) was applied the same day - 1000 and `0` afterwards, confirmed by `test hardware camera`. Confirmed across a reboot on 2026-09-24 (`install-tower --verify`). **Solved on both boards**; open only for Delta towers the kit has not reached. **2026-09-30:** with the kit applied, tower 121 streamed full resolution in every format at 250 and 450 MB/s for 60 s with nothing lost; 2 incomplete frames in 1,574 at 450 MB/s while encoding. **2026-10-01:** tower 111 (Delta kit applied) likewise, at 200, 250 and 450 MB/s, also while encoding, with nothing lost ([camera-capture-measurements.md](camera-capture-measurements.md)). | +| 20 | **`jxl_distance = 0` is not lossless** | `JpegXlEncoder::encodeToFile` always sets `uses_original_profile = false`; for distance 0 libjxl prints `Set uses_original_profile=true for lossless encoding`, still reports success, and encodes through its lossy colour space. Measured 2026-09-30: decoded pixels differ by up to 1 (raw mosaic) or 2 (RGB8) levels in 1.5% / 7.3% of values ([camera-capture-measurements.md](camera-capture-measurements.md)). Fix: set `uses_original_profile = true` when distance is 0. | ## Camera acquisition (USB3) — root cause + approach @@ -51,6 +52,9 @@ session. Two problems were root-caused (both **host/USB3-side**, reproduced in A - **The host xHCI cannot reliably move large single frames.** Measured (LPM off, paced ~1 fps): RGB8 full-res **61 MB stalls in 3–5 frames**; **≤ ~20 MB frames sustain when *paced*** (BayerRG8 full-res 20 MB → 21 frames/20 s; 3.8 MB → 41/20 s). Freerun at max rate stalls even for medium frames. + *Not reproduced on tower 121 (IOTA, 1800 U-1620c) on 2026-09-30: 48.6 MB RGB8 frames free-ran at + the maximum rate for 60 s at 250 and 450 MB/s with nothing lost - see + [camera-capture-measurements.md](camera-capture-measurements.md).* - Also raise `usbcore.usbfs_memory_mb` to 1000 (persist via kernel cmdline; resets to 16 on reboot). **Approach (mode-file-driven, see `[Camera]` in [configuration.md](configuration.md)):** RGB8 + diff --git a/docs/lattepanda-iota.md b/docs/lattepanda-iota.md index 6257e46..68ae35b 100644 --- a/docs/lattepanda-iota.md +++ b/docs/lattepanda-iota.md @@ -180,7 +180,7 @@ above is from the RP2040 datasheet, not from LattePanda. | 4 | Does the header COM1 enumerate as `ttyS0` under Linux, and is it enabled in BIOS? | IMU device path. | | 5 | Can the Quectel RM520N-GL (M.2 B-key) be rehomed? | No B-key slot on the IOTA. | | 6 | USB3 LPM and `usbfs_memory_mb` behaviour on the N150 xHCI. | Known issue #19 — the Alvium stalls without these. | -| 7 | Sustained JPEG XL encode throughput at the production frame size. | Sets the achievable capture rate. | +| 7 | Sustained JPEG XL encode throughput at the production frame size. | Sets the achievable capture rate. **Measured 2026-09-30 on tower 121:** a full-resolution (5328×3040) colour frame at distance 0.8, effort 4 takes 0.67 s with 3 threads (0.73 s while the camera streams), 0.50 s with 4; on a sharp, detailed frame 0.74 s with 4 threads, and one save worker keeps up with about one frame per 0.8 s. The 3 Delta (tower 111) is about 55% slower per pixel ([camera-capture-measurements.md](camera-capture-measurements.md)). | --- diff --git a/docs/raw12-pipeline-implementation-plan.md b/docs/raw12-pipeline-implementation-plan.md new file mode 100644 index 0000000..cb9d40a --- /dev/null +++ b/docs/raw12-pipeline-implementation-plan.md @@ -0,0 +1,207 @@ +# Implementation plan — raw capture, colour on the tower, 12-bit JPEG XL + +Based on [raw12-pipeline-measurements.md](raw12-pipeline-measurements.md) (towers 121 and 111, +2026-10-02/04). Not started; decisions marked **[decide]** are the user's. + +## Goal + +The camera sends raw Bayer (`BayerRG12p`, or `BayerRG8` where chosen) at 250 MB/s, so the stream +runs 2–3× faster than RGB8 and auto-exposure settles in roughly half the time. The tower unpacks, +converts to colour (edge-aware) and saves JPEG XL at the frame's bit depth. A waypoint's frame is +kept when the image has settled, not after a fixed wait. Profiles that ask for RGB8 keep working +exactly as today. + +## What the measurements fix + +| Decision | Value | Why (measured) | +|---|---|---| +| Pixel format | from the profile XML: `BayerRG12p` (proposed) or `BayerRG8`; `RGB8` still accepted | 9.86 / 14.75 / 4.95 fps; no losses at 250 MB/s on either tower | +| Colour conversion | OpenCV edge-aware, 16-bit for 12-bit input | 30–35 ms; SSIM 0.973 vs 0.968 bilinear at the same size | +| Save | colour JXL, **distance 1.0, effort 3, 4 threads**; 12-bit via `JxlEncoderSetFrameBitDepth` | 2.7 MB/frame; 0.57 s (IOTA), 0.85 s (Delta) on the real frame | +| Workers | 1 × 4 threads on the IOTA, **2 × 2 on the Delta** | IOTA keeps up at 0.7 s/frame; Delta at 1.0 s only with 2 workers | +| Queue | bounded, ~4 frames (24 MB each raw) | unbounded today; pile-up measured at 23–97 frames | +| Keep rule | brightness in the camera's target band and steady (≤2%) over 3 frames, with a time limit | right in 21 of 24 steps; "exposure unchanged" 0 of 24 | +| Lossless | must set `uses_original_profile` | known issue #20 (production's distance 0 is lossy) | + +## Processing budget the design must fit + +Measured on the **identical frame** on both computers (5328×3040, real scene; 4 threads; see the +report's [Where the time goes](raw12-pipeline-measurements.md#where-the-time-goes)). + +### Time per saved frame, step by step + +| Step | IOTA (121) | 3 Delta (111) | Where it runs in the new code | +|---|---|---|---| +| Receive frame (SDK buffer → copy) | 1.6 ms (8-bit raw, measured); ~2.4 ms for 12p (estimated from size) | not measured | `VimbaCameraSource::FrameReceived` | +| Unpack `BayerRG12p` → 16-bit | 10 ms | 17 ms | `RawImage::unpack12p` (core) | +| De-Bayer, edge-aware, 16-bit (bilinear) | 28 ms (17) | 33 ms (22) | `ImagePipeline::process` (OpenCV) | +| Rotate (only if `rotation_deg` ≠ 0) | ~30 ms (measured on RGB8 8-bit) | not measured | `ImagePipeline::process` | +| JPEG XL encode, 12-bit colour, d1.0 e3 | 480 ms | 765 ms | `JpegXlEncoder` | +| Write file (2.7 MB) | 1 ms | 4 ms | `JpegXlEncoder` | +| **Total per frame** | **~520 ms** | **~820 ms** | one save-worker job | +| Today, for comparison (RGB8 → JXL d0.8 e4) | 567 ms | 872 ms | | + +Unpacking and colour conversion are ~7% of a save; the encoder is ~90%. Nothing in the new steps +needs optimising before the encoder does. + +### Space + +| Saved as | Per frame | Per day, one frame / 2 s | / 5 s | / 10 s | +|---|---|---|---|---| +| Today: RGB8 → JXL d0.8 e4 (camera RGB8) | 3.3–4.0 MB | 141–172 GB | 57–69 GB | 28–34 GB | +| **12-bit colour JXL d1.0 e3 (proposed)** | **2.65 MB** | **114 GB** | **46 GB** | **23 GB** | +| 12-bit colour JXL d0.5 | 5.15 MB | 222 GB | 89 GB | 44 GB | +| 12-bit raw planes, lossless (option) | 16.8–17.7 MB | 727–763 GB | 291–305 GB | 145–153 GB | +| 12-bit colour, lossless | 44.8 MB | 1.9 TB | 774 GB | 387 GB | +| Working memory per save worker (12-bit colour) | ~0.35 GB; 0.7 GB for 1 worker, 1.1 GB for 2 (measured) | | | | +| Queued raw frame | 24.3 MB each | | | | + +The towers have 11–22 GB free: saved images must leave the tower continuously. + +### Pace + +| | IOTA (121) | 3 Delta (111) | +|---|---|---| +| Camera frame interval at 250 MB/s (12-bit / 8-bit raw / RGB8) | 0.10 / 0.07 / 0.20 s | same camera, same | +| Exposure settle after a large change (12-bit) | ~3 s (2.2–6.6) | same camera, same | +| Saves per second, 1 worker × 4 threads | 1.79 (0.56 s each) | 1.19 (0.84 s each) | +| Saves per second, 2 workers × 2 threads | 2.15 (0.47 s) | 1.53 (0.65 s) | +| Keeps up with one frame every 0.7 s | yes, 1 worker | only with 2 workers | + +A waypoint takes ~1.5–5 s (slew + exposure settle), so saving occupies 10–55% of each tower's +capacity at scan pace. That is why the design keeps saving in the background, sizes workers per +computer, and still bounds the queue for bursts. + +## Phases + +Each phase builds and passes `ctest` on its own; RGB8 profiles behave as today after every phase. + +### 1. Encoder: correct lossless, 12-bit, configurable threads + +`src/camera/JpegXlEncoder.cpp`, `include/fgc/JpegXlEncoder.h` +- One entry point taking `bits` (8 or 12) as well as channels: 16-bit input words when `bits > 8`, + `bits_per_sample = bits`, and `JxlEncoderSetFrameBitDepth(JXL_BIT_DEPTH_FROM_CODESTREAM)`. +- `uses_original_profile = JXL_TRUE` when `distance == 0` (fixes #20). +- `threads` from config instead of the hard-coded 3 (3 is slower than 2 on the IOTA). +- The 8-bit lossy path must stay byte-identical to today's output (the benchmark checked its copy of + these settings against production). +- Reuse: `encodeMem` in `tools/camera-bench/raw12_bench.cpp` is the tested version of all of this. + +### 2. Frames that say what they hold + +`include/fgc/ICameraSource.h`, `src/camera/VimbaCameraSource.cpp`, `include/fgc/mock/MockCameraSource.h` +- `Frame` gains `PixelLayout layout` (`RGB8`, `Mono8`, `BayerRG8`, `BayerRG12p`) and the per-frame + camera data: exposure (µs), gain (dB), white-balance ratios, camera timestamp, and a brightness + mean and clipped share. +- `FrameReceived` takes the layout from `pFrame->GetPixelFormat()` instead of `channels = sz/(w*h)`, + which gives 1 for 12p (wrong). It reads the chunk data (`ChunkExposureTime`, `ChunkGain`, + `ChunkBalanceRatioRed/Blue`) and samples brightness from the buffer (every ~1000th byte: under 1 ms, + as `sample_brightness` in `camera_bench.py`). +- Optional: keep the newest frame in the SDK buffer instead of copying every frame (measured + 1.6–4 ms per frame, 2–4% of a core; low priority). +- `MockCameraSource` can produce `BayerRG8` and `BayerRG12p` frames, so the pipeline runs without a + camera. + +### 3. Raw handling in the core library (testable without OpenCV) + +New `include/fgc/RawImage.h`, `src/core/RawImage.cpp` (in `fgc_core`) +- `unpack12p(const uint8_t*, uint16_t*, n)` (GenICam 12p, two pixels in three bytes). +- `BrightnessSample` (mean on a 0–255 scale, clipped share) per layout. +- The camera's `BayerRG` is OpenCV's `BayerBG` constant; that mapping and the layouts live here. + +### 4. Pipeline: convert, encode, several workers, bounded queue + +`src/camera/ImagePipeline.cpp`, `include/fgc/ImagePipeline.h` +- `process()` by layout: RGB8 as today; Bayer → (unpack) → `cv::cvtColor(..., COLOR_BayerBG2RGB_EA)` + in 8 or 16 bit → rotate (16-bit works) → encode with the frame's bit depth. +- Optional `save_as = raw_planes`: the mosaic rearranged into four colour planes, lossless + (17–18 MB/frame; for a detector that wants raw values). **[decide]** whether to offer it at all. +- N worker threads (`save_workers`), each encoding with `jxl_threads`. +- Bounded queue (`save_queue_max`). **[decide]** what happens when it is full: + - *backpressure (recommended)*: `ImagePipeline::canAccept()`, and `CaptureScheduler` waits before + moving to the next waypoint, so no waypoint is lost and the scan slows down; + - *drop*: the frame is discarded with a warning and a counter. +- Pose still sampled at `submit()`. +- Peak memory per worker: ~0.35 GB (16-bit colour), measured 0.7 GB for 1 worker, 1.1 GB for 2. + +### 5. Keep the right frame + +`include/fgc/CaptureScheduler.h`, `src/core/CaptureScheduler.cpp`, `ICameraSource` +- After standstill, wait a short mechanical settle (`settle_wait_ms`, now a minimum; default ~200 ms), + then keep the first frame for which the last 3 frames are all in the camera's target band + (`IntensityControllerTarget ± IntensityControllerTolerance`, read from the camera when a profile + is applied) and brightness changed ≤2% frame to frame. +- Exposure and gain at their limits with brightness still out of band count as settled (waiting + cannot help); log it. +- `frame_timeout_ms` stays the upper bound; then keep the best frame seen (closest to the band), + not nothing. +- Needs the camera source to expose recent frames' data (`recentFrames(since)` or a callback); the + rule itself goes in `fgc_core` (`SettleRule`), so it can be tested on recorded series. +- Falls back to today's fixed wait when frames carry no brightness (mock without data, old profile). + +### 6. Configuration + +`include/fgc/Config.h`, `src/core/Config.cpp`, `config/config.example.ini`, hardware kit `hardware.ini` +- `[Camera]`: `jxl_distance = 1.0`, `jxl_effort = 3` (new defaults, **[decide]**), `jxl_threads`, + `demosaic = ea | bilinear`, `save_as = colour | raw_planes`. +- `hardware.ini` (what the tower *is*): `save_workers` (1 on IOTA, 2 on Delta), `jxl_threads` + (4 / 2), `save_queue_max`. These depend on the computer, not on what the tower does. +- `[Camera..]`: `settle_wait_ms` becomes the minimum mechanical settle; add + `settle_rule = brightness | fixed` so a tower can be switched back. +- Validation and tests in `tests/test_config.cpp`, as for the existing keys. + +### 7. Camera profiles + +`scripts/export_camera_profile.py` (recipes), `config/camera_profiles/rgb/README.md` +- `standard`: `PixelFormat = BayerRG12p` (**[decide]** vs `BayerRG8`), binning 1, full sensor, + `DeviceLinkThroughputLimit = 250000000`, `AcquisitionFrameRate` = max, + `ChunkModeActive` with ExposureTime, Gain, BalanceRatioRed/Blue, Timestamp enabled (enable each + item before activating chunk mode), `ExposureAutoMax = 20 ms`. +- `night`: `BayerRG12p`, long exposure allowed, where 12 bits carry real shadow detail. +- White balance: **[decide]** keep fixed ratios, or `BalanceWhiteAuto = Continuous` (converges in + 2–4 frames, stable within ±3%). +- Profiles stay the only thing that writes camera settings (the existing rule). + +### 8. Downstream and docs + +- **[decide]** The saved files become 12-bit JPEG XL. Anything that reads them (ground station, + detection) must decode at 16 bits; check before switching a tower. Consider adding the bit depth + to `CamEvent` (`docs/mqtt-api.md`). +- `docs/architecture.md` (Image capture), `docs/configuration.md` (new keys), + `docs/known-issues.md` (#16 settle, #20 lossless), profile README. + +## Tests + +- `fgc_core` unit tests: `unpack12p` against hand-packed bytes; brightness sampling per layout; the + settle rule replayed on the 24 recorded auto-exposure series in + `tools/camera-bench/results-*/ae.jsonl` (it must pick a correct frame in ≥ 21 of 24); config keys; + scheduler backpressure with the mock pipeline. +- A small separate test target linking OpenCV + libjxl (`fgc_pipeline_tests`): 12-bit lossless round + trip is exact; distance-0 8-bit is exact (#20); de-Bayer colour order matches a known patch; 8-bit + lossy output is byte-identical to the old encoder for a fixed input. +- Mock run: the program in auto mode with `MockCameraSource` producing `BayerRG12p`. + +## Verification on the towers + +1. Deploy to 121 with the new `standard` profile, run auto mode for 30 min in daylight: per-waypoint + log (move, standstill, settle-rule wait, save time), queue depth, CPU temperature. +2. Same on 111 with `save_workers = 2`. +3. Compare with the benchmark: save time per frame within ~10% of `raw12_bench pipeline`, no backlog, + temperatures as measured (53 °C IOTA, 66 °C Delta at one frame per 2 s). +4. Check saved files decode at 12 bits and match the camera's colour (channel means). +5. One dusk or night run to confirm the 12-bit shadow benefit before deciding the night profile. + +## Order and size + +| Phase | Depends on | Rough size | +|---|---|---| +| 1 Encoder | – | small | +| 2 Frames | – | medium | +| 3 Raw core | – | small | +| 4 Pipeline | 1, 2, 3 | medium | +| 5 Keep rule | 2, 3 | medium | +| 6 Config | 4, 5 | small | +| 7 Profiles | 2 | small (plus a tower session per profile) | +| 8 Downstream, docs | all | small; downstream check is outside this repo | + +Phases 1–3 are independent and can go first; 4 and 5 can then proceed in parallel. diff --git a/docs/raw12-pipeline-measurements.md b/docs/raw12-pipeline-measurements.md new file mode 100644 index 0000000..dc99313 --- /dev/null +++ b/docs/raw12-pipeline-measurements.md @@ -0,0 +1,561 @@ +# 12-bit raw capture pipeline — final report + +**Question:** should the towers stop asking the camera for 8-bit colour (RGB8) and take raw +12-bit Bayer (`BayerRG12p`) instead, with the LattePanda converting it to colour and saving 12-bit +JPEG XL? The aim is a higher frame rate, so auto-exposure and auto white balance settle faster, +and more tonal detail for the smoke-detection algorithms. Can the IOTA (tower 121) and the +3 Delta (tower 111) carry the work, how long does each step take, and how much space do the +images need? + +**Campaign:** 2–5 October 2026. Camera tests on tower 121 (2 Oct) and tower 111 (4 Oct) **with the +same camera** (Alvium 1800 U-1620c, 5328×3040), moved from 121 to 111 and looking at the same +scene, so the two towers differ only in their computer. Processing tests repeated on **one +identical frame** on both towers (4 Oct). Earlier blurred-scene measurements: +[camera-capture-measurements.md](camera-capture-measurements.md). What to build from this: +[raw12-pipeline-implementation-plan.md](raw12-pipeline-implementation-plan.md). + +## Conclusions + +| Topic | Finding | Evidence | +|---|---|---| +| Frame rate | `BayerRG12p` runs at **9.86 fps**, 2× RGB8's 4.95 fps, at the 250 MB/s cap; `BayerRG8` 14.75 fps. Unpacked `BayerRG12` only 7.4 fps: use the packed format. | [streaming](#frame-rate-and-streaming) | +| Link | No lost or incomplete frames in any format on either tower, also while saving 12-bit flat out. | [streaming](#frame-rate-and-streaming) | +| Auto-exposure | From **under-exposure** (1/16, brightness ~11 of 255) or **over-exposure** (×16, 97–100% blown out), settling takes **~25–35 frames**, so time scales with 1/fps: raw 8-bit 1.8–2.7 s, **12-bit 2.2–6.6 s (typically ~3 s)**, RGB8 4.6–6.6 s. | [method](#method), [auto-exposure](#auto-exposure) | +| Which frame to keep | **"Brightness in the camera's target band and steady for 3 frames"**: right in 21 of 24 steps. "Exposure and gain unchanged" was never right. | [settle rules](#which-frame-to-keep) | +| Auto white balance | Recovers in **2–4 frames** (0.2–0.8 s) at every rate; stable within ±1–3% on a still scene. Exposure, not white balance, sets the wait. | [white balance](#auto-white-balance) | +| Processing cost | Same frame on both computers: unpack + colour conversion **38 ms (IOTA) / 50 ms (Delta)**; JPEG XL encode 480 / 765 ms. **A 12-bit save (518 / 820 ms) costs a little less than today's RGB8 save (567 / 872 ms).** | [stages](#where-the-time-goes) | +| IOTA vs 3 Delta | **The Delta takes 1.6× as long** on the identical frame (820 vs 518 ms for a 12-bit save); it saves 1.19 frames/s with one worker, 1.53 with two (IOTA 1.79 / 2.15). | [stages](#where-the-time-goes) | +| Pace | The camera side sets the pace: a waypoint needs ~1.5–5 s (move + exposure), a 12-bit save 0.56 s (IOTA) or 0.65 s (Delta, two workers). On the shared frame the IOTA keeps up with one frame every **0.7 s** with one worker; the Delta needs **two workers** for that. | [pace](#overall-pace-how-fast-can-a-tower-capture), [pile-up](#keeping-up-with-the-scan) | +| Heat | One frame every 2 s: IOTA **53 °C / 5.1 W**, Delta **66 °C / 5.7 W**. Flat out: ~80 °C on both; the IOTA slowed 16% over 10 min, the Delta held steady at its 8 W power limit. | [heat](#heat-and-power) | +| Storage | 12-bit colour at distance 1.0: **2.7 MB/frame**, less than today's 3.7–4.0 MB. Lossless 12-bit: 44–47 MB as colour, **17–18 MB as raw planes**. | [quality](#quality-setting-size-and-quality) | +| **Value of 12 bits** | Sensor noise in this sunlit scene is **2–6 steps of 8-bit even at 0 dB gain**, so 8-bit already holds what the sensor delivers in mid and bright tones. 12-bit adds information only below about **300 of 4095** (deep shadow), which daylight scenes barely have and dusk or night scenes do. | [bit depth](#what-12-bit-adds) | + +### Recommendation + +1. **Take raw from the camera** (`BayerRG12p`, 250 MB/s) and convert to colour on the tower, + edge-aware. It doubles the frame rate and roughly halves the settle time, at no extra CPU, + heat or storage cost compared with today, on either computer. +2. **Save 12-bit colour JPEG XL at distance 1.0, effort 3, 4 threads.** 2.7 MB per frame. +3. **Workers per computer:** one worker on the IOTA; two workers × 2 threads on the Delta, which + then keeps up with a frame every 1.0 s. With the measured settle times (~2–3 s per waypoint), + the scan pace stays below either limit. The save queue still needs a bound. +4. **Keep the frame by "target band and steady for 3 frames",** with a time limit. +5. **12-bit vs raw 8-bit is a choice, not a cost.** `BayerRG8` (14.75 fps) settles about 1 s faster + in daylight and loses nothing measurable there; `BayerRG12p` keeps the deep shadows at dusk and + night. The pixel format lives in the camera profile, so the code should accept both, and + `standard` and `night` can each use the right one. +6. **Store raw 12-bit planes losslessly (17–18 MB) only if the detection pipeline wants exact sensor + values**; it is 6× the storage. + +## Setup + +| | Tower 121 | Tower 111 | +|---|---|---| +| Computer | LattePanda IOTA, Intel N150, 4 cores, up to 3.6 GHz, 7 GB | LattePanda 3 Delta, Celeron N5105, 4 cores, up to 2.9 GHz, 7.6 GB | +| Storage write | eMMC, 218 MB/s | eMMC, 76 MB/s | +| Libraries | libjxl 0.11.1, OpenCV 4.10 | libjxl 0.11.0, OpenCV 4.5.4 | +| Date, light | 2 Oct, 15:01–16:50, sun | 4 Oct, 16:19–18:19, sun then evening | +| Camera | Alvium 1800 U-1620c `DEV_1AB22C0B7D4E`, 5328×3040, link cap 250 MB/s | same camera | +| Exposure | capped at 20 ms (the `standard` profile), so ~15 dB gain in this scene | same | + +The scene, through the camera: spruce, birch trunks, leaves, a building, ~13% blown-out sky. + +![The scene](images/raw12/examples/121/thumbs/c12_ea_lossless.jpg) + +The tower program was stopped during every test, and both cameras went back to their `standard` +profiles afterwards. + +## Method + +Everything ran on the towers themselves, with the tower program stopped (the camera accepts one +program at a time, and the CPU figures should have a single cause). The test programs are in +[tools/camera-bench](../tools/camera-bench/README.md); the raw results, one JSON line per +measurement, are linked at the end. + +### How the camera is driven + +`camera_bench.py` talks to the camera through Allied Vision's Python SDK (vmbpy), the same +GenICam interface the tower program uses through C++. Every setting change is a **GenICam feature +write** (`feature.set(value)`), sent over the USB control channel **while the camera keeps +streaming**. `ExposureAuto`, `GainAuto` and `BalanceWhiteAuto` are switches with the values `Off`, +`Once` and `Continuous`. Setting one to `Off` freezes the value the automatic control had reached; +`ExposureTime`, `Gain` and `BalanceRatio` can then be written directly and take effect from the next +frame the camera exposes. No profile, user set or file on the camera is changed; afterwards the +camera is put back by reloading its `standard.xml` profile, exactly as the tower program does. + +Common settings for every camera test: + +| Setting | Value | Why | +|---|---|---| +| Resolution | 5328×3040, binning 1 | full resolution, as decided | +| `DeviceLinkThroughputLimit` | 250 MB/s | the agreed conservative link cap | +| `AcquisitionFrameRate` | the camera's maximum for the format | 9.86 fps 12-bit, 14.75 fps 8-bit raw, 4.95 fps RGB8 | +| `ExposureAutoMax` | 20 ms | as in the `standard` profile; keeps long exposures from limiting the frame rate | +| `IntensityAutoPrecedence` | `MinimizeNoise` | as the camera was left (favours exposure over gain; with exposure capped at 20 ms, the camera then uses gain) | +| `IntensityControllerTarget` / `Tolerance` / `Rate` | 50% / 5% / 100 | camera defaults; rate 100 is the fastest | +| `GainAutoMax` | as left in the camera (≥ 43.5 dB) | the bright steps reached 43.5 dB | +| Chunk data | on: `ExposureTime`, `Gain`, `BalanceRatioRed`, `BalanceRatioBlue`, `Timestamp` | each frame carries the exact settings it was exposed with | +| Buffers | 5 | as production | + +**Per-frame record.** For every frame the script keeps the host arrival time, the camera's frame +number, complete or not, the chunk data above, and the frame's **brightness and blown-out share**, +sampled from the image buffer itself: every 997th byte for RGB8 and 8-bit raw (mixed colours), the +high byte of every 331st pixel pair for `BayerRG12p` (green and blue sites), each scaled to 0–255. +"Blown out" means a sample ≥ 250 of 255. + +### Frame rate and streaming + +For each format: set it, read the camera's maximum frame rate (the camera computes it from the +link cap and frame size), stream 60 s at that rate, and count frames received, incomplete frames +(the SDK's frame status) and lost frames (gaps in the camera's frame counter). Once more for +`BayerRG12p` while the tower saved 12-bit images flat out in parallel. + +### Auto-exposure step test + +One run, per format and direction (`camera_bench.py ae`): + +| Phase | Duration | Exactly what is done | +|---|---|---| +| 1. Settle | 8 s | Stream with `ExposureAuto = Continuous`, `GainAuto = Continuous`. "Before" = median exposure and gain of the last 5 frames (chunk data). The brightness the camera steers is exposure × linear gain: E₀ = exposure × 10^(gain/20). | +| 2. Force | 2 s | `ExposureAuto = Off`, `GainAuto = Off` (freezes them). Then write a new exposure and gain. **Dark:** `ExposureTime` = max(`ExposureAutoMin`, E₀/16), `Gain` = 0 dB. **Bright:** `ExposureTime` = min(20 ms, 16·E₀), and the rest as gain: `Gain` = 20·log10(16·E₀ / exposure), up to `GainAutoMax`. Streaming continues; the camera delivers 10–30 frames at the forced settings. | +| 3. Release | 15 s | Time zero is taken, then `ExposureAuto = Continuous` and `GainAuto = Continuous` are written. Every frame for 15 s is recorded. | + +With exposure at its 20 ms cap before the step, the forced values came out as: + +| | Before (settled) | Forced "dark" (start) | Forced "bright" (start) | +|---|---|---|---| +| Exposure | 20 ms (at its cap) | 7.4–11.5 ms | 20 ms | +| Gain | 15.4–19.5 dB | 0 dB | 39.6–43.5 dB (before + 24.1 dB) | +| Brightness (0–255), first frame after release | 117–122 | **9–13: under-exposed** | **254–255: over-exposed** | +| Blown-out share | 10–15% (sky) | 0% | **97–100%** | + +So settling was measured from **both sides, under-exposure and over-exposure**. The bright start is +fully saturated: the camera cannot see how far over it is and can only step down until detail +reappears. + +Each format × direction was run twice: 12 runs per tower, 24 in all. **"Settled"** is judged on the +image: the time from release after which the frame's brightness stays within ±10% of where it +finally settles (median brightness of the run's last 2 s), with the number of frames that took. +The first 1–5 frames after release still carry the forced values (frames already in the +camera-to-host pipeline); they count towards the settle time. + +**Keep rules** were then scored offline on the same 24 per-frame logs: for each rule, the first frame +at which it would have kept a picture, and whether that frame's brightness was within 10% of the +final level ("correct") or not ("early"). The camera's target band is 50% ± 5% of full scale, i.e. +114.8–140.3 on the 0–255 scale used here. + +### Auto white balance step test + +One run (`camera_bench.py awb`), per format and direction: + +| Phase | Duration | Exactly what is done | +|---|---|---| +| 1. Exposure | 6 s | Auto-exposure on, settle; then `ExposureAuto = Off`, `GainAuto = Off`, and `ExposureTime`/`Gain` written to the median of the last 5 frames, so only white balance moves from here on. `BalanceWhiteAutoRate = 100` (fastest). | +| 2. Reference | 8 s | `BalanceWhiteAuto = Continuous`. The reference = median red and blue ratios of the frames in the last 3 s (the camera's own answer for this scene: red 2.34–2.46, blue 1.86–1.97). | +| 3. Force | 2 s | `BalanceWhiteAuto = Off`; `BalanceRatioSelector = Red`, `BalanceRatio` = reference × factor; the same for Blue (green is the camera's fixed reference). **×0.5**: red ≈ 1.2, blue ≈ 0.95 (image too green); **×2**: red ≈ 4.8, blue ≈ 3.8 (too magenta). | +| 4. Release | 15 s | Time zero, then `BalanceWhiteAuto = Continuous`; every frame's ratios recorded (chunk data). | + +"Settled" = both ratios within ±5% of the reference and staying there. Each case twice; also the +spread of the reference over phase 2 on the still scene. + +### Processing tests + +`raw12_bench.cpp`, built on each tower against its own libjxl and OpenCV, runs the steps the tower +program would run for one saved frame and **times each with a monotonic clock**: + +| Step | What it does | +|---|---| +| Unpack | `BayerRG12p` (two 12-bit pixels in three bytes) → one 16-bit word per pixel | +| Colour conversion | OpenCV `cvtColor`, `COLOR_BayerBG2RGB` (bilinear) or `…_EA` (edge-aware), 8- or 16-bit; the camera's `BayerRG` is OpenCV's `BayerBG` (checked against the camera's own RGB8) | +| JPEG XL encode | libjxl into memory; threads via libjxl's thread pool; effort and distance as stated; 12-bit input as 16-bit words with the bit depth taken from the file header; lossless with `uses_original_profile` set | +| File write | `fopen`/`fwrite`/`fflush`/`fclose` to the tower's eMMC | + +Each figure is the median of 3 runs (2 in the quality sweep). Before every run on every tower the +program checks that its 8-bit encoder produces **byte for byte** the file production's encoder +produces, that 12-bit lossless decodes to **exactly** the input, and that its colour order matches +the camera's. + +**Input frames.** +- **Shared frame, identical on both towers (the tower comparison uses this):** tower 121's frame of + 2 Oct (15:07, 20 ms, ~15 dB), rebuilt bit-exactly from its lossless 12-bit file and repacked to + `BayerRG12p`; its 8-bit raw is the 12-bit value shifted down 4 bits; its "RGB8" is the edge-aware + colour conversion of the 12-bit frame shifted to 8 bits (the camera's own RGB8 of that instant + does not exist). Copied to both towers; MD5 checksums identical. +- **Synthetic frame, identical on both towers:** a "dead leaves" texture (the camera texture chart + of ISO 19567-2) with sensor noise, same generator and seed: sharper than any real scene, the worst + case for the encoder. +- Each tower's own camera capture from its test day. + +**Quality.** Every 12-bit result is decoded and compared with the exact lossless 12-bit colour image +of the same frame (mosaic results are converted to colour the same way first): PSNR on the 12-bit +scale, and SSIM on luma, which follows structure and is less fooled by noise. + +**Load.** *Back to back:* workers save the same frame from memory continuously for 60 s. +*Paced:* a producer queues one frame every 2.0, 1.5, 1.0, 0.7 or 0.5 s for 2 minutes, like a scan +keeping one frame per waypoint; each job runs every step above; recorded: time from queueing to file +written, frames still queued at the end, peak memory. *Heat:* 10 minutes flat out and 10 minutes at +one frame every 2 s, each after a 5-minute cool-down, with once a second: CPU package temperature +(`x86_pkg_temp`), average core clock, CPU busy share (`/proc/stat`) and package power (Intel RAPL +energy counter, read through a root helper). + +### Noise test + +To see whether the 12th bit carries information, sensor noise was measured on the raw 12-bit +values: the green sites split into 24×24-pixel patches; in each, noise = the robust spread (median +absolute deviation) of differences between horizontal neighbours ÷ √2; patches binned by brightness; +the quietest 10% in each bin taken as the noise floor (texture only adds to it). Once for the +~15 dB frame and once for a frame taken at **0 dB gain** (124 ms exposure, 4 Oct 16:19, exposure +cap temporarily raised to 200 ms). + +### Campaign log + +| When | Tower | What | Duration | Notes | +|---|---|---|---|---| +| 2 Oct 15:01–15:22 | 121 | camera tests | 21 min | sunny | +| 2 Oct 15:22–16:50 | 121 | processing, pile-up, heat on its own frame and the synthetic frame | 88 min | | +| 2 Oct 19:01 | 121 | attempt at a 0 dB frame | – | too dark (24 dB needed); discarded | +| 4 Oct 16:19 | 111 | 0 dB frame (camera moved from 121) | 2 min | | +| 4 Oct 16:20–16:41 | 111 | camera tests | 21 min | 111's own 1240c camera connected but idle | +| 4 Oct 16:41–18:19 | 111 | processing, pile-up, heat | 98 min | | +| 4 Oct 23:03–23:32 | 121 + 111 | processing on the shared frame, both towers at once | 29 min | no camera needed | + +## Frame rate and streaming + +![Frame rate per format](images/raw12/fps_per_format.png) + +Same camera on both towers, so identical maximum rates. 60 s at the maximum: + +| Format | Frame | Max fps | 121: complete / incomplete / lost | 111: complete / incomplete / lost | +|---|---|---|---|---| +| BayerRG12p | 24.3 MB | 9.86 | 593 / 0 / 0 | 592 / 0 / 0 | +| BayerRG12 (unpacked) | 32.4 MB | 7.41 | 445 / 0 / 0 | 445 / 0 / 0 | +| BayerRG10p | 20.2 MB | 11.81 | 711 / 0 / 0 | 709 / 0 / 0 | +| BayerRG8 | 16.2 MB | 14.75 | 887 / 0 / 0 | 887 / 0 / 0 | +| RGB8 (today) | 48.6 MB | 4.95 | 298 / 0 / 0 | 297 / 0 / 0 | +| **BayerRG12p while saving 12-bit flat out** | 24.3 MB | 9.86 | 591 / 0 / 0 | 592 / 0 / 0 | + +On tower 111 the camera shares the USB 3 controller with the 5G modem, a USB network adapter and +the tower's second camera (idle during the test). + +## Auto-exposure + +![Brightness after a step, 121](images/raw12/ae_brightness_121.png) +![Brightness after a step, 111](images/raw12/ae_brightness_111.png) + +With exposure at its 20 ms cap, auto-exposure works through gain, 1–2 dB per frame, and sometimes +pauses mid-way (the flat stretches, up to 2.8 s while still far too bright). + +Time until brightness stays within ±10% of where it settles (two runs each): + +| Format | fps | Step | Tower 121 | Tower 111 | +|---|---|---|---|---| +| BayerRG8 | 14.75 | dark ×1/16 | 1.94 s (28 frames), 2.21 s (32) | 1.79 s (26), 1.87 s (27) | +| BayerRG8 | 14.75 | bright ×16 | 2.69 s (39), 2.55 s (37) | 2.00 s (29), 1.86 s (27) | +| **BayerRG12p** | 9.86 | dark ×1/16 | 2.90 s (28), 3.00 s (29) | 6.64 s (65), 4.51 s (44) | +| **BayerRG12p** | 9.86 | bright ×16 | 6.45 s (63), 2.69 s (26) | 2.18 s (21), 5.73 s (56) | +| RGB8 (today) | 4.95 | dark ×1/16 | 4.56 s (22), 5.17 s (25) | 5.97 s (29), 5.77 s (28) | +| RGB8 (today) | 4.95 | bright ×16 | 5.17 s (25), 5.17 s (25) | 6.57 s (32), 4.96 s (24) | + +Typically ~25–35 frames, so **settling time scales with 1/fps**. The longer 12-bit runs (44–65 +frames) are the mid-way pauses; they happened at every rate but hit the 12-bit runs more often here. + +### Which frame to keep + +Each rule applied to the per-frame log of all 24 steps. "Correct" means the frame it picks is within +10% of the brightness the camera finally settles at. + +| Rule | 121 correct / early | 111 correct / early | Median time when correct (121 / 111) | +|---|---|---|---| +| Exposure and gain unchanged for 3 frames | 0 / 12 | 0 / 12 | – | +| Brightness in the camera's target band (target 50% ± 5%), 2 frames | 6 / 6 | 6 / 6 | 3.30 / 6.10 s | +| Target band 2 frames and exposure/gain unchanged 2 frames | 6 / 6 | 7 / 5 | 3.35 / 5.02 s | +| Target band 3 frames | 8 / 4 | 8 / 4 | 3.40 / 4.84 s | +| **Target band and brightness steady (≤2% change) over 3 frames** | **11 / 1** | **10 / 2** | 3.61 / 5.45 s | + +Coming down from too bright, brightness enters the band about 15% above where it settles, so "in +band" alone fires early; requiring it to have stopped changing fixes most of that. + +## Auto white balance + +The profiles fix white balance today (red 2.87, blue 2.0). With `BalanceWhiteAuto = Continuous`, the +camera settled this scene at red ≈ 2.36–2.46, blue ≈ 1.87–1.97. + +![Auto white balance, 121](images/raw12/awb_121.png) +![Auto white balance, 111](images/raw12/awb_111.png) + +| Format | fps | Within ±5% and stays, 121 | Within ±5% and stays, 111 | +|---|---|---|---| +| BayerRG12p | 9.86 | 0.21–0.32 s (2–3 frames); once 1.33 s | 0.22–0.41 s (2–4) | +| RGB8 | 4.95 | 0.57–0.77 s (2–3) | 0.55–0.76 s (2–3) | +| BayerRG8 | 14.75 | 0.26–0.27 s (3) | 0.19–0.26 s (2–3) | + +On a still scene the auto ratios held within ±0.8–3.1%. + +## Where the time goes + +### Same frame on both computers + +The fairest comparison: the **identical shared frame** (checksums verified on both towers), every +step timed on its own, 4 threads. Colour paths at distance 1.0 and effort 3; today's RGB8 path at +its production settings (distance 0.8, effort 4). + +![Same frame, time per step](images/raw12/stages_common.png) + +| Step | IOTA (121) | 3 Delta (111) | Delta / IOTA | +|---|---|---|---| +| Unpack 12-bit packed → 16-bit | 10 ms | 17 ms | 1.7× | +| Colour conversion, 8-bit raw, bilinear / edge-aware | 8 / 9 ms | 11 / 12 ms | 1.3× | +| Colour conversion, 12-bit raw, bilinear / edge-aware | 17 / 28 ms | 22 / 33 ms | 1.2× | +| JPEG XL encode, 12-bit colour, d1.0 e3 | 480 ms | 765 ms | 1.6× | +| JPEG XL encode, today's RGB8, d0.8 e4 | 566 ms | 867 ms | 1.5× | +| File write, 2.7 MB colour file | 1 ms | 4 ms | | +| File write, 17–21 MB lossless raw file | 7–8 ms | 162–173 ms | (storage: 218 vs 76 MB/s) | + +Per saved frame, all steps together, and the file each produces (sizes are identical on both +computers, as they must be for the same input): + +| Path | IOTA total | Delta total | File | +|---|---|---|---| +| Today: RGB8 → JPEG XL, d0.8 e4 | 567 ms | 872 ms | 3.27 MB | +| Raw 8 → colour → JPEG XL, d1.0 e3 | 468 ms | 741 ms | 2.66 MB | +| Raw 12 → colour (bilinear) → JPEG XL, d1.0 e3 | 476 ms | 778 ms | 2.53 MB | +| **Raw 12 → colour (edge-aware) → JPEG XL, d1.0 e3** | **518 ms** | **820 ms** | **2.65 MB** | +| Raw 12 planes, lossless (e1) | 91 ms | 297 ms | 17.65 MB | +| Raw 12 mosaic, lossless (e1) | 87 ms | 303 ms | 20.94 MB | + +Unpacking and colour conversion together cost **38 ms on the IOTA and 50 ms on the Delta**, 7% of +the save. The JPEG XL encode is 90% of it. + +The "RGB8" input here is the edge-aware colour conversion of the same frame (the camera's own RGB8 +of that instant does not exist), so its file matches the raw 8-bit path's. The camera's real RGB8 +files were a little larger in the per-tower runs below (3.7–4.0 MB). + +### Size and time per quality setting, same frame + +![Same frame, size and time by quality](images/raw12/sizes_common.png) + +| Distance | File (12-bit colour) | IOTA | Delta | File (raw 12 planes) | +|---|---|---|---|---| +| lossless | 44.8 MB | 1,225 ms | 2,906 ms | 16.8 MB (455 / 860 ms) | +| 0.3 | 7.19 MB | 742 ms | 1,197 ms | 7.31 MB | +| 0.5 | 5.15 MB | 678 ms | 1,033 ms | 5.99 MB | +| 0.8 | 3.16 MB | 570 ms | 848 ms | 4.32 MB | +| **1.0** | **2.65 MB** | **558 ms** | **847 ms** | 3.78 MB | +| 1.5 | 1.63 MB | 515 ms | 751 ms | 2.44 MB | +| 2.0 | 1.32 MB | 558 ms | 738 ms | 1.96 MB | + +(Effort 3, 4 threads, file write included. A separate pass of 2 runs per point, so it differs from +the stage table above by run-to-run variation, up to ~40 ms.) + +### Each tower's own capture + +Same camera and scene, different day and light; consistent with the shared-frame result. + +![Stage times, real frame](images/raw12/stages_real.png) +![Stage times, synthetic frame](images/raw12/stages_synth.png) + +| Path (effort 3, 4 threads, d1.0) | IOTA, own frame / synthetic | Delta, own frame / synthetic | +|---|---|---| +| Today: camera RGB8 | 463 / 652 ms | 788 / 1,071 ms | +| Raw 12 → colour (edge-aware) | 569 / 727 ms | 849 / 1,088 ms | +| Raw 12 planes, lossless (e1) | 90 / 96 ms | 276 / 298 ms | + +### Effort and threads + +![Effort × threads, 121](images/raw12/effort_threads_121.png) +![Effort × threads, 111](images/raw12/effort_threads_111.png) + +- Effort 1–4 are within ~15% of each other; **effort 3 gives the smallest files**. +- On the IOTA, **3 threads is slower than 2** at every effort (e.g. 761 / 582 / 672 / 512 ms for + 1–4 threads at effort 1); the Delta scales normally (1,440 / 985 / 886 / 802 ms). 4 is fastest on + both computers. + +## Quality setting: size and quality + +![Size and quality, real frame](images/raw12/distance_real.png) + +Against the exact lossless 12-bit colour image of the same frame (real plant frame, ~15 dB gain; +the two towers agree within 0.3 dB): + +| Saved as | File 121 / 111 | PSNR | SSIM | +|---|---|---|---| +| 12-bit colour, distance 0.3 | 7.2 / 7.4 MB | 40.3 dB | 0.994 | +| 12-bit colour, distance 0.5 | 5.1 / 5.3 MB | 38.8 dB | 0.989 | +| 12-bit colour, distance 0.8 | 3.2 / 3.3 MB | 37.1 dB | 0.978 | +| **12-bit colour, distance 1.0** | **2.6 / 2.7 MB** | 36.6 dB | 0.973 | +| 12-bit colour, distance 1.5 | 1.6 / 1.7 MB | 35.4 dB | 0.960 | +| 12-bit colour, distance 2.0 | 1.3 / 1.3 MB | 34.8 dB | 0.953 | +| 12-bit colour, bilinear, distance 1.0 | 2.5 / 2.6 MB | 36.6 dB | 0.968 | +| 12-bit raw planes, lossy 0.5 | 6.0 / 6.0 MB | **47.2 dB** | 0.990 | +| 12-bit raw planes, lossless | 17.6 / 17.7 MB | exact | 1 | +| 12-bit raw mosaic, lossless | 20.9 / 19.0 MB | exact | 1 | +| 12-bit colour, lossless | 47.4 / 46.3 MB | exact | 1 | + +- **Colour JPEG XL is tuned to the eye:** it drops sensor noise and fine colour, so PSNR is + moderate while SSIM (structure) stays high. +- **Raw planes keep the sensor values much closer** at the same size (47 vs 39 dB, equal SSIM). + They suit a detector that wants raw values and can convert to colour itself. Lossless, they take + a third of the space of lossless colour. + +## Example images + +Every compression type, from tower 121's frame: a 100% crop of spruce needles over bright leaves; +its difference from lossless (×8, black = identical); and a darker crop brightened 2 stops. +Full-size `.jxl` files are kept outside git in `tools/camera-bench/examples-2026-10-02-tower121/`, +`examples-2026-10-04-tower111/` and `examples-2026-10-04-lowgain/` (~115 MB each). +Crop positions: ![crops](images/raw12/examples/121/crops.jpg) + +| Saved as | File, time (121) | Detail (100%) | Difference ×8 | Shadow, +2 stops | +|---|---|---|---|---| +| Camera RGB8, d0.8 e4 (today; another capture) | 3.8 MB, 0.62 s | ![](images/raw12/examples/121/detail/rgb8_d0.8_e4_today.png) | – | ![](images/raw12/examples/121/shadow/rgb8_d0.8_e4_today.png) | +| Raw 8 → colour, d1.0 (another capture) | 2.7 MB, 0.45 s | ![](images/raw12/examples/121/detail/c8_ea_d1.0.png) | – | ![](images/raw12/examples/121/shadow/c8_ea_d1.0.png) | +| 12-bit colour, lossless (reference) | 47.4 MB, 0.23 s | ![](images/raw12/examples/121/detail/c12_ea_lossless.png) | ![](images/raw12/examples/121/diff/c12_ea_lossless.png) | ![](images/raw12/examples/121/shadow/c12_ea_lossless.png) | +| 12-bit colour, d0.3 | 7.2 MB, 0.77 s | ![](images/raw12/examples/121/detail/c12_ea_d0.3.png) | ![](images/raw12/examples/121/diff/c12_ea_d0.3.png) | ![](images/raw12/examples/121/shadow/c12_ea_d0.3.png) | +| 12-bit colour, d0.5 | 5.1 MB, 0.69 s | ![](images/raw12/examples/121/detail/c12_ea_d0.5.png) | ![](images/raw12/examples/121/diff/c12_ea_d0.5.png) | ![](images/raw12/examples/121/shadow/c12_ea_d0.5.png) | +| 12-bit colour, d0.8 | 3.2 MB, 0.58 s | ![](images/raw12/examples/121/detail/c12_ea_d0.8.png) | ![](images/raw12/examples/121/diff/c12_ea_d0.8.png) | ![](images/raw12/examples/121/shadow/c12_ea_d0.8.png) | +| **12-bit colour, d1.0** | 2.6 MB, 0.53 s | ![](images/raw12/examples/121/detail/c12_ea_d1.0.png) | ![](images/raw12/examples/121/diff/c12_ea_d1.0.png) | ![](images/raw12/examples/121/shadow/c12_ea_d1.0.png) | +| 12-bit colour, d1.5 | 1.6 MB, 0.46 s | ![](images/raw12/examples/121/detail/c12_ea_d1.5.png) | ![](images/raw12/examples/121/diff/c12_ea_d1.5.png) | ![](images/raw12/examples/121/shadow/c12_ea_d1.5.png) | +| 12-bit colour, d2.0 | 1.3 MB, 0.42 s | ![](images/raw12/examples/121/detail/c12_ea_d2.0.png) | ![](images/raw12/examples/121/diff/c12_ea_d2.0.png) | ![](images/raw12/examples/121/shadow/c12_ea_d2.0.png) | +| 12-bit colour, bilinear, d1.0 | 2.5 MB, 0.48 s | ![](images/raw12/examples/121/detail/c12_bil_d1.0.png) | ![](images/raw12/examples/121/diff/c12_bil_d1.0.png) | ![](images/raw12/examples/121/shadow/c12_bil_d1.0.png) | +| 12-bit raw planes, d0.5 | 6.0 MB, 0.54 s | ![](images/raw12/examples/121/detail/raw12_planes_d0.5.png) | ![](images/raw12/examples/121/diff/raw12_planes_d0.5.png) | ![](images/raw12/examples/121/shadow/raw12_planes_d0.5.png) | +| 12-bit raw planes, lossless | 17.6 MB, 0.09 s | ![](images/raw12/examples/121/detail/raw12_planes_lossless.png) | ![](images/raw12/examples/121/diff/raw12_planes_lossless.png) | ![](images/raw12/examples/121/shadow/raw12_planes_lossless.png) | +| 12-bit raw mosaic, lossless | 20.9 MB, 0.09 s | ![](images/raw12/examples/121/detail/raw12_lossless.png) | ![](images/raw12/examples/121/diff/raw12_lossless.png) | ![](images/raw12/examples/121/shadow/raw12_lossless.png) | + +The two 8-bit rows are separate captures seconds apart (branches move), so they have no difference +map. Times are single runs at 4 threads. Tower 111's examples (same camera and scene) and the +0 dB examples are kept as full-size `.jxl` only; `tools/camera-bench/make_figures.py` makes the same +crops from them. + +## What 12-bit adds + +To see whether the extra 4 bits carry information or noise, a frame was also taken at **0 dB gain** +(124 ms exposure, 4 Oct). The noise floor, from the quietest patches at each brightness: + +![Noise vs brightness](images/raw12/noise_vs_signal.png) + +| Signal (12-bit, green) | Noise at 0 dB | Noise at ~15 dB | In 8-bit steps (0 dB / 15 dB) | +|---|---|---|---| +| ~700 (darkest in this scene) | 26 | 42 | 1.6 / 2.6 | +| ~1,500 | 35 | 58 | 2.2 / 3.6 | +| ~3,000 | 37 | 80 | 2.3 / 5.0 | + +Rounding to 8 bits adds an error of about 4.6 (12-bit units): small next to this noise. **In +daylight, 8-bit already holds what the sensor delivers.** By extrapolation, 12-bit starts to carry +information below about 300 (0 dB) or 100 (15 dB) of 4095, i.e. in deep shadow: dusk, night, the +dark side of a ridge. That is where a 12-bit night profile would pay off. Not measured at night. + +Same frame, a darker crop brightened 2 stops, 8-bit (left, the 12-bit value >> 4) vs 12-bit (right), +at ~15 dB and at 0 dB. 3,138 vs 227 distinct levels at 15 dB, yet no visible difference: + +![8 vs 12 bit at 15 dB](images/raw12/examples/121/shadow_8_vs_12.png) +![8 vs 12 bit at 0 dB](images/raw12/examples/lowgain/shadow_8_vs_12.png) + +At 0 dB the same settings give **25–30% smaller files and ~3 dB higher PSNR** (less noise to +encode): 12-bit colour at distance 1.0 is 1.9 MB, PSNR 40.0 dB, SSIM 0.987. + +## Overall pace: how fast can a tower capture? + +A waypoint goes through these steps; saving runs in the background, overlapping the next move: + +| Step | 12-bit raw (proposed) | Raw 8-bit | RGB8 (today) | Source | +|---|---|---|---|---| +| Camera delivers a frame every | 0.10 s (9.86 fps) | 0.07 s (14.75 fps) | 0.20 s (4.95 fps) | measured | +| Slew to the next waypoint | ~1–1.5 s (6° step at 5–8°/s) | same | same | estimate from the scan speed; not measured here | +| Mechanical settle after standstill | ~0.2 s | same | same | to be measured in the field | +| Exposure settle after a **large** change (×16) | ~3 s (2.2–6.6) | ~2 s (1.8–2.7) | ~5 s (4.6–6.6) | measured, 24 steps per tower | +| Exposure settle after a small change (neighbouring waypoints) | well under the above | | | not measured | +| **Save, one worker × 4 threads (IOTA / Delta)** | **0.56 / 0.84 s** | 0.52 / 0.85 s | 0.59 / 0.88 s | measured, shared frame, back to back | +| **Save, two workers × 2 threads (IOTA / Delta)** | **0.47 / 0.65 s** | 0.42 / 0.60 s | 0.43 / 0.64 s | measured, shared frame, back to back | + +![Saves per second, same frame](images/raw12/pace_common.png) + +- **The camera side sets the pace.** A waypoint needs roughly 1.5–5 s (move, then exposure), + while a 12-bit save takes 0.56 s on the IOTA and 0.65 s on the Delta with two workers. Saving + uses a fraction of the time between waypoints on either computer. +- **Saving becomes the limit only below about one waypoint per 0.6–0.85 s**, which a real scan does + not reach. On the shared frame, the IOTA kept up with a frame every 0.7 s with one worker; the + Delta needed two workers for 0.7 s (with one it fell 27 frames behind in 2 minutes). +- Settling after small changes, the common case between neighbouring waypoints, still needs a field + measurement; it is what decides the real scan pace. + +## Keeping up with the scan + +![Pile-up](images/raw12/pileup.png) + +Synthetic detailed frame (worst case). *ok* = no backlog after 2 minutes; otherwise frames behind +and the worst capture-to-file time: + +| Path | Workers × threads | Tower | Saves/s back to back | One frame every 2.0 / 1.5 / 1.0 / 0.7 / 0.5 s | Peak memory | +|---|---|---|---|---|---| +| Today: camera RGB8, d0.8 e4 | 1 × 4 | 121 | 1.33 | ok / – / ok / **3 behind** / – | 0.5 GB | +| Today: camera RGB8, d0.8 e4 | 1 × 4 | 111 | 0.88 | ok / – / **12 behind** / **65 behind** / – | 0.5 GB | +| **12-bit colour, d1.0 e3** | 1 × 4 | 121 | 1.44 | ok / ok / ok / **ok** / – | 0.7 GB | +| **12-bit colour, d1.0 e3** | 1 × 4 | 111 | 0.93 | ok / **ok** / **8 behind** / 60 behind / – | 0.7 GB | +| **12-bit colour, d1.0 e3** | 2 × 2 | 121 | 1.74 | – / – / ok / ok / **34 behind** | 1.1 GB | +| **12-bit colour, d1.0 e3** | 2 × 2 | 111 | 1.18 | – / – / **ok** / **28 behind** / 97 behind | 1.1 GB | +| 12-bit colour, d1.0 e3 | 4 × 1 | 121 / 111 | 1.99 / 1.37 | (back to back only; each save waits 2–3 s) | 2.0 GB | +| 12-bit colour, d0.5 e3 | 1 × 4 | 121 | 1.19 | – / – / ok / 24 behind / – | 0.8 GB | +| 12-bit colour, d0.5 e3 | 1 × 4 | 111 | 0.77 | – / – / 27 behind / 79 behind / – | 0.7 GB | +| 12-bit raw mosaic, lossless | 1 × 4 | 121 / 111 | 10.2 / 1.8 | ok at 1.0 and 0.7 on both | 0.3 GB | + +On the real plant frame (lighter), one worker kept up with a frame every 1.0 s on both towers +(saved within 0.63 s on 121, 0.96 s on 111). + +## Heat and power + +![Flat out](images/raw12/heat_max.png) +![Paced](images/raw12/heat_paced.png) + +10 minutes each: + +| Load | Tower | Temp mean / max | Clock | CPU busy | Power | Saves per minute, first → last | +|---|---|---|---|---|---|---| +| Idle | 121 | 38 / 40 °C | 0.79 GHz | 2% | 2.2 W | – | +| Idle | 111 | 52 / 54 °C | 0.96 GHz | 4% | 2.9 W | – | +| 12-bit, one frame every 2 s (1 × 4) | 121 | **53 / 84 °C** | 1.71 GHz | 21% | **5.1 W** | 30 → 29 | +| 12-bit, one frame every 2 s (1 × 4) | 111 | **66 / 86 °C** | 1.84 GHz | 33% | **5.7 W** | 30 → 30 | +| 12-bit, flat out (2 × 2) | 121 | 80 / 87 °C | 2.88 GHz | 81% | 12.5 W | 122 → 103 (−16%) | +| 12-bit, flat out (2 × 2) | 111 | 78 / 83 °C | 2.44 GHz | 81% | 8.1 W | 74 → 73 | + +At scan pace both computers stay far from their limits. The Delta runs ~13 °C warmer; flat out it +sits at an 8 W power limit, so it does not slow further, while the IOTA starts fast (12.5 W) and +settles 16% lower as it heats. + +## Storage + +![Storage per day](images/raw12/storage.png) + +Real plant frame sizes, 24 hours of scanning: + +| Saved as | Per frame (121 / 111) | One frame / 2 s | / 5 s | / 10 s | +|---|---|---|---|---| +| Camera RGB8, d0.8 (today's quality) | 3.7 / 4.0 MB | 160–172 GB | 64–69 GB | 32–34 GB | +| **12-bit colour, d1.0** | **2.65 / 2.75 MB** | **114–119 GB** | **46–48 GB** | **23–24 GB** | +| 12-bit colour, d0.5 | 5.2 / 5.3 MB | 222–230 GB | 89–92 GB | 44–46 GB | +| 12-bit raw planes, lossless | 16.8 / 17.0 MB | 727–736 GB | 291–294 GB | 145–147 GB | +| 12-bit colour, lossless | 44.8 / 44.1 MB | ~1.9 TB | ~770 GB | ~385 GB | + +The towers have 11–22 GB free, so only a short buffer can stay on the tower. + +## Scope and caveats + +- **One scene, two afternoons,** ~15 dB gain in the main runs; one 0 dB frame. Night is not measured. +- **Auto-exposure steps on a still camera,** not after real slews. Real waypoint-to-waypoint changes + are usually much smaller than ×16, so real settling should be shorter. +- **The tower program was stopped,** so its own load (camera stream, control loop) is not included. + The stream-while-saving test shows the camera link is unaffected by saving. +- **Detection quality is not measured.** PSNR and SSIM stand in for "what the algorithms receive". +- **Tower 111's own camera (1240c, 12 MP) was not tested here.** Its frames are 25% smaller, so the + Delta's limits above are conservative for it. +- **Slew time and mechanical settle** in the pace table are estimates, not measurements. + +## Raw data and tools + +- Programs and how to rerun every test: [tools/camera-bench](../tools/camera-bench/README.md). +- Camera tests, processing on own frames, heat: [tower 121](../tools/camera-bench/results-2026-10-02-tower121/), + [tower 111](../tools/camera-bench/results-2026-10-04-tower111/). +- Shared-frame processing: [tower 121](../tools/camera-bench/results-common-tower121/), + [tower 111](../tools/camera-bench/results-common-tower111/) (each with `frame_md5.txt`). +- Full-size example `.jxl` files (git-ignored, ~115 MB per set): `tools/camera-bench/examples-*`. +- Charts: `plot_results.py` and `plot_common.py`; example figures: `make_figures.py`; + quality scores: `compare_examples.py`. diff --git a/tools/camera-bench/README.md b/tools/camera-bench/README.md new file mode 100644 index 0000000..85eab3a --- /dev/null +++ b/tools/camera-bench/README.md @@ -0,0 +1,124 @@ +# camera-bench + +Measures a tower's camera and CPU for the capture design: maximum and sustained frame rate per pixel +format, losses while streaming, auto-exposure settle time, whether white balance applies to raw +output, and the per-frame cost of copying, de-Bayering and JPEG XL encoding. The results of the first +run, and what they mean, are in [docs/camera-capture-measurements.md](../../docs/camera-capture-measurements.md). + +Not part of the build: CMake only compiles the tools it names. + +| File | What | +|---|---| +| `camera_bench.py` | drives the camera through vmbpy; one JSON line per measurement | +| `bench.cpp` | CPU cost per frame, using the production `src/camera/JpegXlEncoder.cpp` unchanged | +| `lossless_check.cpp` | encodes at `jxl_distance = 0` through the production encoder, decodes, and compares pixels | +| `encode_bench.cpp` | JPEG XL cost per input format, effort, threads and quality; saving throughput and pile-up; CPU temperature, clock, load and power | +| `make_synthetic.py` | a sharp, detailed stand-in frame (dead-leaves texture with sensor noise) in the camera's formats | +| `run_camera_tests.sh` | the camera sequence: streaming per format and link limit, under load, auto-exposure, white balance (~25 min) | +| `run_encode_tests.sh` | the compression, pile-up and heat sequence (~90 min) | +| `summarize_encode.py` | Markdown tables from a `run_encode_tests.sh` result folder | +| `raw12_bench.cpp` | the 12-bit pipeline: unpack, de-Bayer, encode and write timed per stage; quality against the frame's own lossless image; pile-up, heat; example files (2026-10-02) | +| `run_raw12_camera.sh` / `run_raw12_encode.sh` | the 12-bit camera sequence (~25 min) and processing sequence (~1.5 h) | +| `run_common_frame.sh` | the processing tests on one shared frame (`common-frame/`, git-ignored, rebuilt from 121's lossless 12-bit example), so towers compare exactly (~25–40 min) | +| `plot_common.py` | the shared-frame tables and charts (per-step time, size per quality setting, saves per second); laptop venv | +| `compare_examples.py` | scores every 12-bit example against the lossless 12-bit colour image (PSNR, SSIM); laptop venv | +| `make_figures.py` | example-image figures (crops, difference maps, 8 vs 12-bit shadow) from the example `.jxl` files; laptop venv | +| `plot_results.py` | every chart and table of `docs/raw12-pipeline-measurements.md` from the results folders; laptop venv | +| `results-*/` | raw results of each run, one folder per tower and day | +| `examples-*/` | full-size example `.jxl` files (git-ignored, 100+ MB per tower) | + +## Running it on a tower + +The camera accepts one program at a time, so **the tower program must be stopped**, and the tower +captures nothing until it is started again. + +```bash +ssh ggs@ +sudo systemctl stop fire-gimbal-control +cd ~/projects/fwt_2a/software/tools/camera-bench +PY="env GENICAM_GENTL64_PATH=/opt/VimbaX/cti $HOME/.venvs/vmbpy/bin/python camera_bench.py" + +$PY info # current settings and ranges +$PY maxfps BayerRG8 250000000 # full res; also BayerRG12p, RGB8; also 450000000 +$PY stream BayerRG8 250000000 14.75 60 # FPS = the max from maxfps; 60 s +$PY ae BayerRG8 250000000 14.75 dark 100 # dark|bright step, IntensityControllerRate +$PY wb BayerRG8 250000000 # also RGB8, as the reference +$PY save BayerRG8 250000000 3 frames # also BayerRG12p and RGB8: bench.cpp needs all three +``` + +Then the CPU benchmark, built from the repo so it uses the production encoder: + +```bash +cd ~/projects/fwt_2a/software +g++ -O2 -std=c++17 -Iinclude tools/camera-bench/bench.cpp src/camera/JpegXlEncoder.cpp \ + src/core/Logger.cpp -o /tmp/bench $(pkg-config --cflags --libs libjxl libjxl_threads opencv4) -pthread +cd tools/camera-bench +/tmp/bench all frames /tmp/bench-out # every measurement +/tmp/bench raw8true frames /tmp/bench-out # lossless raw 8-bit through a correct lossless path +/tmp/bench load frames /tmp/bench-out 62 & # encode in a loop, while running `$PY stream ...` +``` + +`lossless_check.cpp` and `encode_bench.cpp` build the same way. `lossless_check` runs from the +directory holding `frames/`. + +### The full sequences (2026-10-01) + +Copy the programs into one working directory on the tower (for example `~/fgc-bench`), with the +binaries `bench` and `encode_bench` built there. Then, with the tower program stopped: + +```bash +nohup sh run_camera_tests.sh results > camera.log 2>&1 & # ~25 min; also saves frames/ +python3 make_synthetic.py synth 5328 3040 # the camera's resolution; needs numpy + OpenCV +nohup sh run_encode_tests.sh results-enc > enc.log 2>&1 & # ~90 min +python3 summarize_encode.py results-enc +``` + +`make_synthetic.py` needs Python's numpy and OpenCV; where the tower lacks them (tower 111), generate +the frame elsewhere and copy it. `run_encode_tests.sh` reads package power through a root loop +(`sudo -n`), which it starts and stops itself. In `run_camera_tests.sh`, auto-exposure is capped at +20 ms (`EXPOSURE_CAP_US`), so long exposures cannot limit the frame rate being measured. + +Finally, put the camera back on its profile, confirm frames flow, and restart the program: + +```bash +$PY restore ~/projects/fwt_2a/software/config/camera_profiles/rgb/standard.xml +sudo systemctl start fire-gimbal-control +rm -rf frames /tmp/bench-out # full-resolution frames are 16-49 MB each +``` + +The script never saves a camera user set or writes an XML. The program reloads its profile at start +anyway, so nothing a run changes survives it. + +### The 12-bit pipeline run (2026-10-02) + +On the tower, with `raw12_bench` built like `bench` and the tower program stopped: + +```bash +python3 make_synthetic.py synth # or copy one in +nohup sh run_raw12_camera.sh results-r12 > camera-r12.log 2>&1 & +nohup sh run_raw12_encode.sh results-r12 > encode-r12.log 2>&1 & # after the first ends +``` + +Then on the laptop, in a venv with matplotlib, numpy, imagecodecs and opencv-python-headless +(`~/.venvs/fgc-plots`), after copying `results-r12/` and `examples/jxl/` off the tower: + +```bash +P=~/.venvs/fgc-plots/bin/python +$P compare_examples.py examples-- results--/examples_compared.json +$P make_figures.py examples-- ../../docs/images/raw12/examples/ [dx,dy sx,sy] +$P plot_results.py ../../docs/images/raw12 121=results-2026-10-02-tower121 [111=...] +``` + +## Things to know + +- **vmbpy 1.2.0 cannot stream under Python 3.14 as installed:** its runtime type check raises + `NameError: name 'Camera' is not defined`. `camera_bench.py` bypasses that check in its own process. +- **Per-frame data:** `ChunkEnable` is writable only while `ChunkModeActive` is off, so the script + enables the items first and activates chunk mode last. +- **Colour order:** the camera's `BayerRG8` is OpenCV's `COLOR_BayerBG2RGB`. +- **The camera is the first real one found.** Set `CAM_ID=DEV_...` to choose one. +- **12-bit JPEG XL from 16-bit words needs `JxlEncoderSetFrameBitDepth(JXL_BIT_DEPTH_FROM_CODESTREAM)`**, + or libjxl rescales the values and "lossless" is not. Both C++ programs set it. +- **The `ae` bright step is limited by the camera's limits:** if gain is already at `GainAutoMax` + and exposure near `ExposureAutoMax`, it cannot force a full ×16. The result records the factor + actually reached (`forced_factor`). diff --git a/tools/camera-bench/bench.cpp b/tools/camera-bench/bench.cpp new file mode 100644 index 0000000..611f97b --- /dev/null +++ b/tools/camera-bench/bench.cpp @@ -0,0 +1,210 @@ +// bench.cpp - tower CPU cost per full-resolution frame, using the production +// JpegXlEncoder.cpp unchanged. Frames come from camera_bench.py `save`. +// +// bench all every measurement, one JSON line each +// bench load encode colour d0.8 e4 in a loop +// (to stream alongside) +#include "fgc/JpegXlEncoder.h" +#include "fgc/Logger.h" + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using Clock = std::chrono::steady_clock; + +// Frame size, from the .json camera_bench.py writes beside each frame. +static int W = 0, H = 0; + +static int jsonInt(const std::string& text, const std::string& key) { + auto at = text.find("\"" + key + "\""); + return at == std::string::npos ? 0 : std::atoi(text.c_str() + text.find(':', at) + 1); +} + +static std::vector load(const std::string& p) { + std::ifstream f(p, std::ios::binary); + return {std::istreambuf_iterator(f), {}}; +} + +static double ms(Clock::time_point a, Clock::time_point b) { + return std::chrono::duration(b - a).count(); +} + +// Median of `reps` runs of fn (fn returns nothing; timed around the call). +static double timeit(int reps, const std::function& fn) { + std::vector t; + for (int i = 0; i < reps; ++i) { + auto a = Clock::now(); + fn(); + t.push_back(ms(a, Clock::now())); + } + std::sort(t.begin(), t.end()); + return t[t.size() / 2]; +} + +static void line(const std::string& what, double t, long long bytes = -1) { + std::printf("{\"test\": \"%s\", \"ms\": %.1f", what.c_str(), t); + if (bytes >= 0) std::printf(", \"bytes\": %lld", bytes); + std::printf("}\n"); + std::fflush(stdout); +} + +// GenICam 12p: two pixels in three bytes, little-endian. +static std::vector unpack12p(const std::vector& in) { + std::vector out(static_cast(W) * H); + for (size_t i = 0, o = 0; o + 1 < out.size(); i += 3, o += 2) { + out[o] = static_cast(in[i] | ((in[i + 1] & 0x0F) << 8)); + out[o + 1] = static_cast((in[i + 1] >> 4) | (in[i + 2] << 4)); + } + return out; +} + +// Bench-only: lossless single-channel 16-bit JXL carrying 12 significant bits. +// The production encoder is 8-bit only. +static bool encodeGray(const std::string& path, const void* img, int bits, int effort, int threads) { + auto enc = JxlEncoderMake(nullptr); + auto runner = JxlThreadParallelRunnerMake(nullptr, threads); + JxlEncoderSetParallelRunner(enc.get(), JxlThreadParallelRunner, runner.get()); + JxlBasicInfo bi; + JxlEncoderInitBasicInfo(&bi); + bi.xsize = W; bi.ysize = H; bi.bits_per_sample = bits; bi.num_color_channels = 1; + bi.uses_original_profile = JXL_TRUE; + if (JxlEncoderSetBasicInfo(enc.get(), &bi) != JXL_ENC_SUCCESS) return false; + JxlColorEncoding ce = {}; + JxlColorEncodingSetToSRGB(&ce, JXL_TRUE); + JxlEncoderSetColorEncoding(enc.get(), &ce); + auto* fs = JxlEncoderFrameSettingsCreate(enc.get(), nullptr); + JxlEncoderFrameSettingsSetOption(fs, JXL_ENC_FRAME_SETTING_EFFORT, effort); + JxlEncoderSetFrameDistance(fs, 0); + JxlEncoderSetFrameLossless(fs, JXL_TRUE); + // 12-bit values in 16-bit words must be read at the header's bit depth, or + // libjxl rescales them from 0..65535 and the result is not lossless. + JxlBitDepth depth = {JXL_BIT_DEPTH_FROM_CODESTREAM, 0, 0}; + JxlEncoderSetFrameBitDepth(fs, &depth); + JxlPixelFormat pf = {1, bits > 8 ? JXL_TYPE_UINT16 : JXL_TYPE_UINT8, JXL_NATIVE_ENDIAN, 0}; + if (JxlEncoderAddImageFrame(fs, &pf, img, static_cast(W) * H * (bits > 8 ? 2 : 1)) != JXL_ENC_SUCCESS) + return false; + JxlEncoderCloseInput(enc.get()); + std::vector buf(1 << 20); + uint8_t* next = buf.data(); + size_t avail = buf.size(); + JxlEncoderStatus r; + while ((r = JxlEncoderProcessOutput(enc.get(), &next, &avail)) == JXL_ENC_NEED_MORE_OUTPUT) { + size_t off = next - buf.data(); + buf.resize(buf.size() * 2); + next = buf.data() + off; + avail = buf.size() - off; + } + if (r != JXL_ENC_SUCCESS) return false; + std::ofstream(path, std::ios::binary).write(reinterpret_cast(buf.data()), + next - buf.data()); + return true; +} + +static long long fsize(const std::string& p) { return static_cast(fs::file_size(p)); } + +int main(int argc, char** argv) { + if (argc < 4) { std::fprintf(stderr, "usage: bench all|load [secs]\n"); return 2; } + const std::string mode = argv[1], dir = argv[2], out = argv[3]; + fs::create_directories(out); + { + std::ifstream j(dir + "/BayerRG8_0.json"); + std::string t{std::istreambuf_iterator(j), {}}; + W = jsonInt(t, "width"); + H = jsonInt(t, "height"); + } + auto rgb = load(dir + "/RGB8_0.bin"); + auto bayer = load(dir + "/BayerRG8_0.bin"); + auto b12 = load(dir + "/BayerRG12p_0.bin"); + if (rgb.size() != size_t(W) * H * 3 || bayer.size() != size_t(W) * H || b12.empty()) { + std::fprintf(stderr, "frames missing or wrong size\n"); + return 1; + } + + if (mode == "raw8true") { + for (int e : {1, 2, 3}) { + std::string p = out + "/raw8true_e" + std::to_string(e) + ".jxl"; + double t = timeit(3, [&] { encodeGray(p, bayer.data(), 8, e, 3); }); + line("raw8_truelossless_e" + std::to_string(e) + "_t3", t, fsize(p)); + } + return 0; + } + + if (mode == "load") { + const double secs = argc > 4 ? std::atof(argv[4]) : 60; + auto t0 = Clock::now(); + int n = 0; + while (ms(t0, Clock::now()) < secs * 1000) { + fgc::JpegXlEncoder::encodeToFile(out + "/load.jxl", rgb.data(), W, H, 3, 0.8f, 4); + ++n; + } + line("load_colour_d0.8_e4_count" + std::to_string(n), ms(t0, Clock::now()) / n); + return 0; + } + + // 1. The copy FrameReceived makes of every delivered frame. + std::vector dst; + line("copy_rgb8_48.6MB", timeit(5, [&] { dst.assign(rgb.begin(), rgb.end()); })); + line("copy_bayer8_16.2MB", timeit(5, [&] { dst.assign(bayer.begin(), bayer.end()); })); + + // 2. De-Bayer. The camera's BayerRG8 is OpenCV's BayerBG (measured: matches + // the camera's own RGB8 channel means). + cv::Mat b(H, W, CV_8UC1, bayer.data()), c; + line("debayer_bilinear", timeit(5, [&] { cv::cvtColor(b, c, cv::COLOR_BayerBG2RGB); })); + line("debayer_ea", timeit(3, [&] { cv::cvtColor(b, c, cv::COLOR_BayerBG2RGB_EA); })); + line("debayer_vng", timeit(3, [&] { cv::cvtColor(b, c, cv::COLOR_BayerBG2RGB_VNG); })); + cv::Mat cam_rgb(H, W, CV_8UC3, rgb.data()), rot; + line("rotate_180_rgb8", timeit(5, [&] { cv::rotate(cam_rgb, rot, cv::ROTATE_180); })); + cv::cvtColor(b, c, cv::COLOR_BayerBG2RGB); + + // 3. Colour JXL, lossy as production: camera RGB8 and tower-de-Bayered. + for (int threads : {3, 4}) + for (int e : {1, 2, 3, 4, 5}) { + std::string p = out + "/rgb_d08_e" + std::to_string(e) + ".jxl"; + double t = timeit(3, [&] { + fgc::JpegXlEncoder::encodeToFile(p, rgb.data(), W, H, 3, 0.8f, e, threads); + }); + line("colour_camRGB8_d0.8_e" + std::to_string(e) + "_t" + std::to_string(threads), t, + fsize(p)); + } + for (int e : {1, 4}) { + std::string p = out + "/deb_d08_e" + std::to_string(e) + ".jxl"; + double t = timeit(3, [&] { + fgc::JpegXlEncoder::encodeToFile(p, c.data, W, H, 3, 0.8f, e, 3); + }); + line("colour_towerDebayer_d0.8_e" + std::to_string(e) + "_t3", t, fsize(p)); + } + + // 4. Raw mosaic, lossless, single channel. + for (int e : {1, 2, 3, 4}) { + std::string p = out + "/raw8_e" + std::to_string(e) + ".jxl"; + double t = timeit(3, [&] { + fgc::JpegXlEncoder::encodeToFile(p, bayer.data(), W, H, 1, 0.0f, e, 3); + }); + line("raw8_lossless_e" + std::to_string(e) + "_t3", t, fsize(p)); + } + std::vector u; + line("unpack_12p", timeit(3, [&] { u = unpack12p(b12); })); + for (int e : {1, 2, 3}) { + std::string p = out + "/raw12_e" + std::to_string(e) + ".jxl"; + double t = timeit(3, [&] { encodeGray(p, u.data(), 12, e, 3); }); + line("raw12_lossless_e" + std::to_string(e) + "_t3", t, fsize(p)); + } + return 0; +} diff --git a/tools/camera-bench/camera_bench.py b/tools/camera-bench/camera_bench.py new file mode 100644 index 0000000..c834a6e --- /dev/null +++ b/tools/camera-bench/camera_bench.py @@ -0,0 +1,565 @@ +#!/usr/bin/env python3 +"""camera_bench.py - measure the tower camera for the capture redesign. + +Subcommands (one JSON object per line on stdout): + info key features, ranges, versions + maxfps FMT LIMIT full-res FMT at link LIMIT (bytes/s): payload, max fps + stream FMT LIMIT FPS SECS continuous stream: delivered, incomplete, id gaps, stalls + ae FMT LIMIT FPS DIR RATE auto-exposure step response (DIR dark|bright, RATE = IntensityControllerRate) + awb FMT LIMIT FPS FACTOR RATE auto white balance step response: force both + ratios to FACTOR x the auto result, then release (RATE = + BalanceWhiteAutoRate) + wb FMT LIMIT does BalanceRatio change the delivered pixels? + save FMT LIMIT N DIR save N settled full-res frames as raw .bin + .json + restore XML load the profile XML (as the program does) and check frames flow + +Never saves a user set or writes an XML. Every run leaves the camera in whatever +state the subcommand set; `restore` puts it back on the profile. +""" +import json +import os +import statistics +import sys +import threading +import time + +import inspect + +# vmbpy looks for the transport layers when it is imported, so the path must be +# set before the import (tower 121 has it system-wide; tower 111 does not). +os.environ.setdefault("GENICAM_GENTL64_PATH", "/opt/VimbaX/cti") + +import vmbpy +import vmbpy.util.runtime_type_check as _rtc + + +# vmbpy 1.2.0's runtime type check resolves its own forward references +# ("Camera") with typing.get_type_hints, which fails under Python 3.14's +# deferred annotations - start_streaming() raises NameError before a frame +# arrives. Skip the check, in this process only. +def _no_hints(self, func, *args, **kwargs): + b = inspect.signature(func).bind(*args, **kwargs) + b.apply_defaults() + return b.arguments, {} + + +_rtc.RuntimeTypeCheckEnable._RuntimeTypeCheckEnable__dismantle_sig = _no_hints + +# Default: the first real camera. The SDK's simulator transport layer offers +# DEV_Cam1..3, which are not cameras (as in scripts/export_camera_profile.py). +CAM_ID = os.environ.get("CAM_ID") +EXPOSURE_CAP_US = float(os.environ.get("EXPOSURE_CAP_US", "20000")) + + +def out(**kw): + print(json.dumps(kw), flush=True) + + +def f(cam, n): + return cam.get_feature_by_name(n) + + +def get(cam, n): + try: + v = f(cam, n).get() + return v if isinstance(v, (int, float, bool, str)) else str(v) + except Exception: + return None + + +def rng(cam, n): + try: + return list(f(cam, n).get_range()) + except Exception: + return None + + +def setf(cam, n, v): + f(cam, n).set(v) + return get(cam, n) + + +def bytes_per_px(fmt): + return {"RGB8": 3, "BGR8": 3, "BayerRG8": 1, "Mono8": 1, "BayerRG12p": 1.5, + "BayerRG10p": 1.25, "BayerRG12": 2, "BayerRG10": 2}[fmt] + + +def full_res(cam, fmt, limit): + """Binning 1, full sensor, FMT, link LIMIT; frame rate at its maximum.""" + setf(cam, "BinningSelector", "Digital") + # Each axis's minimum can depend on the other: try both orders. + for _ in range(2): + for n in ("BinningVertical", "BinningHorizontal"): + try: + setf(cam, n, 1) + except Exception: + pass + if get(cam, "BinningHorizontal") != 1 or get(cam, "BinningVertical") != 1: + raise RuntimeError(f"binning refused: {get(cam, 'BinningHorizontal')}x{get(cam, 'BinningVertical')}") + setf(cam, "OffsetX", 0) + setf(cam, "OffsetY", 0) + setf(cam, "Width", get(cam, "WidthMax")) + setf(cam, "Height", get(cam, "HeightMax")) + setf(cam, "PixelFormat", fmt) + setf(cam, "DeviceLinkThroughputLimitMode", "On") + setf(cam, "DeviceLinkThroughputLimit", int(limit)) + # Cap auto-exposure at 20 ms (tower 121's profile value), so the frame rate + # measures the link, not a long exposure: tower 111's profile allows 9 s. + if get(cam, "ExposureAutoMax") > EXPOSURE_CAP_US: + setf(cam, "ExposureAutoMax", EXPOSURE_CAP_US) + if get(cam, "ExposureTime") > EXPOSURE_CAP_US: + setf(cam, "ExposureTime", EXPOSURE_CAP_US) + setf(cam, "AcquisitionFrameRateEnable", True) + hi = rng(cam, "AcquisitionFrameRate")[1] + return hi + + +def set_fps(cam, fps): + lo, hi = rng(cam, "AcquisitionFrameRate") + return setf(cam, "AcquisitionFrameRate", min(max(fps, lo), hi)) + + +def chunks_on(cam): + # ChunkEnable is writable only while ChunkModeActive is off (measured, fw 14.3). + setf(cam, "ChunkModeActive", False) + for sel in ("ExposureTime", "Gain", "Timestamp", "BalanceRatioRed", "BalanceRatioBlue"): + setf(cam, "ChunkSelector", sel) + setf(cam, "ChunkEnable", True) + setf(cam, "ChunkModeActive", True) + + +def auto_on(cam): + setf(cam, "ExposureAuto", "Continuous") + setf(cam, "GainAuto", "Continuous") + + +# Brightness from a sparse sample of the frame buffer, taken in C (bytes +# slicing and translate/count), so the handler stays well under a millisecond. +# Per format: which bytes to sample, and how they map to 0..255. +# RGB8 / BayerRG8: every byte is a value. +# BayerRG12p: the third byte of each 3-byte pair is the high 8 bits of the +# second pixel (odd columns: G on even rows, B on odd rows - no R sites). +# BayerRG12: little-endian 16-bit; the high byte holds bits 8..11 (x16). +_HI = bytes(1 if v >= 250 else 0 for v in range(256)) +_HI12 = bytes(1 if v >= 15 else 0 for v in range(256)) +_SAMPLE_STRIDE = 997 + + +def sample_brightness(mv, fmt, nbytes): + """(mean on a 0..255 scale, share of samples at or near white) or (None, None).""" + if fmt in ("RGB8", "BGR8", "BayerRG8", "Mono8"): + s, scale, tab = bytes(mv[0:nbytes:_SAMPLE_STRIDE]), 1, _HI + elif fmt == "BayerRG12p": + s, scale, tab = bytes(mv[2:nbytes:3 * 331]), 1, _HI + elif fmt == "BayerRG12": + s, scale, tab = bytes(mv[1:nbytes:2 * 499]), 16, _HI12 + else: + return None, None + if not s: + return None, None + return sum(s) * scale / len(s), s.translate(tab).count(1) / len(s) + + +class Recorder: + """Frame handler: metadata only, buffer straight back - so the measurement + cannot cause the drops it is counting. Saving copies only when asked.""" + + def __init__(self, fmt=None, w=0, h=0): + # [host_t, frame_id, complete, exposure_us, gain_db, dev_ts_ns, + # brightness_mean, clipped_share, wb_red, wb_blue] + self.rows = [] + self.lock = threading.Lock() + self.want_pixels = 0 + self.pixels = [] + self.fmt = fmt + self.nbytes = int(w * h * bytes_per_px(fmt)) if fmt else 0 + + def __call__(self, cam, stream, frame): + t = time.monotonic() + ok = frame.get_status() == vmbpy.FrameStatus.Complete + row = [t, frame.get_id(), int(ok), None, None, frame.get_timestamp(), + None, None, None, None] + if ok: + def cb(feats): + try: + row[3] = feats.get_feature_by_name("ChunkExposureTime").get() + row[4] = feats.get_feature_by_name("ChunkGain").get() + row[8] = feats.get_feature_by_name("ChunkBalanceRatioRed").get() + row[9] = feats.get_feature_by_name("ChunkBalanceRatioBlue").get() + except Exception: + pass + try: + frame.access_chunk_data(cb) + except Exception: + pass + if self.fmt and self.nbytes: + try: + row[6], row[7] = sample_brightness(memoryview(frame.get_buffer()), + self.fmt, self.nbytes) + except Exception: + pass + if self.want_pixels > 0: + self.pixels.append((bytes(memoryview(frame.get_buffer())[: self.nbytes]), + row[3], row[4])) + self.want_pixels -= 1 + with self.lock: + self.rows.append(row) + cam.queue_frame(frame) + + def snapshot(self): + with self.lock: + return list(self.rows) + + +def summarize(rows, secs, payload): + comp = [r for r in rows if r[2]] + ids = [r[1] for r in rows if r[1] is not None] + missing = sum(max(0, b - a - 1) for a, b in zip(ids, ids[1:])) + ts = [r[0] for r in rows] + gaps = [b - a for a, b in zip(ts, ts[1:])] + return dict(frames=len(rows), complete=len(comp), incomplete=len(rows) - len(comp), + missing_ids=missing, + fps=round(len(comp) / secs, 3), + mb_s=round(len(comp) * payload / secs / 1e6, 1), + max_gap_s=round(max(gaps), 3) if gaps else None, + gaps_over_1s=sum(1 for g in gaps if g > 1.0), + median_gap_s=round(statistics.median(gaps), 4) if gaps else None) + + +def run_stream(cam, rec, secs, stall_s=5.0): + """Stream for SECS; stop early if nothing arrives for STALL_S.""" + t0 = time.monotonic() + cam.start_streaming(rec, buffer_count=5) + stalled = False + try: + while time.monotonic() - t0 < secs: + time.sleep(0.2) + rows = rec.snapshot() + last = rows[-1][0] if rows else t0 + if time.monotonic() - last > stall_s and time.monotonic() - t0 > stall_s: + stalled = True + break + finally: + cam.stop_streaming() + return time.monotonic() - t0, stalled + + +def eff(e, g): + """Exposure x linear gain: the brightness the auto-exposure is steering.""" + return e * 10 ** (g / 20.0) + + +def cmd_info(cam): + names = ["DeviceModelName", "DeviceSerialNumber", "DeviceFirmwareVersion", "DeviceTemperature", + "PixelFormat", "SensorBitDepth", "BinningHorizontal", "BinningVertical", + "BinningHorizontalMode", "Width", "Height", "WidthMax", "HeightMax", "PayloadSize", + "DeviceLinkThroughputLimit", "AcquisitionFrameRate", "ExposureTime", "ExposureAuto", + "ExposureAutoMin", "ExposureAutoMax", "Gain", "GainAuto", "GainAutoMax", + "IntensityAutoPrecedence", "IntensityControllerRate", "IntensityControllerTarget", + "IntensityControllerTolerance", "IntensityControllerAlgorithm", + "IntensityControllerRegion", "BalanceWhiteAuto", "BalanceRatio", "ChunkModeActive"] + out(cmd="info", values={n: get(cam, n) for n in names}, + ranges={n: rng(cam, n) for n in ["AcquisitionFrameRate", "DeviceLinkThroughputLimit", + "IntensityControllerRate", "IntensityControllerTarget", + "IntensityControllerTolerance", "BalanceRatio", + "ExposureTime", "Gain"]}, + vmbpy=vmbpy.__version__) + + +def cmd_maxfps(cam, fmt, limit): + auto_on(cam) + hi = full_res(cam, fmt, float(limit)) + out(cmd="maxfps", fmt=fmt, limit=int(float(limit)), width=get(cam, "Width"), + height=get(cam, "Height"), payload=get(cam, "PayloadSize"), max_fps=round(hi, 3), + exposure_us=get(cam, "ExposureTime")) + + +def cmd_stream(cam, fmt, limit, fps, secs): + auto_on(cam) + full_res(cam, fmt, float(limit)) + actual = set_fps(cam, float(fps)) + chunks_on(cam) + payload = get(cam, "PayloadSize") + rec = Recorder(fmt, get(cam, "Width"), get(cam, "Height")) + dur, stalled = run_stream(cam, rec, float(secs)) + rows = rec.snapshot() + s = summarize(rows, dur, payload) + exps = [r[3] for r in rows if r[3] is not None] + out(cmd="stream", fmt=fmt, limit=int(float(limit)), fps_set=round(actual, 3), + secs=round(dur, 1), payload=payload, stalled_early=stalled, + chunk_frames=len(exps), exposure_range=[min(exps), max(exps)] if exps else None, **s) + + +def cmd_ae(cam, fmt, limit, fps, direction, rate): + full_res(cam, fmt, float(limit)) + actual = set_fps(cam, float(fps)) + chunks_on(cam) + setf(cam, "IntensityControllerRate", float(rate)) + auto_on(cam) + emin, emax = get(cam, "ExposureAutoMin"), get(cam, "ExposureAutoMax") + gmax = get(cam, "GainAutoMax") + rec = Recorder(fmt, get(cam, "Width"), get(cam, "Height")) + cam.start_streaming(rec, buffer_count=5) + try: + time.sleep(8.0) # let it settle on the scene + pre = [r for r in rec.snapshot() if r[3] is not None][-5:] + base_e = statistics.median(r[3] for r in pre) + base_g = statistics.median(r[4] for r in pre) + base_mean = statistics.median(r[6] for r in pre) if pre and pre[0][6] is not None else None + base_clip = statistics.median(r[7] for r in pre) if pre and pre[0][7] is not None else None + base = eff(base_e, base_g) + # Force a x16 step in brightness, using gain when exposure hits its limit. + setf(cam, "ExposureAuto", "Off") + setf(cam, "GainAuto", "Off") + if direction == "dark": + e = max(emin, base / 16) + g = 0.0 + else: + e = min(emax, base * 16) + g = min(gmax, max(0.0, 20 * __import__("math").log10(base * 16 / e))) + setf(cam, "ExposureTime", e) + setf(cam, "Gain", g) + forced = eff(e, g) + time.sleep(2.0) + t0 = time.monotonic() + auto_on(cam) + time.sleep(15.0) + finally: + cam.stop_streaming() + rows = [r for r in rec.snapshot() if r[0] >= t0 and r[3] is not None] + # [s since release, exposure_us, gain_db, brightness 0..255, clipped share] + series = [[round(r[0] - t0, 3), round(r[3], 1), round(r[4], 2), + None if r[6] is None else round(r[6], 2), + None if r[7] is None else round(r[7], 4)] for r in rows] + vals = [(r[3], r[4]) for r in rows] + + def same(a, b): + return abs(a[0] - b[0]) <= 0.005 * max(a[0], 1) and abs(a[1] - b[1]) <= 0.05 + + # Rule under test: first frame after which exposure+gain stay put for 3 frames. + first3 = None + for i in range(len(vals) - 2): + if same(vals[i], vals[i + 1]) and same(vals[i + 1], vals[i + 2]): + first3 = i + 2 + break + # Truth: the frame after which nothing changes to the end of the run. + last_change = 0 + for i in range(1, len(vals)): + if not same(vals[i], vals[i - 1]): + last_change = i + first_ok = next((i for i, v in enumerate(vals) + if abs(eff(*v) / base - 1) <= 0.10), None) + fin = eff(*vals[-1]) if vals else None + out(cmd="ae", fmt=fmt, fps_set=round(actual, 3), direction=direction, rate=rate, + base_exposure_us=round(base_e, 1), base_gain_db=round(base_g, 2), + base_brightness=None if base_mean is None else round(base_mean, 2), + base_clipped=None if base_clip is None else round(base_clip, 4), + target=get(cam, "IntensityControllerTarget"), + tolerance=get(cam, "IntensityControllerTolerance"), + forced_factor=round(forced / base, 3), final_vs_base=round(fin / base, 3) if fin else None, + frames=len(vals), + stable3_frame=first3, stable3_s=series[first3][0] if first3 is not None else None, + settled_frame=last_change, settled_s=series[last_change][0] if series else None, + within10pct_frame=first_ok, + within10pct_s=series[first_ok][0] if first_ok is not None else None, + series=series) + + +def cmd_awb(cam, fmt, limit, fps, factor, rate): + """Auto white balance step response. Exposure and gain are locked after + auto-exposure settles, so only white balance moves. The auto result on the + unchanged scene is the reference; both ratios are then forced to FACTOR x + that and released.""" + full_res(cam, fmt, float(limit)) + actual = set_fps(cam, float(fps)) + chunks_on(cam) + auto_on(cam) + setf(cam, "BalanceWhiteAutoRate", float(rate)) + rec = Recorder(fmt, get(cam, "Width"), get(cam, "Height")) + cam.start_streaming(rec, buffer_count=5) + try: + time.sleep(6.0) + pre = [r for r in rec.snapshot() if r[3] is not None][-5:] + setf(cam, "ExposureAuto", "Off") + setf(cam, "GainAuto", "Off") + setf(cam, "ExposureTime", statistics.median(r[3] for r in pre)) + setf(cam, "Gain", statistics.median(r[4] for r in pre)) + setf(cam, "BalanceWhiteAuto", "Continuous") + t_ref = time.monotonic() + time.sleep(8.0) # auto white balance finds its answer on the still scene + ref_rows = [r for r in rec.snapshot() if r[0] >= t_ref + 5.0 and r[8] is not None] + ref_r = statistics.median(r[8] for r in ref_rows) + ref_b = statistics.median(r[9] for r in ref_rows) + spread_r = (max(r[8] for r in ref_rows) - min(r[8] for r in ref_rows)) / ref_r + spread_b = (max(r[9] for r in ref_rows) - min(r[9] for r in ref_rows)) / ref_b + setf(cam, "BalanceWhiteAuto", "Off") + lo, hi = rng(cam, "BalanceRatio") + for sel, ref in (("Red", ref_r), ("Blue", ref_b)): + setf(cam, "BalanceRatioSelector", sel) + setf(cam, "BalanceRatio", min(hi, max(lo, ref * float(factor)))) + time.sleep(2.0) + t0 = time.monotonic() + setf(cam, "BalanceWhiteAuto", "Continuous") + time.sleep(15.0) + finally: + cam.stop_streaming() + rows = [r for r in rec.snapshot() if r[0] >= t0 and r[8] is not None] + series = [[round(r[0] - t0, 3), round(r[8], 4), round(r[9], 4)] for r in rows] + + def within(r, tol): + return abs(r[8] / ref_r - 1) <= tol and abs(r[9] / ref_b - 1) <= tol + + first5 = next((i for i, r in enumerate(rows) if within(r, 0.05)), None) + stay5 = next((i for i in range(len(rows)) if all(within(x, 0.05) for x in rows[i:])), None) + last_change = 0 + for i in range(1, len(rows)): + if rows[i][8] != rows[i - 1][8] or rows[i][9] != rows[i - 1][9]: + last_change = i + out(cmd="awb", fmt=fmt, fps_set=round(actual, 3), factor=float(factor), rate=float(rate), + ref_red=round(ref_r, 4), ref_blue=round(ref_b, 4), + ref_spread_red=round(spread_r, 4), ref_spread_blue=round(spread_b, 4), + frames=len(rows), + within5pct_frame=first5, within5pct_s=series[first5][0] if first5 is not None else None, + stays5pct_frame=stay5, stays5pct_s=series[stay5][0] if stay5 is not None else None, + settled_frame=last_change, settled_s=series[last_change][0] if series else None, + series=series) + + +def px12p(buf, i): + """Pixel i of a BayerRG12p buffer (two pixels in three bytes).""" + j = (i >> 1) * 3 + if i & 1 == 0: + return buf[j] | ((buf[j + 1] & 0x0F) << 8) + return (buf[j + 1] >> 4) | (buf[j + 2] << 4) + + +def mean_sites(buf, w, h, fmt, step=16): + """Mean of R, G, B sites over a sparse grid (no numpy on the tower venv).""" + sums = {"R": [0, 0], "G": [0, 0], "B": [0, 0]} + if fmt == "BayerRG12p": # 12-bit values, reported on the 0..255 scale (/16) + for y in range(0, h - 1, step): + for x in range(0, w - 1, step): + i = y * w + x + sums["R"][0] += px12p(buf, i) / 16; sums["R"][1] += 1 + sums["G"][0] += px12p(buf, i + 1) / 16; sums["G"][1] += 1 + sums["B"][0] += px12p(buf, i + w + 1) / 16; sums["B"][1] += 1 + elif fmt == "BayerRG8": + for y in range(0, h - 1, step): + for x in range(0, w - 1, step): + i = y * w + x + sums["R"][0] += buf[i]; sums["R"][1] += 1 + sums["G"][0] += buf[i + 1]; sums["G"][1] += 1 + sums["B"][0] += buf[i + w + 1]; sums["B"][1] += 1 + else: # RGB8 + for y in range(0, h, step): + for x in range(0, w, step): + i = (y * w + x) * 3 + for k, c in enumerate("RGB"): + sums[c][0] += buf[i + k]; sums[c][1] += 1 + return {c: round(s / n, 2) for c, (s, n) in sums.items()} + + +def grab(cam, fmt, n=3): + w, h = get(cam, "Width"), get(cam, "Height") + rec = Recorder(fmt, w, h) + rec.want_pixels = n + cam.start_streaming(rec, buffer_count=5) + try: + time.sleep(1.0) # skip frames exposed before the change + rec.pixels.clear() + rec.want_pixels = n + t0 = time.monotonic() + while rec.want_pixels > 0 and time.monotonic() - t0 < 15: + time.sleep(0.05) + finally: + cam.stop_streaming() + return rec.pixels, w, h + + +def cmd_wb(cam, fmt, limit): + full_res(cam, fmt, float(limit)) + chunks_on(cam) + auto_on(cam) + rec = Recorder() + cam.start_streaming(rec, buffer_count=5) + time.sleep(6.0) + cam.stop_streaming() + pre = [r for r in rec.snapshot() if r[3] is not None][-3:] + setf(cam, "ExposureAuto", "Off") + setf(cam, "GainAuto", "Off") + setf(cam, "ExposureTime", statistics.median(r[3] for r in pre)) + setf(cam, "Gain", statistics.median(r[4] for r in pre)) + setf(cam, "BalanceWhiteAuto", "Off") + res = {} + for sel in ("Red", "Blue"): + setf(cam, "BalanceRatioSelector", sel) + orig = get(cam, "BalanceRatio") + for ratio in (1.0, 2.0): + setf(cam, "BalanceRatio", ratio) + px, w, h = grab(cam, fmt, 2) + res[f"{sel}={ratio}"] = [mean_sites(p[0], w, h, fmt) for p in px] + setf(cam, "BalanceRatio", orig) + out(cmd="wb", fmt=fmt, exposure_us=get(cam, "ExposureTime"), gain_db=get(cam, "Gain"), means=res) + + +def cmd_save(cam, fmt, limit, n, d): + full_res(cam, fmt, float(limit)) + chunks_on(cam) + auto_on(cam) + rec = Recorder() + cam.start_streaming(rec, buffer_count=5) + time.sleep(6.0) + cam.stop_streaming() + px, w, h = grab(cam, fmt, int(n)) + os.makedirs(d, exist_ok=True) + for i, (buf, e, g) in enumerate(px): + base = os.path.join(d, f"{fmt}_{i}") + with open(base + ".bin", "wb") as fh: + fh.write(buf) + with open(base + ".json", "w") as fh: + json.dump(dict(fmt=fmt, width=w, height=h, exposure_us=e, gain_db=g), fh) + out(cmd="save", fmt=fmt, saved=len(px), width=w, height=h, bytes=len(px[0][0]) if px else 0, + means=[mean_sites(p[0], w, h, fmt) for p in px + if fmt in ("BayerRG8", "BayerRG12p", "RGB8")]) + + +def cmd_restore(cam, xml): + cam.load_settings(xml, vmbpy.PersistType.All) + rec = Recorder() + dur, stalled = run_stream(cam, rec, 10.0) + s = summarize(rec.snapshot(), dur, get(cam, "PayloadSize")) + out(cmd="restore", xml=xml, stalled_early=stalled, + values={n: get(cam, n) for n in ["PixelFormat", "BinningHorizontal", "BinningVertical", + "Width", "Height", "AcquisitionFrameRate", + "DeviceLinkThroughputLimit", "ExposureAuto", "GainAuto", + "IntensityControllerRate", "ChunkModeActive", + "BalanceWhiteAuto"]}, **s) + + +def main(): + cmd, args = sys.argv[1], sys.argv[2:] + with vmbpy.VmbSystem.get_instance() as vmb: + if CAM_ID: + cam = vmb.get_camera_by_id(CAM_ID) + else: + real = [c for c in vmb.get_all_cameras() if not c.get_id().startswith("DEV_Cam")] + if not real: + out(cmd=cmd, error="no real camera found") + return 1 + cam = real[0] + with cam: + t0 = time.monotonic() + try: + {"info": cmd_info, "maxfps": cmd_maxfps, "stream": cmd_stream, "ae": cmd_ae, + "awb": cmd_awb, "wb": cmd_wb, "save": cmd_save, "restore": cmd_restore}[cmd](cam, *args) + except Exception as e: + out(cmd=cmd, args=args, error=f"{type(e).__name__}: {e}") + return 1 + finally: + print(json.dumps(dict(cmd=cmd, elapsed_s=round(time.monotonic() - t0, 1))), + file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/camera-bench/compare_examples.py b/tools/camera-bench/compare_examples.py new file mode 100644 index 0000000..e911fc5 --- /dev/null +++ b/tools/camera-bench/compare_examples.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""compare_examples.py - every 12-bit example scored on one footing. + + ~/.venvs/fgc-plots/bin/python compare_examples.py [] + +Decodes each 12-bit example .jxl from `raw12_bench examples` (needs imagecodecs +and OpenCV), turns the mosaic kinds into colour with the same edge-aware +de-Bayer, and compares every result with the lossless 12-bit colour image of +the same frame (c12_ea_lossless.jxl, exact by construction): + PSNR on the 12-bit scale, all three channels + SSIM on luma, 12-bit, 11x11 Gaussian window (standard constants) +So compression is judged on what the downstream algorithms would receive, not +against each kind's own reference. +""" +import glob +import json +import os +import sys + +import cv2 +import imagecodecs +import numpy as np + +d = sys.argv[1] +ref_path = os.path.join(d, "c12_ea_lossless.jxl") + + +def decode(path): + a = imagecodecs.jpegxl_decode(open(path, "rb").read()) + return a + + +def to_rgb12(name, a): + """Decoded array -> HxWx3 float RGB on the 12-bit scale.""" + if a.ndim == 3: + return a.astype(np.float64) + m = a.astype(np.uint16) + h, w = m.shape + if "planes" in name: # undo the 2x2 tiling of R, G1 / G2, B + hh, hw = h // 2, w // 2 + mo = np.empty_like(m) + mo[0::2, 0::2] = m[:hh, :hw] + mo[0::2, 1::2] = m[:hh, hw:] + mo[1::2, 0::2] = m[hh:, :hw] + mo[1::2, 1::2] = m[hh:, hw:] + m = mo + return cv2.cvtColor(m, cv2.COLOR_BayerBG2RGB_EA).astype(np.float64) + + +def luma(rgb): + return 0.2126 * rgb[..., 0] + 0.7152 * rgb[..., 1] + 0.0722 * rgb[..., 2] + + +def ssim(a, b, peak=4095.0): + c1, c2 = (0.01 * peak) ** 2, (0.03 * peak) ** 2 + g = lambda x: cv2.GaussianBlur(x, (11, 11), 1.5) + mu_a, mu_b = g(a), g(b) + va, vb, cov = g(a * a) - mu_a ** 2, g(b * b) - mu_b ** 2, g(a * b) - mu_a * mu_b + s = ((2 * mu_a * mu_b + c1) * (2 * cov + c2)) / ((mu_a ** 2 + mu_b ** 2 + c1) * (va + vb + c2)) + return float(s.mean()) + + +ref = decode(ref_path) +if ref.max() > 4095: # decoded to 16-bit range: bring back to 12-bit + ref = ref / 16.0 +ref = ref.astype(np.float64) +ref_l = luma(ref) +res = [] +for path in sorted(glob.glob(os.path.join(d, "*.jxl"))): + name = os.path.splitext(os.path.basename(path))[0] + if not any(k in name for k in ("c12", "raw12")): + continue # 8-bit examples are other captures, not comparable pixel by pixel + a = decode(path) + if a.max() > 4095: + a = a / 16.0 + rgb = to_rgb12(name, a) + mse = float(np.mean((rgb - ref) ** 2)) + psnr = 99.0 if mse == 0 else 10 * np.log10(4095.0 ** 2 / mse) + r = dict(name=name, bytes=os.path.getsize(path), psnr_db=round(psnr, 2), + ssim=round(ssim(luma(rgb), ref_l), 5)) + res.append(r) + print(json.dumps(r)) +if len(sys.argv) > 2: + json.dump(res, open(sys.argv[2], "w"), indent=1) diff --git a/tools/camera-bench/encode_bench.cpp b/tools/camera-bench/encode_bench.cpp new file mode 100644 index 0000000..190e3ee --- /dev/null +++ b/tools/camera-bench/encode_bench.cpp @@ -0,0 +1,521 @@ +// encode_bench.cpp - what JPEG XL encoding costs the tower, per input format, +// and whether one save worker keeps up without piling frames up or overheating. +// Colour lossy encodes go through the production src/camera/JpegXlEncoder.cpp +// unchanged; lossless and 16-bit encodes use a bench-only path that sets +// uses_original_profile correctly (production's distance 0 is not lossless, +// known-issues #20). Frames come from camera_bench.py `save`. +// +// encode_bench matrix +// encode time / size for every input x effort x threads, a distance sweep +// with PSNR, and the de-Bayer/unpack step on its own. One JSON line each. +// encode_bench pipeline [effort] [distance] +// kind: rgb8 | bilinear | ea | raw8 | raw12 +// period_ms 0: workers process back to back (maximum throughput). +// period_ms > 0: one frame is queued every period_ms, as a scan keeps one +// per waypoint; reports latency and backlog. Samples CPU temperature, +// clock, load and (if readable) package power once a second. +// encode_bench matrix12 +// only the 12-bit parts of matrix. +// encode_bench idle +// the same samples with nothing running: the baseline. +#include "fgc/JpegXlEncoder.h" + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using Clock = std::chrono::steady_clock; + +static int W = 0, H = 0; + +static int jsonInt(const std::string& text, const std::string& key) { + auto at = text.find("\"" + key + "\""); + return at == std::string::npos ? 0 : std::atoi(text.c_str() + text.find(':', at) + 1); +} + +static std::vector load(const std::string& p) { + std::ifstream f(p, std::ios::binary); + return {std::istreambuf_iterator(f), {}}; +} + +static double msSince(Clock::time_point a) { + return std::chrono::duration(Clock::now() - a).count(); +} + +static long long fsize(const std::string& p) { + std::error_code ec; + auto s = fs::file_size(p, ec); + return ec ? -1 : static_cast(s); +} + +static std::vector unpack12p(const std::vector& in) { + std::vector out(static_cast(W) * H); + for (size_t i = 0, o = 0; o + 1 < out.size(); i += 3, o += 2) { + out[o] = static_cast(in[i] | ((in[i + 1] & 0x0F) << 8)); + out[o + 1] = static_cast((in[i + 1] >> 4) | (in[i + 2] << 4)); + } + return out; +} + +// Bench-only encoder: any channel count, 8 or 16-bit container, lossless when +// distance is 0 (with uses_original_profile set, which lossless requires). +static bool encodeBench(const std::string& path, const void* img, int channels, int bits, + float distance, int effort, int threads) { + auto enc = JxlEncoderMake(nullptr); + auto runner = JxlThreadParallelRunnerMake(nullptr, threads); + JxlEncoderSetParallelRunner(enc.get(), JxlThreadParallelRunner, runner.get()); + JxlBasicInfo bi; + JxlEncoderInitBasicInfo(&bi); + bi.xsize = W; bi.ysize = H; bi.bits_per_sample = bits; bi.num_color_channels = channels; + bi.uses_original_profile = distance == 0.0f ? JXL_TRUE : JXL_FALSE; + if (JxlEncoderSetBasicInfo(enc.get(), &bi) != JXL_ENC_SUCCESS) return false; + JxlColorEncoding ce = {}; + JxlColorEncodingSetToSRGB(&ce, channels < 3 ? JXL_TRUE : JXL_FALSE); + JxlEncoderSetColorEncoding(enc.get(), &ce); + auto* fs = JxlEncoderFrameSettingsCreate(enc.get(), nullptr); + JxlEncoderFrameSettingsSetOption(fs, JXL_ENC_FRAME_SETTING_EFFORT, effort); + if (distance == 0.0f) { + JxlEncoderSetFrameDistance(fs, 0); + JxlEncoderSetFrameLossless(fs, JXL_TRUE); + } else { + JxlEncoderSetFrameDistance(fs, distance); + } + // 12-bit values in 16-bit words: without this, libjxl reads them as the full + // 0..65535 range and rescales, so "lossless" keeps only the top bits + // (measured: every pixel changed, by up to 3839). + JxlBitDepth depth = {JXL_BIT_DEPTH_FROM_CODESTREAM, 0, 0}; + JxlEncoderSetFrameBitDepth(fs, &depth); + const bool wide = bits > 8; + JxlPixelFormat pf = {static_cast(channels), wide ? JXL_TYPE_UINT16 : JXL_TYPE_UINT8, + JXL_NATIVE_ENDIAN, 0}; + const size_t bytes = static_cast(W) * H * channels * (wide ? 2 : 1); + if (JxlEncoderAddImageFrame(fs, &pf, img, bytes) != JXL_ENC_SUCCESS) return false; + JxlEncoderCloseInput(enc.get()); + std::vector buf(1 << 22); + uint8_t* next = buf.data(); + size_t avail = buf.size(); + JxlEncoderStatus r; + while ((r = JxlEncoderProcessOutput(enc.get(), &next, &avail)) == JXL_ENC_NEED_MORE_OUTPUT) { + size_t off = next - buf.data(); + buf.resize(buf.size() * 2); + next = buf.data() + off; + avail = buf.size() - off; + } + if (r != JXL_ENC_SUCCESS) return false; + std::ofstream(path, std::ios::binary) + .write(reinterpret_cast(buf.data()), next - buf.data()); + return true; +} + +// PSNR (dB) of a decoded 8-bit colour file against the original pixels. +static double psnr8(const std::string& path, const uint8_t* ref) { + auto data = load(path); + auto dec = JxlDecoderMake(nullptr); + JxlDecoderSubscribeEvents(dec.get(), JXL_DEC_FULL_IMAGE); + JxlDecoderSetInput(dec.get(), data.data(), data.size()); + JxlDecoderCloseInput(dec.get()); + JxlPixelFormat pf = {3, JXL_TYPE_UINT8, JXL_NATIVE_ENDIAN, 0}; + std::vector px; + for (;;) { + auto s = JxlDecoderProcessInput(dec.get()); + if (s == JXL_DEC_NEED_IMAGE_OUT_BUFFER) { + size_t n; + JxlDecoderImageOutBufferSize(dec.get(), &pf, &n); + px.resize(n); + JxlDecoderSetImageOutBuffer(dec.get(), &pf, px.data(), n); + } else if (s == JXL_DEC_FULL_IMAGE || s == JXL_DEC_SUCCESS) { + break; + } else { + return -1; + } + } + double se = 0; + for (size_t i = 0; i < px.size(); ++i) { + double d = double(px[i]) - double(ref[i]); + se += d * d; + } + double mse = se / px.size(); + return mse == 0 ? 99.0 : 10 * std::log10(255.0 * 255.0 / mse); +} + +struct Frames { + std::vector rgb, bayer, b12; + std::vector u12; +}; + +// One full save job of the given kind: the conversion it needs, then the encode +// and the file write. Returns the conversion time in ms through `prep_ms`. +static bool runJob(const Frames& f, const std::string& kind, const std::string& path, int effort, + float distance, int threads, double* prep_ms = nullptr) { + auto t0 = Clock::now(); + if (kind == "rgb8") { + if (prep_ms) *prep_ms = 0; + return fgc::JpegXlEncoder::encodeToFile(path, f.rgb.data(), W, H, 3, distance, effort, + threads); + } + if (kind == "bilinear" || kind == "ea") { + cv::Mat b(H, W, CV_8UC1, const_cast(f.bayer.data())), c; + // The camera's BayerRG8 is OpenCV's BayerBG (measured 2026-09-30). + cv::cvtColor(b, c, kind == "ea" ? cv::COLOR_BayerBG2RGB_EA : cv::COLOR_BayerBG2RGB); + if (prep_ms) *prep_ms = msSince(t0); + return fgc::JpegXlEncoder::encodeToFile(path, c.data, W, H, 3, distance, effort, threads); + } + if (kind == "raw8") { + if (prep_ms) *prep_ms = 0; + return encodeBench(path, f.bayer.data(), 1, 8, 0.0f, effort, threads); + } + if (kind == "raw12") { + auto u = unpack12p(f.b12); + if (prep_ms) *prep_ms = msSince(t0); + return encodeBench(path, u.data(), 1, 12, 0.0f, effort, threads); + } + if (kind == "colour12") { // 12-bit raw -> 16-bit de-Bayer -> lossy colour + auto u = unpack12p(f.b12); + cv::Mat b(H, W, CV_16UC1, u.data()), c; + cv::cvtColor(b, c, cv::COLOR_BayerBG2RGB); + if (prep_ms) *prep_ms = msSince(t0); + return encodeBench(path, c.data, 3, 12, distance, effort, threads); + } + return false; +} + +// ---- samples: temperature, clock, CPU load, package power ------------------ + +static double readNum(const std::string& p, double fallback = -1) { + std::ifstream f(p); + double v; + return (f >> v) ? v : fallback; +} + +static double cpuTempC() { + double best = -1; + for (auto& z : fs::directory_iterator("/sys/class/thermal")) { + if (z.path().filename().string().rfind("thermal_zone", 0) != 0) continue; + std::ifstream t(z.path() / "type"); + std::string type; + t >> type; + double v = readNum((z.path() / "temp").string()); + if (type == "x86_pkg_temp") return v / 1000; + best = std::max(best, v / 1000); + } + return best; +} + +static double avgMhz() { + double sum = 0; + int n = 0; + for (int i = 0; i < 64; ++i) { + double v = readNum("/sys/devices/system/cpu/cpu" + std::to_string(i) + + "/cpufreq/scaling_cur_freq"); + if (v < 0) break; + sum += v / 1000; + ++n; + } + return n ? sum / n : -1; +} + +struct CpuTimes { unsigned long long busy = 0, total = 0; }; +static CpuTimes cpuTimes() { + std::ifstream f("/proc/stat"); + std::string cpu; + unsigned long long v[8] = {}; + f >> cpu >> v[0] >> v[1] >> v[2] >> v[3] >> v[4] >> v[5] >> v[6] >> v[7]; + CpuTimes t; + for (auto x : v) t.total += x; + t.busy = t.total - v[3] - v[4]; + return t; +} + +// The package energy counter is root-only. FGC_RAPL names a copy of it that a +// root loop refreshes (see README), so the bench itself need not run as root. +static const std::string kRapl = std::getenv("FGC_RAPL") + ? std::getenv("FGC_RAPL") + : "/sys/class/powercap/intel-rapl:0/energy_uj"; + +struct Sampler { + std::atomic stop{false}; + std::thread th; + std::function extra; // appends ", \"k\": v" fields + std::vector temps, mhz, watts, load; + + void start(double secs_total) { + th = std::thread([this, secs_total] { + auto t0 = Clock::now(); + CpuTimes prev = cpuTimes(); + double e0 = readNum(kRapl); + auto tp = Clock::now(); + while (!stop && msSince(t0) < secs_total * 1000 + 500) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + CpuTimes now = cpuTimes(); + double pct = now.total > prev.total + ? 100.0 * (now.busy - prev.busy) / (now.total - prev.total) + : 0; + prev = now; + double e1 = readNum(kRapl); + double w = (e0 >= 0 && e1 >= e0) ? (e1 - e0) / 1e6 / (msSince(tp) / 1000) : -1; + e0 = e1; + tp = Clock::now(); + double tc = cpuTempC(), mh = avgMhz(); + temps.push_back(tc); mhz.push_back(mh); load.push_back(pct); + if (w >= 0) watts.push_back(w); + std::string x; + if (extra) extra(x); + std::printf("{\"t\": %.0f, \"temp_c\": %.1f, \"mhz\": %.0f, \"cpu_pct\": %.1f, " + "\"watts\": %.2f%s}\n", + msSince(t0) / 1000, tc, mh, pct, w, x.c_str()); + std::fflush(stdout); + } + }); + } + void finish() { stop = true; if (th.joinable()) th.join(); } +}; + +static double mean(const std::vector& v) { + double s = 0; + for (double x : v) s += x; + return v.empty() ? -1 : s / v.size(); +} +static double maxOf(const std::vector& v) { + return v.empty() ? -1 : *std::max_element(v.begin(), v.end()); +} +static double minOf(const std::vector& v) { + return v.empty() ? -1 : *std::min_element(v.begin(), v.end()); +} +static double pct(std::vector v, double p) { + if (v.empty()) return -1; + std::sort(v.begin(), v.end()); + return v[std::min(v.size() - 1, static_cast(p * (v.size() - 1) + 0.5))]; +} + +static double cpuSecondsSelf() { + rusage r{}; + getrusage(RUSAGE_SELF, &r); + return r.ru_utime.tv_sec + r.ru_utime.tv_usec / 1e6 + r.ru_stime.tv_sec + r.ru_stime.tv_usec / 1e6; +} + +// ---- modes ------------------------------------------------------------------- + +static void matrix(const Frames& f, const std::string& out, bool only12) { + auto timeJob = [&](const std::string& kind, int e, float d, int t, int reps, + const std::string& path) { + std::vector tot, prep; + for (int i = 0; i < reps; ++i) { + double p = 0; + auto t0 = Clock::now(); + runJob(f, kind, path, e, d, t, &p); + tot.push_back(msSince(t0)); + prep.push_back(p); + } + std::sort(tot.begin(), tot.end()); + std::sort(prep.begin(), prep.end()); + return std::make_pair(tot[tot.size() / 2], prep[prep.size() / 2]); + }; + auto emit = [&](const std::string& kind, int e, float d, int t, double ms, double prep, + long long bytes, double ps = -1) { + std::printf("{\"kind\": \"%s\", \"effort\": %d, \"distance\": %.1f, \"threads\": %d, " + "\"ms\": %.1f, \"prep_ms\": %.1f, \"bytes\": %lld", + kind.c_str(), e, d, t, ms, prep, bytes); + if (ps >= 0) std::printf(", \"psnr_db\": %.2f", ps); + std::printf("}\n"); + std::fflush(stdout); + }; + + // Effort x threads at today's distance, for every colour input. + if (!only12) + for (const std::string kind : {"rgb8", "bilinear", "ea"}) + for (int t : {1, 2, 3, 4}) + for (int e : {1, 2, 3, 4, 5}) { + const std::string p = out + "/m_" + kind + ".jxl"; + auto r = timeJob(kind, e, 0.8f, t, e == 5 && t < 3 ? 1 : 3, p); + emit(kind, e, 0.8f, t, r.first, r.second, fsize(p)); + } + // Distance sweep at effort 4, 4 threads, with the loss it costs. + if (!only12) + for (const std::string kind : {"rgb8", "ea"}) { + cv::Mat ref; + if (kind == "ea") { + cv::Mat b(H, W, CV_8UC1, const_cast(f.bayer.data())); + cv::cvtColor(b, ref, cv::COLOR_BayerBG2RGB_EA); + } + const uint8_t* refp = kind == "rgb8" ? f.rgb.data() : ref.data; + for (float d : {0.5f, 0.8f, 1.0f, 1.5f, 2.0f, 3.0f}) { + const std::string p = out + "/d_" + kind + ".jxl"; + auto r = timeJob(kind, 4, d, 4, 3, p); + emit(kind, 4, d, 4, r.first, r.second, fsize(p), psnr8(p, refp)); + } + } + // Lossless raw mosaic, 8 and 12 bit. + for (const std::string kind : {"raw8", "raw12"}) { + if (only12 && kind == "raw8") continue; + for (int t : {1, 2, 3, 4}) + for (int e : {1, 2, 3}) { + const std::string p = out + "/m_" + kind + ".jxl"; + auto r = timeJob(kind, e, 0.0f, t, 3, p); + emit(kind, e, 0.0f, t, r.first, r.second, fsize(p)); + } + } + // 12-bit colour, for comparison with the 8-bit colour paths. + for (int e : {1, 4}) { + const std::string p = out + "/m_colour12.jxl"; + auto r = timeJob("colour12", e, 0.8f, 4, 3, p); + emit("colour12", e, 0.8f, 4, r.first, r.second, fsize(p)); + } +} + +static int pipeline(const Frames& f, const std::string& out, const std::string& kind, int workers, + int threads, double secs, int period_ms, int effort, float distance) { + std::mutex mu; + std::condition_variable cv; + std::deque queue; // enqueue times + std::atomic done{0}; + std::atomic producing{true}, quit{false}; + std::vector lat; // enqueue -> written, ms + size_t max_q = 0; + const bool paced = period_ms > 0; + + Sampler s; + s.extra = [&](std::string& x) { + size_t q; + { std::lock_guard l(mu); q = queue.size(); } + x = ", \"done\": " + std::to_string(done.load()) + ", \"queue\": " + std::to_string(q); + }; + const double cpu0 = cpuSecondsSelf(); + auto t0 = Clock::now(); + s.start(secs); + + std::vector ws; + for (int w = 0; w < workers; ++w) + ws.emplace_back([&, w] { + const std::string path = out + "/p_" + kind + "_" + std::to_string(w) + ".jxl"; + for (;;) { + Clock::time_point enq; + if (paced) { + std::unique_lock l(mu); + cv.wait(l, [&] { return !queue.empty() || quit; }); + if (quit) return; + enq = queue.front(); + queue.pop_front(); + } else { + if (msSince(t0) >= secs * 1000) return; + enq = Clock::now(); + } + runJob(f, kind, path, effort, distance, threads); + ++done; + std::lock_guard l(mu); + lat.push_back(msSince(enq)); + } + }); + + if (paced) { + auto next = Clock::now(); + while (msSince(t0) < secs * 1000) { + { + std::lock_guard l(mu); + queue.push_back(Clock::now()); + max_q = std::max(max_q, queue.size()); + } + cv.notify_one(); + next += std::chrono::milliseconds(period_ms); + std::this_thread::sleep_until(next); + } + std::lock_guard l(mu); + quit = true; + } + cv.notify_all(); + size_t backlog; + { std::lock_guard l(mu); backlog = queue.size(); } + for (auto& w : ws) w.join(); + const double wall = msSince(t0) / 1000; + s.finish(); + const double cpu = cpuSecondsSelf() - cpu0; + + std::lock_guard l(mu); + std::printf("{\"summary\": \"pipeline\", \"kind\": \"%s\", \"workers\": %d, \"threads\": %d, " + "\"effort\": %d, \"distance\": %.1f, \"period_ms\": %d, \"secs\": %.1f, " + "\"frames\": %lld, \"frames_per_s\": %.3f, \"latency_ms_p50\": %.0f, " + "\"latency_ms_p95\": %.0f, \"latency_ms_max\": %.0f, \"max_queue\": %zu, " + "\"backlog_at_end\": %zu, \"cpu_s_per_frame\": %.3f, \"temp_c_mean\": %.1f, " + "\"temp_c_max\": %.1f, \"mhz_mean\": %.0f, \"mhz_min\": %.0f, \"cpu_pct_mean\": %.1f, " + "\"watts_mean\": %.2f}\n", + kind.c_str(), workers, threads, effort, distance, period_ms, wall, + done.load(), done.load() / wall, pct(lat, 0.5), pct(lat, 0.95), maxOf(lat), + max_q, backlog, done ? cpu / done.load() : -1, mean(s.temps), maxOf(s.temps), + mean(s.mhz), minOf(s.mhz), mean(s.load), mean(s.watts)); + return 0; +} + +int main(int argc, char** argv) { + if (argc < 4) { + std::fprintf(stderr, "usage: encode_bench matrix|pipeline|idle ...\n"); + return 2; + } + const std::string mode = argv[1], dir = argv[2], out = argv[3]; + fs::create_directories(out); + + if (mode == "idle") { + Sampler s; + const double secs = argc > 4 ? std::atof(argv[4]) : 60; + s.start(secs); + std::this_thread::sleep_for(std::chrono::milliseconds(static_cast(secs * 1000))); + s.finish(); + std::printf("{\"summary\": \"idle\", \"secs\": %.0f, \"temp_c_mean\": %.1f, " + "\"temp_c_max\": %.1f, \"mhz_mean\": %.0f, \"cpu_pct_mean\": %.1f, " + "\"watts_mean\": %.2f}\n", + secs, mean(s.temps), maxOf(s.temps), mean(s.mhz), mean(s.load), mean(s.watts)); + return 0; + } + + { + std::ifstream j(dir + "/BayerRG8_0.json"); + std::string t{std::istreambuf_iterator(j), {}}; + W = jsonInt(t, "width"); + H = jsonInt(t, "height"); + } + Frames f; + f.rgb = load(dir + "/RGB8_0.bin"); + f.bayer = load(dir + "/BayerRG8_0.bin"); + f.b12 = load(dir + "/BayerRG12p_0.bin"); + if (!W || f.rgb.size() != size_t(W) * H * 3 || f.bayer.size() != size_t(W) * H || + f.b12.size() != size_t(W) * H * 3 / 2) { + std::fprintf(stderr, "frames missing or wrong size\n"); + return 1; + } + + if (mode == "matrix" || mode == "matrix12") { + matrix(f, out, mode == "matrix12"); + return 0; + } + if (mode == "pipeline" && argc >= 9) { + return pipeline(f, out, argv[4], std::atoi(argv[5]), std::atoi(argv[6]), + std::atof(argv[7]), std::atoi(argv[8]), + argc > 9 ? std::atoi(argv[9]) : 4, + argc > 10 ? static_cast(std::atof(argv[10])) : 0.8f); + } + std::fprintf(stderr, "bad arguments\n"); + return 2; +} diff --git a/tools/camera-bench/lossless_check.cpp b/tools/camera-bench/lossless_check.cpp new file mode 100644 index 0000000..8f02592 --- /dev/null +++ b/tools/camera-bench/lossless_check.cpp @@ -0,0 +1,39 @@ +// Does the production encoder's distance-0 path succeed, and is the file lossless? +#include "fgc/JpegXlEncoder.h" +#include +#include +#include +#include +#include +#include +#include +#include +static std::vector load(const char* p){std::ifstream f(p,std::ios::binary);return {std::istreambuf_iterator(f),{}};} +static bool decode(const char* path, int ch, std::vector& px, bool& lossy_flag){ + auto data=load(path); if(data.empty()) return false; + auto dec=JxlDecoderMake(nullptr); + JxlDecoderSubscribeEvents(dec.get(), JXL_DEC_BASIC_INFO|JXL_DEC_FULL_IMAGE); + JxlDecoderSetInput(dec.get(), data.data(), data.size()); JxlDecoderCloseInput(dec.get()); + JxlPixelFormat pf={(uint32_t)ch,JXL_TYPE_UINT8,JXL_NATIVE_ENDIAN,0}; + for(;;){ auto s=JxlDecoderProcessInput(dec.get()); + if(s==JXL_DEC_BASIC_INFO){JxlBasicInfo bi; JxlDecoderGetBasicInfo(dec.get(),&bi); lossy_flag=!bi.uses_original_profile;} + else if(s==JXL_DEC_NEED_IMAGE_OUT_BUFFER){size_t n; JxlDecoderImageOutBufferSize(dec.get(),&pf,&n); px.resize(n); JxlDecoderSetImageOutBuffer(dec.get(),&pf,px.data(),n);} + else if(s==JXL_DEC_FULL_IMAGE||s==JXL_DEC_SUCCESS) return true; + else return false; } +} +int main(){ + std::ifstream j("frames/BayerRG8_0.json"); std::string t{std::istreambuf_iterator(j),{}}; + auto num=[&](const char* k){auto a=t.find(k); return a==std::string::npos?0:std::atoi(t.c_str()+t.find(':',a)+1);}; + const int W=num("\"width\""),H=num("\"height\""); + struct T{const char* in; int ch;} ts[]={{"frames/BayerRG8_0.bin",1},{"frames/RGB8_0.bin",3}}; + for(auto t:ts){ + auto img=load(t.in); std::remove("check.jxl"); + bool ok=fgc::JpegXlEncoder::encodeToFile("check.jxl",img.data(),W,H,t.ch,0.0f,1,3); + std::ifstream f("check.jxl",std::ios::binary|std::ios::ate); long long sz=f?(long long)f.tellg():-1; + std::vector px; bool xyb=false; bool dok=sz>0&&decode("check.jxl",t.ch,px,xyb); + long long diff=0; int maxd=0; + if(dok) for(size_t i=0;imaxd)maxd=d;}} + std::printf("{\"input\":\"%s\",\"encode_returned\":%s,\"file_bytes\":%lld,\"decoded\":%s,\"xyb_lossy_colour\":%s,\"pixels_differing\":%lld,\"max_diff\":%d}\n", + t.in, ok?"true":"false", sz, dok?"true":"false", xyb?"true":"false", diff, maxd); + } +} diff --git a/tools/camera-bench/make_figures.py b/tools/camera-bench/make_figures.py new file mode 100644 index 0000000..3bd7a3b --- /dev/null +++ b/tools/camera-bench/make_figures.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""make_figures.py - image figures for docs/raw12-pipeline-measurements.md. + + ~/.venvs/fgc-plots/bin/python make_figures.py [detail_x,detail_y shadow_x,shadow_y] + + holds the example .jxl files `raw12_bench examples` wrote (its jxl/ +folder, copied off the tower). Decodes them with imagecodecs and OpenCV, so it +runs on the laptop venv. The reference is c12_ea_lossless.jxl (the 12-bit +frame, exact); its >>4 is the 8-bit view of the same moment. Writes: + thumbs/.jpg downscaled full frame (1/4) + detail/.png 480x320 100% crop of a detailed region (8-bit view) + diff/.png |decoded - reference| in that crop, x8 (12-bit examples only) + shadow/.png 480x320 100% crop of a dark region, pushed +2 stops + shadow_8_vs_12.png that shadow, 8-bit vs 12-bit of one frame, +2 stops + crops.jpg where the crops are + levels.json crop positions, distinct levels in the shadow +Crops are chosen automatically unless given: detail = most structure after +blurring away pixel noise, without clipped sky; shadow = darkest textured region. +""" +import glob +import json +import os +import sys + +import cv2 +import imagecodecs +import numpy as np + +src, out = sys.argv[1], sys.argv[2] +for d in ("thumbs", "detail", "diff", "shadow"): + os.makedirs(os.path.join(out, d), exist_ok=True) +CW, CH = 480, 320 + + +def load(path): + """Decoded example as HxWx3 RGB on the 12-bit scale (float).""" + a = imagecodecs.jpegxl_decode(open(path, "rb").read()).astype(np.float64) + name = os.path.basename(path) + eight = name.startswith(("rgb8", "c8")) + if a.max() > 4095.5: + a = a / 16.0 + if eight: + a = a * 4095.0 / 255.0 + if a.ndim == 2: + m = np.round(a).astype(np.uint16) + if "planes" in name: + h, w = m.shape + hh, hw = h // 2, w // 2 + mo = np.empty_like(m) + mo[0::2, 0::2], mo[0::2, 1::2] = m[:hh, :hw], m[:hh, hw:] + mo[1::2, 0::2], mo[1::2, 1::2] = m[hh:, :hw], m[hh:, hw:] + m = mo + a = cv2.cvtColor(m, cv2.COLOR_BayerBG2RGB_EA).astype(np.float64) + return a, eight + + +def view8(rgb12, gain=1.0): + return cv2.cvtColor(np.clip(rgb12 * gain / 4095.0 * 255.0, 0, 255).astype(np.uint8), + cv2.COLOR_RGB2BGR) + + +ref, _ = load(os.path.join(src, "c12_ea_lossless.jxl")) +h, w = ref.shape[:2] +lum = 0.2126 * ref[..., 0] + 0.7152 * ref[..., 1] + 0.0722 * ref[..., 2] + +if len(sys.argv) > 4: + dx, dy = map(int, sys.argv[3].split(",")) + sx, sy = map(int, sys.argv[4].split(",")) +else: + smooth = cv2.GaussianBlur(lum.astype(np.float32), (0, 0), 2.0) + structure = np.abs(cv2.Laplacian(smooth, cv2.CV_32F)) + best_d, best_s = (-1, None), (1e9, None) + for y in range(CH, h - 2 * CH, CH // 2): + for x in range(CW, w - 2 * CW, CW // 2): + win = lum[y:y + CH, x:x + CW] + st = float(structure[y:y + CH, x:x + CW].mean()) + if (win >= 4000).mean() < 0.01 and st > best_d[0]: + best_d = (st, (x, y)) + if st > 0.5 * float(structure.mean()) and win.mean() < best_s[0]: + best_s = (float(win.mean()), (x, y)) + (dx, dy), (sx, sy) = best_d[1], best_s[1] + +ref_detail = ref[dy:dy + CH, dx:dx + CW] +for path in sorted(glob.glob(os.path.join(src, "*.jxl"))): + name = os.path.splitext(os.path.basename(path))[0] + img, eight = load(path) + small = cv2.resize(img, (w // 4, h // 4), interpolation=cv2.INTER_AREA) + cv2.imwrite(os.path.join(out, "thumbs", name + ".jpg"), view8(small), + [cv2.IMWRITE_JPEG_QUALITY, 88]) + cv2.imwrite(os.path.join(out, "detail", name + ".png"), view8(img[dy:dy + CH, dx:dx + CW])) + cv2.imwrite(os.path.join(out, "shadow", name + ".png"), view8(img[sy:sy + CH, sx:sx + CW], 4.0)) + if not eight: # 8-bit examples are other captures: no pixel-by-pixel difference + diff = np.abs(img[dy:dy + CH, dx:dx + CW] - ref_detail).mean(axis=2) + cv2.imwrite(os.path.join(out, "diff", name + ".png"), + np.clip(diff * 8 / 16, 0, 255).astype(np.uint8)) + +# 8-bit vs 12-bit of the same frame: the 8-bit view is the 12-bit >> 4. +s12 = ref[sy:sy + CH, sx:sx + CW] +s8 = np.floor(s12 / 16.0) * 16.0 +gap = np.full((CH, 8, 3), 255, np.uint8) +cv2.imwrite(os.path.join(out, "shadow_8_vs_12.png"), + np.hstack([view8(s8, 4.0), gap, view8(s12, 4.0)])) +# Where the crops are. +th = view8(cv2.resize(ref, (w // 4, h // 4), interpolation=cv2.INTER_AREA)) +for (x, y), c in (((dx, dy), (0, 0, 255)), ((sx, sy), (255, 160, 0))): + cv2.rectangle(th, (x // 4, y // 4), ((x + CW) // 4, (y + CH) // 4), c, 2) +cv2.imwrite(os.path.join(out, "crops.jpg"), th, [cv2.IMWRITE_JPEG_QUALITY, 88]) + +g12 = np.round(s12[..., 1]).astype(int) +levels = dict( + detail_crop=dict(x=dx, y=dy, w=CW, h=CH), shadow_crop=dict(x=sx, y=sy, w=CW, h=CH), + shadow_mean_12bit=round(float(s12.mean()), 1), + shadow_distinct_levels_8bit=int(len(np.unique(g12 >> 4))), + shadow_distinct_levels_12bit=int(len(np.unique(g12))), + shadow_green_range_12bit=[int(g12.min()), int(g12.max())], +) +json.dump(levels, open(os.path.join(out, "levels.json"), "w"), indent=1) +print(json.dumps(levels)) diff --git a/tools/camera-bench/make_synthetic.py b/tools/camera-bench/make_synthetic.py new file mode 100644 index 0000000..9fabe57 --- /dev/null +++ b/tools/camera-bench/make_synthetic.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""make_synthetic.py - a sharp, detailed stand-in frame for encode_bench. + +The tower cameras have so far only seen blurred, dim indoor scenes, which +compress far faster and smaller than a sharp outdoor view. This writes a +"dead leaves" texture - overlapping discs with power-law radii, the texture +chart of ISO 19567-2 - with mild sensor noise, in the three formats +camera_bench.py `save` produces (RGB8, BayerRG8, BayerRG12p), so the encode +numbers can be bounded from above. + + python3 make_synthetic.py [width] [height] + +Needs numpy and OpenCV (the tower's system python3 has both). +""" +import json +import os +import sys + +import cv2 +import numpy as np + +out = sys.argv[1] +w = int(sys.argv[2]) if len(sys.argv) > 2 else 5328 +h = int(sys.argv[3]) if len(sys.argv) > 3 else 3040 +rng = np.random.default_rng(20261001) + +img = np.full((h, w, 3), 128, np.uint8) +rmin, rmax = 2.0, 600.0 +# Enough discs to cover the frame several times over: with this radius +# distribution one disc covers ~143 px on average, the frame 16.2 M px. +n = 600000 +# Radii with density ~ r^-3 (scale-invariant, as in natural images). +u = rng.random(n) +radii = 1.0 / np.sqrt(u * (1 / rmin**2 - 1 / rmax**2) + 1 / rmax**2) +for r in radii: + c = (int(rng.integers(0, w)), int(rng.integers(0, h))) + col = tuple(int(x) for x in rng.integers(10, 246, 3)) + cv2.circle(img, c, int(r), col, -1, lineType=cv2.LINE_AA) + +# Mild sensor noise, in 12-bit units, then the three delivered formats. +lin = img.astype(np.float32) * 16 + rng.normal(0, 24, img.shape).astype(np.float32) +lin12 = np.clip(lin, 0, 4095).astype(np.uint16) +rgb8 = (lin12 >> 4).astype(np.uint8) + +# The camera's BayerRG: R at (even row, even col), B at (odd, odd). +m12 = np.empty((h, w), np.uint16) +m12[0::2, 0::2] = lin12[0::2, 0::2, 0] +m12[0::2, 1::2] = lin12[0::2, 1::2, 1] +m12[1::2, 0::2] = lin12[1::2, 0::2, 1] +m12[1::2, 1::2] = lin12[1::2, 1::2, 2] +m8 = (m12 >> 4).astype(np.uint8) + +# GenICam 12p: two pixels in three bytes, little-endian. +flat = m12.reshape(-1) +p0, p1 = flat[0::2].astype(np.uint32), flat[1::2].astype(np.uint32) +packed = np.empty((p0.size, 3), np.uint8) +packed[:, 0] = p0 & 0xFF +packed[:, 1] = ((p0 >> 8) & 0x0F) | ((p1 & 0x0F) << 4) +packed[:, 2] = (p1 >> 4) & 0xFF + +os.makedirs(out, exist_ok=True) +for fmt, data in (("RGB8", rgb8), ("BayerRG8", m8), ("BayerRG12p", packed)): + data.tofile(os.path.join(out, f"{fmt}_0.bin")) + with open(os.path.join(out, f"{fmt}_0.json"), "w") as f: + json.dump(dict(fmt=fmt, width=w, height=h, synthetic="dead leaves + noise"), f) +cv2.imwrite(os.path.join(out, "preview.jpg"), + cv2.cvtColor(cv2.resize(rgb8, (w // 4, h // 4), interpolation=cv2.INTER_AREA), + cv2.COLOR_RGB2BGR), [cv2.IMWRITE_JPEG_QUALITY, 85]) +print(json.dumps(dict(out=out, width=w, height=h, mean=float(rgb8.mean())))) diff --git a/tools/camera-bench/plot_common.py b/tools/camera-bench/plot_common.py new file mode 100644 index 0000000..5976556 --- /dev/null +++ b/tools/camera-bench/plot_common.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""plot_common.py - the shared-frame comparison between towers. + + ~/.venvs/fgc-plots/bin/python plot_common.py