#pragma once #include #include #include #include namespace fgc { // Watching the tower computer's CPU heat and load, and pacing the scan to it. // // In summer the sun and the gimbal heat the LattePanda until its CPU slows itself // down (throttles). Saving a 12-bit picture costs ~1.4 CPU-seconds, so a slowed // CPU may stop keeping up. The scan already holds while saving is behind // (CaptureScheduler::setSaveGate), so nothing piles up in memory - but a computer // held at its limit stays hot, and starves the modem and the control loop with it. // So the guard backs off before that: // // Normal - full pace. // Slowed - pictures further apart: slow_factor times the interval, or more // when saving needs it - 1.5 x the measured save time per worker, so // the saver keeps some headroom instead of running flat out. Entered // when the CPU is at slow_c or more, when it is throttling, or when // saving is the bottleneck: the scan spent held_share_pct of the last // minute waiting for it, or held_slow_s in one go. Back to Normal once // all of that has been clear for calm_s. // Paused - no new pictures: the gimbal parks where it is and the camera stream // stops, so the CPU all but idles. Entered when the CPU has stayed at // pause_c or more for pause_after_s, or saving is still the // bottleneck held_pause_s after slowing down (or kept the scan waiting // held_pause_s in one go). Resumes (to Slowed) after min_pause_s, once // the CPU is at resume_c or below and nothing is waiting to be saved. // // Why a share and not only one long wait: a CPU that saves a picture in 4.6 s // against a 2 s interval never makes the scan wait long - the save gate lets it // on after 2-3 s each time - yet it runs pegged for as long as the scan does // (measured on the desk with a CPU quota, docs/thermal-throttling.md). // // It never stops the program and never changes the operating mode: auto stays // auto, so the scan picks up by itself once the computer has cooled. Picture // quality is never touched - only the pace. // // Pure: fed one reading at a time with its own clock, no I/O, so every rule here // is unit-tested (test_thermalguard.cpp). Application samples the host every // sample_ms and logs what update() returns. struct ThermalParams { bool enabled = true; std::string sysfs_root = "/"; // fake /sys + /proc tree for a simulation int sample_ms = 2000; double slow_c = 85.0; double pause_c = 92.0; double resume_c = 75.0; double hysteresis_c = 5.0; // Slowed -> Normal needs slow_c - this int pause_after_s = 20; int calm_s = 120; int min_pause_s = 300; int held_slow_s = 30; int held_pause_s = 120; double held_share_pct = 25.0; // of the last minute spent waiting for saving double slow_factor = 2.0; // Throttling is also inferred from the clock (see ThrottleWatch): below this // share of the chip's top clock while at least this busy. double clock_low_pct = 50.0; double busy_high_pct = 50.0; }; // One sample of the host. Each part has its own ok flag: a reading that is // missing is left out of the rules, never taken as zero. struct HostReading { long long now_ms = 0; bool temp_ok = false; double cpu_c = 0.0; std::string zone; // hottest thermal zone's name bool clock_ok = false; double mhz = 0.0; // mean over the cores double hw_max_mhz = 0.0; // the chip's top clock bool count_ok = false; long long throttle_count = 0; // kernel thermal_throttle counters, summed double busy_pct = -1.0; // since the last sample; -1 = unknown bool held_for_save = false; // the scan is waiting for saving long long held_total_ms = 0; // all the time it has waited so far (only rises) int save_pending = 0; // frames waiting or being saved // For the slowed pace: what the scan is configured to, and what saving takes. double base_interval_ms = 0.0; // 0 = unknown double save_ms_mean = 0.0; // last minute; 0 = unknown int save_workers = 1; }; enum class GuardLevel { Info, Warn }; using GuardLog = std::pair; // "93 °C", "1.2 GHz" - for log lines and the UI. std::string celsiusText(double c); std::string ghzText(double mhz); // Is the CPU slowing itself down? Two signs: // - the kernel's thermal_throttle counters rose since the last sample (heat); // - the clock is below clock_low_pct of the chip's top clock while the CPU is // at least busy_high_pct busy. This catches the power-limit slow-downs the // counters never count, and a clock capped for a simulation (cap-cpu.sh). // Logs one line when it starts, at most one summary a minute while it lasts, // and one when it has been clear for a minute. class ThrottleWatch { public: explicit ThrottleWatch(const ThermalParams& p = {}) : p_(p) {} void setParams(const ThermalParams& p) { p_ = p; } void update(const HostReading& r, std::vector& log); bool throttling() const { return throttling_; } // Samples that showed throttling in the last minute, and counter rises since start. int samplesLastMinute() const { return static_cast(recent_.size()); } long long eventsTotal() const { return events_total_; } long long eventsLastMinute() const; // "counters" / "clock" / "counters and clock" - what said so last time. const std::string& cause() const { return cause_; } private: ThermalParams p_; bool have_count_ = false; long long last_count_ = 0; long long events_total_ = 0; // (time, counter rise) of each sample that showed throttling, last 60 s. std::deque> recent_; bool throttling_ = false; long long last_seen_ms_ = 0; // last sample that showed throttling long long last_summary_ms_ = 0; int low_clock_run_ = 0; // consecutive samples with a low, busy clock std::string cause_; }; enum class GuardState { Normal, Slowed, Paused }; const char* guardStateName(GuardState s); // "normal" / "slowed" / "paused" class ThermalGuard { public: explicit ThermalGuard(const ThermalParams& p = {}); // One sample in, the log lines it produced out (throttling start/summary/end // and state changes). Call every sample_ms. std::vector update(const HostReading& r); // What the scan should do now. GuardState state() const { return state_; } // 1 when Normal; when Slowed or Paused, slow_factor or what saving needs // (see the top of this file), at most 10. double intervalScale() const { return state_ == GuardState::Normal ? 1.0 : scale_; } bool pauseCapture() const { return state_ == GuardState::Paused; } // Why it is not Normal, in plain words ("" when Normal). const std::string& reason() const { return reason_; } // `thermal resume`: back to Normal now, every timer restarted. The rules then // apply afresh - a CPU still at pause_c pauses again after pause_after_s. std::vector forceResume(long long now_ms); // `thermal off|on`: off = always Normal; the readings and the throttling log go on. std::vector setEnabled(bool on, long long now_ms); bool enabled() const { return enabled_; } const ThermalParams& params() const { return p_; } const HostReading& last() const { return last_; } bool haveReading() const { return have_last_; } const ThrottleWatch& throttle() const { return watch_; } long long heldForSaveMs() const; // how long the scan has been waiting long long pausedForMs() const; // Share of the last minute the scan spent waiting for saving, in %; -1 until // there is half a minute of history to judge from. double heldSharePct() const; private: void enter(GuardState s, const std::string& why, std::vector& log); void updateScale(const HostReading& r); std::string paceText() const; // "one picture every 4.0 s instead of 2.0" ThermalParams p_; ThrottleWatch watch_; bool enabled_ = true; GuardState state_ = GuardState::Normal; std::string reason_; HostReading last_; bool have_last_ = false; long long hot_since_ms_ = -1; // CPU at pause_c or more since (-1 = not) long long held_since_ms_ = -1; // the scan waiting for saving since (-1 = not) long long calm_since_ms_ = -1; // Slowed, with every reason clear, since (-1 = not) long long paused_at_ms_ = 0; long long slowed_at_ms_ = 0; long long overload_since_ms_ = -1; // Slowed, and saving still the bottleneck, since // (time, held_total_ms) per sample over the last minute, for heldSharePct(). std::deque> held_hist_; double scale_ = 1.0; double last_save_ms_ = 0.0; // the last save time seen (0 = none yet) double need_scale_ = 0.0; // the scale saving needs; <= 1: the normal pace fits }; } // namespace fgc