197 lines
9.2 KiB
C++
197 lines
9.2 KiB
C++
#pragma once
|
|
|
|
#include <deque>
|
|
#include <string>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
namespace fgc {
|
|
|
|
// Watching the tower computer's CPU heat and load, and pacing the scan to it.
|
|
//
|
|
// In summer the sun and the gimbal heat the LattePanda until its CPU slows itself
|
|
// down (throttles). Saving a 12-bit picture costs ~1.4 CPU-seconds, so a slowed
|
|
// CPU may stop keeping up. The scan already holds while saving is behind
|
|
// (CaptureScheduler::setSaveGate), so nothing piles up in memory - but a computer
|
|
// held at its limit stays hot, and starves the modem and the control loop with it.
|
|
// So the guard backs off before that:
|
|
//
|
|
// Normal - full pace.
|
|
// Slowed - pictures further apart: slow_factor times the interval, or more
|
|
// when saving needs it - 1.5 x the measured save time per worker, so
|
|
// the saver keeps some headroom instead of running flat out. Entered
|
|
// when the CPU is at slow_c or more, when it is throttling, or when
|
|
// saving is the bottleneck: the scan spent held_share_pct of the last
|
|
// minute waiting for it, or held_slow_s in one go. Back to Normal once
|
|
// all of that has been clear for calm_s.
|
|
// Paused - no new pictures: the gimbal parks where it is and the camera stream
|
|
// stops, so the CPU all but idles. Entered when the CPU has stayed at
|
|
// pause_c or more for pause_after_s, or saving is still the
|
|
// bottleneck held_pause_s after slowing down (or kept the scan waiting
|
|
// held_pause_s in one go). Resumes (to Slowed) after min_pause_s, once
|
|
// the CPU is at resume_c or below and nothing is waiting to be saved.
|
|
//
|
|
// Why a share and not only one long wait: a CPU that saves a picture in 4.6 s
|
|
// against a 2 s interval never makes the scan wait long - the save gate lets it
|
|
// on after 2-3 s each time - yet it runs pegged for as long as the scan does
|
|
// (measured on the desk with a CPU quota, docs/thermal-throttling.md).
|
|
//
|
|
// It never stops the program and never changes the operating mode: auto stays
|
|
// auto, so the scan picks up by itself once the computer has cooled. Picture
|
|
// quality is never touched - only the pace.
|
|
//
|
|
// Pure: fed one reading at a time with its own clock, no I/O, so every rule here
|
|
// is unit-tested (test_thermalguard.cpp). Application samples the host every
|
|
// sample_ms and logs what update() returns.
|
|
|
|
struct ThermalParams {
|
|
bool enabled = true;
|
|
std::string sysfs_root = "/"; // fake /sys + /proc tree for a simulation
|
|
int sample_ms = 2000;
|
|
double slow_c = 85.0;
|
|
double pause_c = 92.0;
|
|
double resume_c = 75.0;
|
|
double hysteresis_c = 5.0; // Slowed -> Normal needs slow_c - this
|
|
int pause_after_s = 20;
|
|
int calm_s = 120;
|
|
int min_pause_s = 300;
|
|
int held_slow_s = 30;
|
|
int held_pause_s = 120;
|
|
double held_share_pct = 25.0; // of the last minute spent waiting for saving
|
|
double slow_factor = 2.0;
|
|
// Throttling is also inferred from the clock (see ThrottleWatch): below this
|
|
// share of the chip's top clock while at least this busy.
|
|
double clock_low_pct = 50.0;
|
|
double busy_high_pct = 50.0;
|
|
};
|
|
|
|
// One sample of the host. Each part has its own ok flag: a reading that is
|
|
// missing is left out of the rules, never taken as zero.
|
|
struct HostReading {
|
|
long long now_ms = 0;
|
|
bool temp_ok = false;
|
|
double cpu_c = 0.0;
|
|
std::string zone; // hottest thermal zone's name
|
|
bool clock_ok = false;
|
|
double mhz = 0.0; // mean over the cores
|
|
double hw_max_mhz = 0.0; // the chip's top clock
|
|
bool count_ok = false;
|
|
long long throttle_count = 0; // kernel thermal_throttle counters, summed
|
|
double busy_pct = -1.0; // since the last sample; -1 = unknown
|
|
bool held_for_save = false; // the scan is waiting for saving
|
|
long long held_total_ms = 0; // all the time it has waited so far (only rises)
|
|
int save_pending = 0; // frames waiting or being saved
|
|
// For the slowed pace: what the scan is configured to, and what saving takes.
|
|
double base_interval_ms = 0.0; // 0 = unknown
|
|
double save_ms_mean = 0.0; // last minute; 0 = unknown
|
|
int save_workers = 1;
|
|
};
|
|
|
|
enum class GuardLevel { Info, Warn };
|
|
using GuardLog = std::pair<GuardLevel, std::string>;
|
|
|
|
// "93 °C", "1.2 GHz" - for log lines and the UI.
|
|
std::string celsiusText(double c);
|
|
std::string ghzText(double mhz);
|
|
|
|
// Is the CPU slowing itself down? Two signs:
|
|
// - the kernel's thermal_throttle counters rose since the last sample (heat);
|
|
// - the clock is below clock_low_pct of the chip's top clock while the CPU is
|
|
// at least busy_high_pct busy. This catches the power-limit slow-downs the
|
|
// counters never count, and a clock capped for a simulation (cap-cpu.sh).
|
|
// Logs one line when it starts, at most one summary a minute while it lasts,
|
|
// and one when it has been clear for a minute.
|
|
class ThrottleWatch {
|
|
public:
|
|
explicit ThrottleWatch(const ThermalParams& p = {}) : p_(p) {}
|
|
void setParams(const ThermalParams& p) { p_ = p; }
|
|
|
|
void update(const HostReading& r, std::vector<GuardLog>& log);
|
|
|
|
bool throttling() const { return throttling_; }
|
|
// Samples that showed throttling in the last minute, and counter rises since start.
|
|
int samplesLastMinute() const { return static_cast<int>(recent_.size()); }
|
|
long long eventsTotal() const { return events_total_; }
|
|
long long eventsLastMinute() const;
|
|
// "counters" / "clock" / "counters and clock" - what said so last time.
|
|
const std::string& cause() const { return cause_; }
|
|
|
|
private:
|
|
ThermalParams p_;
|
|
bool have_count_ = false;
|
|
long long last_count_ = 0;
|
|
long long events_total_ = 0;
|
|
// (time, counter rise) of each sample that showed throttling, last 60 s.
|
|
std::deque<std::pair<long long, long long>> recent_;
|
|
bool throttling_ = false;
|
|
long long last_seen_ms_ = 0; // last sample that showed throttling
|
|
long long last_summary_ms_ = 0;
|
|
int low_clock_run_ = 0; // consecutive samples with a low, busy clock
|
|
std::string cause_;
|
|
};
|
|
|
|
enum class GuardState { Normal, Slowed, Paused };
|
|
const char* guardStateName(GuardState s); // "normal" / "slowed" / "paused"
|
|
|
|
class ThermalGuard {
|
|
public:
|
|
explicit ThermalGuard(const ThermalParams& p = {});
|
|
|
|
// One sample in, the log lines it produced out (throttling start/summary/end
|
|
// and state changes). Call every sample_ms.
|
|
std::vector<GuardLog> update(const HostReading& r);
|
|
|
|
// What the scan should do now.
|
|
GuardState state() const { return state_; }
|
|
// 1 when Normal; when Slowed or Paused, slow_factor or what saving needs
|
|
// (see the top of this file), at most 10.
|
|
double intervalScale() const { return state_ == GuardState::Normal ? 1.0 : scale_; }
|
|
bool pauseCapture() const { return state_ == GuardState::Paused; }
|
|
// Why it is not Normal, in plain words ("" when Normal).
|
|
const std::string& reason() const { return reason_; }
|
|
|
|
// `thermal resume`: back to Normal now, every timer restarted. The rules then
|
|
// apply afresh - a CPU still at pause_c pauses again after pause_after_s.
|
|
std::vector<GuardLog> forceResume(long long now_ms);
|
|
// `thermal off|on`: off = always Normal; the readings and the throttling log go on.
|
|
std::vector<GuardLog> setEnabled(bool on, long long now_ms);
|
|
bool enabled() const { return enabled_; }
|
|
|
|
const ThermalParams& params() const { return p_; }
|
|
const HostReading& last() const { return last_; }
|
|
bool haveReading() const { return have_last_; }
|
|
const ThrottleWatch& throttle() const { return watch_; }
|
|
long long heldForSaveMs() const; // how long the scan has been waiting
|
|
long long pausedForMs() const;
|
|
// Share of the last minute the scan spent waiting for saving, in %; -1 until
|
|
// there is half a minute of history to judge from.
|
|
double heldSharePct() const;
|
|
|
|
private:
|
|
void enter(GuardState s, const std::string& why, std::vector<GuardLog>& log);
|
|
void updateScale(const HostReading& r);
|
|
std::string paceText() const; // "one picture every 4.0 s instead of 2.0"
|
|
|
|
ThermalParams p_;
|
|
ThrottleWatch watch_;
|
|
bool enabled_ = true;
|
|
GuardState state_ = GuardState::Normal;
|
|
std::string reason_;
|
|
HostReading last_;
|
|
bool have_last_ = false;
|
|
|
|
long long hot_since_ms_ = -1; // CPU at pause_c or more since (-1 = not)
|
|
long long held_since_ms_ = -1; // the scan waiting for saving since (-1 = not)
|
|
long long calm_since_ms_ = -1; // Slowed, with every reason clear, since (-1 = not)
|
|
long long paused_at_ms_ = 0;
|
|
long long slowed_at_ms_ = 0;
|
|
long long overload_since_ms_ = -1; // Slowed, and saving still the bottleneck, since
|
|
// (time, held_total_ms) per sample over the last minute, for heldSharePct().
|
|
std::deque<std::pair<long long, long long>> held_hist_;
|
|
double scale_ = 1.0;
|
|
double last_save_ms_ = 0.0; // the last save time seen (0 = none yet)
|
|
double need_scale_ = 0.0; // the scale saving needs; <= 1: the normal pace fits
|
|
};
|
|
|
|
} // namespace fgc
|