diff --git a/apps/arstack_studio/qml/Main.qml b/apps/arstack_studio/qml/Main.qml index e58d5171f..c3d738d23 100644 --- a/apps/arstack_studio/qml/Main.qml +++ b/apps/arstack_studio/qml/Main.qml @@ -210,6 +210,14 @@ ApplicationWindow { function onDeviceMessage(message) { root.showMessage(message, false) } + function onSvStopReasonChanged() { + if (!device.running && device.svStopReason !== "" && + device.svStopReason !== "AWAITING_BOARD_CAUSE") { + root.showMessage( + "ESP32-P4 stopped SV: " + device.svStopReason + + ". PTP state is independent. Open Copy diagnostics.", true) + } + } function onDeviceVerifiedChanged() { root.pendingApply = true root.offlinePreviewRunning = false @@ -771,11 +779,14 @@ ApplicationWindow { Label { text: !device.deviceVerified ? "No ESP32 board detected · Preview mode only" - : device.portName + " · ESP32-P4" + (workflowBar.injectionRunning ? " · Injecting" : " · Ready") - color: device.deviceVerified ? studioTheme.textSoft : studioTheme.muted + : device.portName + " · ESP32-P4 · " + workflowBar.displayState + color: workflowBar.unexpectedSvStop ? studioTheme.red + : device.deviceVerified ? studioTheme.textSoft : studioTheme.muted + Layout.maximumWidth: root.compactLayout ? 255 : 410 + elide: Text.ElideRight font.family: root.uiFont font.pixelSize: 9 - font.weight: Font.Medium + font.weight: workflowBar.unexpectedSvStop ? Font.DemiBold : Font.Medium } Rectangle { width: 1; height: 12; color: studioTheme.lineSoft } Label { diff --git a/apps/arstack_studio/qml/WorkflowBar.qml b/apps/arstack_studio/qml/WorkflowBar.qml index b113be911..06f5d88c6 100644 --- a/apps/arstack_studio/qml/WorkflowBar.qml +++ b/apps/arstack_studio/qml/WorkflowBar.qml @@ -190,7 +190,18 @@ SurfacePanel { } } + // Never display green READY immediately after an unsolicited board STOP. + // The board-reported reason is distinct from the physical PTP role. + readonly property bool unexpectedSvStop: Boolean( + ribbon.device && ribbon.device.deviceVerified && + !ribbon.device.running && ribbon.device.svStopReason.length > 0 && + !ribbon.startQueued) readonly property string displayState: { + if (ribbon.unexpectedSvStop) { + if (ribbon.device.svStopReason === "LEASE_EXPIRED") return "SV stopped · USB control lease expired" + if (ribbon.device.svStopReason === "RUNTIME_FAULT") return "SV stopped · board runtime fault" + return "SV stopped · checking board cause" + } if (smartSession.state === "WAITING FOR DEVICE") return "Connect device" if (smartSession.state === "DEVICE FOUND" || smartSession.state === "CONNECTING") return "Connecting" if (smartSession.state === "CHECKING DEVICE") return "Checking device" @@ -209,6 +220,7 @@ SurfacePanel { } readonly property color smartStateColor: { + if (ribbon.unexpectedSvStop) return theme.red if (smartSession.state === "RUNNING" || smartSession.state === "READY") return theme.green if (smartSession.state === "CONNECTING" || smartSession.state === "CHECKING DEVICE" || smartSession.state === "FIRMWARE RECOVERY" || smartSession.state === "PREPARING 4I+4V" || diff --git a/apps/arstack_studio/src/DeterministicSessionHarness.cpp b/apps/arstack_studio/src/DeterministicSessionHarness.cpp index b8adce749..11af07dd7 100644 --- a/apps/arstack_studio/src/DeterministicSessionHarness.cpp +++ b/apps/arstack_studio/src/DeterministicSessionHarness.cpp @@ -64,6 +64,7 @@ class DeterministicSessionHarness final { {"SV STOP remains responsive with full command queue", svPriorityStopBypassesFullQueue()}, {"control lease heartbeat coalesces outside saturated exclusive queue", heartbeatKeepsBoundedSafetySlotUnderExclusiveLoad()}, {"board fail-safe STOP is not mislabelled as operator STOP ACK", forcedFirmwareStopRetainsCauseAndIsNotOperatorAck()}, + {"latched board STOP cause overrides READY and survives console truncation", boardStopCausePersistsUntilVerifiedRestart()}, {"PTP STOP requires firmware ACK and stopped readback", ptpStopConfirmationCannotBeGuessed()}, {"PTP STOP timeout blocks restart pending manual recovery", ptpUnverifiedStopBlocksRestart()}, {"App teardown must stop PTP without SV output", ptpOnlyShutdownIntent()}, @@ -980,6 +981,40 @@ class DeterministicSessionHarness final { return cancelled && worker.svStopPending_ && worker.commandQueue_.isEmpty(); } + static bool boardStopCausePersistsUntilVerifiedRestart() { + Fixture f; + auto& d = f.device; + d.setRunning(true); + d.processLine(QStringLiteral( + "I ar_smv_ctrl: state=STOPPED generation=3 signal_frequency=50000 mHz")); + if (d.running() || d.svStopReason_ != QStringLiteral("AWAITING_BOARD_CAUSE")) + return false; + d.processLine(QStringLiteral( + "I ar_smv_ctrl: SVLIFE state=STOPPED stop_cause=LEASE_EXPIRED " + "stop_seq=1 stop_uptime_ms=33000 stop_hb_age_ms=2500 " + "hb_age_ms=2600 hb_seen=1 lease_ms=2500 watchdog=1")); + if (d.svStopReason_ != QStringLiteral("LEASE_EXPIRED") || + d.svLastStopCause_ != QStringLiteral("LEASE_EXPIRED") || + !d.logText_.contains(QStringLiteral("BOARD_STOP_READBACK"))) + return false; + // Repeated polling of the same event must be idempotent. + d.processLine(QStringLiteral( + "SVLIFE state=STOPPED stop_cause=LEASE_EXPIRED stop_seq=1")); + if (d.svStopReason_ != QStringLiteral("LEASE_EXPIRED")) + return false; + // An already restarted board can still return its last-stop history. + d.processLine(QStringLiteral( + "SVLIFE state=RUNNING stop_cause=LEASE_EXPIRED stop_seq=1")); + if (!d.svStopReason_.isEmpty() || + d.svLastStopCause_ != QStringLiteral("LEASE_EXPIRED")) return false; + // Operator STOP must never be reported as spontaneous lease loss. + d.setRunning(true); + d.operatorStopPending_ = true; + d.processLine(QStringLiteral( + "SVLIFE state=STOPPED stop_cause=OPERATOR_STOP stop_seq=2")); + return d.svStopReason_.isEmpty(); + } + static bool forcedFirmwareStopRetainsCauseAndIsNotOperatorAck() { Fixture f; auto& d = f.device; diff --git a/apps/arstack_studio/src/DeviceController.cpp b/apps/arstack_studio/src/DeviceController.cpp index bcdf1af98..405ad07ca 100644 --- a/apps/arstack_studio/src/DeviceController.cpp +++ b/apps/arstack_studio/src/DeviceController.cpp @@ -823,6 +823,7 @@ bool DeviceController::startWithScenario(const QVariantMap& scenario) { return true; } bool DeviceController::stop() { + operatorStopPending_ = true; appendLog(QStringLiteral("SVTX"), QStringLiteral( "OPERATOR_STOP_REQUEST session=%1 boot=%2 running=%3 cfg_pending=%4") .arg(sessionGeneration_).arg(identity_.bootId) @@ -1751,6 +1752,12 @@ void DeviceController::failScenarioStart(const QString& message) { void DeviceController::setRunning(const bool value) { if (running_ == value) return; + if (!value && running_ && !operatorStopPending_ && svStopReason_.isEmpty()) { + // Do not silently turn a physical STOP into a green READY state. + // SHOW/SVLIFE will replace this interim state with the board cause. + svStopReason_ = QStringLiteral("AWAITING_BOARD_CAUSE"); + emit svStopReasonChanged(); + } running_ = value; emit runningChanged(); } @@ -1918,6 +1925,11 @@ void DeviceController::processLine(const QString& rawLine) { failScenarioStart(QStringLiteral("Unexpected SV START ACK identity/hash.")); return; } + operatorStopPending_ = false; + if (!svStopReason_.isEmpty()) { + svStopReason_.clear(); + emit svStopReasonChanged(); + } // Publish verified RUNNING before clearing the pending START phase. // Otherwise QML can briefly see neither RUNNING nor STARTING and // bounce the button back to START during a successful ACK handoff. @@ -1943,6 +1955,11 @@ void DeviceController::processLine(const QString& rawLine) { .arg(forced ? QStringLiteral("RUNTIME_FAIL_SAFE_STOP_OBSERVED") : QStringLiteral("STOP_ACK_OBSERVED")) .arg(sessionGeneration_).arg(identity_.bootId).arg(running_ ? 1 : 0)); + if (forced && svStopReason_ != QStringLiteral("RUNTIME_FAULT")) { + svStopReason_ = QStringLiteral("RUNTIME_FAULT"); + emit svStopReasonChanged(); + } + operatorStopPending_ = false; setRunning(false); } if (line.contains(QStringLiteral("Control session lease expired"), Qt::CaseInsensitive) || @@ -1953,6 +1970,44 @@ void DeviceController::processLine(const QString& rawLine) { appendLog(QStringLiteral("SVTX"), QStringLiteral( "FIRMWARE_STOP_CAUSE session=%1 boot=%2 detail=%3") .arg(sessionGeneration_).arg(identity_.bootId).arg(line)); + const QString reason = line.contains(QStringLiteral("Control session lease expired"), Qt::CaseInsensitive) + ? QStringLiteral("LEASE_EXPIRED") : QStringLiteral("RUNTIME_FAULT"); + if (svLastStopCause_ != reason) { + svLastStopCause_ = reason; + emit svStopReasonChanged(); + } + if (svStopReason_ != reason) { + svStopReason_ = reason; + emit svStopReasonChanged(); + } + } + + // Board-owned latched event survives the small console log window and + // is returned with SHOW even after the operator has restarted SV. + static const QRegularExpression kSvLife( + QStringLiteral("SVLIFE state=(RUNNING|STOPPED) stop_cause=([A-Z_]+) stop_seq=(\\d+)")); + const auto svLife = kSvLife.match(line); + if (svLife.hasMatch()) { + bool seqOk = false; + const auto seq = svLife.captured(3).toULongLong(&seqOk); + if (seqOk && (seq >= lastSvStopSeq_ || !deviceVerified_)) { + lastSvStopSeq_ = seq; + const QString lastReason = svLife.captured(2); + if (seq > 0U && svLastStopCause_ != lastReason) { + svLastStopCause_ = lastReason; + emit svStopReasonChanged(); + } + const bool boardStopped = svLife.captured(1) == QStringLiteral("STOPPED"); + const QString effective = boardStopped && lastReason != QStringLiteral("OPERATOR_STOP") && + lastReason != QStringLiteral("NONE") ? lastReason : QString{}; + if (svStopReason_ != effective) { + svStopReason_ = effective; + appendLog(QStringLiteral("SVTX"), QStringLiteral( + "BOARD_STOP_READBACK session=%1 boot=%2 cause=%3 seq=%4") + .arg(sessionGeneration_).arg(identity_.bootId).arg(lastReason).arg(seq)); + emit svStopReasonChanged(); + } + } } auto match = kStateExpression.match(line); @@ -2146,6 +2201,13 @@ void DeviceController::processLine(const QString& rawLine) { } void DeviceController::resetTelemetry() { + operatorStopPending_ = false; + lastSvStopSeq_ = 0; + svLastStopCause_.clear(); + if (!svStopReason_.isEmpty()) { + svStopReason_.clear(); + emit svStopReasonChanged(); + } fps_ = QStringLiteral("—"); missed_ = QStringLiteral("—"); txFailures_ = QStringLiteral("—"); diff --git a/apps/arstack_studio/src/DeviceController.hpp b/apps/arstack_studio/src/DeviceController.hpp index 70a555882..aa55c036d 100644 --- a/apps/arstack_studio/src/DeviceController.hpp +++ b/apps/arstack_studio/src/DeviceController.hpp @@ -51,6 +51,8 @@ class DeviceController : public QObject { Q_PROPERTY(QStringList capabilities READ capabilities NOTIFY deviceIdentityChanged) Q_PROPERTY(bool connected READ connected NOTIFY connectedChanged) Q_PROPERTY(bool running READ running NOTIFY runningChanged) + Q_PROPERTY(QString svStopReason READ svStopReason NOTIFY svStopReasonChanged) + Q_PROPERTY(QString svLastStopCause READ svLastStopCause NOTIFY svStopReasonChanged) Q_PROPERTY(bool scenarioApplying READ scenarioApplying NOTIFY scenarioStateChanged) Q_PROPERTY(QString portName READ portName NOTIFY connectedChanged) Q_PROPERTY(QString lastError READ lastError NOTIFY lastErrorChanged) @@ -140,6 +142,8 @@ class DeviceController : public QObject { [[nodiscard]] DeviceIdentity deviceIdentity() const; [[nodiscard]] bool connected() const noexcept; [[nodiscard]] bool running() const noexcept; + [[nodiscard]] QString svStopReason() const { return svStopReason_; } + [[nodiscard]] QString svLastStopCause() const { return svLastStopCause_; } [[nodiscard]] bool scenarioApplying() const noexcept; [[nodiscard]] QString portName() const; [[nodiscard]] QString lastError() const; @@ -295,6 +299,7 @@ class DeviceController : public QObject { void deviceIdentityChanged(); void connectedChanged(); void runningChanged(); + void svStopReasonChanged(); void lastErrorChanged(); void logTextChanged(); void syncDiagnosticsChanged(); @@ -416,6 +421,10 @@ class DeviceController : public QObject { int identifyAttempts_{0}; int highConfidenceCount_{0}; bool running_{false}; + bool operatorStopPending_{false}; + QString svStopReason_; + QString svLastStopCause_; + quint64 lastSvStopSeq_{0}; bool connected_{false}; bool discovering_{false}; bool deviceVerified_{false}; diff --git a/apps/arstack_studio/src/SmartSessionController.cpp b/apps/arstack_studio/src/SmartSessionController.cpp index aca4cd7a6..c699d538f 100644 --- a/apps/arstack_studio/src/SmartSessionController.cpp +++ b/apps/arstack_studio/src/SmartSessionController.cpp @@ -979,6 +979,9 @@ QString SmartSessionController::diagnosticsText() const { add(QStringLiteral("Profile generation"), device_->profileGeneration()); add(QStringLiteral("Profile armed"), device_->profileArmed() ? QStringLiteral("yes") : QStringLiteral("no")); add(QStringLiteral("SMV running"), device_->running() ? QStringLiteral("yes") : QStringLiteral("no")); + add(QStringLiteral("Current SV stop condition (board readback)"), device_->svStopReason()); + add(QStringLiteral("Last SV stop cause (same board, persists across restart)"), + device_->svLastStopCause()); add(QStringLiteral("SMV FPS reported"), device_->fps()); add(QStringLiteral("SMV missed reported"), device_->missed()); add(QStringLiteral("SMV TX failures reported"), device_->txFailures()); diff --git a/apps/arstack_studio/tests/check_operator_workspace.py b/apps/arstack_studio/tests/check_operator_workspace.py index 795f09028..6714f689b 100644 --- a/apps/arstack_studio/tests/check_operator_workspace.py +++ b/apps/arstack_studio/tests/check_operator_workspace.py @@ -489,6 +489,27 @@ def main() -> None: 'ptpRuntimeRunningSeen_' in device_control_cpp and 'if (!ptpRuntimeRoleSeen_ || !ptpRuntimeRunningSeen_' in device_control_cpp, "PTP errors must originate on board and two-order runtime proofs must be joined") + # Control-session lease is a safety authority, not a QML watchdog or + # clock qualification. One boot-lifetime periodic timer eliminates the + # old racing one-shot stop/rearm from the ESP console and timer callback. + life_header = (STUDIO / "src/DeviceController.hpp").read_text(encoding="utf-8") + life_firmware = (STUDIO.parents[1] / "embedded/esp32p4_smv_injector/main/live_control.cpp").read_text(encoding="utf-8") + life_qml = source("WorkflowBar.qml") + require("esp_timer_start_periodic(" in life_firmware + and "kControlWatchdogPeriodUs = 250000ULL" in life_firmware + and "void enforce_control_lease() noexcept" in life_firmware + and "g_control_lease_mux" in life_firmware + and "esp_timer_start_once(" not in life_firmware + and "SVLIFE state=%s stop_cause=%s" in life_firmware + and "SvStopCause::lease_expired" in life_firmware + and "SvStopCause::runtime_fault" in life_firmware + and "Q_PROPERTY(QString svStopReason READ svStopReason NOTIFY svStopReasonChanged)" in life_header + and "ribbon.unexpectedSvStop" in life_qml + and "workflowBar.displayState" in source("Main.qml") + and "function onSvStopReasonChanged()" in source("Main.qml"), + "SV output must fail-closed on measured heartbeat loss, preserve board STOP cause, " + "and never show misleading green READY after an unrequested stop") + clipboard_header = (STUDIO / "src/StudioClipboard.hpp").read_text(encoding="utf-8") clipboard_cpp = (STUDIO / "src/StudioClipboard.cpp").read_text(encoding="utf-8") main_cpp = (STUDIO / "src/main.cpp").read_text(encoding="utf-8") diff --git a/embedded/esp32p4_smv_injector/main/live_control.cpp b/embedded/esp32p4_smv_injector/main/live_control.cpp index cd4bd47fe..ccf02a667 100644 --- a/embedded/esp32p4_smv_injector/main/live_control.cpp +++ b/embedded/esp32p4_smv_injector/main/live_control.cpp @@ -42,8 +42,22 @@ constexpr std::array kChannelNames{ "IA", "IB", "IC", "IN", "UA", "UB", "UC", "UN"}; constexpr std::string_view kTokenDelimiters{" \t\r\n"}; constexpr std::uint32_t kControlLeaseTimeoutMs = 2500U; -constexpr std::uint64_t kControlLeaseTimeoutUs = - static_cast(kControlLeaseTimeoutMs) * 1000ULL; +// Fixed-rate watchdog never stops/rearms from console heartbeat and timer +// callback concurrently. An expired lease is detected in <=timeout+250ms. +constexpr std::uint64_t kControlWatchdogPeriodUs = 250000ULL; + +enum class SvStopCause : std::uint32_t { + none = 0U, operator_request, lease_expired, runtime_fault +}; + +const char* sv_stop_cause_name(const SvStopCause cause) noexcept { + switch (cause) { + case SvStopCause::operator_request: return "OPERATOR_STOP"; + case SvStopCause::lease_expired: return "LEASE_EXPIRED"; + case SvStopCause::runtime_fault: return "RUNTIME_FAULT"; + default: return "NONE"; + } +} SvLiveSignalBank g_signal_bank; SvScenarioTransaction g_scenario_txn; @@ -53,6 +67,12 @@ std::atomic g_publisher_task{nullptr}; std::atomic g_control_heartbeat_seen{false}; std::atomic g_control_lease_active{false}; std::atomic g_last_heartbeat_ms{0U}; +std::atomic g_control_watchdog_ready{false}; +std::atomic g_last_sv_stop_cause{SvStopCause::none}; +std::atomic g_sv_stop_seq{0U}; +std::atomic g_last_sv_stop_ms{0U}; +std::atomic g_sv_stop_heartbeat_age_ms{0U}; +portMUX_TYPE g_control_lease_mux = portMUX_INITIALIZER_UNLOCKED; esp_timer_handle_t g_control_lease_timer{nullptr}; std::uint64_t g_boot_id{0U}; @@ -75,96 +95,87 @@ bool heartbeat_is_fresh() noexcept { return static_cast(now - last) <= kControlLeaseTimeoutMs; } -bool start_control_lease_timer(const std::uint64_t delay_us) noexcept { - if (g_control_lease_timer == nullptr) return false; - static_cast(esp_timer_stop(g_control_lease_timer)); - const auto result = esp_timer_start_once(g_control_lease_timer, delay_us == 0U ? 1U : delay_us); - if (result != ESP_OK) { - ESP_LOGE(kTag, "Unable to arm control-session lease timer: %d", static_cast(result)); - return false; - } - return true; +// Preserve stop provenance in board RAM even if serial output scrolls away. +// SHOW reports the same reason after the operator has restarted the publisher. +void record_sv_stop(const SvStopCause cause, const std::uint32_t age_ms) noexcept { + g_last_sv_stop_cause.store(cause, std::memory_order_release); + g_sv_stop_heartbeat_age_ms.store(age_ms, std::memory_order_release); + g_last_sv_stop_ms.store(monotonic_ms(), std::memory_order_release); + g_sv_stop_seq.fetch_add(1U, std::memory_order_acq_rel); } void disarm_control_lease() noexcept { + // The 4 Hz watchdog keeps running for the board lifetime. Disarm only + // the OUTPUT lease, so no concurrent timer restart/stop race exists. + portENTER_CRITICAL(&g_control_lease_mux); g_control_lease_active.store(false, std::memory_order_release); - if (g_control_lease_timer != nullptr) { - static_cast(esp_timer_stop(g_control_lease_timer)); - } + portEXIT_CRITICAL(&g_control_lease_mux); } void clear_control_session() noexcept { - disarm_control_lease(); + portENTER_CRITICAL(&g_control_lease_mux); + g_control_lease_active.store(false, std::memory_order_release); g_control_heartbeat_seen.store(false, std::memory_order_release); g_last_heartbeat_ms.store(0U, std::memory_order_release); + portEXIT_CRITICAL(&g_control_lease_mux); } -void stop_for_lease_failure(const char* reason) noexcept { - bool expected = true; - if (g_running.compare_exchange_strong(expected, false, std::memory_order_acq_rel)) { - g_start_request.store(false, std::memory_order_release); - g_control_lease_active.store(false, std::memory_order_release); - wake_publisher(); - ESP_LOGW(kTag, "%s; SV transmission stopped", reason); - } -} - +// A control lease is NOT a PTP clock lease. The board must stop publishing +// when the Studio connection actually disappears; loss of PTP qualification +// only deasserts smpSynch and does not silently stop the SV publisher. void enforce_control_lease() noexcept { - if (!g_running.load(std::memory_order_acquire) || - !g_control_lease_active.load(std::memory_order_acquire)) { - return; - } - - const auto now = monotonic_ms(); - const auto last = g_last_heartbeat_ms.load(std::memory_order_acquire); - const auto elapsed = static_cast(now - last); - if (elapsed < kControlLeaseTimeoutMs) { - const auto remaining_us = - static_cast(kControlLeaseTimeoutMs - elapsed) * 1000ULL; - if (!start_control_lease_timer(remaining_us)) { - stop_for_lease_failure("Control session lease timer failed"); - } - return; - } - - g_control_lease_active.store(false, std::memory_order_release); - g_start_request.store(false, std::memory_order_release); - const bool was_running = g_running.exchange(false, std::memory_order_acq_rel); - if (was_running) { + std::uint32_t expired_age_ms = 0U; + bool expired = false; + portENTER_CRITICAL(&g_control_lease_mux); + if (g_running.load(std::memory_order_acquire) && + g_control_lease_active.load(std::memory_order_acquire)) { + const auto now = monotonic_ms(); + const auto last = g_last_heartbeat_ms.load(std::memory_order_acquire); + const auto elapsed = static_cast(now - last); + if (elapsed >= kControlLeaseTimeoutMs) { + g_control_lease_active.store(false, std::memory_order_release); + g_start_request.store(false, std::memory_order_release); + expired = g_running.exchange(false, std::memory_order_acq_rel); + expired_age_ms = elapsed; + if (expired) record_sv_stop(SvStopCause::lease_expired, elapsed); + } + } + portEXIT_CRITICAL(&g_control_lease_mux); + if (expired) { wake_publisher(); ESP_LOGW(kTag, "Control session lease expired after %lu ms; SV transmission stopped", - static_cast(elapsed)); + static_cast(expired_age_ms)); } } void control_lease_timer_callback(void*) noexcept { - enforce_control_lease(); + enforce_control_lease(); // 4 Hz bounded age check; NEVER rearm the timer. } void record_control_heartbeat() noexcept { + portENTER_CRITICAL(&g_control_lease_mux); g_last_heartbeat_ms.store(monotonic_ms(), std::memory_order_release); g_control_heartbeat_seen.store(true, std::memory_order_release); - if (!g_running.load(std::memory_order_acquire)) return; - - if (!start_control_lease_timer(kControlLeaseTimeoutUs)) { - stop_for_lease_failure("Control session heartbeat could not rearm the lease timer"); - return; - } - g_control_lease_active.store(true, std::memory_order_release); + portEXIT_CRITICAL(&g_control_lease_mux); + // No esp_timer_stop/start from this control task, and no lease renewal + // after the watchdog has already revoked the RUNNING session. } bool arm_control_lease_if_fresh() noexcept { - if (!heartbeat_is_fresh()) { - g_control_lease_active.store(false, std::memory_order_release); - return false; + if (!g_control_watchdog_ready.load(std::memory_order_acquire)) return false; + bool fresh = false; + portENTER_CRITICAL(&g_control_lease_mux); + if (g_control_heartbeat_seen.load(std::memory_order_acquire)) { + const auto elapsed = static_cast( + monotonic_ms() - + g_last_heartbeat_ms.load(std::memory_order_acquire)); + fresh = elapsed < kControlLeaseTimeoutMs; + if (fresh) + g_control_lease_active.store(true, std::memory_order_release); } - if (!start_control_lease_timer(kControlLeaseTimeoutUs)) { - g_control_lease_active.store(false, std::memory_order_release); - return false; - } - g_control_lease_active.store(true, std::memory_order_release); - return true; + portEXIT_CRITICAL(&g_control_lease_mux); + return fresh; } void uppercase_ascii(char* text) noexcept { @@ -451,6 +462,23 @@ void print_state() noexcept { g_running.load(std::memory_order_acquire) ? "RUNNING" : "STOPPED", static_cast(state.generation), static_cast(state.frequency_millihz)); + const std::uint32_t now_ms = monotonic_ms(); + const std::uint32_t hb_last = + g_last_heartbeat_ms.load(std::memory_order_acquire); + const bool hb_seen = g_control_heartbeat_seen.load(std::memory_order_acquire); + const auto stop_reason = g_last_sv_stop_cause.load(std::memory_order_acquire); + ESP_LOGI(kTag, + "SVLIFE state=%s stop_cause=%s stop_seq=%lu stop_uptime_ms=%lu " + "stop_hb_age_ms=%lu hb_age_ms=%lu hb_seen=%u lease_ms=%lu watchdog=%u", + g_running.load(std::memory_order_acquire) ? "RUNNING" : "STOPPED", + sv_stop_cause_name(stop_reason), + static_cast(g_sv_stop_seq.load(std::memory_order_acquire)), + static_cast(g_last_sv_stop_ms.load(std::memory_order_acquire)), + static_cast(g_sv_stop_heartbeat_age_ms.load(std::memory_order_acquire)), + static_cast(hb_seen ? now_ms - hb_last : 0U), + hb_seen ? 1U : 0U, + static_cast(kControlLeaseTimeoutMs), + g_control_watchdog_ready.load(std::memory_order_acquire) ? 1U : 0U); for (std::size_t index = 0U; index < state.channels.size(); ++index) { const auto& channel = state.channels[index]; ESP_LOGI(kTag, @@ -591,9 +619,10 @@ void handle_line(char* line) noexcept { ESP_LOGE(kTag, "Usage: STOP"); return; } - g_running.store(false, std::memory_order_release); + const bool was_running = g_running.exchange(false, std::memory_order_acq_rel); g_scenario_txn.reset(); disarm_control_lease(); + if (was_running) record_sv_stop(SvStopCause::operator_request, 0U); wake_publisher(); ESP_LOGI(kTag, "STOP accepted: SV transmission suppressed"); return; @@ -807,6 +836,10 @@ void live_control_initialize( g_running.store(false, std::memory_order_release); g_start_request.store(false, std::memory_order_release); clear_control_session(); + g_last_sv_stop_cause.store(SvStopCause::none, std::memory_order_release); + g_sv_stop_seq.store(0U, std::memory_order_release); + g_last_sv_stop_ms.store(0U, std::memory_order_release); + g_sv_stop_heartbeat_age_ms.store(0U, std::memory_order_release); const std::uint64_t random_high = static_cast(esp_random()); const std::uint64_t random_low = static_cast(esp_random()); @@ -817,11 +850,21 @@ void live_control_initialize( esp_timer_create_args_t timer_args{}; timer_args.callback = &control_lease_timer_callback; timer_args.name = "ar_ctrl_lease"; - const auto result = esp_timer_create(&timer_args, &g_control_lease_timer); + esp_err_t result = esp_timer_create(&timer_args, &g_control_lease_timer); + if (result == ESP_OK) { + result = esp_timer_start_periodic( + g_control_lease_timer, kControlWatchdogPeriodUs); + } if (result != ESP_OK) { - g_control_lease_timer = nullptr; - ESP_LOGE(kTag, "Unable to create control-session lease timer: %d", static_cast(result)); + if (g_control_lease_timer != nullptr) { + static_cast(esp_timer_delete(g_control_lease_timer)); + g_control_lease_timer = nullptr; + } + ESP_LOGE(kTag, "Unable to start periodic control lease watchdog: %d", + static_cast(result)); } + g_control_watchdog_ready.store( + g_control_lease_timer != nullptr, std::memory_order_release); } } @@ -830,9 +873,10 @@ void live_control_bind_publisher_task(const TaskHandle_t task) noexcept { } void live_control_force_stop() noexcept { - g_running.store(false, std::memory_order_release); + const bool was_running = g_running.exchange(false, std::memory_order_acq_rel); g_start_request.store(false, std::memory_order_release); clear_control_session(); + if (was_running) record_sv_stop(SvStopCause::runtime_fault, 0U); wake_publisher(); ESP_LOGW(kTag, "STOP accepted: SV transmission suppressed by fail-closed runtime"); }