diff --git a/docs/help/help.json b/docs/help/help.json index 1642a4cb..02feafde 100644 --- a/docs/help/help.json +++ b/docs/help/help.json @@ -4059,6 +4059,14 @@ "fps": { "line": "Frames a second asked of a capture device, which streams the nearest rate it offers." }, + "noiseFloor": { + "line": "Source bytes at or below this read as black.", + "more": "A grabber's black is rarely a 0: a dark frame decodes to a 1 or 2, which no screen shows and every LED does. Raise **noiseFloor** to the level its noise reaches." + }, + "staleMs": { + "line": "Gap tolerated before the picture is dropped, in milliseconds.", + "more": "A source that stops leaves its last picture on the lights, which reads as the room lit from a frozen frame. Past **staleMs** the picture goes, and the lights with it." + }, "width": { "line": "The picture's width in pixels, and the most a capture scales its frames to." }, diff --git a/esp32/main/Kconfig.projbuild b/esp32/main/Kconfig.projbuild index c10b14cc..f207d1d4 100644 --- a/esp32/main/Kconfig.projbuild +++ b/esp32/main/Kconfig.projbuild @@ -46,6 +46,19 @@ menu "MoonLight" MIDI service with no computer. Set by the S3 images with flash to spare (n8r8, n16r8); the 4 MB S3-Zero leaves it off. + config MM_USB_VIDEO + bool "USB video capture from a UVC grabber (ESP32-P4 only)" + # Constrain the VALUE, not the visibility, for the reason MM_HLS gives above: a `depends on` + # hides the symbol, and idf_component.yml gates usb_host_uvc on $CONFIG{MM_USB_VIDEO}, which + # the solver cannot read when hidden. hasUsbVideo requires the P4 too, so y elsewhere is inert. + default n + help + Pulls the espressif/usb_host_uvc managed component and compiles the + UVC host and the JPEG decode, so an HDMI grabber plugged into the + board feeds the Picture service. P4 only: no other chip here has a + High-Speed USB host, and at Full Speed MJPEG starves. It owns the USB + host library, so never set it with MM_USB_MIDI_HOST in one image. + config MM_AUDIO_CODEC bool "Audio codecs in front of a microphone (ES8311, ES7210)" default y if IDF_TARGET_ESP32P4 || IDF_TARGET_ESP32S31 diff --git a/esp32/main/idf_component.yml b/esp32/main/idf_component.yml index 850d7c32..e048940c 100644 --- a/esp32/main/idf_component.yml +++ b/esp32/main/idf_component.yml @@ -55,6 +55,13 @@ dependencies: # microphone behind a codec. Which board has one, and on which pins, is AudioService's runtime # configuration from the device model; the Kconfig gate keeps the component out of every image # that does not set MM_AUDIO_CODEC. Referenced only from platform_esp32_es8311.cpp. + # usb_host_uvc: the UVC (webcam) host a grabber's frames are read through. It declares + # espressif/usb as a public dependency, so the host stack comes with it. Gated on MM_USB_VIDEO + # rather than on the target, so a P4 image that leaves the capture out carries neither. + espressif/usb_host_uvc: + version: "^2.5.2" + rules: + - if: "$CONFIG{MM_USB_VIDEO} == True" espressif/esp_codec_dev: version: "^1.3.0" rules: diff --git a/esp32/sdkconfig.defaults.esp32p4rev1-eth b/esp32/sdkconfig.defaults.esp32p4rev1-eth index 60549f29..f96167d1 100644 --- a/esp32/sdkconfig.defaults.esp32p4rev1-eth +++ b/esp32/sdkconfig.defaults.esp32p4rev1-eth @@ -94,3 +94,12 @@ CONFIG_HEAP_HAS_EXEC_HEAP=y # nothing (3 reboots in 70 s against 4), which is what a blocked network cascade looks like, not an # FFT fault. Lives in the board fragment because every P4 image (rev1/rev3, eth/eth-wifi) layers on it. # CONFIG_DSP_ANSI=y + +# The capture: the UVC host and the JPEG decode, on the only chip here with a High-Speed USB host. +CONFIG_MM_USB_VIDEO=y + +# USB host control-transfer buffer. A UVC configuration descriptor lists every format and frame +# size, 1-3 KB on a grabber, and enum.c rejects a descriptor longer than this buffer before any +# class driver is consulted: at the default 256 an MS2130 never enumerates. Costs 4 KB of heap per +# attached device. +CONFIG_USB_HOST_CONTROL_TRANSFER_MAX_SIZE=4096 diff --git a/src/core/module/Scheduler.h b/src/core/module/Scheduler.h index a336c4ee..5e3e1ed0 100644 --- a/src/core/module/Scheduler.h +++ b/src/core/module/Scheduler.h @@ -66,9 +66,9 @@ class Scheduler { void prepareTree(); /// Ask for a rebuild at the next frame boundary, which is safe from any task. - void requestPrepareTree() { prepareRequested_.store(true, std::memory_order_relaxed); } + void requestPrepareTree() MM_NONBLOCKING { prepareRequested_.store(true, std::memory_order_relaxed); } /// Ask for one subtree's rebuild at the next frame boundary, safe from any task: a few subtrees per frame, one more widening the request to the tree. - void requestPrepare(MoonModule* scope) { + void requestPrepare(MoonModule* scope) MM_NONBLOCKING { if (!scope) { requestPrepareTree(); return; } for (auto& slot : preparePending_) { MoonModule* expected = nullptr; @@ -113,7 +113,7 @@ class Scheduler { MoonModule* topOfType(const char* typeName) const; /// The single live Scheduler, reachable so a factory-created module can drive a control. - static Scheduler* instance() { return instance_; } + static Scheduler* instance() MM_NONBLOCKING { return instance_; } /// A Scheduler that ends without release() still stops being the live one, so instance() never dangles. ~Scheduler() { if (instance_ == this) instance_ = nullptr; } diff --git a/src/core/services/PictureService.h b/src/core/services/PictureService.h index 46911816..bf39e6cc 100644 --- a/src/core/services/PictureService.h +++ b/src/core/services/PictureService.h @@ -1,6 +1,7 @@ #pragma once #include "core/module/MoonModule.h" +#include "core/module/Scheduler.h" // requestPrepare: a grabber arriving or leaving re-opens the capture #include "core/util/ActiveInstance.h" // the one-active-source election #include "core/util/color.h" // RGB #include "core/util/format.h" // formatTo: the status line built off snprintf @@ -34,6 +35,12 @@ class PictureService : public MoonModule { uint8_t source = kTestPattern; /// Frames a second asked of a capture device, which streams the nearest rate it offers. uint8_t fps = 30; + /// Source bytes at or below this read as black. + /// @guide A grabber's black is rarely a 0: a dark frame decodes to a 1 or 2, which no screen shows and every LED does. Raise **noiseFloor** to the level its noise reaches. + uint8_t noiseFloor = 1; + /// Gap tolerated before the picture is dropped, in milliseconds. + /// @guide A source that stops leaves its last picture on the lights, which reads as the room lit from a frozen frame. Past **staleMs** the picture goes, and the lights with it. + uint16_t staleMs = 2000; /// The picture's width in pixels, and the most a capture scales its frames to. uint16_t width = 64; /// The picture's height in pixels, under the same ceiling as the width. @@ -42,8 +49,14 @@ class PictureService : public MoonModule { /// Publish the source and the picture's size. void defineControls() override { static constexpr const char* kSources[] = {"test pattern", "usb capture"}; - controls_.addSelect("source", source, kSources, kSourceCount); - if (platform::hasUsbVideo) controls_.addControl("fps", fps, 1, 120); + // Only where the transport links, so a board without it shows the one source it has, as the capture's own controls do. + controls_.addSelect("source", source, kSources, + static_cast(platform::hasUsbVideo ? kSourceCount : 1)); + if (platform::hasUsbVideo) { + controls_.addControl("fps", fps, 1, 120); + controls_.addControl("noiseFloor", noiseFloor, 0, 64); + controls_.addControl("staleMs", staleMs, 100, 10000); + } controls_.addControl("width", width, 1, kMaxSide); controls_.addControl("height", height, 1, kMaxSide); } @@ -55,6 +68,7 @@ class PictureService : public MoonModule { void prepare() override { seat_.claim(); stopCapture(); // before the pictures move under a decode still writing one + if (source != kUsbCapture) platform::videoCaptureShutdown(); // what sees devices goes with the source if (!prepareBuffers(kRunsInTask[source])) return; if (source == kUsbCapture) { startCapture(); return; } drawTestPattern(front_, width, height); // the frame prepareBuffers published, filled before any reader ticks @@ -73,11 +87,13 @@ class PictureService : public MoonModule { front_ = a_.data(); bufW_ = width; bufH_ = height; + floor_ = noiseFloor; back_ = writerTask && secondFits(bytes) && b_.resize(bytes) ? b_.data() : nullptr; if (!back_) b_.resize(0); if (writerTask && !back_) setStatus("one picture buffer: not enough memory, a capture may skip frames", Severity::Warning); else clearStatus(); - frame_ = PictureFrame{front_, width, height, frame_.generation + 1}; + // A writer's picture is whatever it last decoded, so only its own publish lights it: a rebuild here would re-light the frame the drop has put out. + frame_ = PictureFrame{writerTask ? nullptr : front_, width, height, frame_.generation + 1}; return true; } @@ -114,22 +130,57 @@ class PictureService : public MoonModule { /// Vacate the seat for any survivor. void release() override { stopCapture(); + platform::videoCaptureShutdown(); seat_.vacate(); MoonModule::release(); } - /// Name the capture's health once a second: the size it publishes, the frames that landed and those lost. + /// Name the capture's health once a second: the size it publishes, the rate it holds and whatever is going wrong. void tick1s() MM_NONBLOCKING override { - if (source != kUsbCapture || !task_.impl) return; + if (source != kUsbCapture) return; + dropStalePicture(); // first, so a device that went away puts the lights out on this tick and not the next + // A grabber arriving or leaving rewrites the device list: an unplug closes the stream, which frees the port for a replug, and a replug opens it again. + const uint32_t gen = platform::videoCaptureFormatGeneration(); + if (gen != openedGen_) { + openedGen_ = gen; // asked once; the open the prepare runs records it again + if (Scheduler* s = Scheduler::instance()) s->requestPrepare(this); // this service alone: the picture keeps its size, so no reader re-prepares + return; + } + if (!task_.impl) { + // Nothing to open over while the last teardown unwinds, so the open waits for it rather than failing in a way worth naming. + if (platform::videoCaptureSettling()) + if (Scheduler* s = Scheduler::instance()) s->requestPrepare(this); + return; + } if (!lock_.tryAcquire()) return; // the decode task is publishing: the next second reads it - const unsigned w = frame_.width, h = frame_.height; const bool one = !back_; // the writer swaps back_ under the lock lock_.release(); + const unsigned w = bufW_, h = bufH_; // the size a writer fills, which a dropped picture does not carry const platform::VideoCaptureStats st = platform::videoCaptureStats(); - const uint32_t lost = st.dropped + st.failed; - mm::formatTo(status_, sizeof(status_), "%u x %u, %u frames, %u lost%s", w, h, - static_cast(st.decoded), static_cast(lost), one ? ", one picture" : ""); - setStatus(status_, lost || one ? Severity::Warning : Severity::Status); + // Per second, not since boot: a running total says nothing about whether the capture is live now. + const uint32_t used = st.decoded - lastDecoded_, skipped = st.dropped - lastDropped_; + const uint32_t failed = st.failed - lastFailed_; + lastDecoded_ = st.decoded; + lastDropped_ = st.dropped; + lastFailed_ = st.failed; + // Only what is happening, and skipped frames are the device offering more than the render takes, so only a refused frame warns. + char skipText[24] = {}, tail[56] = {}; + if (skipped) mm::formatTo(skipText, sizeof(skipText), ", %u skipped", static_cast(skipped)); + mm::formatTo(tail, sizeof(tail), "%s%s%s", skipText, failed ? ", frames failing" : "", + one ? ", one picture" : ""); + if (used) mm::formatTo(status_, sizeof(status_), "live, %u x %u at %u fps%s", w, h, + static_cast(used), tail); + else mm::formatTo(status_, sizeof(status_), "no frames, %u x %u%s", w, h, tail); + setStatus(status_, failed || one || !used ? Severity::Warning : Severity::Status); + } + + /// A gap of one frame is normal, since a source runs at its own rate. A long one means it stopped, and holding the picture would light the room from a frozen frame. + void dropStalePicture() MM_NONBLOCKING { + if (!staleMs || platform::millis() - lastFrameMs_.load(std::memory_order_relaxed) <= staleMs) return; + if (!lock_.tryAcquire()) return; // a writer is publishing, so the next second decides + // Under the one hold, since the writer assigns the whole frame. The generation moves too, or an effect's redraw guard skips the change and the lights stay lit. + if (frame_.rgb) frame_ = PictureFrame{nullptr, 0, 0, frame_.generation + 1}; + lock_.release(); } /// What a consumer reads with no source: a class constant, since a function-local static needs a guard on first use. @@ -247,15 +298,25 @@ class PictureService : public MoonModule { /// Open the grabber and decode on a task of its own, so the render loop never waits on the device. void startCapture() { + openedGen_ = platform::videoCaptureFormatGeneration(); // the device list this open saw, which tick1s watches + // A teardown still unwinding is the one failure worth waiting on, since nothing installs over it; tick1s asks again until it has. + if (platform::videoCaptureSettling()) { + setStatus("looking for a grabber", Severity::Status); + return; + } if (!platform::videoCaptureInit(capture_, width, height, fps)) { - // The format list outlives a failed open, so a device that enumerated but refused is a different fault. - setStatus(platform::videoCaptureFormatGeneration() ? "the grabber offers no mode that fits" : "no grabber attached", Severity::Warning); + // The step the platform stopped at, since a device that enumerated and still refused is a different fault from an empty port. + const char* why = platform::videoCaptureFault(); + setStatus(why ? why : "no grabber attached", Severity::Warning); return; } if (back_) clearStatus(); // before the task swaps back_; with one picture, prepareBuffers' warning stays until the first status line + // The open zeroes the device's counters, so the baselines go with them, and the frame clock starts now rather than at boot. + lastDecoded_ = lastDropped_ = lastFailed_ = 0; + lastFrameMs_.store(platform::millis(), std::memory_order_relaxed); stop_.store(false, std::memory_order_release); - // Core 1, beside the USB host's own tasks, so a decode never lands on the render core. - if (!platform::spawnPinnedTask(task_, "mmPicCap", &captureTrampoline, this, 4096, 3, 1)) { + // Core 1, so a decode never lands on the render core, at priority 6 above the UVC driver (5) and the pump (4). A decode preempted mid-frame holds the only pending frame, and 60 fps has no room for that. + if (!platform::spawnPinnedTask(task_, "mmPicCap", &captureTrampoline, this, 4096, 6, 1)) { platform::videoCaptureDeinit(capture_); setStatus("no task for the capture", Severity::Error); return; @@ -280,7 +341,11 @@ class PictureService : public MoonModule { if (!platform::videoCaptureAwaitFrame(capture_, kFrameWaitMs)) continue; uint16_t w = 0, h = 0; uint8_t* px = beginWrite(); - if (platform::videoCaptureDecodeInto(capture_, px, bufW_, bufH_, w, h)) endWrite(w, h); // the size the pictures hold, never a control a write changed since + // The size and the floor the pictures were prepared with, never a control a write changed since. + if (platform::videoCaptureDecodeInto(capture_, px, bufW_, bufH_, w, h, floor_)) { + lastFrameMs_.store(platform::millis(), std::memory_order_relaxed); + endWrite(w, h); + } else abortWrite(); // a refused frame: the lights keep the one they have rather than a new generation of nothing } } @@ -293,16 +358,20 @@ class PictureService : public MoonModule { return free == 0 || (free > platform::HEAP_RESERVE && free - platform::HEAP_RESERVE >= bytes); } + uint32_t openedGen_ = 0; ///< the device list the last open saw, so a grabber arriving or leaving re-opens ScratchBuffer a_{*this}, b_{*this}; ///< the pictures, three bytes a pixel; b_ only for a writer task uint8_t* front_ = nullptr; ///< what readers see uint8_t* back_ = nullptr; ///< what a writer fills, or null with one picture PictureFrame frame_; ///< the front picture, as a Reader hands it out uint16_t bufW_ = 0, bufH_ = 0; ///< the size the pictures were allocated at, which a writer fills; the controls may already say otherwise + uint8_t floor_ = 0; ///< the noise floor the decode reads, taken with the size so the task owns both TryLock lock_; ///< held by a reader for its gather, by the writer to swap or to fill the only picture platform::VideoCaptureHandle capture_; ///< the open grabber, or an empty handle platform::WorkerTask task_; ///< the decode task, or an empty handle std::atomic stop_{false}; ///< tells the decode task to return - char status_[64] = {}; ///< the capture's status line, rebuilt once a second + uint32_t lastDecoded_ = 0, lastDropped_ = 0, lastFailed_ = 0; ///< the counters a second ago, for the rates + std::atomic lastFrameMs_{0}; ///< when a picture last landed, for the stale drop + char status_[96] = {}; ///< the capture's status line, rebuilt once a second ActiveInstance seat_{*this}; }; diff --git a/src/core/util/AreaSample.h b/src/core/util/AreaSample.h index 96d483ad..79ffdf26 100644 --- a/src/core/util/AreaSample.h +++ b/src/core/util/AreaSample.h @@ -26,8 +26,16 @@ inline Span cellSpan(uint32_t i, uint32_t out, uint32_t src) { return Span{static_cast(lo), static_cast(hi)}; } -/// The rectangle's mean in linear light (square, average, root: gamma 2 for the display curve), from at most four reads per axis. -inline RGB areaMean(const uint8_t* rgb, size_t stride, Span cx, Span ry) { +/// Where a sampler writes: the buffer, the picture size it fills, and the floor below which a source byte reads as black. +struct Sink { + uint8_t* rgb = nullptr; ///< the destination, `width` x `height` x 3 bytes, top row first + uint16_t width = 0; ///< the picture's width, which a sampler fills exactly + uint16_t height = 0; ///< the picture's height, likewise + uint8_t floor = 0; ///< source bytes at or below this read as 0; 0 passes every byte through +}; + +/// The rectangle's mean in linear light (square, average, root: gamma 2 for the display curve), from at most four reads per axis. A byte at or below `floor` counts as 0, which a noisy source's black needs. +inline RGB areaMean(const uint8_t* rgb, size_t stride, Span cx, Span ry, uint8_t floor = 0) { const uint32_t sx = (cx.hi - cx.lo + kAreaTapsPerAxis - 1u) / kAreaTapsPerAxis; const uint32_t sy = (ry.hi - ry.lo + kAreaTapsPerAxis - 1u) / kAreaTapsPerAxis; uint32_t sum[3] = {0, 0, 0}, taps = 0; @@ -35,7 +43,10 @@ inline RGB areaMean(const uint8_t* rgb, size_t stride, Span cx, Span ry) { const uint8_t* row = rgb + static_cast(y) * stride; for (uint32_t x = cx.lo; x < cx.hi; x += sx) { const uint8_t* p = row + static_cast(x) * 3u; - for (int c = 0; c < 3; c++) sum[c] += static_cast(p[c]) * p[c]; + for (int c = 0; c < 3; c++) { + const uint32_t v = p[c] > floor ? p[c] : 0u; + sum[c] += v * v; + } taps++; } } diff --git a/src/platform/desktop/platform_desktop.cpp b/src/platform/desktop/platform_desktop.cpp index 5abbba6c..1bf0cb39 100644 --- a/src/platform/desktop/platform_desktop.cpp +++ b/src/platform/desktop/platform_desktop.cpp @@ -2939,6 +2939,7 @@ namespace { struct TestGrabber { std::vector formats; ///< written by a test while no capture runs std::atomic generation{0}; + std::atomic attached{false}; uint16_t openW = 0, openH = 0; ///< the mode videoCaptureInit settled on, before the task starts uint8_t openFps = 0; ///< and the rate it delivers at std::atomic r{0}, g{0}, b{0}; ///< what a decode writes into every pixel @@ -2947,15 +2948,20 @@ struct TestGrabber { std::atomic decoded{0}, failed{0}; std::vector frame; ///< the synthesized source, rebuilt only when its size or color moves uint8_t frameRgb[3] = {}; + const char* fault = nullptr; ///< which step refused the last open, or null where none has } g_grabber; // How far `have` is from `want`, so the nearest advertised mode can be picked without . long apart(long have, long want) { return have > want ? have - want : want - have; } } // namespace -uint32_t videoCaptureFormatGeneration() MM_NONBLOCKING { return g_grabber.generation.load(); } +// 0 while nothing is attached, as a device reports no list; the counter itself only grows. +uint32_t videoCaptureFormatGeneration() MM_NONBLOCKING { return g_grabber.attached.load() ? g_grabber.generation.load() : 0; } bool videoCaptureInit(VideoCaptureHandle& h, uint16_t width, uint16_t height, uint8_t fps) { - if (g_grabber.formats.empty() || g_grabber.open) return false; // one owner at a time, as a device is claimed once + // One owner at a time, as a device is claimed once. + if (g_grabber.formats.empty()) { g_grabber.fault = "no grabber attached"; return false; } + if (g_grabber.open) { g_grabber.fault = "a capture is already open"; return false; } + g_grabber.fault = nullptr; // this open answers for itself // The nearest advertised mode by area then by rate, as a grabber settles a request it cannot meet. const VideoCaptureFormat* best = nullptr; long bestCost = 0; @@ -2982,7 +2988,7 @@ bool videoCaptureAwaitFrame(VideoCaptureHandle& h, uint32_t timeoutMs) { } bool videoCaptureDecodeInto(VideoCaptureHandle& h, uint8_t* dst, uint16_t maxWidth, uint16_t maxHeight, - uint16_t& outWidth, uint16_t& outHeight) { + uint16_t& outWidth, uint16_t& outHeight, uint8_t floor) { if (!h.impl || !g_grabber.open) return false; if (g_grabber.decodeFails) { g_grabber.failed++; return false; } if (!maxWidth || !maxHeight) { g_grabber.failed++; return false; } @@ -2995,18 +3001,25 @@ bool videoCaptureDecodeInto(VideoCaptureHandle& h, uint8_t* dst, uint16_t maxWid for (size_t i = 0; i < need; i += 3) std::memcpy(g_grabber.frame.data() + i, rgb, 3); std::memcpy(g_grabber.frameRgb, rgb, 3); } - usbvideo::areaAverage(g_grabber.frame.data(), static_cast(w) * 3, w, hh, maxWidth, maxHeight, dst); + usbvideo::areaAverage(g_grabber.frame.data(), static_cast(w) * 3, w, hh, + Sink{dst, maxWidth, maxHeight, floor}); outWidth = maxWidth; outHeight = maxHeight; g_grabber.decoded++; return true; } +const char* videoCaptureFault() MM_NONBLOCKING { return g_grabber.fault; } + +bool videoCaptureSettling() MM_NONBLOCKING { return false; } // the stand-in tears down inside the call + void videoCaptureDeinit(VideoCaptureHandle& h) { if (h.impl == &g_grabber) g_grabber.open = false; // only the owning handle closes it h.impl = nullptr; } +void videoCaptureShutdown() {} // the stand-in installs nothing + VideoCaptureStats videoCaptureStats() MM_NONBLOCKING { VideoCaptureStats s; s.decoded = g_grabber.decoded.load(); @@ -3016,8 +3029,9 @@ VideoCaptureStats videoCaptureStats() MM_NONBLOCKING { void setTestVideoCaptureFormats(const VideoCaptureFormat* formats, size_t count) { g_grabber.formats.assign(formats, formats + count); - g_grabber.generation = count ? g_grabber.generation + 1 : 0; - if (!count) g_grabber.open = false; + g_grabber.generation++; // a device enumerating or leaving, as the bus reports it + g_grabber.attached = count != 0; + if (!count) g_grabber.open = false; // an unplug ends the stream } void setTestVideoCaptureColor(uint8_t r, uint8_t g, uint8_t b) { diff --git a/src/platform/esp32/platform_config.h b/src/platform/esp32/platform_config.h index a1fc7e0d..3b654499 100644 --- a/src/platform/esp32/platform_config.h +++ b/src/platform/esp32/platform_config.h @@ -210,8 +210,8 @@ constexpr bool hasUsbMidiHost = true; constexpr bool hasUsbMidiHost = false; #endif -// The UVC host and the JPEG decoder cost flash, so only the images that set the symbol carry them. -#if defined(CONFIG_MM_USB_VIDEO) +// The UVC host and the JPEG decoder cost flash, so only the images that set the symbol carry them. The P4 is named here as well as in the implementation, so the flag cannot promise what links to a stub. +#if defined(CONFIG_MM_USB_VIDEO) && defined(CONFIG_IDF_TARGET_ESP32P4) constexpr bool hasUsbVideo = true; #else constexpr bool hasUsbVideo = false; diff --git a/src/platform/esp32/platform_esp32_usbvideo.cpp b/src/platform/esp32/platform_esp32_usbvideo.cpp index 2f60a31b..bff2d95c 100644 --- a/src/platform/esp32/platform_esp32_usbvideo.cpp +++ b/src/platform/esp32/platform_esp32_usbvideo.cpp @@ -1,19 +1,552 @@ -/// USB video capture on the ESP32: the stubs every image without `CONFIG_MM_USB_VIDEO` links, which report no device. +/// USB video capture on the ESP32-P4: the UVC stream and the JPEG decode behind the capture seam. #include "platform/platform.h" +#include "platform/usbvideo_frames.h" // the hardware-free half: headers, mode choice, the shrink + +#include "sdkconfig.h" + +#if defined(CONFIG_IDF_TARGET_ESP32P4) && defined(CONFIG_MM_USB_VIDEO) + +#include "driver/jpeg_decode.h" +#include "esp_heap_caps.h" +#include "esp_log.h" +#include "usb/usb_host.h" +#include "usb/uvc_host.h" + +#include "freertos/FreeRTOS.h" +#include "freertos/semphr.h" +#include "freertos/task.h" + +#include +#include +#include // std::nothrow: the format list is optional, a failed install says so namespace mm::platform { +namespace { + +constexpr char kTag[] = "usbvideo"; +// A grabber lists its modes largest first, so a cap that is too low hides exactly the cheap ones. +constexpr size_t kMaxFormats = 64; + +struct Capture { + uvc_host_stream_hdl_t stream = nullptr; + jpeg_decoder_handle_t jpeg = nullptr; + /// The frame the UVC callback handed over, or null; exchanged so the callback never blocks. + std::atomic pending{nullptr}; + SemaphoreHandle_t wake = nullptr; ///< the UVC callback to whoever awaits a frame + /// Decoded RGB888 at the negotiated size, from jpeg_alloc_decoder_mem for its DMA alignment. + uint8_t* rgb = nullptr; + size_t rgbCap = 0; + uint16_t negW = 0, negH = 0; ///< the size the device agreed to stream + /// Set by the event callback when the driver pauses the stream, cleared by the capture task that restarts it. + std::atomic restart{false}; + /// One log per fault per open, beside the counters they explain, so a re-open names a recurring fault again. + bool saidInfo = false, saidOversize = false, saidDecode = false, saidStride = false, saidShrink = false; +}; + +/// A seqlock whose generation is odd during a write, so a reader seeing it move across the copy retries. +struct FormatBank { + std::atomic width[kMaxFormats]; + std::atomic height[kMaxFormats]; + std::atomic fps[kMaxFormats]; + std::atomic interval[kMaxFormats]; ///< as published, since `fps` is rounded for the UI + std::atomic count{0}; +}; + +/// What the capture keeps while it is installed, about 3 KB: on the heap, so a board that never selects the capture holds none of it. +struct Bus { + uvc_host_frame_info_t frameList[kMaxFormats]; // too big for the driver task's stack + FormatBank bank; + usbvideo::Mode snapModes[kMaxFormats]; ///< the bank's last snapshot, taken on the render thread only + uint32_t snapIntervals[kMaxFormats]; +}; +Bus* bus = nullptr; // allocated by the install, freed by videoCaptureShutdown +// Only grows, so a snapshot of one device can never be confirmed by another's; odd while a write is in progress. +std::atomic formatGen{0}; +// The device the bank describes, so the open reaches THAT one rather than whichever the driver finds first. +std::atomic formatDevAddr{UVC_HOST_ANY_DEV_ADDR}; +constexpr uint8_t kStreamIndex = 0; // only a device's first streaming function is ever opened + +// Which step refused the last open, so the status names the cause rather than guessing at the mode list. +const char* openFault = nullptr; +bool fail(const char* why) { + openFault = why; + return false; +} + +bool hostReady = false; +bool uvcReady = false; +std::atomic pumpRunning{false}; // until the pump has uninstalled the host library +// Set when a shutdown hands the teardown to the pump, cleared when the pump has finished it. Distinct from pumpRunning, which holds for every working capture. +std::atomic tearingDown{false}; +bool claimed = false; // one owner at a time, as a device is claimed once + +// Counted on the UVC task, read from the render tick: relaxed is enough for a diagnostic. +std::atomic statDecoded{0}, statDropped{0}, statFailed{0}; + +// usb_host_lib_handle_events() is where enumeration and the port state machine run, and it blocks. The UVC driver is the only client, so its uninstall ends the pump, which uninstalls the library, as the MIDI host does. +void pumpTask(void*) { + bool noClients = false; + for (;;) { + uint32_t flags = 0; + usb_host_lib_handle_events(portMAX_DELAY, &flags); + if (flags & USB_HOST_LIB_EVENT_FLAGS_NO_CLIENTS) { + noClients = true; + // Power the port off first: a grabber still on it is enumerated again while the stack is torn down, and the uninstall aborts. + usb_host_lib_set_root_port_power(false); + if (usb_host_device_free_all() == ESP_OK) break; // nothing left to free + } + if ((flags & USB_HOST_LIB_EVENT_FLAGS_ALL_FREE) && noClients) break; + } + if (usb_host_uninstall() != ESP_OK) ESP_LOGE(kTag, "USB host did not uninstall"); + tearingDown.store(false, std::memory_order_release); // the next install may proceed + pumpRunning.store(false, std::memory_order_release); + vTaskDelete(nullptr); +} + +// Installed while the capture is the source, so the format list fills as a device enumerates and an open fails fast; videoCaptureShutdown removes it. +bool ensureUsbHost() { + if (hostReady) return true; + if (tearingDown.load(std::memory_order_acquire)) return fail("the USB host is still shutting down"); + if (!bus) bus = new (std::nothrow) Bus(); + if (!bus) { + ESP_LOGE(kTag, "no memory for the format list"); + return fail("no memory for the format list"); + } + usb_host_config_t hostCfg = {}; + hostCfg.intr_flags = ESP_INTR_FLAG_LOWMED; + if (usb_host_install(&hostCfg) != ESP_OK) { + ESP_LOGE(kTag, "usb_host_install failed"); + return fail("the USB host did not install"); + } + // Core 1, with the UVC driver and the capture task, so no USB work lands on the render core. + pumpRunning.store(true, std::memory_order_release); + if (xTaskCreatePinnedToCore(pumpTask, "usbpump", 4 * 1024, nullptr, 4, nullptr, 1) != pdPASS) { + pumpRunning.store(false, std::memory_order_release); + ESP_LOGE(kTag, "no USB event task"); + usb_host_uninstall(); + return fail("no task for the USB host"); + } + hostReady = true; + return true; +} + +// Runs on the UVC driver task, so the ordinary API is safe; it only hands the frame over. +bool onFrame(const uvc_host_frame_t* frame, void* ctx) { + auto* cap = static_cast(ctx); + uvc_host_frame_t* expected = nullptr; + // Take the slot only if free. Returning false keeps the frame, so the loser must return true or the driver runs dry. + if (!cap->pending.compare_exchange_strong(expected, const_cast(frame))) { + statDropped.fetch_add(1, std::memory_order_relaxed); + return true; + } + xSemaphoreGive(cap->wake); + return false; +} + +// One row per (resolution, rate) pair: a device listing a resolution once with several intervals would otherwise offer only its default. +void addAdvertised(const uvc_host_frame_info_t& info, uint32_t interval, size_t& n) { + if (interval == 0) return; + if (n >= kMaxFormats) { + // Named here rather than on the descriptor count, which a device's rates multiply before this fills. + ESP_LOGW(kTag, "mode list full at %u rates; the rest are not offered", static_cast(kMaxFormats)); + return; + } + const uint16_t w = static_cast(info.h_res), h = static_cast(info.v_res); + const uint8_t fps = usbvideo::fpsFrom(interval); + bus->bank.width[n].store(w, std::memory_order_relaxed); + bus->bank.height[n].store(h, std::memory_order_relaxed); + bus->bank.fps[n].store(fps, std::memory_order_relaxed); + bus->bank.interval[n].store(interval, std::memory_order_relaxed); + n++; + ESP_LOGI(kTag, "offers MJPEG %ux%u @ %u fps", w, h, fps); +} + +// A continuous-range device names no rate of its own, so each rate a picture asks for becomes a rung. +void addRange(const uvc_host_frame_info_t& info, size_t& n) { + for (uint8_t rate : usbvideo::kOfferedRates) { + const uint32_t rung = usbvideo::rungFor(rate, info.interval_min, info.interval_max, info.interval_step); + if (rung) addAdvertised(info, rung, n); + } +} + +// Every MJPEG row of the read frame list, expanded one row per rate, answering how many were kept. +size_t publishMjpegRows(size_t count) { + size_t n = 0; + for (size_t i = 0; i < count; i++) { + const uvc_host_frame_info_t& info = bus->frameList[i]; + if (info.format != UVC_VS_FORMAT_MJPEG) continue; // nothing else is decodable here + if (info.interval_type == 0) { // a continuous range: the rungs a picture asks for, not only its ends + addRange(info, n); + continue; + } + const uint8_t rates = info.interval_type < CONFIG_UVC_INTERVAL_ARRAY_SIZE ? info.interval_type + : CONFIG_UVC_INTERVAL_ARRAY_SIZE; + for (uint8_t j = 0; j < rates; j++) addAdvertised(info, info.interval[j], n); + } + return n; +} + +// Runs on the UVC driver task when a device enumerates, before a stream is opened, which is what makes the list available even when the open then fails. +void onDriverEvent(const uvc_host_driver_event_data_t* event, void*) { + if (event->type != UVC_HOST_DRIVER_EVENT_DEVICE_CONNECTED) return; + if (event->device_connected.uvc_stream_index != kStreamIndex) return; + + const uint8_t devAddr = event->device_connected.dev_addr; + const size_t offered = event->device_connected.frame_info_num; + if (offered > kMaxFormats) ESP_LOGW(kTag, "device offers %u modes; keeping the first %u", + static_cast(offered), static_cast(kMaxFormats)); + if (!bus) return; // a shutdown freed the bank: this device belongs to the install that follows + std::memset(bus->frameList, 0, sizeof bus->frameList); // a short list must not read the last device's rows + size_t count = kMaxFormats; + if (uvc_host_get_frame_list(devAddr, kStreamIndex, + reinterpret_cast(bus->frameList), &count) != ESP_OK) { + ESP_LOGW(kTag, "device connected but its frame list could not be read"); + return; + } + + formatGen.fetch_add(1, std::memory_order_release); // odd: a rewrite is in progress + formatDevAddr.store(devAddr, std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_release); + const size_t n = publishMjpegRows(count); + bus->bank.count.store(n, std::memory_order_relaxed); + formatGen.fetch_add(1, std::memory_order_release); // even again: the list is settled +} + +// Every way a stream dies or starves. A paused stream never restarts, so it is named rather than left to read as healthy; an unplug empties the bank so the next open fails fast instead of waiting. +void onEvent(const uvc_host_stream_event_data_t* event, void* ctx) { + switch (event->type) { + case UVC_HOST_DEVICE_DISCONNECTED: + ESP_LOGW(kTag, "capture device disconnected"); + // A rewrite to an empty list: the generation moves, so the Picture service closes the stream and the port recycles for a replug. + formatGen.fetch_add(1, std::memory_order_release); + formatDevAddr.store(UVC_HOST_ANY_DEV_ADDR, std::memory_order_relaxed); + if (bus) bus->bank.count.store(0, std::memory_order_relaxed); + formatGen.fetch_add(1, std::memory_order_release); + break; + case UVC_HOST_TRANSFER_ERROR: + statFailed.fetch_add(1, std::memory_order_relaxed); + ESP_LOGW(kTag, "transfer error: the stream is paused, and the capture task restarts it"); + // The restart runs on the capture task: starting a stream from the driver's own callback would re-enter it. + if (auto* cap = static_cast(ctx)) cap->restart.store(true, std::memory_order_release); + break; + case UVC_HOST_FRAME_BUFFER_OVERFLOW: + statFailed.fetch_add(1, std::memory_order_relaxed); + break; + case UVC_HOST_FRAME_BUFFER_UNDERFLOW: + statDropped.fetch_add(1, std::memory_order_relaxed); + break; + default: + break; + } +} + +// RGB888 bytes for a w x h JPEG, both axes padded to the widest MCU, which is what sizes the buffer. +size_t decodedBytes(uint16_t w, uint16_t h) { + return static_cast(usbvideo::alignUp(w, 16)) * usbvideo::alignUp(h, 16) * 3u; +} + +// Warn on the FIRST of each kind only: a drop repeats at frame rate, and the counters carry the rate. +void warnOnce(bool& said, const char* what) { + if (said) return; + said = true; + ESP_LOGW(kTag, "%s - first occurrence; see the Picture status for the running count", what); +} + +// The decoder takes no scale factor and writes only into its own aligned buffer. So a frame lands there full size, then averages down into the caller's, which is where the fit to its ceiling happens. +bool decodeAndShrink(Capture& cap, uvc_host_frame_t* frame, const Sink& sink) { + const size_t len = usbvideo::stripPayloadHeaders(frame->data, frame->data_len); + // Dimensions from the bitstream, not from the request: a device may negotiate something else. Read here rather than by the decoder's header call, which allocates per frame. + usbvideo::JpegShape shape; + if (!usbvideo::jpegShape(frame->data, len, shape)) { + statFailed.fetch_add(1, std::memory_order_relaxed); + warnOnce(cap.saidInfo, "frame is not readable JPEG (payload stride mis-detected?)"); + return false; + } + const uint16_t w = shape.width, h = shape.height; + if (decodedBytes(w, h) > cap.rgbCap) { + statFailed.fetch_add(1, std::memory_order_relaxed); + warnOnce(cap.saidOversize, "frame exceeds the buffer sized at open"); + return false; + } + // The picture is exactly the size asked for, as the test pattern's is. The ranges below tile the frame whatever the ratio, so a 4:3 source fills a wider picture rather than leaving it short. + if (!sink.width || !sink.height) { + statFailed.fetch_add(1, std::memory_order_relaxed); + warnOnce(cap.saidShrink, "the picture has no room for a frame: raise its size"); + return false; + } + jpeg_decode_cfg_t decodeCfg = {}; + decodeCfg.output_format = JPEG_DECODE_OUT_FORMAT_RGB888; + decodeCfg.rgb_order = JPEG_DEC_RGB_ELEMENT_ORDER_RGB; + decodeCfg.conv_std = JPEG_YUV_RGB_CONV_STD_BT601; + uint32_t outSize = 0; + if (jpeg_decoder_process(cap.jpeg, &decodeCfg, frame->data, len, cap.rgb, cap.rgbCap, &outSize) + != ESP_OK) { + statFailed.fetch_add(1, std::memory_order_relaxed); + warnOnce(cap.saidDecode, "the decoder refused a bitstream whose header it accepted"); + return false; + } + const size_t stride = usbvideo::strideFor(w, h, shape.mcuW, shape.mcuH, outSize); + if (!stride) { + statFailed.fetch_add(1, std::memory_order_relaxed); + warnOnce(cap.saidStride, "the decoder wrote a size its sampling does not explain"); + return false; + } + usbvideo::areaAverage(cap.rgb, stride, w, h, sink); + statDecoded.fetch_add(1, std::memory_order_relaxed); + return true; +} + +// --- Open and close ------------------------------------------------------------------------------ -#if !defined(CONFIG_MM_USB_VIDEO) +// Installed once, like the host library: the format list belongs to the bus rather than to one open. +bool ensureUvcHost() { + if (uvcReady) return true; + uvc_host_driver_config_t driverCfg = {}; + driverCfg.driver_task_stack_size = 4 * 1024; + driverCfg.driver_task_priority = 5; + driverCfg.xCoreID = 1; // core 1, with the pump and the capture task + driverCfg.create_background_task = true; + driverCfg.event_cb = onDriverEvent; // fills the format bank as soon as a device enumerates + if (uvc_host_install(&driverCfg) != ESP_OK) { + ESP_LOGE(kTag, "uvc_host_install failed"); + return fail("the UVC driver did not install"); + } + uvcReady = true; + return true; +} -// An image built without the capture reports no device, which the Picture service shows as a status. +bool createJpeg(Capture& cap) { + jpeg_decode_engine_cfg_t jpegCfg = {}; + // The timeout aborts the 2D-DMA mid-frame, and a 1080p RGB frame takes about 34 ms to write, so the bound is an order above that. + jpegCfg.timeout_ms = 200; + if (jpeg_new_decoder_engine(&jpegCfg, &cap.jpeg) == ESP_OK) return true; + ESP_LOGE(kTag, "no JPEG decoder engine"); + return false; +} + +uint8_t snapDevAddr = UVC_HOST_ANY_DEV_ADDR; ///< the device the snapshot describes, read under the same hold + +// One snapshot of the bank under its seqlock, so a mode is never chosen from a half-rewritten list. Taken on the render thread in init, the only caller, so the snapshot needs no lock of its own. +size_t snapshotModes() { + for (int tries = 0; tries < 8; tries++) { + const uint32_t before = formatGen.load(std::memory_order_acquire); + if (before & 1u) continue; // a rewrite in flight + size_t n = bus->bank.count.load(std::memory_order_relaxed); + if (n == 0) return 0; // no device listed + if (n > kMaxFormats) n = kMaxFormats; + for (size_t i = 0; i < n; i++) { + bus->snapModes[i].width = bus->bank.width[i].load(std::memory_order_relaxed); + bus->snapModes[i].height = bus->bank.height[i].load(std::memory_order_relaxed); + bus->snapModes[i].fps = bus->bank.fps[i].load(std::memory_order_relaxed); + bus->snapIntervals[i] = bus->bank.interval[i].load(std::memory_order_relaxed); + } + snapDevAddr = formatDevAddr.load(std::memory_order_relaxed); + std::atomic_thread_fence(std::memory_order_acquire); + if (formatGen.load(std::memory_order_relaxed) == before) return n; // no write overlapped + } + return 0; +} + +// The snapshot's own interval as the float the driver compares against; 0 asks for the device default. +float exactFpsFor(size_t modes, uint16_t w, uint16_t h, uint8_t fps) { + for (size_t i = 0; i < modes; i++) + if (bus->snapModes[i].width == w && bus->snapModes[i].height == h && bus->snapModes[i].fps == fps + && bus->snapIntervals[i]) + return 10000000.0f / static_cast(bus->snapIntervals[i]); + return 0.0f; +} + +bool openStream(Capture& cap, size_t modes, uint16_t width, uint16_t height, uint8_t fps) { + uvc_host_stream_config_t streamCfg = {}; + streamCfg.event_cb = onEvent; + streamCfg.frame_cb = onFrame; + streamCfg.user_ctx = ∩ + streamCfg.usb.dev_addr = snapDevAddr; + streamCfg.usb.vid = UVC_HOST_ANY_VID; + streamCfg.usb.pid = UVC_HOST_ANY_PID; + streamCfg.usb.uvc_stream_index = kStreamIndex; + streamCfg.vs_format.h_res = width; + streamCfg.vs_format.v_res = height; + // The device's own interval, not the rounded fps: the driver matches within 0.0001 fps, so a 59.94 mode never matches a requested 60. + streamCfg.vs_format.fps = exactFpsFor(modes, width, height, fps); + streamCfg.vs_format.format = UVC_VS_FORMAT_MJPEG; + // urb_size left at 0 (4x MPS), the driver's default. number_of_urbs has none: 0 is malloc(0). + streamCfg.advanced.number_of_urbs = 4; + streamCfg.advanced.number_of_frame_buffers = 3; + streamCfg.advanced.frame_size = 0; // 0 takes dwMaxVideoFrameSize from the negotiation: the device's own figure, not a guess + streamCfg.advanced.frame_heap_caps = MALLOC_CAP_SPIRAM; // keep the internal heap for USB and WiFi + + // A device has enumerated by now, so the driver has only to claim it, and half a second bounds a device that went away since the snapshot. + if (uvc_host_stream_open(&streamCfg, pdMS_TO_TICKS(500), &cap.stream) != ESP_OK) { + ESP_LOGW(kTag, "no UVC device offering MJPEG %ux%u", width, height); + return false; + } + return true; +} + +// Sized from what the device agreed to, so no frame needs more room than the buffer has. +bool sizeBuffer(Capture& cap) { + uvc_host_stream_format_t got = {}; + if (uvc_host_stream_format_get(cap.stream, &got) != ESP_OK) { + ESP_LOGE(kTag, "cannot read the negotiated format"); + return false; + } + ESP_LOGI(kTag, "streaming MJPEG %ux%u @ %.1f fps", got.h_res, got.v_res, got.fps); + cap.negW = static_cast(got.h_res); + cap.negH = static_cast(got.v_res); + jpeg_decode_memory_alloc_cfg_t memCfg = {}; + memCfg.buffer_direction = JPEG_DEC_ALLOC_OUTPUT_BUFFER; + cap.rgb = static_cast( + jpeg_alloc_decoder_mem(decodedBytes(cap.negW, cap.negH), &memCfg, &cap.rgbCap)); + if (cap.rgb) return true; + cap.rgbCap = 0; + ESP_LOGE(kTag, "no memory for the %ux%u decode buffer", cap.negW, cap.negH); + return false; +} + +// The counters the status reads, zeroed per open. +void resetStats() { + statDecoded.store(0, std::memory_order_relaxed); + statDropped.store(0, std::memory_order_relaxed); + statFailed.store(0, std::memory_order_relaxed); +} + +// In this order on purpose: the decode engine and the signal first, then the stream, then its buffer. +bool bringUp(Capture& cap, size_t modes, uint16_t width, uint16_t height, uint8_t fps) { + cap.wake = xSemaphoreCreateBinary(); + if (!cap.wake) return fail("no semaphore for the capture"); + if (!createJpeg(cap)) return fail("the JPEG decoder did not start"); + if (!openStream(cap, modes, width, height, fps)) return fail("the device refused the stream"); + if (!sizeBuffer(cap)) return fail("no memory for the decoded frame"); + if (uvc_host_stream_start(cap.stream) != ESP_OK) return fail("the stream did not start"); + return true; +} + +} // namespace + +bool videoCaptureInit(VideoCaptureHandle& handle, uint16_t width, uint16_t height, uint8_t fps) { + if (handle.impl) return true; + openFault = nullptr; // this open answers for itself + if (claimed) return fail("a capture is already open"); // one owner at a time, as the seam documents + if (!ensureUsbHost() || !ensureUvcHost()) return false; + // No device has enumerated, so there is nothing to open: failing here keeps a board without a grabber from stalling on the open's timeout. + const size_t modes = snapshotModes(); + if (!modes) return fail("no grabber attached"); + auto* cap = new Capture(); + handle.impl = cap; // every failure below unwinds through videoCaptureDeinit + resetStats(); // per open, as the seam documents them, so a re-open inherits no losses + // Nothing advertised covers the picture, so the open would spend its whole timeout on a mode no device offers. + if (!usbvideo::pickMode(bus->snapModes, modes, width, height, fps)) { + ESP_LOGW(kTag, "no advertised mode covers %ux%u", width, height); + videoCaptureDeinit(handle); + return fail("no advertised mode covers the picture"); + } + if (bringUp(*cap, modes, width, height, fps)) { + claimed = true; + return true; + } + videoCaptureDeinit(handle); + return false; +} + +bool videoCaptureAwaitFrame(VideoCaptureHandle& handle, uint32_t timeoutMs) { + auto* cap = static_cast(handle.impl); + if (!cap) return false; + // A transfer error leaves the stream paused, and only a start brings frames back. + if (cap->restart.exchange(false, std::memory_order_acq_rel) + && uvc_host_stream_start(cap->stream) != ESP_OK) + ESP_LOGW(kTag, "the paused stream refused to restart"); + // A token left by a frame already taken would otherwise return at once with nothing to decode. + if (cap->pending.load(std::memory_order_acquire)) { + xSemaphoreTake(cap->wake, 0); + return true; + } + if (xSemaphoreTake(cap->wake, pdMS_TO_TICKS(timeoutMs)) != pdTRUE) return false; + return cap->pending.load(std::memory_order_acquire) != nullptr; +} + +bool videoCaptureDecodeInto(VideoCaptureHandle& handle, uint8_t* dst, uint16_t maxWidth, uint16_t maxHeight, + uint16_t& outWidth, uint16_t& outHeight, uint8_t floor) { + auto* cap = static_cast(handle.impl); + if (!cap) return false; + uvc_host_frame_t* frame = cap->pending.exchange(nullptr, std::memory_order_acq_rel); + if (!frame) return false; + const bool ok = decodeAndShrink(*cap, frame, Sink{dst, maxWidth, maxHeight, floor}); + uvc_host_frame_return(cap->stream, frame); // returned either way, or the driver runs out + if (ok) { outWidth = maxWidth; outHeight = maxHeight; } // the sampler tiles the frame, so the picture is the size asked for + return ok; +} + +const char* videoCaptureFault() MM_NONBLOCKING { return openFault; } + +bool videoCaptureSettling() MM_NONBLOCKING { return tearingDown.load(std::memory_order_acquire); } + +uint32_t videoCaptureFormatGeneration() MM_NONBLOCKING { + if (!bus || bus->bank.count.load(std::memory_order_relaxed) == 0) return 0; // no device listed + return formatGen.load(std::memory_order_acquire); +} + +void videoCaptureDeinit(VideoCaptureHandle& handle) { + auto* cap = static_cast(handle.impl); + if (!cap) return; + // Stop the stream first, so no frame lands mid-teardown; the caller's task has stopped calling in. + if (cap->stream) uvc_host_stream_stop(cap->stream); + if (uvc_host_frame_t* frame = cap->pending.exchange(nullptr)) uvc_host_frame_return(cap->stream, frame); + if (cap->stream) uvc_host_stream_close(cap->stream); + if (cap->jpeg) jpeg_del_decoder_engine(cap->jpeg); + if (cap->wake) vSemaphoreDelete(cap->wake); + free(cap->rgb); // free(nullptr) is a no-op, so a failed open is fine + delete cap; + handle.impl = nullptr; + claimed = false; // the device is free for the next open +} + +// Starts the teardown and returns: the pump runs it to completion, and the next open waits on pumpRunning rather than installing over it. Nothing here blocks, since prepare runs on the render thread. +void videoCaptureShutdown() { + if (claimed) return; // a capture is open: its owner deinits first + if (uvcReady) { + if (uvc_host_uninstall() != ESP_OK) { + ESP_LOGW(kTag, "the UVC driver did not uninstall"); + return; + } + uvcReady = false; + } + // The host goes with the driver, and the pump unwinds the rest: until it has, tearingDown refuses an install over it. + if (pumpRunning.load(std::memory_order_acquire)) tearingDown.store(true, std::memory_order_release); + hostReady = false; + // Freed here, with the driver that filled it uninstalled and the render thread the only other reader, so the next install starts from an empty bank. + delete bus; + bus = nullptr; + formatDevAddr.store(UVC_HOST_ANY_DEV_ADDR, std::memory_order_relaxed); +} + +VideoCaptureStats videoCaptureStats() MM_NONBLOCKING { + VideoCaptureStats s; + s.decoded = statDecoded.load(std::memory_order_relaxed); + s.dropped = statDropped.load(std::memory_order_relaxed); + s.failed = statFailed.load(std::memory_order_relaxed); + return s; +} + +} // namespace mm::platform + +#else // no High-Speed USB host, no JPEG decoder, or the capture left out of this image + +namespace mm::platform { + +// An image without the capture reports no device, which the Picture service shows as a status. uint32_t videoCaptureFormatGeneration() MM_NONBLOCKING { return 0; } +const char* videoCaptureFault() MM_NONBLOCKING { return nullptr; } +bool videoCaptureSettling() MM_NONBLOCKING { return false; } bool videoCaptureInit(VideoCaptureHandle&, uint16_t, uint16_t, uint8_t) { return false; } bool videoCaptureAwaitFrame(VideoCaptureHandle&, uint32_t) { return false; } -bool videoCaptureDecodeInto(VideoCaptureHandle&, uint8_t*, uint16_t, uint16_t, uint16_t&, uint16_t&) { return false; } +bool videoCaptureDecodeInto(VideoCaptureHandle&, uint8_t*, uint16_t, uint16_t, uint16_t&, uint16_t&, + uint8_t) { return false; } void videoCaptureDeinit(VideoCaptureHandle&) {} +void videoCaptureShutdown() {} VideoCaptureStats videoCaptureStats() MM_NONBLOCKING { return VideoCaptureStats{}; } -#endif +} // namespace mm::platform -} // namespace mm::platform +#endif diff --git a/src/platform/platform.h b/src/platform/platform.h index e26c4477..9888936f 100644 --- a/src/platform/platform.h +++ b/src/platform/platform.h @@ -1497,7 +1497,7 @@ struct VideoCaptureFormat { uint8_t fps = 0; ///< frames a second offered at that size }; -/// Bumped whenever that list is rewritten, which is when a device enumerates; 0 until then. +/// Moves whenever that list is rewritten, a device enumerating or leaving, and never returns to a value it had; 0 while no device is listed. uint32_t videoCaptureFormatGeneration() MM_NONBLOCKING; /// Claim the first grabber and stream MJPEG near this mode; false when none is attached or none fits. @@ -1506,13 +1506,22 @@ bool videoCaptureInit(VideoCaptureHandle& h, uint16_t width, uint16_t height, ui /// Wait up to `timeoutMs` for the next frame; true once one is ready to decode. bool videoCaptureAwaitFrame(VideoCaptureHandle& h, uint32_t timeoutMs); -/// Decode the frame videoCaptureAwaitFrame announced into `dst`, scaled down to fit `maxWidth` x `maxHeight` x 3 bytes, top row first. +/// Decode the frame videoCaptureAwaitFrame announced into `dst`, no larger than `maxWidth` x `maxHeight` x 3 bytes, counting a source byte at or below `floor` as black. bool videoCaptureDecodeInto(VideoCaptureHandle& h, uint8_t* dst, uint16_t maxWidth, uint16_t maxHeight, - uint16_t& outWidth, uint16_t& outHeight); + uint16_t& outWidth, uint16_t& outHeight, uint8_t floor); + +/// Which step the last open failed at, or null where none has failed since; a static string the caller may show. +const char* videoCaptureFault() MM_NONBLOCKING; /// Release the device and free its buffers. void videoCaptureDeinit(VideoCaptureHandle& h); +/// Start uninstalling what the capture installed to see devices, once no capture is open. It returns before the teardown finishes, so an init is refused while videoCaptureSettling() holds. +void videoCaptureShutdown(); + +/// Whether a teardown is still unwinding, so an open would be refused rather than failing for a reason worth naming. +bool videoCaptureSettling() MM_NONBLOCKING; + /// Why frames did not reach the lights, cumulative since the device was opened. struct VideoCaptureStats { uint32_t decoded = 0; ///< frames decoded into a picture diff --git a/src/platform/teensy/platform_teensy.cpp b/src/platform/teensy/platform_teensy.cpp index fdbd3fa2..6179c6b3 100644 --- a/src/platform/teensy/platform_teensy.cpp +++ b/src/platform/teensy/platform_teensy.cpp @@ -779,9 +779,13 @@ bool usbMidiWrite(const uint8_t (*)[4], size_t) { return false; } // No USB host on this chip, so a capture source finds no device and says so in its status. uint32_t videoCaptureFormatGeneration() MM_NONBLOCKING { return 0; } bool videoCaptureInit(VideoCaptureHandle&, uint16_t, uint16_t, uint8_t) { return false; } +const char* videoCaptureFault() MM_NONBLOCKING { return nullptr; } +bool videoCaptureSettling() MM_NONBLOCKING { return false; } bool videoCaptureAwaitFrame(VideoCaptureHandle&, uint32_t) { return false; } -bool videoCaptureDecodeInto(VideoCaptureHandle&, uint8_t*, uint16_t, uint16_t, uint16_t&, uint16_t&) { return false; } +bool videoCaptureDecodeInto(VideoCaptureHandle&, uint8_t*, uint16_t, uint16_t, uint16_t&, uint16_t&, + uint8_t) { return false; } void videoCaptureDeinit(VideoCaptureHandle&) {} +void videoCaptureShutdown() {} VideoCaptureStats videoCaptureStats() MM_NONBLOCKING { return VideoCaptureStats{}; } } // namespace mm::platform diff --git a/src/platform/usbvideo_frames.h b/src/platform/usbvideo_frames.h index 2f3302a4..582eb4a3 100644 --- a/src/platform/usbvideo_frames.h +++ b/src/platform/usbvideo_frames.h @@ -102,6 +102,21 @@ inline uint8_t fpsFrom(uint32_t interval) { return interval ? static_cast((kTicksPerSecond + interval / 2) / interval) : 0; } +/// The rates a continuous range is asked for, since such a device names no rate of its own and a picture wants a familiar one. +constexpr uint8_t kOfferedRates[] = {60, 50, 30, 25, 24, 15, 10, 5}; + +/// The frame interval, in 100 ns ticks, that asks for `fps`. +inline uint32_t intervalFor(uint8_t fps) { return fps ? 10000000u / fps : 0; } + +/// The rung of a continuous range that holds `fps`, or 0 where the range does not reach it. Rounded away from `min`, so the rate lands at or under the one asked for rather than over it. +inline uint32_t rungFor(uint8_t fps, uint32_t min, uint32_t max, uint32_t step) { + const uint32_t want = intervalFor(fps); + if (!want || !min || max < min || want < min || want > max) return 0; + if (!step) return want; // a range with no step reaches every interval in it + const uint32_t rung = min + ((want - min + step - 1) / step) * step; + return rung <= max ? rung : max; +} + /// Of two advertised rates, the one to ask for: the fastest at or below `wanted`, else the slowest. inline uint8_t betterFps(uint8_t have, uint8_t row, uint8_t wanted) { if (!have) return row; @@ -142,6 +157,45 @@ inline bool pickMode(const Mode* modes, size_t count, uint16_t& width, uint16_t& // --- The decoded frame -------------------------------------------------------------------------- +/// A JPEG's size and the MCU its luma sampling pads to: 16x16 for 4:2:0, 16x8 for 4:2:2, 8x8 for 4:4:4. +struct JpegShape { + uint16_t width = 0; ///< pixels across, from the frame header + uint16_t height = 0; ///< pixels down + uint8_t mcuW = 8; ///< the MCU's width + uint8_t mcuH = 8; ///< and its height +}; + +/// The shape from one frame header's fields, `s` at its precision byte and `seg` its length; false when too short to hold one component. +inline bool readFrameHeader(const uint8_t* s, size_t seg, JpegShape& out) { + if (seg < 11) return false; // precision, height, width, count and one component + out.height = static_cast((s[1] << 8) | s[2]); + out.width = static_cast((s[3] << 8) | s[4]); + out.mcuW = static_cast(8 * (s[7] >> 4)); // the first component's horizontal sampling + out.mcuH = static_cast(8 * (s[7] & 0x0F)); // and its vertical + return out.width && out.height && out.mcuW && out.mcuH; +} + +/// Whether `d` opens with a JPEG's start-of-image marker. +inline bool startsImage(const uint8_t* d, size_t len) { return len >= 4 && d[0] == 0xFF && d[1] == 0xD8; } +/// Whether a marker is a baseline, extended or progressive frame header (SOF0 to SOF2). +inline bool isFrameHeader(uint8_t marker) { return marker >= 0xC0 && marker <= 0xC2; } + +/// The shape from a frame header, walking the segments before it; false for anything that is not a JPEG with one before its scan. +inline bool jpegShape(const uint8_t* d, size_t len, JpegShape& out) { + if (!startsImage(d, len)) return false; + size_t i = 2; + while (i + 4 <= len) { + if (d[i] != 0xFF) return false; + const uint8_t marker = d[i + 1]; + if (marker == 0xFF) { i++; continue; } // a fill byte before the marker + const size_t seg = (static_cast(d[i + 2]) << 8) | d[i + 3]; // the segment's length, its own two bytes included + if (seg < 2 || i + 2 + seg > len || marker == 0xDA) return false; // cut short, or the scan began without a frame header + if (isFrameHeader(marker)) return readFrameHeader(d + i + 4, seg, out); + i += 2 + seg; + } + return false; +} + /// `v` rounded up to a multiple of `to`, the MCU boundary a decoder pads a frame's axis to. inline uint32_t alignUp(uint32_t v, uint8_t to) { return (v + to - 1u) / to * to; } @@ -155,13 +209,12 @@ inline size_t strideFor(uint16_t w, uint16_t h, uint8_t mcuW, uint8_t mcuH, uint } /// Each output pixel the mean of its rectangle, through the shared sampler so one fix reaches both paths. -inline void areaAverage(const uint8_t* src, size_t stride, uint16_t w, uint16_t h, uint16_t ow, - uint16_t oh, uint8_t* dst) { - for (uint16_t y = 0; y < oh; y++) { - const Span ry = cellSpan(y, oh, h); - for (uint16_t x = 0; x < ow; x++) { - const RGB mean = areaMean(src, stride, cellSpan(x, ow, w), ry); - uint8_t* out = dst + (static_cast(y) * ow + x) * 3; +inline void areaAverage(const uint8_t* src, size_t stride, uint16_t w, uint16_t h, const Sink& sink) { + for (uint16_t y = 0; y < sink.height; y++) { + const Span ry = cellSpan(y, sink.height, h); + for (uint16_t x = 0; x < sink.width; x++) { + const RGB mean = areaMean(src, stride, cellSpan(x, sink.width, w), ry, sink.floor); + uint8_t* out = sink.rgb + (static_cast(y) * sink.width + x) * 3; out[0] = mean.r; out[1] = mean.g; out[2] = mean.b; diff --git a/test/unit/core/unit_AreaSample.cpp b/test/unit/core/unit_AreaSample.cpp index 8617aa1f..eae5bbe0 100644 --- a/test/unit/core/unit_AreaSample.cpp +++ b/test/unit/core/unit_AreaSample.cpp @@ -70,6 +70,15 @@ TEST_CASE("a uniform frame averages to itself") { for (uint8_t v : dst) CHECK(v == 123); } +// The floor's edge: a byte at the floor reads as black, one step above it passes through. +TEST_CASE("a byte at the noise floor reads as black, one above it passes through") { + const uint8_t at[3] = {4, 4, 4}, above[3] = {5, 5, 5}; + const mm::RGB a = mm::areaMean(at, 3, mm::Span{0, 1}, mm::Span{0, 1}, /*floor=*/4); + const mm::RGB b = mm::areaMean(above, 3, mm::Span{0, 1}, mm::Span{0, 1}, /*floor=*/4); + CHECK(a.r == 0); + CHECK(b.r == 5); +} + // Averaging in linear light rather than on the bytes, so a shrink does not darken the picture. TEST_CASE("a half-black half-white frame averages above the arithmetic mean") { const uint16_t w = 2, h = 2; diff --git a/test/unit/core/unit_PictureService.cpp b/test/unit/core/unit_PictureService.cpp index f1652219..0b69e789 100644 --- a/test/unit/core/unit_PictureService.cpp +++ b/test/unit/core/unit_PictureService.cpp @@ -3,6 +3,7 @@ /// The picture source: the test pattern it publishes, the generation that tells a consumer it changed, and the empty frame when nothing publishes. #include "doctest.h" +#include "core/module/Scheduler.h" #include "core/services/PictureService.h" #include @@ -50,6 +51,15 @@ TEST_CASE("the test pattern publishes the test card") { CHECK(is(5, 2, 128, 128, 128)); // a grid square's gray } +// The capture needs the transport to link, so a build without it offers the test pattern alone rather than a source that cannot open. +TEST_CASE("the source list holds only what the build can open") { + Picture p(16, 9); + uint8_t options = 0; + for (uint8_t i = 0; i < p.svc.controls().count(); i++) + if (std::strcmp(p.svc.controls()[i].name, "source") == 0) options = p.svc.controls()[i].max; + CHECK(options == (mm::platform::hasUsbVideo ? mm::PictureService::kSourceCount : 1)); +} + // A consumer compares generations to skip a picture it already sampled, so a new size is a new generation. TEST_CASE("a new picture size publishes a new generation") { Picture p(16, 9); @@ -177,6 +187,14 @@ bool startCapture(Picture& p, uint32_t& was) { was = current().generation; // prepareBuffers published the resized picture; a frame comes after it return publishedPast(was); } + +// A capture publishing a picture with a short stale window, which both drop tests open from. +void openWithShortStale(Picture& p) { + uint32_t was = 0; + REQUIRE(startCapture(p, was)); + REQUIRE(current().rgb != nullptr); + p.svc.staleMs = 100; +} } // namespace // The whole picture, as the test pattern fills it. A 4:3 frame fills a 2:1 picture rather than leaving it short. @@ -195,6 +213,23 @@ TEST_CASE("the usb capture source publishes the grabber's frames at the size the p.svc.release(); } +// The platform names the step a failed open stopped at, and the status shows it rather than guessing at the mode list. +TEST_CASE("a failed open reports the step the platform stopped at") { + const mm::platform::VideoCaptureFormat f{640, 480, 30}; + mm::platform::setTestVideoCaptureFormats(&f, 1); + mm::platform::VideoCaptureHandle held; + REQUIRE(mm::platform::videoCaptureInit(held, 64, 48, 30)); // the one owner, taken before the service asks + Picture p(64, 48); + p.svc.source = mm::PictureService::kUsbCapture; + p.svc.prepare(); + REQUIRE(p.svc.status() != nullptr); + CHECK(std::string(p.svc.status()) == "a capture is already open"); + CHECK(p.svc.severity() == mm::MoonModule::Severity::Warning); + p.svc.release(); + mm::platform::videoCaptureDeinit(held); + mm::platform::setTestVideoCaptureFormats(nullptr, 0); +} + // A board whose USB port has nothing on it says so, rather than failing or waiting on a device. TEST_CASE("the usb capture source without a grabber reports it") { mm::platform::setTestVideoCaptureFormats(nullptr, 0); @@ -203,6 +238,7 @@ TEST_CASE("the usb capture source without a grabber reports it") { p.svc.prepare(); REQUIRE(p.svc.status() != nullptr); CHECK(std::string(p.svc.status()) == "no grabber attached"); + CHECK(p.svc.severity() == mm::MoonModule::Severity::Warning); p.svc.release(); } @@ -221,16 +257,18 @@ TEST_CASE("a frame the decoder refuses publishes nothing, and recovery publishes p.svc.release(); } -// The status names the size it publishes and what it lost, so a climbing loss is visible without a log. -TEST_CASE("the capture status counts the frames that landed and those lost") { +// The status says whether the capture is live right now, which a total since boot cannot. +TEST_CASE("the capture status reports a live rate, not a total") { Grabber g(320, 240, 30, 5, 5, 5); Picture p(64, 48); uint32_t was = 0; REQUIRE(startCapture(p, was)); const std::string line = captureStatus(p.svc); - CHECK(line.rfind("64 x 48, ", 0) == 0); // the picture's own size, whatever the grabber streams - CHECK(line.find(" frames, 0 lost") != std::string::npos); - CHECK(line.find(" 0 frames") == std::string::npos); // the picture that arrived is counted + CHECK(line.rfind("live, 64 x 48 at ", 0) == 0); // the picture's size, whatever the grabber streams + CHECK(line.find(" at ") != std::string::npos); + CHECK(line.find("fps") != std::string::npos); + CHECK(line.find("skipped") == std::string::npos); // nothing skipped, so the line says nothing of it + CHECK(line.find("frames failing") == std::string::npos); CHECK(line.find("one picture") == std::string::npos); CHECK(p.svc.severity() == mm::MoonModule::Severity::Status); p.svc.release(); @@ -279,3 +317,143 @@ TEST_CASE("switching away from the capture stops its task") { CHECK(px[2] == 128); p.svc.release(); } + +// A frame the decoder refuses is a fault and must warn, where a spare frame must not. +TEST_CASE("a failed frame warns where a spare one does not") { + Grabber g(320, 240, 30, 5, 5, 5); + Picture p(64, 48); + uint32_t was = 0; + REQUIRE(startCapture(p, was)); + REQUIRE(p.svc.severity() == mm::MoonModule::Severity::Status); + mm::platform::setTestVideoCaptureDecodeFails(true); + // The rate is a per-tick delta, so read it on the tick that sees the failures, not after. + bool failing = false; + for (int i = 0; i < 200 && !failing; i++) { + failing = captureStatus(p.svc).find("frames failing") != std::string::npos; + if (!failing) std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + CHECK(failing); + CHECK(p.svc.severity() == mm::MoonModule::Severity::Warning); + p.svc.release(); +} + +// A source that stops leaves its last picture lighting the room. Past staleMs it goes, generation and all. +TEST_CASE("a capture that stops sending has its picture dropped") { + Grabber g(320, 240, 30, 9, 9, 9); + Picture p(64, 48); + openWithShortStale(p); + mm::platform::setTestVideoCaptureDecodeFails(true); // the grabber goes quiet: the gap opens + const uint32_t held = current().generation; + bool dropped = false; + for (int i = 0; i < 200 && !dropped; i++) { + p.svc.tick1s(); + dropped = current().rgb == nullptr; + if (!dropped) std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + CHECK(dropped); + CHECK(current().generation != held); // or an effect's redraw guard would skip the change + p.svc.release(); +} + +// The open zeroes the device's counters, so a re-open that kept its baselines would read a rate of four billion. +TEST_CASE("a re-opened capture reports a plain rate, not an underflowed one") { + Grabber g(320, 240, 30, 5, 5, 5); + Picture p(64, 48); + uint32_t was = 0; + REQUIRE(startCapture(p, was)); + REQUIRE(captureStatus(p.svc).rfind("live, ", 0) == 0); // a first run, so the baselines are set + p.svc.prepare(); // the same source again: stop, then open + const std::string line = captureStatus(p.svc); + CHECK(line.find("4294") == std::string::npos); // no wrapped delta anywhere on the line + CHECK(line.find("frames failing") == std::string::npos); + p.svc.release(); +} + +// A capture whose frames are landing holds its picture through the stale guard, whatever the board's uptime. +TEST_CASE("a capture that has just opened keeps its first picture") { + Grabber g(320, 240, 30, 9, 9, 9); + Picture p(64, 48); + uint32_t was = 0; + REQUIRE(startCapture(p, was)); + p.svc.tick1s(); // the stale guard runs here, with staleMs at its default + CHECK(current().rgb != nullptr); + p.svc.release(); +} + +// A grabber that leaves takes its task with it, so the drop cannot wait on one: its last picture would stay on the lights. +TEST_CASE("a capture whose device is unplugged has its picture dropped") { + mm::Scheduler sched; + Grabber g(320, 240, 30, 9, 9, 9); + Picture p(64, 48); + sched.addModule(&p.svc); + sched.setup(); // the instance the service asks for a rebuild + openWithShortStale(p); + mm::platform::setTestVideoCaptureFormats(nullptr, 0); // the grabber is pulled off the bus + bool dropped = false; + for (int i = 0; i < 200 && !dropped; i++) { + p.svc.tick1s(); + sched.tick(); // the rebuild the tick asked for, run where the Scheduler runs it + dropped = current().rgb == nullptr; + if (!dropped) std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + CHECK(dropped); + // The rebuild must not light the frame again: a writer's picture is published by its own decode. + sched.tick(); + CHECK(current().rgb == nullptr); + p.svc.release(); +} + +// A gap of one frame is ordinary, since the capture runs at its own rate: it must not strobe. +TEST_CASE("an ordinary gap between frames keeps the picture") { + Grabber g(320, 240, 30, 9, 9, 9); + Picture p(64, 48); + uint32_t was = 0; + REQUIRE(startCapture(p, was)); + p.svc.staleMs = 2000; // far longer than the 33 ms the grabber paces at + for (int i = 0; i < 20; i++) { + p.svc.tick1s(); + CHECK(current().rgb != nullptr); + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + p.svc.release(); +} + +// A grabber's black is rarely a 0, and a 1 no screen shows is a lit LED. The floor cuts it. +TEST_CASE("the noise floor cuts a grabber's near-black to black") { + Grabber g(320, 240, 30, 2, 2, 2); // a frame of 2s, which is what a dark source decodes to + Picture p(8, 8); + uint32_t was = 0; + p.svc.noiseFloor = 0; + REQUIRE(startCapture(p, was)); + CHECK(*pixel(current(), 4, 4) == 2); // off: the byte arrives as sent + p.svc.noiseFloor = 4; + p.svc.prepare(); + REQUIRE(publishedPast(current().generation)); + CHECK(*pixel(current(), 4, 4) == 0); // at or under the floor: black, so the lights stay off + p.svc.release(); +} + +// A grabber unplugged mid-stream left the port in recovery, since the stream was never closed, and a replug never enumerated: the device list moving is what closes and re-opens it. +TEST_CASE("a grabber unplugged and plugged back is opened again") { + mm::Scheduler sched; + Grabber g(640, 480, 30, 10, 200, 90); + Picture p(64, 48); + sched.addModule(&p.svc); + sched.setup(); // the instance a service asks for a rebuild + uint32_t was = 0; + REQUIRE(startCapture(p, was)); + + mm::platform::setTestVideoCaptureFormats(nullptr, 0); // unplugged + p.svc.tick1s(); + sched.tick(); // the rebuild it asked for closes the stream + REQUIRE(p.svc.status() != nullptr); + CHECK(std::string(p.svc.status()) == "no grabber attached"); + + const mm::platform::VideoCaptureFormat f{640, 480, 30}; + mm::platform::setTestVideoCaptureFormats(&f, 1); // plugged back + p.svc.tick1s(); + was = current().generation; + sched.tick(); // and the next one opens it again + CHECK(publishedPast(was)); + p.svc.release(); +} diff --git a/test/unit/core/unit_usbvideo_frames.cpp b/test/unit/core/unit_usbvideo_frames.cpp index f770f6eb..bf879cc7 100644 --- a/test/unit/core/unit_usbvideo_frames.cpp +++ b/test/unit/core/unit_usbvideo_frames.cpp @@ -84,6 +84,28 @@ TEST_CASE("the rate asked for wins over a faster one the device offers") { CHECK(betterFps(60, 90, 30) == 60); // neither fits, so the slower } +// A continuous-range device names no rate, so a rung is computed for each rate a picture asks for. +TEST_CASE("a continuous range yields the rung for a rate it covers") { + const uint32_t fast = intervalFor(60), slow = intervalFor(5); // 100 ns ticks: a shorter interval is a faster rate + CHECK(rungFor(30, fast, slow, 1) == intervalFor(30)); + CHECK(rungFor(5, fast, slow, 1) == slow); + CHECK(rungFor(60, fast, slow, 1) == fast); + CHECK(rungFor(90, fast, slow, 1) == 0); // above the range: the device cannot reach it + CHECK(rungFor(1, fast, slow, 1) == 0); // below it + CHECK(rungFor(30, fast, slow, 0) == intervalFor(30)); // no step: every interval in the range +} + +// A step the request falls between must round to the slower rung, since betterFps prefers a rate at or under the one asked for. +TEST_CASE("a rung lands at or under the rate asked for") { + const uint32_t fast = intervalFor(60), slow = intervalFor(10); + const uint32_t step = 100000; // 10 ms rungs, so 30 fps (333333) is between two of them + const uint32_t rung = rungFor(30, fast, slow, step); + REQUIRE(rung != 0); + CHECK((rung - fast) % step == 0); // on the device's ladder + CHECK(rung >= intervalFor(30)); // at or slower than asked + CHECK(fpsFrom(rung) <= 30); +} + // The smallest mode at or above the picture, at the rate asked for: the decode and the shrink both scale with it. TEST_CASE("the mode asked for is the smallest that covers the picture") { const Mode rows[] = {{1920, 1080, 30}, {1280, 720, 60}, {1280, 720, 30}, {640, 480, 60}, {640, 480, 30}}; @@ -140,7 +162,7 @@ TEST_CASE("a frame of any shape fills the whole picture that was asked for") { const uint16_t w = 640, h = 480, ow = 60, oh = 34; std::vector src(static_cast(w) * h * 3, 77); std::vector dst(static_cast(ow) * oh * 3 + 3, 0xEE); - areaAverage(src.data(), static_cast(w) * 3, w, h, ow, oh, dst.data()); + areaAverage(src.data(), static_cast(w) * 3, w, h, mm::Sink{dst.data(), ow, oh}); for (size_t i = 0; i < static_cast(ow) * oh * 3; i++) CHECK(dst[i] == 77); for (size_t i = static_cast(ow) * oh * 3; i < dst.size(); i++) CHECK(dst[i] == 0xEE); } @@ -150,7 +172,7 @@ TEST_CASE("a picture larger than the frame stays inside the source") { const uint16_t w = 4, h = 3, ow = 9, oh = 7; std::vector src(static_cast(w) * h * 3, 200); std::vector dst(static_cast(ow) * oh * 3, 0); - areaAverage(src.data(), static_cast(w) * 3, w, h, ow, oh, dst.data()); + areaAverage(src.data(), static_cast(w) * 3, w, h, mm::Sink{dst.data(), ow, oh}); for (uint8_t v : dst) CHECK(v == 200); for (uint32_t i = 0; i < ow; i++) { // no cell may name a column the source does not have const mm::Span s = mm::cellSpan(i, ow, w); @@ -158,3 +180,40 @@ TEST_CASE("a picture larger than the frame stays inside the source") { CHECK(s.lo < s.hi); } } + +namespace { +// A JPEG's start: SOI, an APP0 segment to walk past, then a baseline frame header for `w` x `h` whose luma samples at `sampling`. +std::vector jpegHead(uint16_t w, uint16_t h, uint8_t sampling) { + std::vector d = {0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x06, 'J', 'F', 'I', 'F'}; + const uint8_t sof[] = {0xFF, 0xC0, 0x00, 0x11, 8, uint8_t(h >> 8), uint8_t(h), uint8_t(w >> 8), uint8_t(w), 3, + 1, sampling, 0, 2, 0x11, 1, 3, 0x11, 1}; + d.insert(d.end(), sof, sof + sizeof sof); + return d; +} +} // namespace + +// Read from the bitstream, since the decoder's own header call allocates on every frame. +TEST_CASE("a JPEG's size and MCU come from its frame header") { + JpegShape s; + std::vector d = jpegHead(640, 480, 0x21); // 4:2:2 + REQUIRE(jpegShape(d.data(), d.size(), s)); + CHECK(s.width == 640); + CHECK(s.height == 480); + CHECK(s.mcuW == 16); + CHECK(s.mcuH == 8); + d = jpegHead(1920, 1080, 0x22); // 4:2:0 + REQUIRE(jpegShape(d.data(), d.size(), s)); + CHECK(s.mcuW == 16); + CHECK(s.mcuH == 16); +} + +// A frame cut short or one that is not a JPEG at all is refused rather than read past its end. +TEST_CASE("a truncated or foreign frame has no JPEG shape") { + JpegShape s; + std::vector d = jpegHead(640, 480, 0x21); + CHECK_FALSE(jpegShape(d.data(), d.size() - 4, s)); // the frame header cut short + const uint8_t png[] = {0x89, 'P', 'N', 'G', 0x0D, 0x0A}; + CHECK_FALSE(jpegShape(png, sizeof png, s)); + const uint8_t scanFirst[] = {0xFF, 0xD8, 0xFF, 0xDA, 0x00, 0x02}; + CHECK_FALSE(jpegShape(scanFirst, sizeof scanFirst, s)); +}