diff --git a/OpenParsec/NetworkHandler.swift b/OpenParsec/NetworkHandler.swift index e9db31b..98670a2 100644 --- a/OpenParsec/NetworkHandler.swift +++ b/OpenParsec/NetworkHandler.swift @@ -14,6 +14,8 @@ class NetworkWatcher { private var offline = false private(set) var noNetwork = false private var known = false + private var interface: NWInterface.InterfaceType? + private(set) var interfaceChanges = 0 private var waiting: [() -> Void] = [] var onReturn: (() -> Void)? @@ -24,9 +26,16 @@ class NetworkWatcher { monitor.pathUpdateHandler = { [weak self] path in let status = path.status + let type = [NWInterface.InterfaceType.wifi, .cellular, .wiredEthernet].first { path.usesInterfaceType($0) } DispatchQueue.main.async { guard let self = self else { return } self.noNetwork = status == .unsatisfied + if status == .satisfied, let type = type { + if let old = self.interface, old != type { + self.interfaceChanges += 1 + } + self.interface = type + } if !self.known { self.known = true let runs = self.waiting diff --git a/OpenParsec/ParsecBackgroundManager.swift b/OpenParsec/ParsecBackgroundManager.swift index 2540708..1ff3ed1 100644 --- a/OpenParsec/ParsecBackgroundManager.swift +++ b/OpenParsec/ParsecBackgroundManager.swift @@ -32,6 +32,12 @@ class ParsecBackgroundManager { var reconnectStartTime = Date() var resumedAt: Date? var isPaused = false + private var holdTask: UIBackgroundTaskIdentifier = .invalid + private var backgroundAt: Date? + private var endedInBackground = false + var onResumeEndedSession: (() -> Void)? + var onEndedInBackground: (() -> Void)? + var onNextFrame: (() -> Void)? weak var glkViewController: RenderLoop? var onShouldReconnect: ((String) -> Void)? @@ -106,6 +112,55 @@ class ParsecBackgroundManager { } } + func holdSessionInBackground() { + guard holdTask == .invalid else { return } + backgroundAt = Date() + holdTask = UIApplication.shared.beginBackgroundTask(withName: "stream") { [weak self] in + self?.endSessionInBackground("background time ran out") + } + guard holdTask != .invalid else { + endSessionInBackground("no background time") + return + } + appNote("[background] holding stream") + } + + private func endSessionInBackground(_ reason: String) { + if hasActiveConnection && !endedInBackground { + appNote("[background] \(reason), ending stream") + CParsec.disconnect() + isPaused = true + endedInBackground = true + onEndedInBackground?() + RunLoop.main.run(until: Date(timeIntervalSinceNow: 0.2)) + for scene in UIApplication.shared.connectedScenes { + UIApplication.shared.requestSceneSessionRefresh(scene.session) + } + } + endHoldTask() + } + + private func endHoldTask() { + guard holdTask != .invalid else { return } + UIApplication.shared.endBackgroundTask(holdTask) + holdTask = .invalid + } + + func sceneWillEnterForeground() { + guard backgroundAt != nil else { return } + backgroundAt = nil + endHoldTask() + if endedInBackground { + endedInBackground = false + isPaused = false + appNote("[foreground] stream ended in the background") + glkViewController?.isPaused = false + onResumeEndedSession?() + } else if hasActiveConnection { + appNote("[foreground] stream held, resuming") + } + } + func markForReconnect() { guard lastPeerId != nil else { return } didDisconnectDueToBackground = true @@ -114,6 +169,7 @@ class ParsecBackgroundManager { func endSession() { didDisconnectDueToBackground = false endedByUser = true + endedInBackground = false isReconnecting = false reconnectAttempts = 0 resumeOnLaunch = false diff --git a/OpenParsec/ParsecMetalRenderer.swift b/OpenParsec/ParsecMetalRenderer.swift index 6237a5b..551b15c 100644 --- a/OpenParsec/ParsecMetalRenderer.swift +++ b/OpenParsec/ParsecMetalRenderer.swift @@ -84,6 +84,10 @@ class ParsecMetalRenderer: NSObject, MTKViewDelegate { } fileprivate func upload(_ f: ParsecFrame, _ image: UnsafeRawPointer) { + if let next = ParsecBackgroundManager.shared.onNextFrame { + ParsecBackgroundManager.shared.onNextFrame = nil + next() + } let w = Int(f.width), h = Int(f.height), stride = Int(f.fullWidth), rows = Int(f.fullHeight) guard f.isWholeNV12 else { return } let slot = (current + 1) % planes.count diff --git a/OpenParsec/ParsecSDKBridge.swift b/OpenParsec/ParsecSDKBridge.swift index 0e358b3..ba9086d 100644 --- a/OpenParsec/ParsecSDKBridge.swift +++ b/OpenParsec/ParsecSDKBridge.swift @@ -227,6 +227,7 @@ class ParsecSDKBridge: ParsecService { didSetResolution = false ParsecClientDisconnect(_parsec) audio_clear(&_audio) + audio_hold(false, _audioPtr) ParsecBackgroundManager.shared.connectionDidEnd() let status = connect(peerID) if status != PARSEC_OK && status != PARSEC_CONNECTING { diff --git a/OpenParsec/ParsecView.swift b/OpenParsec/ParsecView.swift index 313616a..55bc8d9 100644 --- a/OpenParsec/ParsecView.swift +++ b/OpenParsec/ParsecView.swift @@ -10,18 +10,25 @@ struct ParsecStatusBar: View { @Binding var DCAlertText: String @Binding var isReconnecting: Bool @Binding var connectionLost: Bool + @Binding var waitingForNetwork: Bool + @Binding var waitingForStream: Bool @State var networkFailureSince: Date? + @State var frameHeld = false @State var parsecViewController: ParsecViewController? @State var wasDisconnected: Bool = true + @State var liveSince: CFTimeInterval? + @State var seenInterfaceChanges = NetworkWatcher.shared.interfaceChanges let timer = Timer.publish(every: 0.5, on: .main, in: .common).autoconnect() let onSilentDisconnect: () -> Void - init(showMenu: Binding, showDCAlert: Binding, DCAlertText: Binding, parsecViewController: ParsecViewController, isReconnecting: Binding, connectionLost: Binding, onSilentDisconnect: @escaping () -> Void) { + init(showMenu: Binding, showDCAlert: Binding, DCAlertText: Binding, parsecViewController: ParsecViewController, isReconnecting: Binding, connectionLost: Binding, waitingForNetwork: Binding, waitingForStream: Binding, onSilentDisconnect: @escaping () -> Void) { _showMenu = showMenu _showDCAlert = showDCAlert _DCAlertText = DCAlertText _isReconnecting = isReconnecting _connectionLost = connectionLost + _waitingForNetwork = waitingForNetwork + _waitingForStream = waitingForStream self.parsecViewController = parsecViewController self.onSilentDisconnect = onSilentDisconnect } @@ -50,8 +57,31 @@ struct ParsecStatusBar: View { } } + func holdFrame(_ hold: Bool) { + var pumping = false + if #available(iOS 15.0, *) { pumping = PictureInPictureManager.shared.isPumping } + if hold { + frameHeld = true + if !pumping { ParsecBackgroundManager.shared.glkViewController?.isPaused = true } + } else if frameHeld { + frameHeld = false + if !pumping { ParsecBackgroundManager.shared.glkViewController?.isPaused = false } + } + } + + func pictureBack() { + guard isReconnecting, liveSince != nil else { return } + ParsecBackgroundManager.shared.onNextFrame = nil + liveSince = nil + withAnimation(.easeOut(duration: 0.3)) { + isReconnecting = false + waitingForStream = false + } + appNote("[reconnect] picture back") + } + func poll() { - if showDCAlert || ParsecBackgroundManager.shared.isPaused { + if showDCAlert || ParsecBackgroundManager.shared.isPaused || ParsecBackgroundManager.shared.endedByUser { return } @@ -62,6 +92,37 @@ struct ParsecStatusBar: View { if #available(iOS 15.0, *) { pipActive = PictureInPictureManager.shared.isPiPActive } + let offline = NetworkWatcher.shared.noNetwork + + if waitingForNetwork { + if offline { + holdFrame(true) + return + } + waitingForNetwork = false + ParsecBackgroundManager.shared.resumedAt = Date() + if let peerId = ParsecBackgroundManager.shared.lastPeerId { + let mgr = ParsecBackgroundManager.shared + mgr.reconnectAttempts = 1 + appNote("[reconnect] network back, reconnecting") + isReconnecting = true + parsecViewController?.resetKeyState() + CParsec.reconnect(peerId) + mgr.reconnectStartTime = Date() + return + } + } + + if status == PARSEC_OK && !isReconnecting && !pipActive { + if offline { + appNote("[stream] no network, not waiting on sdk") + status = NETWORK_ERR_INTERRUPTED + } else if SettingsHandler.autoReconnect && NetworkWatcher.shared.interfaceChanges != seenInterfaceChanges { + appNote("[stream] network interface changed, reconnecting") + status = NETWORK_ERR_INTERRUPTED + } + } + seenInterfaceChanges = NetworkWatcher.shared.interfaceChanges if status == PARSEC_OK && pcs.networkFailure && !isReconnecting && !pipActive { if networkFailureSince == nil { networkFailureSince = Date() } @@ -81,17 +142,38 @@ struct ParsecStatusBar: View { } if status != PARSEC_OK { - if ParsecBackgroundManager.shared.isMarkedForReconnect || ParsecBackgroundManager.shared.endedByUser { + liveSince = nil + waitingForStream = false + ParsecBackgroundManager.shared.onNextFrame = nil + holdFrame(true) + if ParsecBackgroundManager.shared.isMarkedForReconnect { + return + } + + if offline && !pipActive { + isReconnecting = false + connectionLost = false + if SettingsHandler.autoReconnect && ParsecBackgroundManager.shared.lastPeerId != nil { + ParsecBackgroundManager.shared.reconnectAttempts = 0 + waitingForNetwork = true + appNote("[stream] no network, waiting") + return + } + appNote("[stream] ended, no network, alert") + DCAlertText = "No network connection" + showDCAlert = true return } - if status == PARSEC_CONNECTING && isReconnecting { + let setupFailure = isReconnecting ? CParsec.setupError.flatMap { $0.at > ParsecBackgroundManager.shared.reconnectStartTime ? $0.line : nil } : nil + + if status == PARSEC_CONNECTING && isReconnecting && setupFailure == nil { if Date().timeIntervalSince(ParsecBackgroundManager.shared.reconnectStartTime) < 15 { return } } - if status == CONNECT_WRN_APPROVAL && isReconnecting { + if status == CONNECT_WRN_APPROVAL && isReconnecting && setupFailure == nil { if Date().timeIntervalSince(ParsecBackgroundManager.shared.reconnectStartTime) < 60 { return } @@ -109,7 +191,7 @@ struct ParsecStatusBar: View { return } - if SettingsHandler.autoReconnect, !status.isPermanentFailure, let peerId = ParsecBackgroundManager.shared.lastPeerId { + if SettingsHandler.autoReconnect, !status.isPermanentFailure || (status == CONNECT_WRN_APPROVAL && setupFailure != nil), let peerId = ParsecBackgroundManager.shared.lastPeerId { let mgr = ParsecBackgroundManager.shared if mgr.reconnectAttempts < 3 { let gap: TimeInterval = mgr.reconnectAttempts == 0 ? 0 : mgr.reconnectAttempts == 1 ? 2 : 5 @@ -117,11 +199,11 @@ struct ParsecStatusBar: View { return } mgr.reconnectAttempts += 1 - appNote("[reconnect] attempt \(mgr.reconnectAttempts) after status \(status.rawValue)") + appNote("[reconnect] attempt \(mgr.reconnectAttempts) after status \(status.rawValue)\(setupFailure.map { ", sdk setup failed: \($0)" } ?? "")") isReconnecting = true - mgr.reconnectStartTime = Date() parsecViewController?.resetKeyState() CParsec.reconnect(peerId) + mgr.reconnectStartTime = Date() return } mgr.reconnectAttempts = 0 @@ -141,19 +223,23 @@ struct ParsecStatusBar: View { return } + holdFrame(connectionLost) + if isReconnecting { - ParsecBackgroundManager.shared.reconnectAttempts = 0 - isReconnecting = false - appNote("[reconnect] live again") - ParsecView.getHostUserData() - var pumping = false - if #available(iOS 15.0, *) { pumping = PictureInPictureManager.shared.isPumping } - if !pumping { - ParsecBackgroundManager.shared.glkViewController?.isPaused = false + if liveSince == nil { + liveSince = CACurrentMediaTime() + withAnimation(.easeInOut(duration: 0.2)) { waitingForStream = true } + ParsecBackgroundManager.shared.onNextFrame = { pictureBack() } + ParsecBackgroundManager.shared.reconnectAttempts = 0 + appNote("[reconnect] live again") + ParsecView.getHostUserData() + // SDK forgets stream dims on reconnect, re-send them so the frame fills not corners + if let vc = parsecViewController { + CParsec.setFrame(vc.view.bounds.width, vc.view.bounds.height, UIScreen.main.scale) + } } - // SDK forgets stream dims on reconnect, re-send them so the frame fills not corners - if let vc = parsecViewController { - CParsec.setFrame(vc.view.bounds.width, vc.view.bounds.height, UIScreen.main.scale) + if CParsec.runningRenderer != .metal || CACurrentMediaTime() - liveSince! > 6 { + pictureBack() } } @@ -215,6 +301,8 @@ struct ParsecView: View { @State var pipAtOpen: Bool = false @State var resolutionAtOpen: ParsecResolution = .client @State var bitrateAtOpen: Int = 0 + @State var waitingForNetwork: Bool = false + @State var waitingForStream: Bool = false @State var resolutions: [ParsecResolution] @State var bitrates: [Int] @@ -255,16 +343,30 @@ struct ParsecView: View { .zIndex(1) .prefersPersistentSystemOverlaysHidden() - ParsecStatusBar(showMenu: $showMenu, showDCAlert: $showDCAlert, DCAlertText: $DCAlertText, parsecViewController: parsecViewController, isReconnecting: $isReconnecting, connectionLost: $connectionLost, onSilentDisconnect: { disconnect() }) + ParsecStatusBar(showMenu: $showMenu, showDCAlert: $showDCAlert, DCAlertText: $DCAlertText, parsecViewController: parsecViewController, isReconnecting: $isReconnecting, connectionLost: $connectionLost, waitingForNetwork: $waitingForNetwork, waitingForStream: $waitingForStream, onSilentDisconnect: { disconnect() }) - if isReconnecting || connectionLost { + if isReconnecting || connectionLost || waitingForNetwork { VStack(spacing: 12) { - ActivityIndicator(isAnimating: .constant(true), style: .large, tint: .white) - Text(isReconnecting ? ParsecBackgroundManager.shared.lastHostname.map { "Reconnecting to \($0)..." } ?? "Reconnecting..." : "Connection lost...") + if waitingForNetwork { + Image(systemName: "wifi.slash") + .font(.system(size: 34, weight: .medium)) + .foregroundColor(.white) + } else { + ActivityIndicator(isAnimating: .constant(true), style: .large, tint: .white) + } + Text(overlayTitle) .font(.system(size: 16, weight: .medium)) .foregroundColor(.white) + .id(waitingForStream) + .transition(.opacity) + if waitingForNetwork { + Text("OpenParsec will reconnect when you're back online") + .font(.system(size: 13)) + .foregroundColor(.white) + .opacity(0.7) + } Button(action: { disconnect() }) { - Text("Disconnect") + Text("Close") .font(.system(size: 15, weight: .semibold)) .foregroundColor(.white) .padding(.horizontal, 20) @@ -279,6 +381,7 @@ struct ParsecView: View { } .frame(maxWidth: .infinity, maxHeight: .infinity) .background(Color.black.opacity(0.6)) + .transition(.asymmetric(insertion: .identity, removal: .opacity)) .zIndex(10) } @@ -374,6 +477,13 @@ struct ParsecView: View { disconnect() return } + if NetworkWatcher.shared.noNetwork { + DispatchQueue.main.async { + DCAlertText = "No network connection" + showDCAlert = true + } + return + } showDCAlert = false reconnectTo(peerId) }), @@ -532,10 +642,36 @@ struct ParsecView: View { // .edgesIgnoringSafeArea(.all) } + var overlayTitle: String { + if waitingForNetwork { + return "No network connection" + } + if !isReconnecting { + return "Connection lost..." + } + if waitingForStream { + return "Reconnected, waiting for the stream..." + } + return ParsecBackgroundManager.shared.lastHostname.map { "Reconnecting to \($0)..." } ?? "Reconnecting..." + } + func post() { ParsecBackgroundManager.shared.onShouldDisconnect = { NotificationCenter.default.post(name: NSNotification.Name("ParsecBackgroundDisconnect"), object: nil) } + ParsecBackgroundManager.shared.onEndedInBackground = { [self] in + ParsecBackgroundManager.shared.onNextFrame = nil + waitingForStream = false + if showDCAlert || waitingForNetwork { return } + isReconnecting = true + } + ParsecBackgroundManager.shared.onResumeEndedSession = { [self] in + if showDCAlert || waitingForNetwork { return } + guard let peerId = ParsecBackgroundManager.shared.lastPeerId else { return } + appNote("[reconnect] back from the background, reconnecting") + reconnectTo(peerId) + ParsecBackgroundManager.shared.reconnectAttempts = 1 + } if #available(iOS 15.0, *) { PictureInPictureManager.shared.onPiPStopped = { [self] in if UIApplication.shared.applicationState != .active { @@ -543,6 +679,9 @@ struct ParsecView: View { CParsec.sendReleaseMessage() CParsec.pause() ParsecBackgroundManager.shared.isPaused = true + if UIApplication.shared.applicationState == .background { + ParsecBackgroundManager.shared.holdSessionInBackground() + } } else { if ParsecBackgroundManager.shared.isReconnecting { return @@ -566,6 +705,9 @@ struct ParsecView: View { CParsec.sendReleaseMessage() CParsec.pause() ParsecBackgroundManager.shared.isPaused = true + if UIApplication.shared.applicationState == .background { + ParsecBackgroundManager.shared.holdSessionInBackground() + } } } } @@ -667,9 +809,9 @@ struct ParsecView: View { func reconnectTo(_ peerId: String) { isReconnecting = true ParsecBackgroundManager.shared.reconnectAttempts = 0 - ParsecBackgroundManager.shared.reconnectStartTime = Date() parsecViewController.resetKeyState() CParsec.reconnect(peerId) + ParsecBackgroundManager.shared.reconnectStartTime = Date() } func disableOverlay() { @@ -733,6 +875,9 @@ struct ParsecView: View { if !isBackgroundDisconnect { ParsecBackgroundManager.shared.endSession() } + ParsecBackgroundManager.shared.onResumeEndedSession = nil + ParsecBackgroundManager.shared.onEndedInBackground = nil + ParsecBackgroundManager.shared.onNextFrame = nil if #available(iOS 15.0, *) { PictureInPictureManager.shared.teardown() diff --git a/OpenParsec/SceneDelegate.swift b/OpenParsec/SceneDelegate.swift index 4337b6e..fb5dca6 100644 --- a/OpenParsec/SceneDelegate.swift +++ b/OpenParsec/SceneDelegate.swift @@ -68,6 +68,7 @@ class SceneDelegate: UIResponder, UIWindowSceneDelegate { } func sceneWillEnterForeground(_ scene: UIScene) { + ParsecBackgroundManager.shared.sceneWillEnterForeground() } func sceneDidEnterBackground(_ scene: UIScene) { @@ -94,6 +95,7 @@ class SceneDelegate: UIResponder, UIWindowSceneDelegate { CParsec.sendReleaseMessage() CParsec.pause() ParsecBackgroundManager.shared.isPaused = true + ParsecBackgroundManager.shared.holdSessionInBackground() } if ParsecBackgroundManager.shared.hasActiveConnection {