diff --git a/server/internal/session/manager.go b/server/internal/session/manager.go index a638595bc..25de3c1b2 100644 --- a/server/internal/session/manager.go +++ b/server/internal/session/manager.go @@ -159,8 +159,9 @@ func (manager *SessionManagerCtx) Delete(id string) error { session.DestroyWebSocketPeer("session deleted") } - if session.State().IsWatching { - session.GetWebRTCPeer().Destroy() + // a closing peer is cleared from another goroutine, so it can be gone already + if webrtcPeer := session.GetWebRTCPeer(); webrtcPeer != nil { + webrtcPeer.Destroy() } manager.emmiter.Emit("deleted", session) @@ -182,8 +183,9 @@ func (manager *SessionManagerCtx) Disconnect(id string) error { session.DestroyWebSocketPeer("session disconnected") } - if session.State().IsWatching { - session.GetWebRTCPeer().Destroy() + // a closing peer is cleared from another goroutine, so it can be gone already + if webrtcPeer := session.GetWebRTCPeer(); webrtcPeer != nil { + webrtcPeer.Destroy() } return nil diff --git a/server/internal/session/session.go b/server/internal/session/session.go index 686c9611a..59469754e 100644 --- a/server/internal/session/session.go +++ b/server/internal/session/session.go @@ -30,7 +30,10 @@ type SessionCtx struct { wsDelayedTimer *time.Timer webrtcPeer types.WebRTCPeer - webrtcMu sync.Mutex + // whether webrtcPeer has connected; unlike state.IsWatching this is never + // left over from a peer that was replaced + webrtcConnected bool + webrtcMu sync.Mutex } func (session *SessionCtx) ID() string { @@ -46,12 +49,27 @@ func (session *SessionCtx) profileChanged() { session.ClearHost() } - if (!session.profile.CanConnect || !session.profile.CanLogin || !session.profile.CanWatch) && session.state.IsWatching { + if !session.profile.CanConnect || !session.profile.CanLogin || !session.profile.CanWatch { // TODO: Needed for legacy implementation. Websocket must die before webrtc and deliver signal close message // otherwise webrtc destroy would trigger websocket reconnect. In case of kick event, webrtc destroy is called // before websocket destroy that delivers the information about the kick. time.AfterFunc(time.Second, func() { - session.GetWebRTCPeer().Destroy() + // The profile can change again before the timer fires. A peer of + // a session that may watch again is not this timer's to destroy. + session.manager.sessionsMu.Lock() + profile := session.profile + session.manager.sessionsMu.Unlock() + + if profile.CanConnect && profile.CanLogin && profile.CanWatch { + return + } + + // A peer that has not connected is destroyed too, or it could + // still connect. There may be no peer at all, or the websocket + // destroyed below took one that had not connected with it. + if webrtcPeer := session.GetWebRTCPeer(); webrtcPeer != nil { + webrtcPeer.Destroy() + } }) } @@ -149,6 +167,8 @@ func (session *SessionCtx) ConnectWebSocketPeer(websocketPeer types.WebSocketPee // but after a delay. If the peer is connected again before the delay, the disconnect will be // cancelled. // +// A WebRTC peer that has not connected is destroyed with the websocket peer. +// // If the peer is not the current peer or the peer is nil, it will be ignored. func (session *SessionCtx) DisconnectWebSocketPeer(websocketPeer types.WebSocketPeer, delayed bool) { session.websocketMu.Lock() @@ -212,6 +232,42 @@ func (session *SessionCtx) DisconnectWebSocketPeer(websocketPeer types.WebSocket session.websocketPeer = nil } session.websocketMu.Unlock() + + // A WebRTC peer that has not connected yet is not waited for. Until the + // client has answered, the peer needs this websocket to connect at all: + // without it the peer would stay half-open for good and keep the capture + // pipelines it subscribed to running with nobody watching. Once the client + // has answered, the peer could still connect without the websocket, but a + // client whose websocket is gone for good has left. + session.destroyPendingWebRTCPeer() +} + +// Destroy the WebRTC peer if it has not connected and no websocket peer is +// left to connect it. A connected peer is left alone: it does not need the +// websocket to keep streaming. +// +// The websocket peer must be cleared before this is called, and a WebRTC peer +// must be set before it. That order makes sure a disconnecting websocket and +// a peer being set see each other, whichever happens first. +func (session *SessionCtx) destroyPendingWebRTCPeer() { + session.webrtcMu.Lock() + webrtcPeer, connected := session.webrtcPeer, session.webrtcConnected + session.webrtcMu.Unlock() + + if webrtcPeer == nil || connected { + return + } + + // Another websocket may have connected by now, and the peer may be one it + // asked for. Looking at the websocket after the peer makes sure a peer + // that was set while a websocket is connected is left to that websocket. + session.websocketMu.Lock() + hasWebSocket := session.websocketPeer != nil + session.websocketMu.Unlock() + + if !hasWebSocket { + webrtcPeer.Destroy() + } } // Destroy WebSocket peer disconnects the peer and destroys it. It ensures that the peer is @@ -250,12 +306,19 @@ func (session *SessionCtx) Send(event string, payload any) { // Set webrtc peer and destroy the old one, if there is old one. func (session *SessionCtx) SetWebRTCPeer(webrtcPeer types.WebRTCPeer) { session.webrtcMu.Lock() + if webrtcPeer != session.webrtcPeer { + session.webrtcConnected = false + } session.webrtcPeer, webrtcPeer = webrtcPeer, session.webrtcPeer session.webrtcMu.Unlock() if webrtcPeer != nil && webrtcPeer != session.webrtcPeer { webrtcPeer.Destroy() } + + // The websocket can go away while a peer is being created for it. Its + // disconnect found no peer to destroy then, so it is done here. + session.destroyPendingWebRTCPeer() } // Set if current webrtc peer is connected or not. Since there might be lefover calls from @@ -267,6 +330,9 @@ func (session *SessionCtx) SetWebRTCPeer(webrtcPeer types.WebRTCPeer) { func (session *SessionCtx) SetWebRTCConnected(webrtcPeer types.WebRTCPeer, connected bool) { session.webrtcMu.Lock() isCurrentPeer := webrtcPeer == session.webrtcPeer + if isCurrentPeer { + session.webrtcConnected = connected + } session.webrtcMu.Unlock() if !isCurrentPeer { diff --git a/server/internal/session/session_test.go b/server/internal/session/session_test.go index 2a2dddac6..1795821e7 100644 --- a/server/internal/session/session_test.go +++ b/server/internal/session/session_test.go @@ -1,7 +1,10 @@ package session import ( + "runtime" "testing" + "testing/synctest" + "time" "github.com/m1k1o/neko/server/internal/config" "github.com/m1k1o/neko/server/pkg/types" @@ -39,3 +42,502 @@ func TestLegacyIsHost(t *testing.T) { }) } } + +// Not empty, so that two of them are never the same pointer. +type fakeWebSocketPeer struct{ _ bool } + +func (fakeWebSocketPeer) Send(event string, payload any) {} +func (fakeWebSocketPeer) Ping() error { return nil } +func (fakeWebSocketPeer) Destroy(reason string) {} + +// fakeWebRTCPeer records whether it was destroyed. Like the real peer it can +// be destroyed more than once. The embedded interface is nil: these tests +// must not reach any other method. +type fakeWebRTCPeer struct { + types.WebRTCPeer + destroyed bool +} + +func (p *fakeWebRTCPeer) Destroy() { p.destroyed = true } + +// A change of profile also sets the paused state of the peer. +type pausableWebRTCPeer struct{ fakeWebRTCPeer } + +func (*pausableWebRTCPeer) SetPaused(isPaused bool) error { return nil } + +// A client can ask for media and vanish before it answers the offer. Its peer +// never connects, so nothing but the session can end it; left alone it keeps +// the capture pipelines it subscribed to running with nobody watching. +func TestWebRTCPeerThatNeverConnectedIsDestroyedWithItsSession(t *testing.T) { + profile := types.MemberProfile{CanLogin: true, CanConnect: true, CanWatch: true} + + tests := []struct { + name string + connected bool + end func(manager *SessionManagerCtx, session types.Session, websocket types.WebSocketPeer) error + wantDestroyed bool + }{ + { + name: "websocket closes before the peer connects", + end: func(_ *SessionManagerCtx, session types.Session, websocket types.WebSocketPeer) error { + session.DisconnectWebSocketPeer(websocket, false) + return nil + }, + wantDestroyed: true, + }, + { + name: "websocket closes while the peer is streaming", + connected: true, + end: func(_ *SessionManagerCtx, session types.Session, websocket types.WebSocketPeer) error { + session.DisconnectWebSocketPeer(websocket, false) + return nil + }, + wantDestroyed: false, + }, + { + name: "websocket closes while the peer is streaming and was set again", + connected: true, + end: func(_ *SessionManagerCtx, session types.Session, websocket types.WebSocketPeer) error { + session.SetWebRTCPeer(session.GetWebRTCPeer()) + session.DisconnectWebSocketPeer(websocket, false) + return nil + }, + wantDestroyed: false, + }, + { + name: "websocket drops and may still reconnect", + end: func(_ *SessionManagerCtx, session types.Session, websocket types.WebSocketPeer) error { + session.DisconnectWebSocketPeer(websocket, true) + return nil + }, + wantDestroyed: false, + }, + { + name: "session is deleted before the peer connects", + end: func(manager *SessionManagerCtx, session types.Session, _ types.WebSocketPeer) error { + return manager.Delete(session.ID()) + }, + wantDestroyed: true, + }, + { + name: "session is disconnected before the peer connects", + end: func(manager *SessionManagerCtx, session types.Session, _ types.WebSocketPeer) error { + return manager.Disconnect(session.ID()) + }, + wantDestroyed: true, + }, + { + name: "session is deleted while the peer is streaming", + connected: true, + end: func(manager *SessionManagerCtx, session types.Session, _ types.WebSocketPeer) error { + return manager.Delete(session.ID()) + }, + wantDestroyed: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + manager := New(&config.Session{}) + session, _, err := manager.Create("member", profile) + if err != nil { + t.Fatalf("create session: %v", err) + } + websocket := &fakeWebSocketPeer{} + session.ConnectWebSocketPeer(websocket) + // stops the timer of a delayed disconnect + t.Cleanup(func() { session.DisconnectWebSocketPeer(websocket, false) }) + peer := &fakeWebRTCPeer{} + session.SetWebRTCPeer(peer) + if tt.connected { + session.SetWebRTCConnected(peer, true) + } + + if err := tt.end(manager, session, websocket); err != nil { + t.Fatalf("end session: %v", err) + } + + if peer.destroyed != tt.wantDestroyed { + t.Errorf("peer destroyed = %t, want %t", peer.destroyed, tt.wantDestroyed) + } + }) + } +} + +// A websocket that dropped is given time to come back, and a peer that has not +// connected is kept for that long. When the time is up the peer is destroyed, +// unless the client is back on a new websocket by then. +func TestWebRTCPeerThatNeverConnectedIsKeptWhileItsWebSocketMayReconnect(t *testing.T) { + profile := types.MemberProfile{CanLogin: true, CanConnect: true, CanWatch: true} + + tests := []struct { + name string + // how long after the drop the client is back, if it comes back + reconnectAfter time.Duration + // how long after the drop the peer is looked at + after time.Duration + wantDestroyed bool + }{ + { + name: "time is not up", + after: WS_DELAYED_DURATION - time.Second, + wantDestroyed: false, + }, + { + name: "time is up", + after: WS_DELAYED_DURATION + time.Second, + wantDestroyed: true, + }, + { + name: "client came back in time", + reconnectAfter: WS_DELAYED_DURATION - time.Second, + after: WS_DELAYED_DURATION + time.Second, + wantDestroyed: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + manager := New(&config.Session{}) + session, _, err := manager.Create("member", profile) + if err != nil { + t.Fatalf("create session: %v", err) + } + websocket := &fakeWebSocketPeer{} + session.ConnectWebSocketPeer(websocket) + // stops the timer of a delayed disconnect + defer session.DisconnectWebSocketPeer(websocket, false) + peer := &fakeWebRTCPeer{} + session.SetWebRTCPeer(peer) + + session.DisconnectWebSocketPeer(websocket, true) + if tt.reconnectAfter != 0 { + time.Sleep(tt.reconnectAfter) + session.ConnectWebSocketPeer(&fakeWebSocketPeer{}) + } + time.Sleep(tt.after - tt.reconnectAfter) + synctest.Wait() + + if peer.destroyed != tt.wantDestroyed { + t.Errorf("peer destroyed = %t, want %t", peer.destroyed, tt.wantDestroyed) + } + }) + }) + } +} + +// Creating a peer takes a while, and the websocket that asked for it can be +// gone by the time the peer is attached to the session. Its disconnect could +// not destroy a peer that did not exist yet. +func TestWebRTCPeerSetAfterItsWebSocketLeftIsDestroyed(t *testing.T) { + profile := types.MemberProfile{CanLogin: true, CanConnect: true, CanWatch: true} + + tests := []struct { + name string + leave func(manager *SessionManagerCtx, session types.Session, websocket types.WebSocketPeer) error + wantDestroyed bool + }{ + { + name: "websocket still connected", + leave: func(*SessionManagerCtx, types.Session, types.WebSocketPeer) error { + return nil + }, + wantDestroyed: false, + }, + { + name: "websocket dropped and may still reconnect", + leave: func(_ *SessionManagerCtx, session types.Session, websocket types.WebSocketPeer) error { + session.DisconnectWebSocketPeer(websocket, true) + return nil + }, + wantDestroyed: false, + }, + { + name: "websocket closed", + leave: func(_ *SessionManagerCtx, session types.Session, websocket types.WebSocketPeer) error { + session.DisconnectWebSocketPeer(websocket, false) + return nil + }, + wantDestroyed: true, + }, + { + name: "session deleted", + leave: func(manager *SessionManagerCtx, session types.Session, _ types.WebSocketPeer) error { + return manager.Delete(session.ID()) + }, + wantDestroyed: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + manager := New(&config.Session{}) + session, _, err := manager.Create("member", profile) + if err != nil { + t.Fatalf("create session: %v", err) + } + websocket := &fakeWebSocketPeer{} + session.ConnectWebSocketPeer(websocket) + // stops the timer of a delayed disconnect + t.Cleanup(func() { session.DisconnectWebSocketPeer(websocket, false) }) + + if err := tt.leave(manager, session, websocket); err != nil { + t.Fatalf("leave: %v", err) + } + peer := &fakeWebRTCPeer{} + session.SetWebRTCPeer(peer) + + if peer.destroyed != tt.wantDestroyed { + t.Errorf("peer destroyed = %t, want %t", peer.destroyed, tt.wantDestroyed) + } + }) + } +} + +// A client that is watching can ask for media again, which replaces its peer. +// The session goes on counting as watching, because the closing of a replaced +// peer is ignored, but the new peer has not connected. +func TestWebRTCPeerThatReplacedAStreamingOneIsDestroyedUnlessItConnected(t *testing.T) { + profile := types.MemberProfile{CanLogin: true, CanConnect: true, CanWatch: true} + + tests := []struct { + name string + connected bool + wantDestroyed bool + }{ + {name: "websocket closes before the new peer connects", wantDestroyed: true}, + {name: "websocket closes while the new peer is streaming", connected: true, wantDestroyed: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + manager := New(&config.Session{}) + session, _, err := manager.Create("member", profile) + if err != nil { + t.Fatalf("create session: %v", err) + } + websocket := &fakeWebSocketPeer{} + session.ConnectWebSocketPeer(websocket) + replaced := &fakeWebRTCPeer{} + session.SetWebRTCPeer(replaced) + session.SetWebRTCConnected(replaced, true) + + peer := &fakeWebRTCPeer{} + session.SetWebRTCPeer(peer) + if tt.connected { + session.SetWebRTCConnected(peer, true) + } + // the replaced peer reports its closing whenever it gets to it + session.SetWebRTCConnected(replaced, false) + + session.DisconnectWebSocketPeer(websocket, false) + + if !replaced.destroyed { + t.Error("replaced peer was not destroyed") + } + if peer.destroyed != tt.wantDestroyed { + t.Errorf("peer destroyed = %t, want %t", peer.destroyed, tt.wantDestroyed) + } + }) + } +} + +// A closing peer is cleared from the session on another goroutine. A session +// that is deleted or disconnected at that moment can still count as watching +// with no peer left to destroy. +func TestSessionThatCountsAsWatchingWithoutAPeerCanBeEnded(t *testing.T) { + profile := types.MemberProfile{CanLogin: true, CanConnect: true, CanWatch: true} + + tests := []struct { + name string + end func(manager *SessionManagerCtx, id string) error + }{ + {name: "session is deleted", end: (*SessionManagerCtx).Delete}, + {name: "session is disconnected", end: (*SessionManagerCtx).Disconnect}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + manager := New(&config.Session{}) + session, _, err := manager.Create("member", profile) + if err != nil { + t.Fatalf("create session: %v", err) + } + session.(*SessionCtx).state.IsWatching = true + + if err := tt.end(manager, session.ID()); err != nil { + t.Fatalf("end session: %v", err) + } + }) + } +} + +// A session that may no longer connect has its peer destroyed a moment after +// its websocket. The websocket takes a peer that has not connected with it, +// so a session that counts as watching can have no peer left by then. +func TestSessionThatCountsAsWatchingWithoutAPeerCanLoseItsLogin(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + profile := types.MemberProfile{CanLogin: true, CanConnect: true, CanWatch: true} + + manager := New(&config.Session{}) + session, _, err := manager.Create("member", profile) + if err != nil { + t.Fatalf("create session: %v", err) + } + session.(*SessionCtx).state.IsWatching = true + + profile.CanLogin = false + if err := manager.Update(session.ID(), profile); err != nil { + t.Fatalf("update session: %v", err) + } + + // the peer is destroyed from a timer + time.Sleep(2 * time.Second) + }) +} + +// A session that may no longer watch keeps its websocket, so a peer that has +// not connected is not taken with it. Left alone it could still connect, and +// the session would watch without being allowed to. +func TestWebRTCPeerThatNeverConnectedIsDestroyedWhenItsSessionMayNoLongerWatch(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + profile := types.MemberProfile{CanLogin: true, CanConnect: true, CanWatch: true} + + manager := New(&config.Session{}) + session, _, err := manager.Create("member", profile) + if err != nil { + t.Fatalf("create session: %v", err) + } + session.ConnectWebSocketPeer(&fakeWebSocketPeer{}) + peer := &pausableWebRTCPeer{} + session.SetWebRTCPeer(peer) + + profile.CanWatch = false + if err := manager.Update(session.ID(), profile); err != nil { + t.Fatalf("update session: %v", err) + } + + // the peer is destroyed from a timer + time.Sleep(2 * time.Second) + synctest.Wait() + if !peer.destroyed { + t.Error("peer was not destroyed") + } + }) +} + +// The peer of a session that may no longer watch is destroyed a moment later. +// A session that may watch again by then keeps the peer it has. +func TestWebRTCPeerIsKeptWhenItsSessionMayWatchAgain(t *testing.T) { + tests := []struct { + name string + connected bool + }{ + {name: "peer that has not connected"}, + {name: "peer that is streaming", connected: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + profile := types.MemberProfile{CanLogin: true, CanConnect: true, CanWatch: true} + + manager := New(&config.Session{}) + session, _, err := manager.Create("member", profile) + if err != nil { + t.Fatalf("create session: %v", err) + } + session.ConnectWebSocketPeer(&fakeWebSocketPeer{}) + + revoked := profile + revoked.CanWatch = false + if err := manager.Update(session.ID(), revoked); err != nil { + t.Fatalf("update session: %v", err) + } + if err := manager.Update(session.ID(), profile); err != nil { + t.Fatalf("update session: %v", err) + } + + peer := &pausableWebRTCPeer{} + session.SetWebRTCPeer(peer) + if tt.connected { + session.SetWebRTCConnected(peer, true) + } + + time.Sleep(2 * time.Second) + synctest.Wait() + if peer.destroyed { + t.Error("peer was destroyed") + } + }) + }) + } +} + +// A client can be back on a new websocket before the old one is done leaving, +// because the session's listeners run in between. A peer the new websocket +// asked for is not the old one's to destroy. +func TestWebRTCPeerOfAWebSocketThatTookOverIsKept(t *testing.T) { + profile := types.MemberProfile{CanLogin: true, CanConnect: true, CanWatch: true} + + manager := New(&config.Session{}) + session, _, err := manager.Create("member", profile) + if err != nil { + t.Fatalf("create session: %v", err) + } + websocket := &fakeWebSocketPeer{} + session.ConnectWebSocketPeer(websocket) + + peer := &fakeWebRTCPeer{} + manager.OnDisconnected(func(session types.Session) { + session.ConnectWebSocketPeer(&fakeWebSocketPeer{}) + session.SetWebRTCPeer(peer) + }) + + session.DisconnectWebSocketPeer(websocket, false) + + if peer.destroyed { + t.Error("peer of the websocket that took over was destroyed") + } +} + +// The new websocket can also connect after the old one was cleared from the +// session, and ask for a peer before the old one looks for a peer to destroy. +func TestWebRTCPeerOfAWebSocketThatConnectedAfterTheOldOneWasClearedIsKept(t *testing.T) { + profile := types.MemberProfile{CanLogin: true, CanConnect: true, CanWatch: true} + + manager := New(&config.Session{}) + created, _, err := manager.Create("member", profile) + if err != nil { + t.Fatalf("create session: %v", err) + } + session := created.(*SessionCtx) + websocket := &fakeWebSocketPeer{} + session.ConnectWebSocketPeer(websocket) + + // holds the old websocket where it looks for a peer to destroy + session.webrtcMu.Lock() + disconnected := make(chan struct{}) + go func() { + defer close(disconnected) + session.DisconnectWebSocketPeer(websocket, false) + }() + for cleared := false; !cleared; { + session.websocketMu.Lock() + cleared = session.websocketPeer == nil + session.websocketMu.Unlock() + runtime.Gosched() + } + + // the lock is held, so the peer is set as SetWebRTCPeer would set it + session.ConnectWebSocketPeer(&fakeWebSocketPeer{}) + peer := &fakeWebRTCPeer{} + session.webrtcPeer, session.webrtcConnected = peer, false + session.webrtcMu.Unlock() + <-disconnected + + if peer.destroyed { + t.Error("peer of the websocket that connected was destroyed") + } +} diff --git a/server/internal/webrtc/manager_test.go b/server/internal/webrtc/manager_test.go new file mode 100644 index 000000000..7411235ed --- /dev/null +++ b/server/internal/webrtc/manager_test.go @@ -0,0 +1,212 @@ +package webrtc + +import ( + "testing" + "time" + + "github.com/m1k1o/neko/server/internal/config" + "github.com/m1k1o/neko/server/internal/session" + "github.com/m1k1o/neko/server/pkg/types" + "github.com/m1k1o/neko/server/pkg/types/codec" + "github.com/m1k1o/neko/server/pkg/utils" +) + +// listenerCountingCapture has one audio and one video stream. The embedded +// interface is nil: these tests must not reach any other method. +type listenerCountingCapture struct { + types.CaptureManager + audio, video *listenerCountingStream +} + +func (c *listenerCountingCapture) Audio() types.StreamSinkManager { return c.audio } +func (c *listenerCountingCapture) Video() types.StreamSelectorManager { return c } + +func (c *listenerCountingCapture) IDs() []string { return []string{c.video.ID()} } +func (c *listenerCountingCapture) Codec() codec.RTPCodec { return c.video.Codec() } +func (c *listenerCountingCapture) GetStream(types.StreamSelector) (types.StreamSinkManager, bool) { + return c.video, true +} + +func (c *listenerCountingCapture) listeners() int { + return c.audio.listeners() + c.video.listeners() +} + +type nopWebSocketPeer struct{ _ bool } + +func (nopWebSocketPeer) Send(event string, payload any) {} +func (nopWebSocketPeer) Ping() error { return nil } +func (nopWebSocketPeer) Destroy(reason string) {} + +// What the signal/request handler does for a client that wants video and audio. +func requestMedia(manager *WebRTCManagerCtx, session types.Session) error { + _, peer, err := manager.CreatePeer(session) + if err != nil { + return err + } + + err = peer.SetVideo(types.PeerVideoRequest{Selector: &types.StreamSelector{ID: "test"}}) + if err != nil { + return err + } + + disabled := false + return peer.SetAudio(types.PeerAudioRequest{Disabled: &disabled}) +} + +// leavingClient is a session with a websocket on a capture of its own, so +// that what one client leaves listening is not counted for the next. +type leavingClient struct { + capture *listenerCountingCapture + manager *WebRTCManagerCtx + sessions *session.SessionManagerCtx + session types.Session + websocket types.WebSocketPeer +} + +func (c *leavingClient) request(t *testing.T) { + t.Helper() + if err := requestMedia(c.manager, c.session); err != nil { + t.Fatalf("request media: %v", err) + } + if c.capture.listeners() == 0 { + t.Fatal("nothing listens after the request") + } +} + +// A real peer is closed on its own goroutines: destroying it only starts that. +// These clients ask for media and leave without answering the offer. In the +// end nothing may listen to the capture streams. +func TestPeerOfAClientThatLeftStopsListening(t *testing.T) { + tests := []struct { + name string + run func(t *testing.T, c *leavingClient) + }{ + { + name: "websocket closes after the offer", + run: func(t *testing.T, c *leavingClient) { + c.request(t) + c.session.DisconnectWebSocketPeer(c.websocket, false) + }, + }, + { + name: "session is deleted after the offer", + run: func(t *testing.T, c *leavingClient) { + c.request(t) + if err := c.sessions.Delete(c.session.ID()); err != nil { + t.Fatalf("delete session: %v", err) + } + }, + }, + { + name: "client asks again and its websocket closes", + run: func(t *testing.T, c *leavingClient) { + c.request(t) + c.request(t) + c.session.DisconnectWebSocketPeer(c.websocket, false) + }, + }, + { + name: "websocket is gone when the peer is set", + run: func(t *testing.T, c *leavingClient) { + c.session.DisconnectWebSocketPeer(c.websocket, false) + // the peer is closed when it is set, which fails the request + if err := requestMedia(c.manager, c.session); err == nil { + t.Error("request media succeeded without a websocket") + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + capture := &listenerCountingCapture{ + audio: &listenerCountingStream{codec: codec.Opus()}, + video: &listenerCountingStream{codec: codec.VP8()}, + } + manager := New(nil, capture, &config.WebRTC{ICETrickle: true}) + sessions := session.New(&config.Session{}) + + // metrics are registered once per session id + id, err := utils.NewUID(16) + if err != nil { + t.Fatalf("create session id: %v", err) + } + profile := types.MemberProfile{CanLogin: true, CanConnect: true, CanWatch: true} + session, _, err := sessions.Create(id, profile) + if err != nil { + t.Fatalf("create session: %v", err) + } + websocket := &nopWebSocketPeer{} + session.ConnectWebSocketPeer(websocket) + + tt.run(t, &leavingClient{capture, manager, sessions, session, websocket}) + + waitUntilNothingListens(t, capture, session) + }) + } +} + +func waitUntilNothingListens(t *testing.T, capture *listenerCountingCapture, session types.Session) { + t.Helper() + + deadline := time.Now().Add(5 * time.Second) + for capture.listeners() != 0 || session.GetWebRTCPeer() != nil { + if time.Now().After(deadline) { + t.Fatalf("%d listeners left, peer left: %t", + capture.listeners(), session.GetWebRTCPeer() != nil) + } + time.Sleep(time.Millisecond) + } +} + +// The first listener of a stream starts its capture pipeline, which takes +// seconds, and its peer is busy for that long. A session is logged out under +// a lock that every login needs, so ending it must not wait for the pipeline. +func TestSessionIsDeletedWithoutWaitingForItsPeerToStartAStream(t *testing.T) { + capture := &listenerCountingCapture{ + audio: &listenerCountingStream{codec: codec.Opus()}, + video: &listenerCountingStream{ + codec: codec.VP8(), + starting: make(chan struct{}), + start: make(chan struct{}), + }, + } + manager := New(nil, capture, &config.WebRTC{ICETrickle: true}) + sessions := session.New(&config.Session{}) + + // metrics are registered once per session id + id, err := utils.NewUID(16) + if err != nil { + t.Fatalf("create session id: %v", err) + } + profile := types.MemberProfile{CanLogin: true, CanConnect: true, CanWatch: true} + session, _, err := sessions.Create(id, profile) + if err != nil { + t.Fatalf("create session: %v", err) + } + session.ConnectWebSocketPeer(&nopWebSocketPeer{}) + + requested := make(chan struct{}) + go func() { + defer close(requested) + // fails or not, depending on how far it got when the peer closed + _ = requestMedia(manager, session) + }() + <-capture.video.starting + + deleted := make(chan error, 1) + go func() { deleted <- sessions.Delete(id) }() + + select { + case err := <-deleted: + if err != nil { + t.Errorf("delete session: %v", err) + } + case <-time.After(5 * time.Second): + t.Error("deleting the session waits for the stream to start") + } + + close(capture.video.start) + <-requested + waitUntilNothingListens(t, capture, session) +} diff --git a/server/internal/webrtc/peer.go b/server/internal/webrtc/peer.go index 3310b624d..ffce16fbf 100644 --- a/server/internal/webrtc/peer.go +++ b/server/internal/webrtc/peer.go @@ -108,10 +108,26 @@ func (peer *WebRTCPeerCtx) SetCandidate(candidate webrtc.ICECandidateInit) error } // TODO: Add shutdown function? +// +// A peer that is busy is not waited for: it is closed once it is free. A peer +// is busy for seconds while its first listener starts a capture pipeline, and +// sessions are ended under a lock that every login needs. func (peer *WebRTCPeerCtx) Destroy() { - peer.mu.Lock() + if !peer.mu.TryLock() { + go func() { + peer.mu.Lock() + defer peer.mu.Unlock() + + peer.close() + }() + return + } defer peer.mu.Unlock() + peer.close() +} + +func (peer *WebRTCPeerCtx) close() { var err error // if peer connection is not closed, close it diff --git a/server/internal/webrtc/track.go b/server/internal/webrtc/track.go index 8ddc7ea29..1fc1b557a 100644 --- a/server/internal/webrtc/track.go +++ b/server/internal/webrtc/track.go @@ -22,10 +22,13 @@ type Track struct { sample chan types.Sample paused bool + closed bool stream types.StreamSinkManager streamMu sync.Mutex } +var ErrTrackClosed = errors.New("track is closed") + type trackOption func(*Track) func WithRtcpChan(rtcp chan []rtcp.Packet) trackOption { @@ -64,6 +67,10 @@ func NewTrack(logger zerolog.Logger, codec codec.RTPCodec, connection *webrtc.Pe } func (t *Track) Shutdown() { + t.streamMu.Lock() + t.closed = true + t.streamMu.Unlock() + t.RemoveStream() close(t.sample) } @@ -119,6 +126,13 @@ func (t *Track) SetStream(stream types.StreamSinkManager) (bool, error) { t.streamMu.Lock() defer t.streamMu.Unlock() + // A peer can be destroyed while it is still being set up. A listener added + // after that would never be removed, and its samples would be written to a + // closed channel. + if t.closed { + return false, ErrTrackClosed + } + // if we already listen to the stream, do nothing if t.stream == stream { return false, nil diff --git a/server/internal/webrtc/track_test.go b/server/internal/webrtc/track_test.go new file mode 100644 index 000000000..896b1b237 --- /dev/null +++ b/server/internal/webrtc/track_test.go @@ -0,0 +1,126 @@ +package webrtc + +import ( + "errors" + "sync" + "testing" + + "github.com/pion/webrtc/v3" + "github.com/rs/zerolog" + + "github.com/m1k1o/neko/server/pkg/types" + "github.com/m1k1o/neko/server/pkg/types/codec" +) + +// listenerCountingStream counts its listeners. The embedded interface is nil: +// these tests must not reach any other method. +type listenerCountingStream struct { + types.StreamSinkManager + codec codec.RTPCodec + + mu sync.Mutex + added, removed int + + // When set, a listener is not added until start is closed, like one that + // has to wait for a capture pipeline. The first to wait closes starting. + starting, start chan struct{} + startingOnce sync.Once +} + +func (s *listenerCountingStream) ID() string { return "test" } + +func (s *listenerCountingStream) Codec() codec.RTPCodec { return s.codec } + +func (s *listenerCountingStream) AddListener(types.SampleListener) error { + if s.start != nil { + s.startingOnce.Do(func() { close(s.starting) }) + <-s.start + } + + s.mu.Lock() + defer s.mu.Unlock() + + s.added++ + return nil +} + +func (s *listenerCountingStream) RemoveListener(types.SampleListener) error { + s.mu.Lock() + defer s.mu.Unlock() + + s.removed++ + return nil +} + +func (s *listenerCountingStream) listeners() int { + s.mu.Lock() + defer s.mu.Unlock() + + return s.added - s.removed +} + +func newTestTrack(t *testing.T) *Track { + t.Helper() + connection, err := webrtc.NewPeerConnection(webrtc.Configuration{}) + if err != nil { + t.Fatalf("create peer connection: %v", err) + } + t.Cleanup(func() { connection.Close() }) + + track, err := NewTrack(zerolog.Nop(), codec.VP8(), connection) + if err != nil { + t.Fatalf("create track: %v", err) + } + return track +} + +func TestTrackStopsListeningOnShutdown(t *testing.T) { + track := newTestTrack(t) + stream := &listenerCountingStream{} + + if _, err := track.SetStream(stream); err != nil { + t.Fatalf("set stream: %v", err) + } + track.Shutdown() + + if stream.added != 1 || stream.removed != 1 { + t.Errorf("listener added %d times and removed %d times, want 1 and 1", stream.added, stream.removed) + } +} + +// A peer can be destroyed between its creation and the request that selects +// its video. The track is shut down by then; listening to a stream after that +// would keep the pipeline running for a track nobody removes. +func TestTrackDoesNotListenAfterShutdown(t *testing.T) { + track := newTestTrack(t) + stream := &listenerCountingStream{} + + track.Shutdown() + + if _, err := track.SetStream(stream); !errors.Is(err, ErrTrackClosed) { + t.Errorf("set stream after shutdown returned %v, want %v", err, ErrTrackClosed) + } + + if stream.added != 0 { + t.Errorf("listener added %d times after shutdown, want 0", stream.added) + } +} + +// The audio track gets its stream while it is paused, when the peer is +// created, and only listens once it is unpaused. That can come after the peer +// was destroyed. +func TestPausedTrackDoesNotListenAfterShutdown(t *testing.T) { + track := newTestTrack(t) + stream := &listenerCountingStream{} + + track.SetPaused(true) + if _, err := track.SetStream(stream); err != nil { + t.Fatalf("set stream: %v", err) + } + track.Shutdown() + track.SetPaused(false) + + if stream.added != 0 { + t.Errorf("listener added %d times after shutdown, want 0", stream.added) + } +} diff --git a/server/pkg/gst/gst.c b/server/pkg/gst/gst.c index f1cc9a14a..468a2688b 100644 --- a/server/pkg/gst/gst.c +++ b/server/pkg/gst/gst.c @@ -216,6 +216,12 @@ void gstreamer_pipeline_destory(GstPipelineCtx *ctx) { // set null state gst_element_set_state(GST_ELEMENT(ctx->pipeline), GST_STATE_NULL); + // the watch holds a reference to the bus, which would keep it and its + // control sockets after the pipeline is gone + GstBus *bus = gst_pipeline_get_bus(GST_PIPELINE(ctx->pipeline)); + gst_bus_remove_watch(bus); + gst_object_unref(bus); + if (ctx->appsink) { gst_object_unref(ctx->appsink); ctx->appsink = NULL; diff --git a/server/pkg/gst/gst_test.go b/server/pkg/gst/gst_test.go index 52ac0bf2b..80c64c423 100644 --- a/server/pkg/gst/gst_test.go +++ b/server/pkg/gst/gst_test.go @@ -1,6 +1,7 @@ package gst import ( + "os" "strings" "testing" "time" @@ -95,6 +96,38 @@ func TestCreatePipelineReleasesLockBeforeSlowCUDAProbe(t *testing.T) { } } +func TestDestroyClosesPipelineFiles(t *testing.T) { + openFiles := func() int { + entries, err := os.ReadDir("/proc/self/fd") + if err != nil { + t.Fatalf("count open files: %v", err) + } + return len(entries) + } + + runPipeline := func() { + p, err := CreatePipeline("fakesrc num-buffers=1 ! fakesink") + if err != nil { + t.Fatalf("CreatePipeline() error = %v", err) + } + p.Play() + p.Destroy() + } + + // the first pipeline opens files that GStreamer keeps for the process + runPipeline() + + const pipelines = 10 + before := openFiles() + for i := 0; i < pipelines; i++ { + runPipeline() + } + + if left := openFiles() - before; left > 0 { + t.Fatalf("%d pipelines left %d files open", pipelines, left) + } +} + func TestNVENCFailureDetail(t *testing.T) { t.Parallel()