Line data Source code
1 : import 'dart:async';
2 :
3 : import 'package:collection/collection.dart';
4 : import 'package:webrtc_interface/webrtc_interface.dart';
5 :
6 : import 'package:matrix/matrix.dart';
7 : import 'package:matrix/src/utils/cached_stream_controller.dart';
8 : import 'package:matrix/src/voip/models/call_membership.dart';
9 : import 'package:matrix/src/voip/models/call_options.dart';
10 : import 'package:matrix/src/voip/utils/stream_helper.dart';
11 : import 'package:matrix/src/voip/utils/user_media_constraints.dart';
12 :
13 : class MeshBackend extends CallBackend {
14 2 : MeshBackend({
15 : super.type = 'mesh',
16 : });
17 :
18 : final List<CallSession> _callSessions = [];
19 :
20 : /// participant:volume
21 : final Map<CallParticipant, double> _audioLevelsMap = {};
22 :
23 : /// The stream is used to prepare for incoming peer calls like registering listeners
24 : StreamSubscription<CallSession>? _callSetupSubscription;
25 :
26 : /// The stream is used to signal the start of an incoming peer call
27 : StreamSubscription<CallSession>? _callStartSubscription;
28 :
29 : Timer? _activeSpeakerLoopTimeout;
30 :
31 : final CachedStreamController<WrappedMediaStream> onStreamAdd =
32 : CachedStreamController();
33 :
34 : final CachedStreamController<WrappedMediaStream> onStreamRemoved =
35 : CachedStreamController();
36 :
37 : final CachedStreamController<GroupCallSession> onGroupCallFeedsChanged =
38 : CachedStreamController();
39 :
40 2 : @override
41 : Map<String, Object?> toJson() {
42 2 : return {
43 2 : 'type': type,
44 : };
45 : }
46 :
47 : CallParticipant? _activeSpeaker;
48 : WrappedMediaStream? _localUserMediaStream;
49 : WrappedMediaStream? _localScreenshareStream;
50 : final List<WrappedMediaStream> _userMediaStreams = [];
51 : final List<WrappedMediaStream> _screenshareStreams = [];
52 :
53 0 : List<WrappedMediaStream> _getLocalStreams() {
54 0 : final feeds = <WrappedMediaStream>[];
55 :
56 0 : if (localUserMediaStream != null) {
57 0 : feeds.add(localUserMediaStream!);
58 : }
59 :
60 0 : if (localScreenshareStream != null) {
61 0 : feeds.add(localScreenshareStream!);
62 : }
63 :
64 : return feeds;
65 : }
66 :
67 0 : Future<MediaStream> _getUserMedia(
68 : GroupCallSession groupCall,
69 : CallType type,
70 : ) async {
71 0 : final mediaConstraints = {
72 : 'audio': UserMediaConstraints.micMediaConstraints,
73 0 : 'video': type == CallType.kVideo
74 : ? UserMediaConstraints.camMediaConstraints
75 : : false,
76 : };
77 :
78 : try {
79 0 : return await groupCall.voip.delegate.mediaDevices
80 0 : .getUserMedia(mediaConstraints);
81 : } catch (e) {
82 0 : groupCall.setState(GroupCallState.localCallFeedUninitialized);
83 : rethrow;
84 : }
85 : }
86 :
87 0 : Future<MediaStream> _getDisplayMedia(GroupCallSession groupCall) async {
88 0 : final mediaConstraints = {
89 : 'audio': false,
90 : 'video': true,
91 : };
92 : try {
93 0 : return await groupCall.voip.delegate.mediaDevices
94 0 : .getDisplayMedia(mediaConstraints);
95 : } catch (e, s) {
96 0 : throw MatrixSDKVoipException('_getDisplayMedia failed', stackTrace: s);
97 : }
98 : }
99 :
100 0 : CallSession? _getCallForParticipant(
101 : GroupCallSession groupCall,
102 : CallParticipant participant,
103 : ) {
104 0 : return _callSessions.singleWhereOrNull(
105 0 : (call) =>
106 0 : call.groupCallId == groupCall.groupCallId &&
107 0 : CallParticipant(
108 0 : groupCall.voip,
109 0 : userId: call.remoteUserId!,
110 0 : deviceId: call.remoteDeviceId,
111 0 : ) ==
112 : participant,
113 : );
114 : }
115 :
116 : /// Register listeners for a peer call to use for the group calls, that is
117 : /// needed before even call is added to `_callSessions`.
118 : /// We do this here for onStreamAdd and onStreamRemoved to make sure we don't
119 : /// miss any events that happen before the call is completely started.
120 0 : void _registerListenersBeforeCallAdd(CallSession call) {
121 0 : call.onStreamAdd.stream.listen((stream) {
122 0 : if (!stream.isLocal()) {
123 0 : onStreamAdd.add(stream);
124 : }
125 : });
126 :
127 0 : call.onStreamRemoved.stream.listen((stream) {
128 0 : if (!stream.isLocal()) {
129 0 : onStreamRemoved.add(stream);
130 : }
131 : });
132 : }
133 :
134 0 : Future<void> _addCall(GroupCallSession groupCall, CallSession call) async {
135 0 : _callSessions.add(call);
136 0 : _initCall(groupCall, call);
137 0 : groupCall.onGroupCallEvent.add(GroupCallStateChange.callsChanged);
138 : }
139 :
140 : /// init a peer call from group calls.
141 0 : void _initCall(GroupCallSession groupCall, CallSession call) {
142 0 : if (call.remoteUserId == null) {
143 0 : throw MatrixSDKVoipException(
144 : 'Cannot init call without proper invitee user and device Id',
145 : );
146 : }
147 :
148 0 : call.onCallStateChanged.stream.listen(
149 0 : ((event) async {
150 0 : await _onCallStateChanged(call, event);
151 : }),
152 : );
153 :
154 0 : call.onCallReplaced.stream.listen((CallSession newCall) async {
155 0 : await _replaceCall(groupCall, call, newCall);
156 : });
157 :
158 0 : call.onCallStreamsChanged.stream.listen((call) async {
159 0 : await call.tryRemoveStopedStreams();
160 0 : await _onStreamsChanged(groupCall, call);
161 : });
162 :
163 0 : call.onCallHangupNotifierForGroupCalls.stream.listen((event) async {
164 0 : await _onCallHangup(groupCall, call);
165 : });
166 : }
167 :
168 0 : Future<void> _replaceCall(
169 : GroupCallSession groupCall,
170 : CallSession existingCall,
171 : CallSession replacementCall,
172 : ) async {
173 0 : final existingCallIndex = _callSessions
174 0 : .indexWhere((element) => element.callId == existingCall.callId);
175 :
176 0 : if (existingCallIndex == -1) {
177 0 : throw MatrixSDKVoipException('Couldn\'t find call to replace');
178 : }
179 :
180 0 : _callSessions.removeAt(existingCallIndex);
181 0 : _callSessions.add(replacementCall);
182 :
183 0 : await _disposeCall(groupCall, existingCall, CallErrorCode.replaced);
184 0 : _registerListenersBeforeCallAdd(replacementCall);
185 0 : _initCall(groupCall, replacementCall);
186 :
187 0 : groupCall.onGroupCallEvent.add(GroupCallStateChange.callsChanged);
188 : }
189 :
190 : /// Removes a peer call from group calls.
191 0 : Future<void> _removeCall(
192 : GroupCallSession groupCall,
193 : CallSession call,
194 : CallErrorCode hangupReason,
195 : ) async {
196 0 : await _disposeCall(groupCall, call, hangupReason);
197 :
198 0 : _callSessions.removeWhere((element) => call.callId == element.callId);
199 :
200 0 : groupCall.onGroupCallEvent.add(GroupCallStateChange.callsChanged);
201 : }
202 :
203 0 : Future<void> _disposeCall(
204 : GroupCallSession groupCall,
205 : CallSession call,
206 : CallErrorCode hangupReason,
207 : ) async {
208 0 : if (call.remoteUserId == null) {
209 0 : throw MatrixSDKVoipException(
210 : 'Cannot init call without proper invitee user and device Id',
211 : );
212 : }
213 :
214 0 : if (call.hangupReason == CallErrorCode.replaced) {
215 : return;
216 : }
217 :
218 0 : if (call.state != CallState.kEnded) {
219 : // no need to emit individual handleCallEnded on group calls
220 : // also prevents a loop of hangup and onCallHangupNotifierForGroupCalls
221 0 : await call.hangup(reason: hangupReason, shouldEmit: false);
222 : }
223 :
224 0 : final usermediaStream = _getUserMediaStreamByParticipantId(
225 0 : CallParticipant(
226 0 : groupCall.voip,
227 0 : userId: call.remoteUserId!,
228 0 : deviceId: call.remoteDeviceId,
229 0 : ).id,
230 : );
231 :
232 : if (usermediaStream != null) {
233 0 : await _removeUserMediaStream(groupCall, usermediaStream);
234 : }
235 :
236 0 : final screenshareStream = _getScreenshareStreamByParticipantId(
237 0 : CallParticipant(
238 0 : groupCall.voip,
239 0 : userId: call.remoteUserId!,
240 0 : deviceId: call.remoteDeviceId,
241 0 : ).id,
242 : );
243 :
244 : if (screenshareStream != null) {
245 0 : await _removeScreenshareStream(groupCall, screenshareStream);
246 : }
247 : }
248 :
249 0 : Future<void> _onStreamsChanged(
250 : GroupCallSession groupCall,
251 : CallSession call,
252 : ) async {
253 0 : if (call.remoteUserId == null) {
254 0 : throw MatrixSDKVoipException(
255 : 'Cannot init call without proper invitee user and device Id',
256 : );
257 : }
258 :
259 0 : final currentUserMediaStream = _getUserMediaStreamByParticipantId(
260 0 : CallParticipant(
261 0 : groupCall.voip,
262 0 : userId: call.remoteUserId!,
263 0 : deviceId: call.remoteDeviceId,
264 0 : ).id,
265 : );
266 :
267 0 : final remoteUsermediaStream = call.remoteUserMediaStream;
268 0 : final remoteStreamChanged = remoteUsermediaStream != currentUserMediaStream;
269 :
270 : if (remoteStreamChanged) {
271 : if (currentUserMediaStream == null && remoteUsermediaStream != null) {
272 0 : await _addUserMediaStream(groupCall, remoteUsermediaStream);
273 : } else if (currentUserMediaStream != null &&
274 : remoteUsermediaStream != null) {
275 0 : await _replaceUserMediaStream(
276 : groupCall,
277 : currentUserMediaStream,
278 : remoteUsermediaStream,
279 : );
280 : } else if (currentUserMediaStream != null &&
281 : remoteUsermediaStream == null) {
282 0 : await _removeUserMediaStream(groupCall, currentUserMediaStream);
283 : }
284 : }
285 :
286 0 : final currentScreenshareStream = _getScreenshareStreamByParticipantId(
287 0 : CallParticipant(
288 0 : groupCall.voip,
289 0 : userId: call.remoteUserId!,
290 0 : deviceId: call.remoteDeviceId,
291 0 : ).id,
292 : );
293 0 : final remoteScreensharingStream = call.remoteScreenSharingStream;
294 : final remoteScreenshareStreamChanged =
295 0 : remoteScreensharingStream != currentScreenshareStream;
296 :
297 : if (remoteScreenshareStreamChanged) {
298 : if (currentScreenshareStream == null &&
299 : remoteScreensharingStream != null) {
300 0 : _addScreenshareStream(groupCall, remoteScreensharingStream);
301 : } else if (currentScreenshareStream != null &&
302 : remoteScreensharingStream != null) {
303 0 : await _replaceScreenshareStream(
304 : groupCall,
305 : currentScreenshareStream,
306 : remoteScreensharingStream,
307 : );
308 : } else if (currentScreenshareStream != null &&
309 : remoteScreensharingStream == null) {
310 0 : await _removeScreenshareStream(groupCall, currentScreenshareStream);
311 : }
312 : }
313 :
314 0 : onGroupCallFeedsChanged.add(groupCall);
315 : }
316 :
317 0 : WrappedMediaStream? _getUserMediaStreamByParticipantId(String participantId) {
318 0 : final stream = _userMediaStreams
319 0 : .where((stream) => stream.participant.id == participantId);
320 0 : if (stream.isNotEmpty) {
321 0 : return stream.first;
322 : }
323 : return null;
324 : }
325 :
326 0 : void _onActiveSpeakerLoop(GroupCallSession groupCall) async {
327 : CallParticipant? nextActiveSpeaker;
328 : // idc about screen sharing atm.
329 : final userMediaStreamsCopyList =
330 0 : List<WrappedMediaStream>.from(_userMediaStreams);
331 0 : for (final stream in userMediaStreamsCopyList) {
332 0 : if (stream.participant.isLocal && stream.pc == null) {
333 : continue;
334 : }
335 :
336 0 : final List<StatsReport> statsReport = await stream.pc!.getStats();
337 : statsReport
338 0 : .removeWhere((element) => !element.values.containsKey('audioLevel'));
339 :
340 : // https://www.w3.org/TR/webrtc-stats/#summary
341 : final otherPartyAudioLevel = statsReport
342 0 : .singleWhereOrNull(
343 0 : (element) =>
344 0 : element.type == 'inbound-rtp' &&
345 0 : element.values['kind'] == 'audio',
346 : )
347 0 : ?.values['audioLevel'];
348 : if (otherPartyAudioLevel != null) {
349 0 : _audioLevelsMap[stream.participant] = otherPartyAudioLevel;
350 : }
351 :
352 : // https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-media-source
353 : // firefox does not seem to have this though. Works on chrome and android
354 : final ownAudioLevel = statsReport
355 0 : .singleWhereOrNull(
356 0 : (element) =>
357 0 : element.type == 'media-source' &&
358 0 : element.values['kind'] == 'audio',
359 : )
360 0 : ?.values['audioLevel'];
361 0 : if (groupCall.localParticipant != null &&
362 : ownAudioLevel != null &&
363 0 : _audioLevelsMap[groupCall.localParticipant] != ownAudioLevel) {
364 0 : _audioLevelsMap[groupCall.localParticipant!] = ownAudioLevel;
365 : }
366 : }
367 :
368 : double maxAudioLevel = double.negativeInfinity;
369 : // TODO: we probably want a threshold here?
370 0 : _audioLevelsMap.forEach((key, value) {
371 0 : if (value > maxAudioLevel) {
372 : nextActiveSpeaker = key;
373 : maxAudioLevel = value;
374 : }
375 : });
376 :
377 0 : if (nextActiveSpeaker != null && _activeSpeaker != nextActiveSpeaker) {
378 0 : _activeSpeaker = nextActiveSpeaker;
379 0 : groupCall.onGroupCallEvent.add(GroupCallStateChange.activeSpeakerChanged);
380 : }
381 0 : _activeSpeakerLoopTimeout?.cancel();
382 0 : _activeSpeakerLoopTimeout = Timer(
383 : CallConstants.activeSpeakerInterval,
384 0 : () => _onActiveSpeakerLoop(groupCall),
385 : );
386 : }
387 :
388 0 : WrappedMediaStream? _getScreenshareStreamByParticipantId(
389 : String participantId,
390 : ) {
391 0 : final stream = _screenshareStreams
392 0 : .where((stream) => stream.participant.id == participantId);
393 0 : if (stream.isNotEmpty) {
394 0 : return stream.first;
395 : }
396 : return null;
397 : }
398 :
399 0 : void _addScreenshareStream(
400 : GroupCallSession groupCall,
401 : WrappedMediaStream stream,
402 : ) {
403 0 : _screenshareStreams.add(stream);
404 0 : onStreamAdd.add(stream);
405 0 : groupCall.onGroupCallEvent
406 0 : .add(GroupCallStateChange.screenshareStreamsChanged);
407 : }
408 :
409 0 : Future<void> _replaceScreenshareStream(
410 : GroupCallSession groupCall,
411 : WrappedMediaStream existingStream,
412 : WrappedMediaStream replacementStream,
413 : ) async {
414 0 : final streamIndex = _screenshareStreams.indexWhere(
415 0 : (stream) => stream.participant.id == existingStream.participant.id,
416 : );
417 :
418 0 : if (streamIndex == -1) {
419 0 : throw MatrixSDKVoipException(
420 : 'Couldn\'t find screenshare stream to replace',
421 : );
422 : }
423 :
424 0 : _screenshareStreams.replaceRange(streamIndex, 1, [replacementStream]);
425 :
426 0 : await existingStream.dispose();
427 0 : groupCall.onGroupCallEvent
428 0 : .add(GroupCallStateChange.screenshareStreamsChanged);
429 : }
430 :
431 0 : Future<void> _removeScreenshareStream(
432 : GroupCallSession groupCall,
433 : WrappedMediaStream stream,
434 : ) async {
435 0 : final streamIndex = _screenshareStreams
436 0 : .indexWhere((stream) => stream.participant.id == stream.participant.id);
437 :
438 0 : if (streamIndex == -1) {
439 0 : throw MatrixSDKVoipException(
440 : 'Couldn\'t find screenshare stream to remove',
441 : );
442 : }
443 :
444 0 : _screenshareStreams.removeWhere(
445 0 : (element) => element.participant.id == stream.participant.id,
446 : );
447 :
448 0 : onStreamRemoved.add(stream);
449 :
450 0 : if (stream.isLocal()) {
451 0 : await stopMediaStream(stream.stream);
452 : }
453 :
454 0 : groupCall.onGroupCallEvent
455 0 : .add(GroupCallStateChange.screenshareStreamsChanged);
456 : }
457 :
458 0 : Future<void> _onCallStateChanged(CallSession call, CallState state) async {
459 0 : final audioMuted = localUserMediaStream?.isAudioMuted() ?? true;
460 0 : if (call.localUserMediaStream != null &&
461 0 : call.isMicrophoneMuted != audioMuted) {
462 0 : await call.setMicrophoneMuted(audioMuted);
463 : }
464 :
465 0 : final videoMuted = localUserMediaStream?.isVideoMuted() ?? true;
466 :
467 0 : if (call.localUserMediaStream != null &&
468 0 : call.isLocalVideoMuted != videoMuted) {
469 0 : await call.setLocalVideoMuted(videoMuted);
470 : }
471 : }
472 :
473 0 : Future<void> _onCallHangup(
474 : GroupCallSession groupCall,
475 : CallSession call,
476 : ) async {
477 0 : if (call.hangupReason == CallErrorCode.replaced) {
478 : return;
479 : }
480 0 : await _onStreamsChanged(groupCall, call);
481 0 : await _removeCall(groupCall, call, call.hangupReason!);
482 : }
483 :
484 0 : Future<void> _addUserMediaStream(
485 : GroupCallSession groupCall,
486 : WrappedMediaStream stream,
487 : ) async {
488 0 : _userMediaStreams.add(stream);
489 0 : onStreamAdd.add(stream);
490 0 : groupCall.onGroupCallEvent
491 0 : .add(GroupCallStateChange.userMediaStreamsChanged);
492 : }
493 :
494 0 : Future<void> _replaceUserMediaStream(
495 : GroupCallSession groupCall,
496 : WrappedMediaStream existingStream,
497 : WrappedMediaStream replacementStream,
498 : ) async {
499 0 : final streamIndex = _userMediaStreams.indexWhere(
500 0 : (stream) => stream.participant.id == existingStream.participant.id,
501 : );
502 :
503 0 : if (streamIndex == -1) {
504 0 : throw MatrixSDKVoipException(
505 : 'Couldn\'t find user media stream to replace',
506 : );
507 : }
508 :
509 0 : _userMediaStreams.replaceRange(streamIndex, 1, [replacementStream]);
510 :
511 0 : await existingStream.dispose();
512 0 : groupCall.onGroupCallEvent
513 0 : .add(GroupCallStateChange.userMediaStreamsChanged);
514 : }
515 :
516 0 : Future<void> _removeUserMediaStream(
517 : GroupCallSession groupCall,
518 : WrappedMediaStream stream,
519 : ) async {
520 0 : final streamIndex = _userMediaStreams.indexWhere(
521 0 : (element) => element.participant.id == stream.participant.id,
522 : );
523 :
524 0 : if (streamIndex == -1) {
525 0 : throw MatrixSDKVoipException(
526 : 'Couldn\'t find user media stream to remove',
527 : );
528 : }
529 :
530 0 : _userMediaStreams.removeWhere(
531 0 : (element) => element.participant.id == stream.participant.id,
532 : );
533 0 : _audioLevelsMap.remove(stream.participant);
534 0 : onStreamRemoved.add(stream);
535 :
536 0 : if (stream.isLocal()) {
537 0 : await stopMediaStream(stream.stream);
538 : }
539 :
540 0 : groupCall.onGroupCallEvent
541 0 : .add(GroupCallStateChange.userMediaStreamsChanged);
542 :
543 0 : if (_activeSpeaker == stream.participant && _userMediaStreams.isNotEmpty) {
544 0 : _activeSpeaker = _userMediaStreams[0].participant;
545 0 : groupCall.onGroupCallEvent.add(GroupCallStateChange.activeSpeakerChanged);
546 : }
547 : }
548 :
549 0 : @override
550 : bool get e2eeEnabled => false;
551 :
552 0 : @override
553 0 : CallParticipant? get activeSpeaker => _activeSpeaker;
554 :
555 0 : @override
556 0 : WrappedMediaStream? get localUserMediaStream => _localUserMediaStream;
557 :
558 0 : @override
559 0 : WrappedMediaStream? get localScreenshareStream => _localScreenshareStream;
560 :
561 0 : @override
562 : List<WrappedMediaStream> get userMediaStreams =>
563 0 : List.unmodifiable(_userMediaStreams);
564 :
565 0 : @override
566 : List<WrappedMediaStream> get screenShareStreams =>
567 0 : List.unmodifiable(_screenshareStreams);
568 :
569 0 : @override
570 : Future<void> updateMediaDeviceForCalls() async {
571 0 : for (final call in _callSessions) {
572 0 : await call.updateMediaDeviceForCall();
573 : }
574 : }
575 :
576 : /// Initializes the local user media stream.
577 : /// The media stream must be prepared before the group call enters.
578 : /// if you allow the user to configure their camera and such ahead of time,
579 : /// you can pass that `stream` on to this function.
580 : /// This allows you to configure the camera before joining the call without
581 : /// having to reopen the stream and possibly losing settings.
582 0 : @override
583 : Future<WrappedMediaStream?> initLocalStream(
584 : GroupCallSession groupCall, {
585 : WrappedMediaStream? stream,
586 : }) async {
587 0 : if (groupCall.state != GroupCallState.localCallFeedUninitialized) {
588 0 : throw MatrixSDKVoipException(
589 0 : 'Cannot initialize local call feed in the ${groupCall.state} state.',
590 : );
591 : }
592 :
593 0 : groupCall.setState(GroupCallState.initializingLocalCallFeed);
594 :
595 : WrappedMediaStream localWrappedMediaStream;
596 :
597 : if (stream == null) {
598 : MediaStream stream;
599 :
600 : try {
601 0 : stream = await _getUserMedia(groupCall, CallType.kVideo);
602 : } catch (error) {
603 0 : groupCall.setState(GroupCallState.localCallFeedUninitialized);
604 : rethrow;
605 : }
606 :
607 0 : localWrappedMediaStream = WrappedMediaStream(
608 : stream: stream,
609 0 : participant: groupCall.localParticipant!,
610 0 : room: groupCall.room,
611 0 : client: groupCall.client,
612 : purpose: SDPStreamMetadataPurpose.Usermedia,
613 0 : audioMuted: stream.getAudioTracks().isEmpty,
614 0 : videoMuted: stream.getVideoTracks().isEmpty,
615 : isGroupCall: true,
616 0 : voip: groupCall.voip,
617 : );
618 : } else {
619 : localWrappedMediaStream = stream;
620 : }
621 :
622 0 : _localUserMediaStream = localWrappedMediaStream;
623 0 : await _addUserMediaStream(groupCall, localWrappedMediaStream);
624 :
625 0 : groupCall.setState(GroupCallState.localCallFeedInitialized);
626 :
627 0 : _activeSpeaker = null;
628 :
629 : return localWrappedMediaStream;
630 : }
631 :
632 0 : @override
633 : Future<void> setDeviceMuted(
634 : GroupCallSession groupCall,
635 : bool muted,
636 : MediaInputKind kind,
637 : ) async {
638 0 : if (!await hasMediaDevice(groupCall.voip.delegate, kind)) {
639 : return;
640 : }
641 :
642 0 : if (localUserMediaStream != null) {
643 : switch (kind) {
644 0 : case MediaInputKind.audioinput:
645 0 : localUserMediaStream!.setAudioMuted(muted);
646 0 : setTracksEnabled(
647 0 : localUserMediaStream!.stream!.getAudioTracks(),
648 : !muted,
649 : );
650 0 : for (final call in _callSessions) {
651 0 : await call.setMicrophoneMuted(muted);
652 : }
653 : break;
654 0 : case MediaInputKind.videoinput:
655 0 : localUserMediaStream!.setVideoMuted(muted);
656 0 : setTracksEnabled(
657 0 : localUserMediaStream!.stream!.getVideoTracks(),
658 : !muted,
659 : );
660 0 : for (final call in _callSessions) {
661 0 : await call.setLocalVideoMuted(muted);
662 : }
663 : break;
664 : }
665 : }
666 :
667 0 : groupCall.onGroupCallEvent.add(GroupCallStateChange.localMuteStateChanged);
668 : return;
669 : }
670 :
671 0 : void _onIncomingCallInMeshSetup(
672 : GroupCallSession groupCall,
673 : CallSession newCall,
674 : ) {
675 : // The incoming calls may be for another room, which we will ignore.
676 0 : if (newCall.room.id != groupCall.room.id) return;
677 :
678 0 : if (newCall.state != CallState.kRinging) {
679 0 : Logs().v(
680 : '[_onIncomingCallInMeshSetup] Incoming call no longer in ringing state. Ignoring.',
681 : );
682 : return;
683 : }
684 :
685 0 : if (newCall.groupCallId == null ||
686 0 : newCall.groupCallId != groupCall.groupCallId) {
687 0 : Logs().v(
688 0 : '[_onIncomingCallInMeshSetup] Incoming call with groupCallId ${newCall.groupCallId} ignored because it doesn\'t match the current group call',
689 : );
690 : return;
691 : }
692 :
693 0 : final existingCall = _getCallForParticipant(
694 : groupCall,
695 0 : CallParticipant(
696 0 : groupCall.voip,
697 0 : userId: newCall.remoteUserId!,
698 0 : deviceId: newCall.remoteDeviceId,
699 : ),
700 : );
701 :
702 : // if it's an existing call, `_registerListenersForCall` will be called in
703 : // `_replaceCall` that is used in `_onIncomingCallStart`.
704 : if (existingCall != null) return;
705 :
706 0 : Logs().v(
707 0 : '[_onIncomingCallInMeshSetup] GroupCallSession: incoming call from: ${newCall.remoteUserId}${newCall.remoteDeviceId}${newCall.remotePartyId}',
708 : );
709 :
710 0 : _registerListenersBeforeCallAdd(newCall);
711 : }
712 :
713 0 : Future<void> _onIncomingCallInMeshStart(
714 : GroupCallSession groupCall,
715 : CallSession newCall,
716 : ) async {
717 : // The incoming calls may be for another room, which we will ignore.
718 0 : if (newCall.room.id != groupCall.room.id) {
719 : return;
720 : }
721 :
722 0 : if (newCall.state != CallState.kRinging) {
723 0 : Logs().v(
724 : '[_onIncomingCallInMeshStart] Incoming call no longer in ringing state. Ignoring.',
725 : );
726 : return;
727 : }
728 :
729 0 : if (newCall.groupCallId == null ||
730 0 : newCall.groupCallId != groupCall.groupCallId) {
731 0 : Logs().v(
732 0 : '[_onIncomingCallInMeshStart] Incoming call with groupCallId ${newCall.groupCallId} ignored because it doesn\'t match the current group call',
733 : );
734 0 : await newCall.reject();
735 : return;
736 : }
737 :
738 0 : final existingCall = _getCallForParticipant(
739 : groupCall,
740 0 : CallParticipant(
741 0 : groupCall.voip,
742 0 : userId: newCall.remoteUserId!,
743 0 : deviceId: newCall.remoteDeviceId,
744 : ),
745 : );
746 :
747 0 : if (existingCall != null && existingCall.callId == newCall.callId) {
748 : return;
749 : }
750 :
751 0 : Logs().v(
752 0 : '[_onIncomingCallInMeshStart] GroupCallSession: incoming call from: ${newCall.remoteUserId}${newCall.remoteDeviceId}${newCall.remotePartyId}',
753 : );
754 :
755 : // Check if the user calling has an existing call and use this call instead.
756 : if (existingCall != null) {
757 0 : await _replaceCall(groupCall, existingCall, newCall);
758 : } else {
759 0 : await _addCall(groupCall, newCall);
760 : }
761 :
762 0 : await newCall.answerWithStreams(_getLocalStreams());
763 : }
764 :
765 0 : @override
766 : Future<void> setScreensharingEnabled(
767 : GroupCallSession groupCall,
768 : bool enabled,
769 : String desktopCapturerSourceId,
770 : ) async {
771 0 : if (enabled == (localScreenshareStream != null)) {
772 : return;
773 : }
774 :
775 : if (enabled) {
776 : try {
777 0 : Logs().v('Asking for screensharing permissions...');
778 0 : final stream = await _getDisplayMedia(groupCall);
779 0 : for (final track in stream.getTracks()) {
780 : // screen sharing should only have 1 video track anyway, so this only
781 : // fires once
782 0 : track.onEnded = () async {
783 0 : await setScreensharingEnabled(groupCall, false, '');
784 : };
785 : }
786 0 : Logs().v(
787 : 'Screensharing permissions granted. Setting screensharing enabled on all calls',
788 : );
789 0 : _localScreenshareStream = WrappedMediaStream(
790 : stream: stream,
791 0 : participant: groupCall.localParticipant!,
792 0 : room: groupCall.room,
793 0 : client: groupCall.client,
794 : purpose: SDPStreamMetadataPurpose.Screenshare,
795 0 : audioMuted: stream.getAudioTracks().isEmpty,
796 0 : videoMuted: stream.getVideoTracks().isEmpty,
797 : isGroupCall: true,
798 0 : voip: groupCall.voip,
799 : );
800 :
801 0 : _addScreenshareStream(groupCall, localScreenshareStream!);
802 :
803 0 : groupCall.onGroupCallEvent
804 0 : .add(GroupCallStateChange.localScreenshareStateChanged);
805 0 : for (final call in _callSessions) {
806 0 : await call.addLocalStream(
807 0 : await localScreenshareStream!.stream!.clone(),
808 0 : localScreenshareStream!.purpose,
809 : );
810 : }
811 :
812 0 : await groupCall.sendMemberStateEvent();
813 :
814 : return;
815 : } catch (e, s) {
816 0 : Logs().e('[VOIP] Enabling screensharing error', e, s);
817 0 : groupCall.onGroupCallEvent.add(GroupCallStateChange.error);
818 : return;
819 : }
820 : } else {
821 0 : for (final call in _callSessions) {
822 0 : await call.removeLocalStream(call.localScreenSharingStream!);
823 : }
824 0 : await stopMediaStream(localScreenshareStream?.stream);
825 0 : await _removeScreenshareStream(groupCall, localScreenshareStream!);
826 0 : _localScreenshareStream = null;
827 :
828 0 : await groupCall.sendMemberStateEvent();
829 :
830 0 : groupCall.onGroupCallEvent
831 0 : .add(GroupCallStateChange.localMuteStateChanged);
832 : return;
833 : }
834 : }
835 :
836 0 : @override
837 : Future<void> dispose(GroupCallSession groupCall) async {
838 0 : if (localUserMediaStream != null) {
839 0 : await _removeUserMediaStream(groupCall, localUserMediaStream!);
840 0 : _localUserMediaStream = null;
841 : }
842 :
843 0 : if (localScreenshareStream != null) {
844 0 : await stopMediaStream(localScreenshareStream!.stream);
845 0 : await _removeScreenshareStream(groupCall, localScreenshareStream!);
846 0 : _localScreenshareStream = null;
847 : }
848 :
849 : // removeCall removes it from `_callSessions` later.
850 0 : final callsCopy = _callSessions.toList();
851 :
852 0 : for (final call in callsCopy) {
853 0 : await _removeCall(groupCall, call, CallErrorCode.userHangup);
854 : }
855 :
856 0 : _activeSpeaker = null;
857 0 : _activeSpeakerLoopTimeout?.cancel();
858 0 : await _callSetupSubscription?.cancel();
859 0 : await _callStartSubscription?.cancel();
860 : }
861 :
862 0 : @override
863 : bool get isLocalVideoMuted {
864 0 : if (localUserMediaStream != null) {
865 0 : return localUserMediaStream!.isVideoMuted();
866 : }
867 :
868 : return true;
869 : }
870 :
871 0 : @override
872 : bool get isMicrophoneMuted {
873 0 : if (localUserMediaStream != null) {
874 0 : return localUserMediaStream!.isAudioMuted();
875 : }
876 :
877 : return true;
878 : }
879 :
880 0 : @override
881 : Future<void> setupP2PCallsWithExistingMembers(
882 : GroupCallSession groupCall,
883 : ) async {
884 0 : for (final call in _callSessions) {
885 0 : _onIncomingCallInMeshSetup(groupCall, call);
886 0 : await _onIncomingCallInMeshStart(groupCall, call);
887 : }
888 :
889 0 : _callSetupSubscription = groupCall.voip.onIncomingCallSetup.stream.listen(
890 0 : (newCall) => _onIncomingCallInMeshSetup(groupCall, newCall),
891 : );
892 :
893 0 : _callStartSubscription = groupCall.voip.onIncomingCallStart.stream.listen(
894 0 : (newCall) => _onIncomingCallInMeshStart(groupCall, newCall),
895 : );
896 :
897 0 : _onActiveSpeakerLoop(groupCall);
898 : }
899 :
900 0 : @override
901 : Future<void> setupP2PCallWithNewMember(
902 : GroupCallSession groupCall,
903 : CallParticipant rp,
904 : CallMembership mem,
905 : ) async {
906 0 : final existingCall = _getCallForParticipant(groupCall, rp);
907 : if (existingCall != null) {
908 0 : if (existingCall.remoteSessionId != mem.membershipId) {
909 0 : await existingCall.hangup(reason: CallErrorCode.unknownError);
910 : } else {
911 0 : Logs().e(
912 0 : '[VOIP] onMemberStateChanged Not updating _participants list, already have a ongoing call with ${rp.id}',
913 : );
914 : return;
915 : }
916 : }
917 :
918 : // Only initiate a call with a participant who has a id that is lexicographically
919 : // less than your own. Otherwise, that user will call you.
920 0 : if (groupCall.localParticipant!.id.compareTo(rp.id) > 0) {
921 0 : Logs().i('[VOIP] Waiting for ${rp.id} to send call invite.');
922 : return;
923 : }
924 :
925 0 : final opts = CallOptions(
926 0 : callId: genCallID(),
927 0 : room: groupCall.room,
928 0 : voip: groupCall.voip,
929 : dir: CallDirection.kOutgoing,
930 0 : localPartyId: groupCall.voip.currentSessionId,
931 0 : groupCallId: groupCall.groupCallId,
932 : type: CallType.kVideo,
933 0 : iceServers: await groupCall.voip.getIceServers(),
934 : );
935 0 : final newCall = groupCall.voip.createNewCall(opts);
936 :
937 : /// both invitee userId and deviceId are set here because there can be
938 : /// multiple devices from same user in a call, so we specifiy who the
939 : /// invite is for
940 : ///
941 : /// MOVE TO CREATENEWCALL?
942 0 : newCall.remoteUserId = mem.userId;
943 0 : newCall.remoteDeviceId = mem.deviceId;
944 : // party id set to when answered
945 0 : newCall.remoteSessionId = mem.membershipId;
946 :
947 0 : _registerListenersBeforeCallAdd(newCall);
948 :
949 0 : await newCall.placeCallWithStreams(
950 0 : _getLocalStreams(),
951 0 : requestScreenSharing: mem.feeds?.any(
952 0 : (element) =>
953 0 : element['purpose'] == SDPStreamMetadataPurpose.Screenshare,
954 : ) ??
955 : false,
956 : );
957 :
958 0 : await _addCall(groupCall, newCall);
959 : }
960 :
961 0 : @override
962 : List<Map<String, String>>? getCurrentFeeds() {
963 0 : return _getLocalStreams()
964 0 : .map(
965 0 : (feed) => ({
966 0 : 'purpose': feed.purpose,
967 : }),
968 : )
969 0 : .toList();
970 : }
971 :
972 0 : @override
973 : bool operator ==(Object other) =>
974 0 : identical(this, other) || (other is MeshBackend && type == other.type);
975 0 : @override
976 0 : int get hashCode => type.hashCode;
977 :
978 : /// get everything is livekit specific mesh calls shouldn't be affected by these
979 0 : @override
980 : Future<void> onCallEncryption(
981 : GroupCallSession groupCall,
982 : String userId,
983 : String deviceId,
984 : Map<String, dynamic> content,
985 : ) async {
986 : return;
987 : }
988 :
989 0 : @override
990 : Future<void> onCallEncryptionKeyRequest(
991 : GroupCallSession groupCall,
992 : String userId,
993 : String deviceId,
994 : Map<String, dynamic> content,
995 : ) async {
996 : return;
997 : }
998 :
999 0 : @override
1000 : Future<void> onLeftParticipant(
1001 : GroupCallSession groupCall,
1002 : List<CallParticipant> anyLeft,
1003 : ) async {
1004 : return;
1005 : }
1006 :
1007 0 : @override
1008 : Future<void> onNewParticipant(
1009 : GroupCallSession groupCall,
1010 : List<CallParticipant> anyJoined,
1011 : ) async {
1012 : return;
1013 : }
1014 :
1015 0 : @override
1016 : Future<void> requestEncrytionKey(
1017 : GroupCallSession groupCall,
1018 : List<CallParticipant> remoteParticipants,
1019 : ) async {
1020 : return;
1021 : }
1022 :
1023 0 : @override
1024 : Future<void> preShareKey(GroupCallSession groupCall) async {
1025 : return;
1026 : }
1027 : }
|
