// SPDX-FileCopyrightText: 2019-Present Christian Kußowski // SPDX-FileCopyrightText: 2019-Present Contributors to FluffyChat // // SPDX-License-Identifier: AGPL-3.0-or-later import 'dart:async'; import 'dart:convert'; import 'dart:math'; import 'package:collection/collection.dart'; import 'package:tatchat/l10n/l10n.dart'; import 'package:tatchat/utils/call_kit_params.dart'; import 'package:tatchat/utils/error_reporter.dart'; import 'package:tatchat/utils/matrix_live_kit_calls/call_keys_event_content.dart'; import 'package:tatchat/utils/matrix_live_kit_calls/matrix_live_kit_call.dart'; import 'package:tatchat/utils/matrix_live_kit_calls/matrix_live_kit_call_member.dart'; import 'package:tatchat/utils/platform_infos.dart'; import 'package:tatchat/utils/position_from_build_context.dart'; import 'package:tatchat/widgets/future_loading_dialog.dart'; import 'package:tatchat/widgets/matrix.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter_callkit_incoming/flutter_callkit_incoming.dart'; import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; import 'package:just_audio/just_audio.dart'; import 'package:livekit_client/livekit_client.dart' as lk; import 'package:material_ui/material_ui.dart'; import 'package:matrix/matrix.dart'; import 'package:wakelock_plus/wakelock_plus.dart'; class CallViewModelState { lk.Room? room; lk.LocalVideoTrack? localVideoTrack; bool startWithAudio = true; bool isLoading = false; Object? error; String? focusedTrack; } class CallViewModel extends ValueNotifier { final Room room; Timeline? timeline; final AudioPlayer _audioPlayer = AudioPlayer(); DateTime? startTime; StreamSubscription? _onCallEncryptionKeysSub, _onCallMembersChanged; Timer? _resendCallMemberState; bool waitForOtherSide = false; String? callKitId; Set _lastSharedParticipants = {}; DateTime? _keyCreatedAt; Uint8List? _lastKey; lk.BaseKeyProvider? _keyProvider; CallViewModel({required this.room}) : super(CallViewModelState()) { _init(); } void setFocusedTrack(String trackId) { Logs().d('Set focus on', trackId); if (value.focusedTrack == trackId) { value.focusedTrack = null; } else { value.focusedTrack = trackId; } notifyListeners(); } Future _onCallEncryptionKeys(CallKeysEvent event) async { final callKeys = event.callKeysContent; final keyProvider = _keyProvider; if (keyProvider == null) { ErrorReporter( null, 'Received a new key but the keyProvider is not ready yet!', ).onErrorCallback(Exception(), StackTrace.current); return; } await keyProvider.setRawKey( base64Decode(callKeys.keys.key), participantId: '${event.sender}:${callKeys.member.claimedDeviceId}', keyIndex: callKeys.keys.index, ); } Future _createKeyAndShare() async { final liveKitRoom = value.room; if (liveKitRoom == null) return; final ownMemberId = '${room.client.userID}:${room.client.deviceID}'; final currentParticipants = room .getActiveMatrixRtcMembers() .map((member) => member.membershipId) .whereType() .toSet() ..remove(ownMemberId); if (setEquals(currentParticipants, _lastSharedParticipants) && currentParticipants.isNotEmpty) { Logs().d( 'Participant list has not changed. No need to share keys again!', ); return; } var index = liveKitRoom.roomOptions.encryption!.keyProvider.getLatestIndex( ownMemberId, ); final keyCreatedAt = _keyCreatedAt; final canJustForwardToNewUsers = keyCreatedAt != null && DateTime.now().difference(keyCreatedAt).inSeconds < 15 && _lastSharedParticipants.difference(currentParticipants).isEmpty; late final Uint8List key; if (_lastKey == null || !canJustForwardToNewUsers) { // Key generation final rng = Random.secure(); key = Uint8List(16); key.setAll(0, Iterable.generate(key.length, (i) => rng.nextInt(256))); if (_lastKey != null) index = (index + 1) % 256; await liveKitRoom.roomOptions.encryption!.keyProvider.setRawKey( key, keyIndex: index, participantId: ownMemberId, ); _keyCreatedAt = DateTime.now(); _lastKey = key; if (_lastKey != null) { await liveKitRoom.e2eeManager?.setKeyIndex( index, participantIdentity: ownMemberId, ); } } else { key = _lastKey!; } final forwardParticipants = canJustForwardToNewUsers ? currentParticipants.difference(_lastSharedParticipants) : currentParticipants; final deviceKeys = []; for (final membershipId in forwardParticipants) { final membershipParts = membershipId.split(':'); final deviceId = membershipParts.removeLast(); final userId = membershipParts.join(':'); final keys = room.client.userDeviceKeys[userId]?.deviceKeys[deviceId]; if (keys == null) { Logs().w('No device keys found for $membershipId'); continue; } deviceKeys.add(keys); } _lastSharedParticipants = currentParticipants; await room.shareMatrixRtcCallKey( key: key, index: index, memberId: ownMemberId, deviceKeys: deviceKeys, ); } Future _init() async { _onCallEncryptionKeysSub = room.client.onCallEncryptionKeys.listen( _onCallEncryptionKeys, ); await WakelockPlus.enable(); await lk.LiveKitClient.initialize(); // kHKDF matches the key derivation used by element-web's LiveKit JS SDK final keyProviderOptions = rtc.KeyProviderOptions( sharedKey: false, ratchetSalt: Uint8List.fromList('LKFrameEncryptionKey'.codeUnits), ratchetWindowSize: 0, discardFrameWhenCryptorNotReady: true, keyDerivationAlgorithm: rtc.KeyDerivationAlgorithm.kHKDF, keyRingSize: 255, ); final nativeKeyProvider = await rtc.frameCryptorFactory .createDefaultKeyProvider(keyProviderOptions); final baseKeyProvider = _keyProvider = lk.BaseKeyProvider( nativeKeyProvider, keyProviderOptions, ); final liveKitRoom = value.room = lk.Room( roomOptions: lk.RoomOptions( adaptiveStream: true, dynacast: true, defaultAudioCaptureOptions: lk.AudioCaptureOptions( echoCancellation: true, noiseSuppression: true, autoGainControl: true, highPassFilter: true, ), defaultScreenShareCaptureOptions: lk.ScreenShareCaptureOptions( useiOSBroadcastExtension: PlatformInfos.isIOS, ), defaultAudioOutputOptions: lk.AudioOutputOptions( speakerOn: PlatformInfos.isMobile, ), encryption: lk.E2EEOptions(keyProvider: baseKeyProvider), ), ); liveKitRoom.events.on((event) async { if (event.participant.identity == liveKitRoom.localParticipant?.identity) { return; } if (waitForOtherSide) { waitForOtherSide = false; await _audioPlayer.stop(); final callKitId = this.callKitId; if (callKitId != null) { await FlutterCallkitIncoming.setCallConnected(callKitId); } } _playJoinSound(); }); liveKitRoom.events.on(_playLeaveSound); liveKitRoom.events.on((event) { if (event.state == .kOk || event.state == .kNew) { Logs().i( 'E2EE event from ${event.participant} (${event.publication})', event.state, ); return; } if (event.state == .kMissingKey) { Logs().d('Waiting for call key...', event); return; } Logs().e('LiveKit E2EE Error', event); }); liveKitRoom.addListener(notifyListeners); if (room.isDirectChat && room.hasActiveMatrixRtcCall) { await connect(); } else { value.localVideoTrack = await lk.LocalVideoTrack.createCameraTrack(); if (room.getActiveMatrixRtcMembers().firstOrNull?.callIntent == .voice) { value.localVideoTrack?.mute(); } value.localVideoTrack?.addListener(notifyListeners); notifyListeners(); } } Future togglePreviewCamera() async { final track = value.localVideoTrack; if (track == null) return; if (!track.muted) { await track.mute(); return; } track.removeListener(notifyListeners); await track.stop(); value.localVideoTrack = await lk.LocalVideoTrack.createCameraTrack(); value.localVideoTrack?.addListener(notifyListeners); notifyListeners(); } void togglePreviewMic() { value.startWithAudio = !value.startWithAudio; notifyListeners(); } bool participantRaisedHand(String matrixId) { final timeline = this.timeline; if (timeline == null) return false; return room.participantRaisedHandInMatrixRtcCall(matrixId, timeline); } Future _onNewTimelineEvent(int i) async { final event = timeline?.events[i]; if (event == null) return; switch (event.type) { case 'io.element.call.reaction': Logs().d( 'Call reactions are not yet handled', event.content.tryGet('emoji'), ); // TODO: handle reaction break; case EventTypes.Reaction: final reactedEvent = await timeline!.getEventById( event.relationshipEventId!, ); if (reactedEvent != null && reactedEvent.type == MatrixRtcCallMember.eventType && event.senderId == reactedEvent.senderId && event.content .tryGetMap('m.relates_to') ?.tryGet('key') == '🖐️') { Logs().d('User raised hand!'); _playRaiseHandSound(); notifyListeners(); } break; case EventTypes.Redaction: notifyListeners(); break; } } Future connect() async { try { value.isLoading = true; notifyListeners(); final playWaitingSound = !room.hasActiveMatrixRtcCall && room.isDirectChat; final intent = room.getActiveMatrixRtcMembers().firstOrNull?.callIntent ?? (value.localVideoTrack?.muted == false ? .video : .voice); if (PlatformInfos.isMobile) { await lk.AudioManager.instance.setSpeakerOutputPreferred( intent == .video, ); } timeline = await room.getTimeline(onInsert: _onNewTimelineEvent); final credentials = await room.joinMatrixRtcCall(intent: intent); _onCallMembersChanged = room.client.onSync.stream .where( (syncUpdate) => syncUpdate.rooms?.join?[room.id]?.timeline?.events?.any( (event) => event.type == MatrixRtcCallMember.eventType, ) ?? false, ) .listen((_) { notifyListeners(); _createKeyAndShare(); }); await _createKeyAndShare(); final startWithVideo = !(value.localVideoTrack?.muted ?? intent == .voice); final startWithAudio = value.startWithAudio; await value.localVideoTrack?.stop(); await value.localVideoTrack?.dispose(); value.localVideoTrack = null; await value.room!.connect( credentials.url, credentials.jwt, fastConnectOptions: lk.FastConnectOptions( microphone: lk.TrackOption( track: startWithAudio ? await lk.LocalAudioTrack.create() : null, ), camera: lk.TrackOption( track: startWithVideo ? await lk.LocalVideoTrack.createCameraTrack() : null, ), ), ); _resendCallMemberState?.cancel(); _resendCallMemberState = Timer.periodic(const Duration(hours: 1), (_) { final ownMembership = room.ownMatrixRtcMembership; if (ownMembership == null) { _resendCallMemberState?.cancel(); return; } room.setMatrixRtcMembershipState( ownMembership.fociPreferred, intent: intent, ); }); startTime = DateTime.now(); if (PlatformInfos.isMobile) { final activeCalls = await FlutterCallkitIncoming.activeCalls(); callKitId = activeCalls .firstWhereOrNull( (call) => call.extra?['roomId'] == room.id && call.extra?['clientName'] == room.client.clientName, ) ?.id; if (callKitId == null) { final params = await buildFluffyChatCallKitParams( room, intent: intent, ); await FlutterCallkitIncoming.startCall(params); callKitId = params.id; } } if (playWaitingSound) { waitForOtherSide = true; _playWaitingSound(); } else { final callKitId = this.callKitId; if (callKitId != null) { await FlutterCallkitIncoming.setCallConnected(callKitId); } _playJoinSound(); } notifyListeners(); } catch (e, s) { Logs().e('Error while connecting', e, s); value.error = e; ErrorReporter(null, 'Unable to connect').onErrorCallback(e, s); } finally { value.isLoading = false; notifyListeners(); } } Future close(BuildContext context) async { _playLeaveSound(); final liveKitRoom = value.room; if (liveKitRoom != null) { liveKitRoom.removeListener(notifyListeners); await showFutureLoadingDialog( context: context, future: () async { if (liveKitRoom.connectionState != lk.ConnectionState.disconnected) { await liveKitRoom.disconnect(); } await liveKitRoom.dispose(); value.room = null; if (room.ownMatrixRtcMembership != null) { await room.leaveMatrixRtcCall(); } }, ); } if (context.mounted) { Matrix.of(context).activeCallRoomId.value = null; } } @override void dispose() { _onCallEncryptionKeysSub?.cancel(); _onCallMembersChanged?.cancel(); _resendCallMemberState?.cancel(); timeline?.cancelSubscriptions(); value.localVideoTrack?.dispose(); final callKitId = this.callKitId; if (callKitId != null) { FlutterCallkitIncoming.endCall(callKitId); } final liveKitRoom = value.room; if (liveKitRoom != null) { liveKitRoom.removeListener(notifyListeners); if (liveKitRoom.connectionState != lk.ConnectionState.disconnected) { liveKitRoom.disconnect(); } liveKitRoom.dispose(); room.leaveMatrixRtcCall(); } _audioPlayer.dispose(); WakelockPlus.disable(); super.dispose(); } Future _playSoundIndex( AudioSource audioSource, { LoopMode loopMode = .off, }) async { if (_audioPlayer.playing) await _audioPlayer.stop(); await _audioPlayer.setAudioSource(audioSource); if (_audioPlayer.loopMode != loopMode) { await _audioPlayer.setLoopMode(loopMode); } try { await _audioPlayer.play(); } catch (e) { Logs().w('Unable to start audio player. Try again...', e); await Future.delayed(const Duration(milliseconds: 300)); await _audioPlayer.play(); } } final _joinSource = AudioSource.asset('assets/sounds/call_join.mp3'); final _leaveSource = AudioSource.asset('assets/sounds/call_leave.mp3'); final _raiseHandSource = AudioSource.asset( 'assets/sounds/call_attention.mp3', ); final _waitSource = AudioSource.asset('assets/sounds/call_wait.mp3'); Future _playJoinSound() => _playSoundIndex(_joinSource); Future _playLeaveSound([_]) => _playSoundIndex(_leaveSource); Future _playRaiseHandSound() => _playSoundIndex(_raiseHandSource); Future _playWaitingSound() => _playSoundIndex(_waitSource, loopMode: .all); Future _selectMediaDevice( BuildContext context, List devices, String? activeDeviceId, ) async { if (!context.mounted) return null; return await showMenu( context: context, position: context.position, items: devices.isEmpty ? [ PopupMenuItem( enabled: false, child: Text(L10n.of(context).noDevicesFound), ), ] : devices .map( (device) => PopupMenuItem( value: device, child: Row( mainAxisSize: .min, spacing: 12, children: [ Icon( device.deviceId == activeDeviceId ? Icons.check_circle_outlined : Icons.circle_outlined, ), Expanded( child: Text( device.label, maxLines: 1, overflow: .ellipsis, ), ), ], ), ), ) .toList(), ); } Future selectCamera(BuildContext context) async { final devices = await lk.Hardware.instance.videoInputs(); if (!context.mounted) return; final source = await _selectMediaDevice( context, devices, value.room?.selectedVideoInputDeviceId, ); if (source == null) return; await value.room?.setVideoInputDevice(source); } Future selectMicrophone(BuildContext context) async { final devices = await lk.Hardware.instance.audioInputs(); if (!context.mounted) return; final source = await _selectMediaDevice( context, devices, value.room?.selectedAudioInputDeviceId, ); if (source == null) return; await value.room?.setAudioInputDevice(source); } Future selectSpeaker(BuildContext context) async { final devices = await lk.Hardware.instance.audioOutputs(); if (!context.mounted) return; if (devices.isEmpty && PlatformInfos.isMobile) { final preferSpeaker = lk.AudioManager.instance.isSpeakerOutputPreferred; final toggle = await showMenu( context: context, position: context.position, items: [ PopupMenuItem( value: true, child: Row( mainAxisSize: .min, spacing: 12, children: [ Icon( preferSpeaker ? Icons.check_circle_outlined : Icons.circle_outlined, ), Expanded( child: Text( L10n.of(context).preferSpeaker, maxLines: 1, overflow: .ellipsis, ), ), ], ), ), ], ); if (toggle == null) return; await lk.AudioManager.instance.setSpeakerOutputPreferred(!preferSpeaker); return; } final source = await _selectMediaDevice( context, devices, value.room?.selectedAudioOutputDeviceId, ); if (source == null) return; await value.room?.setAudioOutputDevice(source); } }