Initial commit

lib/utils/matrix_live_kit_calls/call_keys_event_content.dart

@@ -0,0 +1,125 @@
+// SPDX-FileCopyrightText: 2019-Present Christian Kußowski
+// SPDX-FileCopyrightText: 2019-Present Contributors to FluffyChat
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import 'dart:convert';
+
+import 'package:matrix/matrix.dart';
+
+class CallKeysEvent extends ToDeviceEvent {
+  CallKeysEvent._({
+    required super.sender,
+    required super.type,
+    required super.content,
+  });
+
+  factory CallKeysEvent.fromToDeviceEvent(ToDeviceEvent event) =>
+      CallKeysEvent._(
+        content: event.content,
+        sender: event.sender,
+        type: event.type,
+      );
+
+  CallKeysEventContent get callKeysContent =>
+      CallKeysEventContent.fromJson(super.content);
+
+  String get key =>
+      String.fromCharCodes(base64Decode(callKeysContent.keys.key));
+}
+
+/// To-device event content for encryption key exchange (EncryptionKeysToDeviceEventContent).
+class CallKeysEventContent {
+  static const String eventType = 'io.element.call.encryption_keys';
+
+  final CallKeysEntry keys;
+  final CallKeysMember member;
+  final String roomId;
+  final CallKeysSession session;
+  final int? sentTs;
+
+  const CallKeysEventContent({
+    required this.keys,
+    required this.member,
+    required this.roomId,
+    required this.session,
+    this.sentTs,
+  });
+
+  factory CallKeysEventContent.fromJson(Map<String, Object?> json) =>
+      CallKeysEventContent(
+        keys: CallKeysEntry.fromJson(
+          Map<String, Object?>.from(json['keys'] as Map),
+        ),
+        member: CallKeysMember.fromJson(
+          Map<String, Object?>.from(json['member'] as Map),
+        ),
+        roomId: json['room_id'] as String,
+        session: CallKeysSession.fromJson(
+          Map<String, Object?>.from(json['session'] as Map),
+        ),
+        sentTs: json['sent_ts'] as int?,
+      );
+
+  Map<String, Object?> toJson() => {
+    'keys': keys.toJson(),
+    'member': member.toJson(),
+    'room_id': roomId,
+    'session': session.toJson(),
+    if (sentTs != null) 'sent_ts': sentTs,
+  };
+}
+
+class CallKeysEntry {
+  final int index;
+  final String key;
+
+  const CallKeysEntry({required this.index, required this.key});
+
+  factory CallKeysEntry.fromJson(Map<String, Object?> json) =>
+      CallKeysEntry(index: json['index'] as int, key: json['key'] as String);
+
+  Map<String, Object?> toJson() => {'index': index, 'key': key};
+}
+
+class CallKeysMember {
+  final String id;
+  final String claimedDeviceId;
+
+  const CallKeysMember({required this.id, required this.claimedDeviceId});
+
+  factory CallKeysMember.fromJson(Map<String, Object?> json) => CallKeysMember(
+    id: json['id'] as String,
+    claimedDeviceId: json['claimed_device_id'] as String,
+  );
+
+  Map<String, Object?> toJson() => {
+    'id': id,
+    'claimed_device_id': claimedDeviceId,
+  };
+}
+
+class CallKeysSession {
+  final String application;
+  final String callId;
+  final String scope;
+
+  const CallKeysSession({
+    required this.application,
+    required this.callId,
+    required this.scope,
+  });
+
+  factory CallKeysSession.fromJson(Map<String, Object?> json) =>
+      CallKeysSession(
+        application: json['application'] as String,
+        callId: json['call_id'] as String,
+        scope: json['scope'] as String,
+      );
+
+  Map<String, Object?> toJson() => {
+    'application': application,
+    'call_id': callId,
+    'scope': scope,
+  };
+}

lib/utils/matrix_live_kit_calls/matrix_live_kit_call.dart

@@ -0,0 +1,352 @@
+// SPDX-FileCopyrightText: 2019-Present Christian Kußowski
+// SPDX-FileCopyrightText: 2019-Present Contributors to FluffyChat
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import 'dart:convert';
+import 'dart:typed_data';
+
+import 'package:collection/collection.dart';
+import 'package:tatchat/config/setting_keys.dart';
+import 'package:tatchat/utils/matrix_live_kit_calls/call_keys_event_content.dart';
+import 'package:tatchat/utils/matrix_live_kit_calls/matrix_live_kit_call_member.dart';
+import 'package:matrix/matrix.dart';
+
+extension MatrixRtcClientExtension on Client {
+  Future<List<String>> getLiveKitServiceUrls() async {
+    if (AppSettings.customLiveKitInstance.value.isNotEmpty) {
+      return [AppSettings.customLiveKitInstance.value];
+    }
+    final wellKnown = await getWellknown();
+    final rtcFociMap = wellKnown.additionalProperties
+        .tryGetMap<String, Object?>(
+          'org.matrix.msc4143.rtc_foci',
+          TryGet.silent,
+        ); // Famedly style
+    if (rtcFociMap != null) {
+      return [?rtcFociMap.tryGet<String>('livekit_service_url')];
+    }
+
+    // TODO: Also query /_matrix/client/unstable/org.matrix.msc4143/rtc/transports
+
+    return wellKnown.additionalProperties
+            .tryGetList<Map<String, Object?>>('org.matrix.msc4143.rtc_foci')
+            ?.map((foci) => foci.tryGet<String>('livekit_service_url'))
+            .whereType<String>()
+            .toList() ??
+        [];
+  }
+
+  Stream<CallKeysEvent> get onCallEncryptionKeys => onToDeviceEvent.stream
+      .where((event) => event.type == CallKeysEventContent.eventType)
+      .map(CallKeysEvent.fromToDeviceEvent);
+}
+
+extension MatrixRtcRoomExtension on Room {
+  bool get hasActiveMatrixRtcCall => getActiveMatrixRtcMembers().isNotEmpty;
+
+  MatrixRtcCallIntent? get activeMatrixRtcCallIntent {
+    final members = getActiveMatrixRtcMembers();
+    if (members.isEmpty) return null;
+    if (members.any((member) => member.callIntent == .video)) return .video;
+    return .voice;
+  }
+
+  bool get hasPermissionForMatrixRtcCall =>
+      canChangeStateEvent(MatrixRtcCallMember.eventType);
+
+  List<MatrixRtcCallMember> getActiveMatrixRtcMembers() {
+    final states = this.states[MatrixRtcCallMember.eventType];
+    if (states == null) return [];
+    // Empty content {} means the member has left — filter those out
+    final activeMemberStates = states.values.where((event) {
+      if (event.content.isEmpty) return false;
+      if (event is! Event) return false;
+      try {
+        final validContent = MatrixRtcCallMember.fromJson(
+          event.content,
+          senderId: event.senderId,
+        );
+        if (validContent.focusActive?.type != 'livekit') return false;
+        final expiresAt = (validContent.createdAt ?? event.originServerTs).add(
+          validContent.expires,
+        );
+        if (expiresAt.isBefore(DateTime.now())) return false;
+        return true;
+      } catch (e, s) {
+        Logs().d('Invalid "org.matrix.msc3401.call.member" event!', e, s);
+        return false;
+      }
+    });
+    return activeMemberStates
+        .map(
+          (state) => MatrixRtcCallMember.fromJson(
+            state.content,
+            senderId: state.senderId,
+          ),
+        )
+        .toList();
+  }
+
+  String get _ownMatrixRtcMembershipStateKey =>
+      '_${client.userID}_${client.deviceID}_m.call';
+
+  MatrixRtcCallMember? get ownMatrixRtcMembership {
+    final state = getState(
+      MatrixRtcCallMember.eventType,
+      _ownMatrixRtcMembershipStateKey,
+    );
+    if (state == null || state.content.isEmpty) return null;
+    try {
+      return MatrixRtcCallMember.fromJson(
+        state.content,
+        senderId: state.senderId,
+      );
+    } catch (e, s) {
+      Logs().d(
+        'Unknown format for ${MatrixRtcCallMember.eventType} event',
+        e,
+        s,
+      );
+      return null;
+    }
+  }
+
+  Future<void> setMatrixRtcMembershipState(
+    List<MatrixRtcFocusPreferred> fociPreferred, {
+    MatrixRtcCallIntent intent = MatrixRtcCallIntent.video,
+  }) => client.setRoomStateWithKey(
+    id,
+    MatrixRtcCallMember.eventType,
+    _ownMatrixRtcMembershipStateKey,
+    MatrixRtcCallMember(
+      application: 'm.call',
+      callId: '',
+      deviceId: client.deviceID,
+      expires: Duration(hours: 4),
+      createdAt: DateTime.now(),
+      fociPreferred: fociPreferred,
+      focusActive: MatrixRtcFocusActive(
+        focusSelection: 'oldest_membership',
+        type: 'livekit',
+      ),
+      callIntent: intent,
+      membershipId: '${client.userID}:${client.deviceID}',
+      scope: 'm.room',
+      senderId: null,
+    ).toJson(),
+  );
+
+  bool participantRaisedHandInMatrixRtcCall(
+    String matrixId,
+    Timeline timeline,
+  ) {
+    final stateEvent =
+        states[MatrixRtcCallMember.eventType]?[_ownMatrixRtcMembershipStateKey]
+            as Event?;
+    if (stateEvent == null) return false;
+    final aggregatedEvents = timeline
+        .aggregatedEvents[stateEvent.eventId]?['m.annotation']
+        ?.where(
+          (event) =>
+              event.senderId == matrixId &&
+              event.content
+                      .tryGetMap<String, Object?>('m.relates_to')
+                      ?.tryGet<String>('key') ==
+                  '🖐️',
+        );
+    if (aggregatedEvents == null || aggregatedEvents.isEmpty) return false;
+
+    return true;
+  }
+
+  Future<String?> raiseHandInMatrixRtcCall() async {
+    final ownMemberStateEvent =
+        states[MatrixRtcCallMember.eventType]?[_ownMatrixRtcMembershipStateKey]
+            as Event?;
+    if (ownMemberStateEvent == null) return null;
+    return await sendReaction(ownMemberStateEvent.eventId, '🖐️');
+  }
+
+  Future<String?> lowerHandInMatrixRtcCall(Timeline timeline) async {
+    final ownMemberStateEvent =
+        states[MatrixRtcCallMember.eventType]?[_ownMatrixRtcMembershipStateKey]
+            as Event?;
+    if (ownMemberStateEvent == null) return null;
+    final reaction = timeline.events.firstWhereOrNull(
+      (event) =>
+          event.type == EventTypes.Reaction &&
+          event.relationshipEventId == ownMemberStateEvent.eventId &&
+          event.senderId == client.userID &&
+          event.status.isSent &&
+          event.content
+                  .tryGetMap<String, Object?>('m.relates_to')
+                  ?.tryGet<String>('key') ==
+              '🖐️',
+    );
+    if (reaction == null) {
+      throw Exception('Hand raise reaction event not found!');
+    }
+    return await reaction.redactEvent();
+  }
+
+  Future<void> shareMatrixRtcCallKey({
+    required Uint8List key,
+    required int index,
+    required String memberId,
+    required List<DeviceKeys> deviceKeys,
+  }) async {
+    if (deviceKeys.isEmpty) {
+      Logs().d('No devices to share call keys with');
+      return;
+    }
+    Logs().d(
+      'Share call keys with',
+      deviceKeys.map((key) => '${key.userId}:${key.deviceId}'),
+    );
+
+    final doNotEncryptTo = deviceKeys.where((d) => !d.encryptToDevice);
+    if (doNotEncryptTo.isNotEmpty) {
+      Logs().w(
+        'Not sharing keys with ${doNotEncryptTo.length} device(s) because of key sharing restrictions!',
+      );
+      deviceKeys.removeWhere((d) => !d.encryptToDevice);
+    }
+
+    await client.sendToDeviceEncrypted(
+      deviceKeys,
+      CallKeysEventContent.eventType,
+      CallKeysEventContent(
+        keys: CallKeysEntry(index: index, key: base64Encode(key)),
+        member: CallKeysMember(id: memberId, claimedDeviceId: client.deviceID!),
+        roomId: id,
+        sentTs: DateTime.now().millisecondsSinceEpoch,
+        session: CallKeysSession(
+          application: 'm.call',
+          callId: '',
+          scope: 'm.room',
+        ),
+      ).toJson(),
+    );
+  }
+
+  Future<MatrixRtcCredentials> joinMatrixRtcCall({
+    MatrixRtcCallIntent intent = MatrixRtcCallIntent.video,
+  }) async {
+    await postLoad();
+    if (ownMatrixRtcMembership != null) {
+      Logs().w(
+        'User has already an active rtc membership state in this room. Resetting it now...',
+      );
+    }
+    Logs().d('[Join MatrixRtc Call] (1/5) Get LiveKit Backend Urls...');
+
+    final hasActiveMatrixRtcCall = this.hasActiveMatrixRtcCall;
+
+    final urls = getActiveMatrixRtcMembers()
+        .map(
+          (state) =>
+              state.fociPreferred.map((focus) => focus.livekitServiceUrl),
+        )
+        .fold<List<String>>([], (urls, foci) => [...urls, ...foci]);
+
+    if (urls.isEmpty) {
+      urls.addAll(await client.getLiveKitServiceUrls());
+    }
+
+    Logs().v('Available SFUs', urls);
+    if (urls.isEmpty) {
+      throw Exception('This server does not support livekit calls!');
+    }
+
+    Logs().d(
+      '[Join MatrixRtc Call] (2/5) Set "${MatrixRtcCallMember.eventType}" State event...',
+    );
+    await setMatrixRtcMembershipState(
+      urls
+          .map(
+            (url) => MatrixRtcFocusPreferred(
+              type: 'livekit',
+              livekitServiceUrl: url,
+              livekitAlias: id,
+            ),
+          )
+          .toList(),
+      intent: intent,
+    );
+
+    if (!hasActiveMatrixRtcCall) {
+      final dmUserId = directChatMatrixID;
+      if (dmUserId != null) {
+        await sendRtcNotification(type: .ring, userIds: [dmUserId]);
+      } else {
+        await sendRtcNotification(type: .notification, mentionRoom: true);
+      }
+    }
+
+    Logs().v('[Join MatrixRtc Call] (3/5) Request OpenId Token...');
+    final openIdCredentials = await client.requestOpenIdToken(
+      client.userID!,
+      {},
+    );
+
+    for (final url in urls) {
+      try {
+        Logs().d(
+          '[Join MatrixRtc Call] (4/5) Try authenticate to LiveKit instance $url...',
+        );
+        final response = await client.httpClient.post(
+          Uri.parse('${url.replaceAll(RegExp(r'/+$'), '')}/sfu/get'),
+          headers: {'Content-Type': 'application/json'},
+          body: jsonEncode({
+            'room': id,
+            'openid_token': openIdCredentials.toJson(),
+            'device_id': client.deviceID,
+          }),
+        );
+        if (response.statusCode != 200) {
+          throw Exception(response.reasonPhrase ?? response.body);
+        }
+        final json =
+            jsonDecode(utf8.decode(response.bodyBytes)) as Map<String, Object?>;
+        Logs().d('Use SFU', url);
+        return MatrixRtcCredentials.fromJson(json);
+      } catch (e) {
+        Logs().v(
+          '[Join MatrixRtc Call] (4/4) Unable to authenticate to $url',
+          e,
+        );
+      }
+    }
+    throw Exception(
+      'Unable to authenticate to any of the visible LiveKit instances!',
+    );
+  }
+
+  Future<void> leaveMatrixRtcCall() async {
+    await client.setRoomStateWithKey(
+      id,
+      MatrixRtcCallMember.eventType,
+      _ownMatrixRtcMembershipStateKey,
+      {},
+    );
+  }
+}
+
+class MatrixRtcCredentials {
+  final String url, jwt;
+  const MatrixRtcCredentials({required this.url, required this.jwt});
+  factory MatrixRtcCredentials.fromJson(Map<String, Object?> json) =>
+      MatrixRtcCredentials(
+        url: json['url'] as String,
+        jwt: json['jwt'] as String,
+      );
+}
+
+enum MatrixRtcCallIntent {
+  voice(callKitType: 0),
+  video(callKitType: 1);
+
+  final int callKitType;
+  const MatrixRtcCallIntent({required this.callKitType});
+}

lib/utils/matrix_live_kit_calls/matrix_live_kit_call_member.dart

@@ -0,0 +1,135 @@
+// SPDX-FileCopyrightText: 2019-Present Christian Kußowski
+// SPDX-FileCopyrightText: 2019-Present Contributors to FluffyChat
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+import 'package:collection/collection.dart';
+import 'package:tatchat/utils/matrix_live_kit_calls/matrix_live_kit_call.dart';
+import 'package:matrix/matrix_api_lite/utils/try_get_map_extension.dart';
+
+class MatrixRtcCallMember {
+  static const String eventType = 'org.matrix.msc3401.call.member';
+
+  final String application;
+  final String callId;
+  final String? deviceId;
+  final DateTime? createdAt;
+  final Duration expires;
+  final List<MatrixRtcFocusPreferred> fociPreferred;
+  final MatrixRtcFocusActive? focusActive;
+  final MatrixRtcCallIntent callIntent;
+  final String? membershipId;
+  final String scope;
+
+  /// Not part of JSON content but for convenience
+  final String? senderId;
+
+  const MatrixRtcCallMember({
+    required this.application,
+    required this.callId,
+    required this.deviceId,
+    required this.expires,
+    required this.fociPreferred,
+    required this.focusActive,
+    required this.callIntent,
+    required this.membershipId,
+    required this.scope,
+    required this.createdAt,
+    required this.senderId,
+  });
+
+  factory MatrixRtcCallMember.fromJson(
+    Map<String, Object?> json, {
+    required String? senderId,
+  }) => MatrixRtcCallMember(
+    application: json.tryGet<String>('application') ?? '',
+    callId: json.tryGet<String>('call_id') ?? '',
+    deviceId: json.tryGet<String>('device_id'),
+    createdAt: json.tryGet<int>('created_at') == null
+        ? null
+        : DateTime.fromMillisecondsSinceEpoch(json.tryGet<int>('created_at')!),
+    expires: Duration(milliseconds: json.tryGet<int>('expires') ?? 0),
+    fociPreferred:
+        (json['foci_preferred'] as List?)
+            ?.whereType<Map>()
+            .map(
+              (e) => MatrixRtcFocusPreferred.fromJson(
+                Map<String, Object?>.from(e),
+              ),
+            )
+            .toList() ??
+        [],
+    focusActive: json['focus_active'] is Map
+        ? MatrixRtcFocusActive.fromJson(
+            Map<String, Object?>.from(json['focus_active'] as Map),
+          )
+        : null,
+    callIntent:
+        MatrixRtcCallIntent.values.singleWhereOrNull(
+          (intent) => intent.name == json.tryGet<String>('m.call.intent'),
+        ) ??
+        MatrixRtcCallIntent.video,
+    membershipId: json.tryGet<String>('membershipID'),
+    scope: json.tryGet<String>('scope') ?? 'm.room',
+    senderId: senderId,
+  );
+
+  Map<String, Object?> toJson() => {
+    'application': application,
+    'call_id': callId,
+    if (deviceId != null) 'device_id': deviceId,
+    'expires': expires.inMilliseconds,
+    'foci_preferred': fociPreferred.map((f) => f.toJson()).toList(),
+    if (focusActive != null) 'focus_active': focusActive!.toJson(),
+    'm.call.intent': callIntent.name,
+    if (membershipId != null) 'membershipID': membershipId,
+    'scope': scope,
+    'created_at': ?createdAt?.millisecondsSinceEpoch,
+  };
+}
+
+class MatrixRtcFocusPreferred {
+  final String type;
+  final String livekitServiceUrl;
+  final String livekitAlias;
+
+  const MatrixRtcFocusPreferred({
+    required this.type,
+    required this.livekitServiceUrl,
+    required this.livekitAlias,
+  });
+
+  factory MatrixRtcFocusPreferred.fromJson(Map<String, Object?> json) =>
+      MatrixRtcFocusPreferred(
+        type: json.tryGet<String>('type') ?? '',
+        livekitServiceUrl: json.tryGet<String>('livekit_service_url') ?? '',
+        livekitAlias: json.tryGet<String>('livekit_alias') ?? '',
+      );
+
+  Map<String, Object?> toJson() => {
+    'type': type,
+    'livekit_service_url': livekitServiceUrl,
+    'livekit_alias': livekitAlias,
+  };
+}
+
+class MatrixRtcFocusActive {
+  final String type;
+  final String focusSelection;
+
+  const MatrixRtcFocusActive({
+    required this.type,
+    required this.focusSelection,
+  });
+
+  factory MatrixRtcFocusActive.fromJson(Map<String, Object?> json) =>
+      MatrixRtcFocusActive(
+        type: json.tryGet<String>('type') ?? '',
+        focusSelection: json.tryGet<String>('focus_selection') ?? '',
+      );
+
+  Map<String, Object?> toJson() => {
+    'type': type,
+    'focus_selection': focusSelection,
+  };
+}