fix session out of sync performance issues

This commit is contained in:
otsmr 2026-07-09 16:35:34 +02:00
parent 599882700c
commit 2cee00aff6
4 changed files with 95 additions and 27 deletions

View file

@ -516,6 +516,15 @@ class MessagesDao extends DatabaseAccessor<TwonlyDB> with _$MessagesDaoMixin {
.getSingleOrNull(); .getSingleOrNull();
} }
Stream<MessageAction?> watchLastMessageAction(String messageId) {
return (((select(messageActions)..where(
(t) => t.messageId.equals(messageId),
))
..orderBy([(t) => OrderingTerm.desc(t.actionAt)]))
..limit(1))
.watchSingleOrNull();
}
Future<void> deleteMessagesById(String messageId) { Future<void> deleteMessagesById(String messageId) {
return (delete(messages)..where((t) => t.messageId.equals(messageId))).go(); return (delete(messages)..where((t) => t.messageId.equals(messageId))).go();
} }

View file

@ -56,9 +56,13 @@ Future<void> handleServerMessage(server.ServerToClient msg) async {
Log.info( Log.info(
'Got ${msg.v0.newMessages.newMessages.length} messages from the server.', 'Got ${msg.v0.newMessages.newMessages.length} messages from the server.',
); );
final brokenSessionsInCurrentBatch = <int>{};
for (final newMessage in msg.v0.newMessages.newMessages) { for (final newMessage in msg.v0.newMessages.newMessages) {
try { try {
await handleClient2ClientMessage(newMessage); await handleClient2ClientMessage(
newMessage,
brokenSessionsInCurrentBatch: brokenSessionsInCurrentBatch,
);
} catch (e) { } catch (e) {
Log.error(e); Log.error(e);
} }
@ -83,7 +87,10 @@ DateTime lastPushKeyRequest = clock.now().subtract(const Duration(hours: 1));
final Map<String, Mutex> _messageLocks = {}; final Map<String, Mutex> _messageLocks = {};
Future<void> handleClient2ClientMessage(NewMessage newMessage) async { Future<void> handleClient2ClientMessage(
NewMessage newMessage, {
Set<int>? brokenSessionsInCurrentBatch,
}) async {
final body = Uint8List.fromList(newMessage.body); final body = Uint8List.fromList(newMessage.body);
final message = Message.fromBuffer(body); final message = Message.fromBuffer(body);
final receiptId = message.receiptId; final receiptId = message.receiptId;
@ -97,7 +104,11 @@ Future<void> handleClient2ClientMessage(NewMessage newMessage) async {
} }
await mutex.protect(() async { await mutex.protect(() async {
try { try {
await _handleClient2ClientMessage(newMessage, message); await _handleClient2ClientMessage(
newMessage,
message,
brokenSessionsInCurrentBatch: brokenSessionsInCurrentBatch,
);
} finally { } finally {
_messageLocks.remove(receiptId); _messageLocks.remove(receiptId);
} }
@ -106,11 +117,27 @@ Future<void> handleClient2ClientMessage(NewMessage newMessage) async {
Future<void> _handleClient2ClientMessage( Future<void> _handleClient2ClientMessage(
NewMessage newMessage, NewMessage newMessage,
Message message, Message message, {
) async { Set<int>? brokenSessionsInCurrentBatch,
}) async {
final fromUserId = newMessage.fromUserId.toInt(); final fromUserId = newMessage.fromUserId.toInt();
final receiptId = message.receiptId; final receiptId = message.receiptId;
if (brokenSessionsInCurrentBatch?.contains(fromUserId) == true) {
// This happens when a session goes out of sync (e.g. wrong message order).
// We skip the remaining messages in the batch because each failed decryption
// attempt is extremely slow (~1.2s) and would otherwise freeze the app.
// By returning early, we skip gotReceipt() and error responses.
// The server still deletes the batch since we ACK the entire batch later.
// The sender keeps the message unacknowledged. Once they process our SESSION_OUT_OF_SYNC
// error and establish a new session, their retry logic will automatically re-encrypt
// and re-send these messages with the new keys.
Log.info(
'Skipping message from $fromUserId - session known broken in this batch',
);
return;
}
if (await twonlyDB.receiptsDao.isDuplicated(receiptId)) { if (await twonlyDB.receiptsDao.isDuplicated(receiptId)) {
return; return;
} }
@ -200,6 +227,7 @@ Future<void> _handleClient2ClientMessage(
encryptedContentRaw, encryptedContentRaw,
message.type, message.type,
receiptId, receiptId,
brokenSessionsInCurrentBatch: brokenSessionsInCurrentBatch,
); );
if (plainTextContent != null) { if (plainTextContent != null) {
response = Message( response = Message(
@ -265,13 +293,15 @@ Future<(EncryptedContent?, PlaintextContent?)> handleEncryptedMessageRaw(
int fromUserId, int fromUserId,
Uint8List encryptedContentRaw, Uint8List encryptedContentRaw,
Message_Type messageType, Message_Type messageType,
String receiptId, String receiptId, {
) async { Set<int>? brokenSessionsInCurrentBatch,
}) async {
Log.info('[$receiptId] calling signalDecryptMessage'); Log.info('[$receiptId] calling signalDecryptMessage');
var (encryptedContent, decryptionErrorType) = await signalDecryptMessage( var (encryptedContent, decryptionErrorType) = await signalDecryptMessage(
fromUserId, fromUserId,
encryptedContentRaw, encryptedContentRaw,
messageType.value, messageType.value,
brokenSessionsInCurrentBatch: brokenSessionsInCurrentBatch,
); );
if (encryptedContent == null) { if (encryptedContent == null) {

View file

@ -38,8 +38,9 @@ Future<(EncryptedContent?, PlaintextContent_DecryptionErrorMessage_Type?)>
signalDecryptMessage( signalDecryptMessage(
int fromUserId, int fromUserId,
Uint8List encryptedContentRaw, Uint8List encryptedContentRaw,
int type, int type, {
) async { Set<int>? brokenSessionsInCurrentBatch,
}) async {
// Hold the lock only for the cryptographic operation, not for network I/O // Hold the lock only for the cryptographic operation, not for network I/O
Log.info('Acquiring lockingSignalProtocol for $fromUserId'); Log.info('Acquiring lockingSignalProtocol for $fromUserId');
final ( final (
@ -74,6 +75,7 @@ signalDecryptMessage(
); );
} }
recordResyncAttempt(fromUserId, success: true);
return (EncryptedContent.fromBuffer(plaintext), null, false); return (EncryptedContent.fromBuffer(plaintext), null, false);
} on InvalidKeyIdException catch (e) { } on InvalidKeyIdException catch (e) {
Log.warn(e); Log.warn(e);
@ -113,11 +115,13 @@ signalDecryptMessage(
// Handle session resync OUTSIDE the lock to avoid holding it during // Handle session resync OUTSIDE the lock to avoid holding it during
// network round-trips (which can block for up to 60 seconds) // network round-trips (which can block for up to 60 seconds)
if (needsResync && !resyncedUsers.contains(fromUserId)) { if (needsResync) {
brokenSessionsInCurrentBatch?.add(fromUserId);
if (shouldAttemptResync(fromUserId)) {
if (await handleSessionResync(fromUserId)) { if (await handleSessionResync(fromUserId)) {
// This flag prevents from resyncing the session the client received // This flag prevents from resyncing the session the client received
// multiple new messages from the server he could not decrypt // multiple new messages from the server he could not decrypt
resyncedUsers.add(fromUserId); recordResyncAttempt(fromUserId, success: false);
// This message contains a new PreKeyBundle establishing a new signal // This message contains a new PreKeyBundle establishing a new signal
// session // session
@ -131,6 +135,7 @@ signalDecryptMessage(
); );
} }
} }
}
return (decryptedContent, errorType); return (decryptedContent, errorType);
} }

View file

@ -1,12 +1,36 @@
import 'dart:math';
import 'package:mutex/mutex.dart'; import 'package:mutex/mutex.dart';
/// Unified lock for all Signal protocol operations (encryption, decryption, session management). /// Unified lock for all Signal protocol operations (encryption, decryption, session management).
final lockingSignalProtocol = Mutex(); final lockingSignalProtocol = Mutex();
/// Tracking users who have already been resynced in the current session. /// Tracking users who have already been resynced in the current session.
final resyncedUsers = <int>{}; final Map<int, ({int failureCount, DateTime lastAttempt})> _resyncAttempts = {};
/// Reset the resync tracking set. const int maxResyncAttempts = 3;
void resetResyncedUsers() {
resyncedUsers.clear(); bool shouldAttemptResync(int userId) {
final attempt = _resyncAttempts[userId];
if (attempt == null) return true;
if (attempt.failureCount >= maxResyncAttempts) return false;
final cooldown = Duration(minutes: 5 * pow(5, attempt.failureCount - 1).toInt());
return DateTime.now().difference(attempt.lastAttempt) > cooldown;
}
void recordResyncAttempt(int userId, {required bool success}) {
if (success) {
_resyncAttempts.remove(userId);
} else {
final current = _resyncAttempts[userId];
_resyncAttempts[userId] = (
failureCount: (current?.failureCount ?? 0) + 1,
lastAttempt: DateTime.now(),
);
}
}
/// Reset the resync tracking set (currently unused, backoff handles expiry naturally).
void resetResyncedUsers() {
// No-op. We want the backoff state to persist across reconnects.
} }