import 'dart:async'; import 'dart:convert'; import 'dart:isolate'; import 'dart:math'; import 'package:fixnum/fixnum.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter_secure_storage/flutter_secure_storage.dart'; import 'package:http/http.dart' as http; import 'dart:io'; import 'dart:typed_data'; import 'package:cryptography_plus/cryptography_plus.dart'; import 'package:drift/drift.dart'; import 'package:flutter_image_compress/flutter_image_compress.dart'; import 'package:mutex/mutex.dart'; import 'package:path/path.dart'; import 'package:path_provider/path_provider.dart'; import 'package:twonly/globals.dart'; import 'package:twonly/src/database/tables/media_uploads_table.dart'; import 'package:twonly/src/database/tables/messages_table.dart'; import 'package:twonly/src/database/twonly_database.dart'; import 'package:twonly/src/model/json/message.dart'; import 'package:twonly/src/model/protobuf/api/http/http_requests.pb.dart'; import 'package:twonly/src/model/protobuf/api/websocket/error.pb.dart'; import 'package:twonly/src/services/api/media_received.dart'; import 'package:twonly/src/services/notification.service.dart'; import 'package:twonly/src/services/signal/encryption.signal.dart'; import 'package:twonly/src/utils/log.dart'; import 'package:twonly/src/utils/misc.dart'; import 'package:twonly/src/utils/storage.dart'; import 'package:video_compress/video_compress.dart'; Future isAllowedToSend() async { final user = await getUser(); if (user == null) return null; if (user.subscriptionPlan == "Preview") { return ErrorCode.PlanNotAllowed; } if (user.subscriptionPlan == "Free") { if (user.lastImageSend != null && user.todaysImageCounter != null) { if (isToday(user.lastImageSend!)) { if (user.todaysImageCounter == 3) { return ErrorCode.PlanLimitReached; } user.todaysImageCounter = user.todaysImageCounter! + 1; } else { user.todaysImageCounter = 1; } } else { user.todaysImageCounter = 1; } user.lastImageSend = DateTime.now(); await updateUser(user); } return null; } /// States: /// when user recorded an video /// 1. Compress video /// when user clicked the send button (direct send) or share with /// 2. Encrypt media files /// 3. Upload media files /// click send button /// 4. Finalize upload by websocket -> get download tokens /// 5. Send all users the message /// Create a new entry in the database Future checkForFailedUploads() async { final messages = await twonlyDB.messagesDao.getAllMessagesPendingUpload(); List mediaUploadIds = []; for (Message message in messages) { if (mediaUploadIds.contains(message.mediaUploadId)) { continue; } int affectedRows = await twonlyDB.mediaUploadsDao.updateMediaUpload( message.mediaUploadId!, MediaUploadsCompanion( state: Value(UploadState.pending), encryptionData: Value( null, // start from scratch e.q. encrypt the files again if already happen ), ), ); if (affectedRows == 0) { Log.error( "The media from message ${message.messageId} already deleted.", ); await twonlyDB.messagesDao.updateMessageByMessageId( message.messageId, MessagesCompanion( errorWhileSending: Value(true), ), ); } else { mediaUploadIds.add(message.mediaUploadId!); } } if (messages.isNotEmpty) { Log.error( "Got ${messages.length} messages (${mediaUploadIds.length} media upload files) that are not correctly uploaded. Trying from scratch again.", ); } return mediaUploadIds.isNotEmpty; // return true if there are affected } final lockingHandleMediaFile = Mutex(); Future retryMediaUpload(bool appRestarted, {int maxRetries = 3}) async { if (maxRetries == 0) { Log.error("retried media upload 3 times. abort retrying"); return; } bool retry = await lockingHandleMediaFile.protect(() async { final mediaFiles = await twonlyDB.mediaUploadsDao.getMediaUploadsForRetry(); if (mediaFiles.isEmpty) { return checkForFailedUploads(); } Log.info("re uploading ${mediaFiles.length} media files."); for (final mediaFile in mediaFiles) { if (mediaFile.messageIds == null || mediaFile.metadata == null) { if (appRestarted) { /// When the app got restarted and the messageIds or the metadata is not /// set then the app was closed before the images was send. await twonlyDB.mediaUploadsDao .deleteMediaUpload(mediaFile.mediaUploadId); Log.info( "upload can be removed, the finalized function was never called...", ); } continue; } if (mediaFile.state == UploadState.readyToUpload) { await handleNextMediaUploadSteps(mediaFile.mediaUploadId); } else { await handlePreProcessingState(mediaFile); } } return false; }); if (retry) { await retryMediaUpload(false, maxRetries: maxRetries - 1); } } Future initMediaUpload() async { return await twonlyDB.mediaUploadsDao .insertMediaUpload(MediaUploadsCompanion()); } Future addVideoToUpload(int mediaUploadId, File videoFilePath) async { String basePath = await getMediaFilePath(mediaUploadId, "send"); await videoFilePath.copy("$basePath.original.mp4"); return await compressVideoIfExists(mediaUploadId); } Future addOrModifyImageToUpload( int mediaUploadId, Uint8List imageBytes) async { Uint8List imageBytesCompressed; try { imageBytesCompressed = await FlutterImageCompress.compressWithList( format: CompressFormat.png, imageBytes, quality: 90, ); if (imageBytesCompressed.length >= 2 * 1000 * 1000) { // if the media file is over 2MB compress it with 60% imageBytesCompressed = await FlutterImageCompress.compressWithList( format: CompressFormat.png, imageBytes, quality: 60, ); } await writeMediaFile(mediaUploadId, "png", imageBytesCompressed); } catch (e) { Log.error("$e"); // as a fall back use the original image await writeMediaFile(mediaUploadId, "png", imageBytes); imageBytesCompressed = imageBytes; } /// in case the media file was already encrypted of even uploaded /// remove the data so it will be done again. await twonlyDB.mediaUploadsDao.updateMediaUpload( mediaUploadId, MediaUploadsCompanion( encryptionData: Value(null), ), ); return imageBytesCompressed; } Future handlePreProcessingState(MediaUpload media) async { try { final imageHandler = readMediaFile(media.mediaUploadId, "png"); final videoHandler = compressVideoIfExists(media.mediaUploadId); await encryptMediaFiles( media.mediaUploadId, imageHandler, videoHandler, ); } catch (e) { Log.error("${media.mediaUploadId} got error in pre processing: $e"); await handleUploadError(media); } } Future encryptMediaFiles( int mediaUploadId, Future imageHandler, Future? videoHandler, ) async { Log.info("$mediaUploadId encrypting files"); Uint8List dataToEncrypt = await imageHandler; /// if there is a video wait until it is finished with compression if (videoHandler != null) { if (await videoHandler) { Uint8List compressedVideo = await readMediaFile(mediaUploadId, "mp4"); dataToEncrypt = combineUint8Lists(dataToEncrypt, compressedVideo); } } var state = MediaEncryptionData(); final xchacha20 = Xchacha20.poly1305Aead(); SecretKeyData secretKey = await (await xchacha20.newSecretKey()).extract(); state.encryptionKey = secretKey.bytes; state.encryptionNonce = xchacha20.newNonce(); final secretBox = await Isolate.run( () => xchacha20.encrypt( dataToEncrypt, secretKey: secretKey, nonce: state.encryptionNonce, ), ); state.encryptionMac = secretBox.mac.bytes; final algorithm = Sha256(); state.sha2Hash = (await algorithm.hash(secretBox.cipherText)).bytes; final encryptedBytes = Uint8List.fromList(secretBox.cipherText); await writeMediaFile( mediaUploadId, "encrypted", encryptedBytes, ); await twonlyDB.mediaUploadsDao.updateMediaUpload( mediaUploadId, MediaUploadsCompanion( state: Value(UploadState.readyToUpload), encryptionData: Value(state), ), ); handleNextMediaUploadSteps(mediaUploadId); } Future finalizeUpload(int mediaUploadId, List contactIds, bool isRealTwonly, bool isVideo, bool mirrorVideo, int maxShowTime) async { MediaUploadMetadata metadata = MediaUploadMetadata(); metadata.contactIds = contactIds; metadata.isRealTwonly = isRealTwonly; metadata.messageSendAt = DateTime.now(); metadata.isVideo = isVideo; metadata.maxShowTime = maxShowTime; metadata.mirrorVideo = mirrorVideo; List messageIds = []; for (final contactId in contactIds) { int? messageId = await twonlyDB.messagesDao.insertMessage( MessagesCompanion( contactId: Value(contactId), kind: Value(MessageKind.media), sendAt: Value(metadata.messageSendAt), downloadState: Value(DownloadState.pending), mediaUploadId: Value(mediaUploadId), contentJson: Value( jsonEncode( MediaMessageContent( maxShowTime: maxShowTime, isRealTwonly: isRealTwonly, isVideo: isVideo, mirrorVideo: mirrorVideo, ).toJson(), ), ), ), ); // de-archive contact when sending a new message await twonlyDB.contactsDao.updateContact( contactId, ContactsCompanion( archived: Value(false), ), ); if (messageId != null) { messageIds.add(messageId); } else { Log.error("Error inserting media upload message in database."); } } await twonlyDB.mediaUploadsDao.updateMediaUpload( mediaUploadId, MediaUploadsCompanion( messageIds: Value(messageIds), metadata: Value(metadata), ), ); handleNextMediaUploadSteps(mediaUploadId); } final lockingHandleNextMediaUploadStep = Mutex(); Future handleNextMediaUploadSteps(int mediaUploadId) async { bool rerun = await lockingHandleNextMediaUploadStep.protect(() async { var mediaUpload = await twonlyDB.mediaUploadsDao .getMediaUploadById(mediaUploadId) .getSingleOrNull(); if (mediaUpload == null) return false; if (mediaUpload.state == UploadState.receiverNotified) { /// Upload done and all users are notified :) Log.info("$mediaUploadId is already done"); return false; } try { /// Stage 1: media files are not yet encrypted... if (mediaUpload.encryptionData == null) { // when set this function will be called again by encryptAndPreUploadMediaFiles... return false; } if (mediaUpload.messageIds == null || mediaUpload.metadata == null) { /// the finalize function was not called yet... return false; } return await handleMediaUpload(mediaUpload); } catch (e) { Log.error("Non recoverable error while sending media file: $e"); await handleUploadError(mediaUpload); } return false; }); if (rerun) { handleNextMediaUploadSteps(mediaUploadId); } } /// /// -- private functions -- /// /// /// Future handleUploadError(MediaUpload mediaUpload) async { // if the messageIds are already there notify the user about this error... if (mediaUpload.messageIds != null) { for (int messageId in mediaUpload.messageIds!) { await twonlyDB.messagesDao.updateMessageByMessageId( messageId, MessagesCompanion( errorWhileSending: Value(true), ), ); } } await twonlyDB.mediaUploadsDao.deleteMediaUpload(mediaUpload.mediaUploadId); } Future handleMediaUpload(MediaUpload media) async { Uint8List bytesToUpload = await readMediaFile(media.mediaUploadId, "encrypted"); if (media.messageIds == null) return false; List downloadTokens = createDownloadTokens(media.messageIds!.length); List messagesOnSuccess = []; for (var i = 0; i < media.messageIds!.length; i++) { MessageJson msg = MessageJson( kind: MessageKind.media, messageId: media.messageIds![i], content: MediaMessageContent( downloadToken: downloadTokens[i], maxShowTime: media.metadata!.maxShowTime, isRealTwonly: media.metadata!.isRealTwonly, isVideo: media.metadata!.isVideo, mirrorVideo: media.metadata!.mirrorVideo, encryptionKey: media.encryptionData!.encryptionKey, encryptionMac: media.encryptionData!.encryptionMac, encryptionNonce: media.encryptionData!.encryptionNonce, ), timestamp: media.metadata!.messageSendAt, ); Uint8List plaintextContent = Uint8List.fromList(gzip.encode(utf8.encode(jsonEncode(msg.toJson())))); Message? message = await twonlyDB.messagesDao .getMessageByMessageId(media.messageIds![i]) .getSingleOrNull(); if (message == null) continue; await twonlyDB.contactsDao.incFlameCounter( message.contactId, false, message.sendAt, ); Uint8List? encryptedBytes = await signalEncryptMessage( message.contactId, plaintextContent, ); if (encryptedBytes == null) continue; final pushKind = (media.metadata!.isRealTwonly) ? PushKind.twonly : (media.metadata!.isVideo) ? PushKind.video : PushKind.image; var messageOnSuccess = TextMessage() ..body = encryptedBytes ..userId = Int64(message.contactId); var pushData = await getPushData(message.contactId, pushKind); if (pushData != null) { messageOnSuccess.pushData = pushData.toList(); } messagesOnSuccess.add(messageOnSuccess); } final uploadRequest = UploadRequest( messagesOnSuccess: messagesOnSuccess, downloadTokens: downloadTokens, encryptedData: bytesToUpload, ); final uploadRequestBytes = uploadRequest.writeToBuffer(); final storage = FlutterSecureStorage(); String? apiAuthToken = await storage.read(key: "api_auth_token"); if (apiAuthToken == null) { Log.error("api auth token not defined."); return false; } String apiUrl = "http${apiService.apiSecure}://${apiService.apiHost}/api/upload"; var requestMultipart = http.MultipartRequest( "POST", Uri.parse(apiUrl), ); requestMultipart.headers['x-twonly-auth-token'] = uint8ListToHex(base64Decode(apiAuthToken)); requestMultipart.files.add(http.MultipartFile.fromBytes( "file", uploadRequestBytes, filename: "upload", )); Log.info("Starting upload from ${media.mediaUploadId}"); try { var streamedResponse = await requestMultipart.send(); final response = await http.Response.fromStream(streamedResponse); if (response.statusCode == 200) { Log.info("Upload was success!"); await twonlyDB.mediaUploadsDao.updateMediaUpload( media.mediaUploadId, MediaUploadsCompanion( state: Value(UploadState.receiverNotified), ), ); for (final messageId in media.messageIds!) { await twonlyDB.messagesDao.updateMessageByMessageId( messageId, MessagesCompanion( acknowledgeByServer: Value(true), errorWhileSending: Value(false), ), ); } return true; } else { if (response.statusCode >= 400 && response.statusCode < 500) { for (final messageId in media.messageIds!) { await twonlyDB.messagesDao.updateMessageByMessageId( messageId, MessagesCompanion( acknowledgeByServer: Value(true), errorWhileSending: Value(true), ), ); } } Log.error("Got error while uploading: ${response.statusCode}"); } } catch (e) { Log.error("Exception during upload: $e"); } return false; } Future compressVideoIfExists(int mediaUploadId) async { String basePath = await getMediaFilePath(mediaUploadId, "send"); File videoOriginalFile = File("$basePath.original.mp4"); File videoCompressedFile = File("$basePath.mp4"); if (videoCompressedFile.existsSync()) { // file is already compressed and exists return true; } if (!videoOriginalFile.existsSync()) { // media upload does not have a video return false; } return await Isolate.run(() async { MediaInfo? mediaInfo; try { mediaInfo = await VideoCompress.compressVideo( videoOriginalFile.path, quality: VideoQuality.Res1280x720Quality, deleteOrigin: false, includeAudio: true, // https://github.com/jonataslaw/VideoCompress/issues/184 ); if (mediaInfo!.filesize! >= 30 * 1000 * 1000) { // if the media file is over 20MB compress it with low quality mediaInfo = await VideoCompress.compressVideo( videoOriginalFile.path, quality: VideoQuality.Res960x540Quality, deleteOrigin: false, includeAudio: true, ); } } catch (e) { Log.error("during video compression: $e"); } if (mediaInfo == null) { Log.error("could not compress video."); // as a fall back use the non compressed version await videoOriginalFile.copy(videoCompressedFile.path); await videoOriginalFile.delete(); } else { await mediaInfo.file!.copy(videoCompressedFile.path); await mediaInfo.file!.delete(); } return true; }); } /// --- helper functions --- Future readMediaFile(int mediaUploadId, String type) async { String basePath = await getMediaFilePath(mediaUploadId, "send"); File file = File("$basePath.$type"); if (!await file.exists()) { throw Exception("$file not found"); } return await file.readAsBytes(); } Future writeMediaFile( int mediaUploadId, String type, Uint8List data) async { String basePath = await getMediaFilePath(mediaUploadId, "send"); File file = File("$basePath.$type"); await file.writeAsBytes(data); } Future deleteMediaFile(int mediaUploadId, String type) async { String basePath = await getMediaFilePath(mediaUploadId, "send"); File file = File("$basePath.$type"); if (await file.exists()) { await file.delete(); } } Future getMediaFilePath(dynamic mediaId, String type) async { final basedir = await getApplicationSupportDirectory(); final mediaSendDir = Directory(join(basedir.path, 'media', type)); if (!await mediaSendDir.exists()) { await mediaSendDir.create(recursive: true); } return join(mediaSendDir.path, '$mediaId'); } Future getMediaBaseFilePath(String type) async { final basedir = await getApplicationSupportDirectory(); final mediaSendDir = Directory(join(basedir.path, 'media', type)); if (!await mediaSendDir.exists()) { await mediaSendDir.create(recursive: true); } return mediaSendDir.path; } /// combines two utf8 list Uint8List combineUint8Lists(Uint8List list1, Uint8List list2) { final combinedLength = 4 + list1.length + list2.length; final combinedList = Uint8List(combinedLength); final byteData = ByteData.sublistView(combinedList); byteData.setInt32( 0, list1.length, Endian.big); // Store size in big-endian format combinedList.setRange(4, 4 + list1.length, list1); combinedList.setRange(4 + list1.length, combinedLength, list2); return combinedList; } List extractUint8Lists(Uint8List combinedList) { final byteData = ByteData.sublistView(combinedList); final sizeOfList1 = byteData.getInt32(0, Endian.big); final list1 = Uint8List.view(combinedList.buffer, 4, sizeOfList1); final list2 = Uint8List.view(combinedList.buffer, 4 + sizeOfList1, combinedList.lengthInBytes - 4 - sizeOfList1); return [list1, list2]; } Future purgeSendMediaFiles() async { final basedir = await getApplicationSupportDirectory(); final directory = Directory(join(basedir.path, 'media', "send")); await purgeMediaFiles(directory); } String uint8ListToHex(List bytes) { return bytes.map((byte) => byte.toRadixString(16).padLeft(2, '0')).join(); } Uint8List hexToUint8List(String hex) => Uint8List.fromList(List.generate( hex.length ~/ 2, (i) => int.parse(hex.substring(i * 2, i * 2 + 2), radix: 16))); List createDownloadTokens(int n) { final Random random = Random(); List tokens = []; for (int i = 0; i < n; i++) { Uint8List token = Uint8List(32); for (int j = 0; j < 32; j++) { token[j] = random.nextInt(256); // Generate a random byte (0-255) } tokens.add(token); } return tokens; }