diff --git a/android/app/build.gradle b/android/app/build.gradle index 674b2bd6..f0777537 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -56,9 +56,12 @@ android { applicationIdSuffix ".testing" manifestPlaceholders = [appName: "twonly [dev]"] } - // profile { - // applicationIdSuffix ".STOP" - // } + profile { + // Profile builds are installed while profiling production-like + // performance. They must never replace the store app. + applicationIdSuffix ".testing" + manifestPlaceholders = [appName: "twonly [profile]"] + } release { shrinkResources false minifyEnabled false diff --git a/android/app/src/main/AndroidManifest.xml b/android/app/src/main/AndroidManifest.xml index 224bb24c..b619df21 100644 --- a/android/app/src/main/AndroidManifest.xml +++ b/android/app/src/main/AndroidManifest.xml @@ -94,12 +94,25 @@ + + + + + + + + diff --git a/android/app/src/main/kotlin/eu/twonly/MyApplication.kt b/android/app/src/main/kotlin/eu/twonly/MyApplication.kt index 268eb43b..aaad9a93 100644 --- a/android/app/src/main/kotlin/eu/twonly/MyApplication.kt +++ b/android/app/src/main/kotlin/eu/twonly/MyApplication.kt @@ -1,5 +1,6 @@ package eu.twonly +import eu.twonly.directmedia.DirectMediaPrepare import io.flutter.app.FlutterApplication import io.crates.keyring.Keyring @@ -13,5 +14,9 @@ class MyApplication : FlutterApplication() { super.onCreate() instance = this Keyring.initializeNdkContext(this) + // Registered from the application rather than the activity: a process + // started by a push or by WorkManager itself must keep the flush alive + // just as much as one the user opened. + DirectMediaPrepare.ensurePeriodicFlush() } } diff --git a/android/app/src/main/kotlin/eu/twonly/directmedia/BootCompletedReceiver.kt b/android/app/src/main/kotlin/eu/twonly/directmedia/BootCompletedReceiver.kt new file mode 100644 index 00000000..a5d10a88 --- /dev/null +++ b/android/app/src/main/kotlin/eu/twonly/directmedia/BootCompletedReceiver.kt @@ -0,0 +1,23 @@ +package eu.twonly.directmedia + +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent + +/** + * A reboot clears the periodic flush, and a phone that was rebooted with unsent + * messages would otherwise wait for the user to open the app. Registering again + * here is what makes "it sends as soon as you are online" true across a + * restart. + */ +class BootCompletedReceiver : BroadcastReceiver() { + override fun onReceive(context: Context, intent: Intent) { + if (intent.action != Intent.ACTION_BOOT_COMPLETED && + intent.action != Intent.ACTION_MY_PACKAGE_REPLACED + ) { + return + } + DirectMediaPrepare.ensurePeriodicFlush() + DirectMediaPrepare.flushNow() + } +} diff --git a/android/app/src/main/kotlin/eu/twonly/directmedia/DirectMediaPrepare.kt b/android/app/src/main/kotlin/eu/twonly/directmedia/DirectMediaPrepare.kt new file mode 100644 index 00000000..c670f48a --- /dev/null +++ b/android/app/src/main/kotlin/eu/twonly/directmedia/DirectMediaPrepare.kt @@ -0,0 +1,113 @@ +package eu.twonly.directmedia + +import androidx.work.BackoffPolicy +import androidx.work.Constraints +import androidx.work.Data +import androidx.work.ExistingPeriodicWorkPolicy +import androidx.work.ExistingWorkPolicy +import androidx.work.NetworkType +import androidx.work.OneTimeWorkRequest +import androidx.work.OutOfQuotaPolicy +import androidx.work.PeriodicWorkRequestBuilder +import androidx.work.WorkManager +import eu.twonly.MyApplication +import java.util.concurrent.TimeUnit + +/** + * Owns the window between the send button and the point where a transfer + * belongs to the OS. + * + * Compressing, encrypting and encrypting-for-each-recipient all run in the app + * process, so swiping the app away mid-send used to strand the message until + * the next launch. Running that work inside WorkManager instead means the + * system restarts it: a job survives the task being removed from recents and is + * re-run after a reboot. + * + * [schedule] is called directly from Rust over JNI. + */ +object DirectMediaPrepare { + /** Prepares one media file. Unique per media id, so a send that is already + * being prepared is never started twice. */ + @JvmStatic + fun schedule(mediaId: String): Boolean = try { + val data = Data.Builder() + .putString(MediaPrepareWorker.MEDIA_ID, mediaId) + .build() + val request = OneTimeWorkRequest.Builder(MediaPrepareWorker::class.java) + .setInputData(data) + // Preparation is what the user is waiting on, and it is short, so it + // asks for an expedited slot and settles for a normal one. + .setExpedited(OutOfQuotaPolicy.RUN_AS_NON_EXPEDITED_WORK_REQUEST) + .setBackoffCriteria(BackoffPolicy.EXPONENTIAL, 30, TimeUnit.SECONDS) + .addTag(TAG) + .build() + WorkManager.getInstance(MyApplication.instance).enqueueUniqueWork( + "$UNIQUE_PREFIX$mediaId", + ExistingWorkPolicy.KEEP, + request, + ) + true + } catch (_: Throwable) { + false + } + + /** + * Keeps a periodic flush registered. + * + * Preparation and the message outbox are both resumed by a connection that + * only exists while something is running. This is what gives a device that + * regained a network hours after the app was closed somewhere to resume + * from. Registered from [eu.twonly.MyApplication] and after a reboot. + */ + @JvmStatic + fun ensurePeriodicFlush() { + try { + val request = PeriodicWorkRequestBuilder( + FLUSH_INTERVAL_MINUTES, + TimeUnit.MINUTES, + ) + .setConstraints( + Constraints.Builder().setRequiredNetworkType(NetworkType.CONNECTED).build(), + ) + .setBackoffCriteria(BackoffPolicy.EXPONENTIAL, 5, TimeUnit.MINUTES) + .addTag(TAG) + .build() + WorkManager.getInstance(MyApplication.instance).enqueueUniquePeriodicWork( + UNIQUE_FLUSH, + // KEEP would silently ignore a changed interval or constraint. + ExistingPeriodicWorkPolicy.UPDATE, + request, + ) + } catch (_: Throwable) { + // A device that will not schedule the flush still resumes on the + // next launch, which is where this behaviour was before. + } + } + + /** Runs the flush once, now. Used after a reboot and on connectivity + * changes, where waiting for the periodic slot would be pointless. */ + @JvmStatic + fun flushNow() { + try { + val request = OneTimeWorkRequest.Builder(MediaPrepareWorker::class.java) + .setConstraints( + Constraints.Builder().setRequiredNetworkType(NetworkType.CONNECTED).build(), + ) + .setBackoffCriteria(BackoffPolicy.EXPONENTIAL, 1, TimeUnit.MINUTES) + .addTag(TAG) + .build() + WorkManager.getInstance(MyApplication.instance).enqueueUniqueWork( + UNIQUE_FLUSH_NOW, + ExistingWorkPolicy.KEEP, + request, + ) + } catch (_: Throwable) { + } + } + + const val TAG = "twonly-media-prepare" + private const val UNIQUE_PREFIX = "media-prepare-" + private const val UNIQUE_FLUSH = "twonly-outbox-flush" + private const val UNIQUE_FLUSH_NOW = "twonly-outbox-flush-now" + private const val FLUSH_INTERVAL_MINUTES = 30L +} diff --git a/android/app/src/main/kotlin/eu/twonly/directmedia/DirectMediaTransfer.kt b/android/app/src/main/kotlin/eu/twonly/directmedia/DirectMediaTransfer.kt index 751d3929..119ac460 100644 --- a/android/app/src/main/kotlin/eu/twonly/directmedia/DirectMediaTransfer.kt +++ b/android/app/src/main/kotlin/eu/twonly/directmedia/DirectMediaTransfer.kt @@ -13,7 +13,12 @@ import java.util.concurrent.TimeUnit import org.json.JSONObject /** Called directly from Rust/JNI. It persists the descriptor in app-private - * storage and enqueues both OS-owned transfers before reporting success. */ + * storage and enqueues every OS-owned transfer it names before reporting + * success. + * + * A descriptor carries a `media` request and, for a media upload, a `manifest` + * alongside it. A queued message envelope is the single-request form: there is + * nothing to describe beyond the POST itself. */ object DirectMediaTransfer { @JvmStatic fun schedule(descriptorJson: String): Boolean = try { @@ -28,13 +33,15 @@ object DirectMediaTransfer { val manager = WorkManager.getInstance(MyApplication.instance) val media = request(attachmentId, "media", descriptorFile, expiresAt) - val manifest = request(attachmentId, "manifest", descriptorFile, expiresAt) manager.enqueueUniqueWork("direct-media-$attachmentId-media", ExistingWorkPolicy.KEEP, media) - manager.enqueueUniqueWork( - "direct-media-$attachmentId-manifest", - ExistingWorkPolicy.KEEP, - manifest, - ) + if (descriptor.has("manifest")) { + val manifest = request(attachmentId, "manifest", descriptorFile, expiresAt) + manager.enqueueUniqueWork( + "direct-media-$attachmentId-manifest", + ExistingWorkPolicy.KEEP, + manifest, + ) + } true } catch (_: Throwable) { false diff --git a/android/app/src/main/kotlin/eu/twonly/directmedia/DirectMediaUploadWorker.kt b/android/app/src/main/kotlin/eu/twonly/directmedia/DirectMediaUploadWorker.kt index 1e0af529..1571c03e 100644 --- a/android/app/src/main/kotlin/eu/twonly/directmedia/DirectMediaUploadWorker.kt +++ b/android/app/src/main/kotlin/eu/twonly/directmedia/DirectMediaUploadWorker.kt @@ -35,9 +35,11 @@ class DirectMediaUploadWorker( val request = descriptor.getJSONObject(role) val status = upload(request) if (status in 200..299) { - if (role == "media") { - // Best effort only. A 202 is success because server reconciliation - // owns completion when the manifest has not arrived yet. + // Best effort only. A 202 is success because server reconciliation + // owns completion when the manifest has not arrived yet. A + // descriptor with no completion step — a queued message envelope — + // is finished as soon as its POST is accepted. + if (role == "media" && descriptor.has("complete")) { upload(descriptor.getJSONObject("complete")) } Result.success() diff --git a/android/app/src/main/kotlin/eu/twonly/directmedia/MediaPrepareWorker.kt b/android/app/src/main/kotlin/eu/twonly/directmedia/MediaPrepareWorker.kt new file mode 100644 index 00000000..db3f13e0 --- /dev/null +++ b/android/app/src/main/kotlin/eu/twonly/directmedia/MediaPrepareWorker.kt @@ -0,0 +1,95 @@ +package eu.twonly.directmedia + +import android.app.NotificationChannel +import android.app.NotificationManager +import android.content.Context +import android.content.pm.ServiceInfo +import android.os.Build +import androidx.core.app.NotificationCompat +import androidx.work.ForegroundInfo +import androidx.work.Worker +import androidx.work.WorkerParameters +import eu.twonly.R +import org.json.JSONObject + +/** + * Runs one Rust maintenance job. + * + * With no media id this is the flush: resume interrupted preparations, settle + * transfers the OS finished while nothing was listening, and drain the message + * outbox. With one, it prepares that single send. + * + * The Rust call is synchronous and owns its own runtime, so this worker only + * has to hold a foreground service around it — without one, Android stops long + * background work, which is the exact failure this whole path exists to fix. + */ +class MediaPrepareWorker( + appContext: Context, + params: WorkerParameters, +) : Worker(appContext, params) { + override fun doWork(): Result { + val mediaId = inputData.getString(MEDIA_ID) + return try { + if (mediaId != null) { + // Only a user-visible send justifies a foreground service; the + // periodic flush runs as ordinary background work. + setForegroundAsync(foregroundInfo()).get() + } + val directory = applicationContext.filesDir.absolutePath + val response = JSONObject( + NativeMediaPrepareBridge.run(directory, directory, mediaId), + ) + if (response.optBoolean("ok", false)) { + // Rust performed a synchronous status pass before returning. + // Keep this WorkManager job as the durable owner until the + // server says the attachment is terminal; a Tokio task created + // inside NativeMediaPrepareBridge would die with that call. + if (response.optBoolean("pending_uploads", false)) { + Result.retry() + } else { + Result.success() + } + } else if (runAttemptCount < MAX_RETRIES) { + Result.retry() + } else { + // Giving up here loses nothing: the media row still says the + // send is unfinished, so the next flush picks it up again. + Result.failure() + } + } catch (_: Throwable) { + if (runAttemptCount < MAX_RETRIES) Result.retry() else Result.failure() + } + } + + private fun foregroundInfo(): ForegroundInfo { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + applicationContext.getSystemService(NotificationManager::class.java) + .createNotificationChannel( + NotificationChannel( + CHANNEL_ID, + "Media uploads", + NotificationManager.IMPORTANCE_LOW, + ), + ) + } + val notification = NotificationCompat.Builder(applicationContext, CHANNEL_ID) + .setSmallIcon(R.drawable.ic_launcher_foreground) + .setContentTitle("Preparing media") + .setContentText("twonly will finish sending this in the background") + .setOngoing(true) + .setOnlyAlertOnce(true) + .build() + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + ForegroundInfo(NOTIFICATION_ID, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC) + } else { + ForegroundInfo(NOTIFICATION_ID, notification) + } + } + + companion object { + const val MEDIA_ID = "media_id" + private const val CHANNEL_ID = "twonly_direct_media_upload" + private const val NOTIFICATION_ID = 0x7A01 + private const val MAX_RETRIES = 5 + } +} diff --git a/android/app/src/main/kotlin/eu/twonly/directmedia/NativeMediaPrepareBridge.kt b/android/app/src/main/kotlin/eu/twonly/directmedia/NativeMediaPrepareBridge.kt new file mode 100644 index 00000000..75d45157 --- /dev/null +++ b/android/app/src/main/kotlin/eu/twonly/directmedia/NativeMediaPrepareBridge.kt @@ -0,0 +1,19 @@ +package eu.twonly.directmedia + +/** Rust entry point for work the platform's schedulers drive: preparing one + * media file for upload, or flushing everything a terminated process left + * behind. Neither needs a Flutter engine. */ +internal object NativeMediaPrepareBridge { + init { + System.loadLibrary("rust_lib_twonly") + } + + /** A null [mediaId] runs the full flush instead of one preparation. + * Returns `{"ok":bool,"error":string?}`. */ + @JvmStatic + external fun run( + databaseDirectory: String, + dataDirectory: String, + mediaId: String?, + ): String +} diff --git a/android/app/src/main/kotlin/eu/twonly/media/NativeVideoCodec.kt b/android/app/src/main/kotlin/eu/twonly/media/NativeVideoCodec.kt index 911e99d0..350e66d6 100644 --- a/android/app/src/main/kotlin/eu/twonly/media/NativeVideoCodec.kt +++ b/android/app/src/main/kotlin/eu/twonly/media/NativeVideoCodec.kt @@ -89,6 +89,8 @@ object NativeVideoCodec { overlayPath: String?, outputPath: String, removeAudio: Boolean, + trimStartMs: Long, + trimEndMs: Long, mediaId: String, ): Boolean { val context = MyApplication.instance @@ -110,7 +112,14 @@ object NativeVideoCodec { handler.post { try { val effects = buildEffects(overlayPath, source) - val editedItem = EditedMediaItem.Builder(MediaItem.fromUri(File(inputPath).toURI().toString())) + val mediaItem = MediaItem.Builder() + .setUri(File(inputPath).toURI().toString()) + // Transformer applies the cut while it decodes, so trimming + // costs nothing on top of the render that was happening + // anyway, and the recording on disk is left alone. + .setClippingConfiguration(clipping(trimStartMs, trimEndMs)) + .build() + val editedItem = EditedMediaItem.Builder(mediaItem) .setRemoveAudio(removeAudio) .setEffects(effects) .build() @@ -232,6 +241,28 @@ object NativeVideoCodec { * settings, so a source that refuses to be probed still renders — it just * falls back to the 1080p30 defaults. */ + /** + * The slice of the recording the editor's cutter selected. + * + * Bounds arrive as milliseconds, with a negative value meaning the clip + * keeps that end. A pair that does not describe a real slice is dropped + * altogether: sending the untrimmed moment beats sending an empty file. + */ + private fun clipping(trimStartMs: Long, trimEndMs: Long): MediaItem.ClippingConfiguration { + if (trimStartMs <= 0L && trimEndMs <= 0L) { + return MediaItem.ClippingConfiguration.UNSET + } + val start = trimStartMs.coerceAtLeast(0L) + if (trimEndMs > 0L && trimEndMs <= start) { + return MediaItem.ClippingConfiguration.UNSET + } + val builder = MediaItem.ClippingConfiguration.Builder().setStartPositionMs(start) + if (trimEndMs > 0L) { + builder.setEndPositionMs(trimEndMs) + } + return builder.build() + } + private fun probe(inputPath: String): SourceVideo? { val retriever = MediaMetadataRetriever() return try { diff --git a/ios/Runner.xcodeproj/project.pbxproj b/ios/Runner.xcodeproj/project.pbxproj index 2fba3030..bca121e8 100644 --- a/ios/Runner.xcodeproj/project.pbxproj +++ b/ios/Runner.xcodeproj/project.pbxproj @@ -23,6 +23,7 @@ D25D4D1E2EF626E30029F805 /* StoreKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D25D4D1D2EF626E30029F805 /* StoreKit.framework */; }; D25D4D7A2EFF41DB0029F805 /* ShareExtension.appex in Embed Foundation Extensions */ = {isa = PBXBuildFile; fileRef = D25D4D702EFF41DB0029F805 /* ShareExtension.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; D3A100022F70000100D1A001 /* DirectMediaTransfer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D3A100012F70000100D1A001 /* DirectMediaTransfer.swift */; }; + D3A100062F70000100D1A006 /* BackgroundWork.swift in Sources */ = {isa = PBXBuildFile; fileRef = D3A100052F70000100D1A005 /* BackgroundWork.swift */; }; D3A100032F70000100D1A002 /* NativeImageCodec.swift in Sources */ = {isa = PBXBuildFile; fileRef = D3A100042F70000100D1A002 /* NativeImageCodec.swift */; }; D3A100052F70000100D1A003 /* NativeVideoCodec.swift in Sources */ = {isa = PBXBuildFile; fileRef = D3A100062F70000100D1A003 /* NativeVideoCodec.swift */; }; D3A100072F70000100D1A004 /* NativeGallery.swift in Sources */ = {isa = PBXBuildFile; fileRef = D3A100082F70000100D1A004 /* NativeGallery.swift */; }; @@ -114,6 +115,7 @@ D25D4D702EFF41DB0029F805 /* ShareExtension.appex */ = {isa = PBXFileReference; explicitFileType = "wrapper.app-extension"; includeInIndex = 0; path = ShareExtension.appex; sourceTree = BUILT_PRODUCTS_DIR; }; D25D4D802EFF437F0029F805 /* RunnerDebug.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = RunnerDebug.entitlements; sourceTree = ""; }; D3A100012F70000100D1A001 /* DirectMediaTransfer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectMediaTransfer.swift; sourceTree = ""; }; + D3A100052F70000100D1A005 /* BackgroundWork.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BackgroundWork.swift; sourceTree = ""; }; D3A100042F70000100D1A002 /* NativeImageCodec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NativeImageCodec.swift; sourceTree = ""; }; D3A100062F70000100D1A003 /* NativeVideoCodec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NativeVideoCodec.swift; sourceTree = ""; }; D3A100082F70000100D1A004 /* NativeGallery.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NativeGallery.swift; sourceTree = ""; }; @@ -248,6 +250,7 @@ isa = PBXGroup; children = ( D3A100012F70000100D1A001 /* DirectMediaTransfer.swift */, + D3A100052F70000100D1A005 /* BackgroundWork.swift */, D3A100042F70000100D1A002 /* NativeImageCodec.swift */, D3A100062F70000100D1A003 /* NativeVideoCodec.swift */, D3A100082F70000100D1A004 /* NativeGallery.swift */, @@ -647,6 +650,7 @@ buildActionMask = 2147483647; files = ( D3A100022F70000100D1A001 /* DirectMediaTransfer.swift in Sources */, + D3A100062F70000100D1A006 /* BackgroundWork.swift in Sources */, D3A100032F70000100D1A002 /* NativeImageCodec.swift in Sources */, D3A100052F70000100D1A003 /* NativeVideoCodec.swift in Sources */, D3A100072F70000100D1A004 /* NativeGallery.swift in Sources */, @@ -776,7 +780,7 @@ DEVELOPMENT_TEAM = CN332ZUGRP; ENABLE_BITCODE = NO; INFOPLIST_FILE = Runner/Info.plist; - INFOPLIST_KEY_CFBundleDisplayName = twonly; + INFOPLIST_KEY_CFBundleDisplayName = "twonly [profile]"; INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.social-networking"; IPHONEOS_DEPLOYMENT_TARGET = 15.6; LD_RUNPATH_SEARCH_PATHS = ( @@ -784,7 +788,7 @@ "@executable_path/Frameworks", ); MARKETING_VERSION = 0.0.5; - PRODUCT_BUNDLE_IDENTIFIER = eu.twonly; + PRODUCT_BUNDLE_IDENTIFIER = eu.twonly.testing; PRODUCT_NAME = "$(TARGET_NAME)"; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; SUPPORTS_MACCATALYST = NO; @@ -975,7 +979,7 @@ DEVELOPMENT_TEAM = CN332ZUGRP; ENABLE_BITCODE = NO; INFOPLIST_FILE = Runner/Info.plist; - INFOPLIST_KEY_CFBundleDisplayName = twonly; + INFOPLIST_KEY_CFBundleDisplayName = "twonly [dev]"; INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.social-networking"; IPHONEOS_DEPLOYMENT_TARGET = 15.6; LD_RUNPATH_SEARCH_PATHS = ( @@ -983,7 +987,7 @@ "@executable_path/Frameworks", ); MARKETING_VERSION = 0.0.5; - PRODUCT_BUNDLE_IDENTIFIER = eu.twonly; + PRODUCT_BUNDLE_IDENTIFIER = eu.twonly.testing; PRODUCT_NAME = "$(TARGET_NAME)"; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; SUPPORTS_MACCATALYST = NO; @@ -1061,7 +1065,7 @@ MARKETING_VERSION = 1.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = eu.twonly.NotificationService; + PRODUCT_BUNDLE_IDENTIFIER = eu.twonly.testing.NotificationService; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; SWIFT_ACTIVE_COMPILATION_CONDITIONS = "DEBUG $(inherited)"; @@ -1139,7 +1143,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = eu.twonly.NotificationService; + PRODUCT_BUNDLE_IDENTIFIER = eu.twonly.testing.NotificationService; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; SWIFT_EMIT_LOC_STRINGS = YES; @@ -1179,7 +1183,7 @@ MARKETING_VERSION = 1.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = eu.twonly.ShareExtension; + PRODUCT_BUNDLE_IDENTIFIER = eu.twonly.testing.ShareExtension; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; STRING_CATALOG_GENERATE_SYMBOLS = YES; @@ -1265,7 +1269,7 @@ LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MARKETING_VERSION = 1.0; MTL_FAST_MATH = YES; - PRODUCT_BUNDLE_IDENTIFIER = eu.twonly.ShareExtension; + PRODUCT_BUNDLE_IDENTIFIER = eu.twonly.testing.ShareExtension; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; STRING_CATALOG_GENERATE_SYMBOLS = YES; diff --git a/ios/Runner/AppDelegate.swift b/ios/Runner/AppDelegate.swift index cacce4a0..40421803 100644 --- a/ios/Runner/AppDelegate.swift +++ b/ios/Runner/AppDelegate.swift @@ -12,10 +12,20 @@ import flutter_sharing_intent didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? ) -> Bool { UNUserNotificationCenter.current().delegate = self + // Must happen before launching finishes: BGTaskScheduler refuses an + // identifier registered any later. + BackgroundWork.register() return super.application(application, didFinishLaunchingWithOptions: launchOptions) } + override func applicationDidEnterBackground(_ application: UIApplication) { + // The app is about to be suspended, and the socket with it. Reserve a later + // slot so anything still queued is sent without the user coming back. + BackgroundWork.scheduleFlush() + super.applicationDidEnterBackground(application) + } + override func application( _ application: UIApplication, handleEventsForBackgroundURLSession identifier: String, diff --git a/ios/Runner/BackgroundWork.swift b/ios/Runner/BackgroundWork.swift new file mode 100644 index 00000000..9feea5c5 --- /dev/null +++ b/ios/Runner/BackgroundWork.swift @@ -0,0 +1,158 @@ +import BackgroundTasks +import Foundation +import UIKit + +/// Keeps a send alive while iOS is taking the app away. +/// +/// Everything between the send button and the point where a background +/// `URLSession` owns the transfer — transcoding, encryption, one Signal ratchet +/// step per recipient — runs inside this process. iOS suspends a backgrounded +/// app within seconds unless something has asked it not to, which used to leave +/// a send stranded until the next launch. +/// +/// Two mechanisms, because iOS offers no single one that covers both cases: +/// +/// * A `UIApplication` background task assertion, taken by Rust for the length +/// of a preparation. It buys the roughly thirty seconds an app gets after +/// being backgrounded, which is what an ordinary send needs. +/// * A `BGProcessingTask`, scheduled whenever the app goes away with work +/// still outstanding. The system runs it later, on its own schedule, and it +/// is the only thing that resumes a send after the app has been suspended for +/// real. Nothing survives a force quit; that case is picked up on next launch. +enum BackgroundWork { + static let processingTaskIdentifier = "eu.twonly.outbox-flush" + + /// Registered from `didFinishLaunchingWithOptions`. iOS requires every + /// identifier to be registered before the app finishes launching. + static func register() { + BGTaskScheduler.shared.register( + forTaskWithIdentifier: processingTaskIdentifier, + using: nil + ) { task in + guard let task = task as? BGProcessingTask else { + task.setTaskCompleted(success: false) + return + } + run(task) + } + } + + /// Asks the system for a later chance to finish whatever is still queued. + /// Submitting again replaces the pending request rather than stacking up. + static func scheduleFlush() { + let request = BGProcessingTaskRequest(identifier: processingTaskIdentifier) + request.requiresNetworkConnectivity = true + request.requiresExternalPower = false + do { + try BGTaskScheduler.shared.submit(request) + } catch { + // Simulators and devices with background refresh disabled refuse this. + // The next launch still resumes everything, which is where this + // behaviour was before. + NSLog("twonly: could not schedule the outbox flush: \(error.localizedDescription)") + } + } + + /// The App Group container the app and its extensions share, which is where + /// the Rust databases live. + private static func runtimeDirectory() -> String? { + guard + let container = FileManager.default.containerURL( + forSecurityApplicationGroupIdentifier: runtimeAppGroup + ) + else { return nil } + let directory = container.appendingPathComponent("runtime", isDirectory: true) + do { + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + return directory.path + } catch { + NSLog("twonly: could not open the runtime directory: \(error.localizedDescription)") + return nil + } + } + + private static let runtimeAppGroup = "group.eu.twonly.runtime" + + private static func run(_ task: BGProcessingTask) { + // The system may reclaim the task at any point; ask Rust to stop by ending + // the run, and reschedule so the work is not simply dropped. + let queue = DispatchQueue(label: "eu.twonly.outbox-flush", qos: .utility) + var finished = false + let finish: (Bool) -> Void = { success in + guard !finished else { return } + finished = true + task.setTaskCompleted(success: success) + } + task.expirationHandler = { + scheduleFlush() + finish(false) + } + queue.async { + guard let directory = runtimeDirectory() else { + finish(false) + return + } + let response = directory.withCString { databaseDirectory in + directory.withCString { dataDirectory in + twonly_background_run(databaseDirectory, dataDirectory, nil) + } + } + let json = response.map { pointer -> String in + let value = String(cString: pointer) + twonly_background_string_free(pointer) + return value + } + let ok = json?.contains("\"ok\":true") ?? false + // Keep a rolling reservation: as long as anything is unsent, there is a + // scheduled chance to send it. + scheduleFlush() + finish(ok) + } + } +} + +/// The background task assertion Rust takes around a media preparation. +/// +/// Called over the C ABI so the preparation does not have to know it is on iOS. +/// Returns zero when no assertion could be taken, which Rust reads as "run +/// unprotected" rather than as a failure. +/// Rust calls this from one of its own worker threads, never from the main +/// one, so `UIApplication` is reached through the main queue rather than +/// directly. A `sync` hop cannot deadlock here for that same reason, and the +/// caller needs the identifier back before it can start working. +private func onMain(_ work: @escaping () -> T) -> T { + if Thread.isMainThread { + return work() + } + return DispatchQueue.main.sync(execute: work) +} + +@_cdecl("twonly_begin_background_task") +func twonlyBeginBackgroundTask() -> UInt64 { + let identifier: UIBackgroundTaskIdentifier = onMain { + var identifier = UIBackgroundTaskIdentifier.invalid + identifier = UIApplication.shared.beginBackgroundTask(withName: "eu.twonly.media-preparation") + { + // The system is reclaiming the time. Ending the assertion here is + // required; the preparation itself is resumed by the flush task or by + // the next launch. + if identifier != .invalid { + UIApplication.shared.endBackgroundTask(identifier) + identifier = .invalid + } + } + return identifier + } + guard identifier != .invalid else { return 0 } + // A preparation that outlives its assertion has to be picked up later. + BackgroundWork.scheduleFlush() + return UInt64(identifier.rawValue) +} + +@_cdecl("twonly_end_background_task") +func twonlyEndBackgroundTask(_ identifier: UInt64) { + guard identifier != 0, let raw = Int(exactly: identifier) else { return } + onMain { + UIApplication.shared.endBackgroundTask(UIBackgroundTaskIdentifier(rawValue: raw)) + } +} diff --git a/ios/Runner/DirectMediaTransfer.swift b/ios/Runner/DirectMediaTransfer.swift index 37e078b8..99364757 100644 --- a/ios/Runner/DirectMediaTransfer.swift +++ b/ios/Runner/DirectMediaTransfer.swift @@ -8,12 +8,14 @@ private struct DirectMediaRequest: Codable { let bodyPath: String } +/// A media upload names all three requests. A queued message envelope is the +/// single-request form: there is nothing to describe beyond the POST itself. private struct DirectMediaDescriptor: Codable { let attachmentId: String let expiresAt: Int64 let media: DirectMediaRequest - let manifest: DirectMediaRequest - let complete: DirectMediaRequest + let manifest: DirectMediaRequest? + let complete: DirectMediaRequest? } /// Thin transport-only adapter. Rust supplies immutable request files and all @@ -52,19 +54,29 @@ final class DirectMediaTransfer: NSObject, URLSessionTaskDelegate, URLSessionDel let data = json.data(using: .utf8), let descriptor = try? decoder.decode(DirectMediaDescriptor.self, from: data), descriptor.expiresAt > Int64(Date().timeIntervalSince1970), - FileManager.default.fileExists(atPath: descriptor.media.bodyPath), - FileManager.default.fileExists(atPath: descriptor.manifest.bodyPath) + FileManager.default.fileExists(atPath: descriptor.media.bodyPath) else { return false } + if let manifest = descriptor.manifest, + !FileManager.default.fileExists(atPath: manifest.bodyPath) + { + return false + } defaults.set(data, forKey: descriptorPrefix + descriptor.attachmentId) - // Both durable transfers are created before either is resumed, so process - // death cannot leave a media object with no corresponding manifest task. - guard - let mediaTask = makeTask(descriptor.media, attachmentId: descriptor.attachmentId), - let manifestTask = makeTask(descriptor.manifest, attachmentId: descriptor.attachmentId) + // Every durable transfer is created before any of them is resumed, so + // process death cannot leave a media object with no corresponding manifest + // task. + guard let mediaTask = makeTask(descriptor.media, attachmentId: descriptor.attachmentId) else { return false } + var manifestTask: URLSessionUploadTask? + if let manifest = descriptor.manifest { + guard let task = makeTask(manifest, attachmentId: descriptor.attachmentId) else { + return false + } + manifestTask = task + } mediaTask.resume() - manifestTask.resume() + manifestTask?.resume() return true } @@ -121,7 +133,11 @@ final class DirectMediaTransfer: NSObject, URLSessionTaskDelegate, URLSessionDel let success = error == nil && (200...299).contains(status) if success && role == "media" { - makeTask(descriptor.complete, attachmentId: attachmentId)?.resume() + // A descriptor with no completion step — a queued message envelope — is + // finished as soon as its POST is accepted. + if let complete = descriptor.complete { + makeTask(complete, attachmentId: attachmentId)?.resume() + } return } if success { @@ -138,6 +154,10 @@ final class DirectMediaTransfer: NSObject, URLSessionTaskDelegate, URLSessionDel } func urlSessionDidFinishEvents(forBackgroundURLSession session: URLSession) { + // The HTTP tasks can finish while Rust is not loaded. Ask the OS for a + // durable reconciliation opportunity rather than relying on an in-process + // callback or websocket receipt. + BackgroundWork.scheduleFlush() DispatchQueue.main.async { [weak self] in let completion = self?.backgroundCompletion self?.backgroundCompletion = nil diff --git a/ios/Runner/Info.plist b/ios/Runner/Info.plist index 26d64017..4d8b0fe0 100644 --- a/ios/Runner/Info.plist +++ b/ios/Runner/Info.plist @@ -86,6 +86,10 @@ UIApplicationSupportsIndirectInputEvents + BGTaskSchedulerPermittedIdentifiers + + eu.twonly.outbox-flush + UIBackgroundModes fetch diff --git a/ios/Runner/NativeVideoCodec.swift b/ios/Runner/NativeVideoCodec.swift index b295e8c3..9f17ca06 100644 --- a/ios/Runner/NativeVideoCodec.swift +++ b/ios/Runner/NativeVideoCodec.swift @@ -41,6 +41,8 @@ enum NativeVideoCodec { overlayPath: String?, outputPath: String, removeAudio: Bool, + trimStartMs: Int64, + trimEndMs: Int64, onProgress: @escaping (Int) -> Void ) -> Bool { let asset = AVURLAsset(url: URL(fileURLWithPath: inputPath)) @@ -79,7 +81,11 @@ enum NativeVideoCodec { ) do { + let trim = trimRange(for: asset, startMs: trimStartMs, endMs: trimEndMs) let reader = try AVAssetReader(asset: asset) + // Every output of a reader is bounded by this, so the cut costs nothing + // beyond the samples it stops the decoder from reading. + reader.timeRange = trim let writer = try AVAssetWriter(outputURL: outputURL, fileType: .mp4) writer.shouldOptimizeForNetworkUse = true @@ -150,15 +156,20 @@ enum NativeVideoCodec { } guard reader.startReading(), writer.startWriting() else { return false } - writer.startSession(atSourceTime: .zero) + // Samples keep the timestamps they had in the recording, so the session + // has to start where the cut does. Starting it at zero would prepend the + // trimmed-off head as an empty edit instead of removing it. + writer.startSession(atSourceTime: trim.start) - let duration = CMTimeGetSeconds(asset.duration) + let startSeconds = CMTimeGetSeconds(trim.start) + let duration = CMTimeGetSeconds(trim.duration) let group = DispatchGroup() pump( input: videoInput, output: videoOutput, label: "video", group: group, + startSeconds: startSeconds, duration: duration, onProgress: onProgress ) @@ -168,6 +179,7 @@ enum NativeVideoCodec { output: audioOutput, label: "audio", group: group, + startSeconds: 0, duration: nil, onProgress: nil ) @@ -267,11 +279,36 @@ enum NativeVideoCodec { ) } + /// The slice of the recording the editor's cutter selected. + /// + /// Bounds arrive as milliseconds, with a negative value meaning the clip + /// keeps that end. Anything that does not describe a real slice - a start + /// past the end of the clip, an inverted pair, a zero-length result - falls + /// back to the whole asset: sending the untrimmed moment beats sending an + /// empty file. + private static func trimRange(for asset: AVAsset, startMs: Int64, endMs: Int64) -> CMTimeRange { + let whole = CMTimeRange(start: .zero, duration: asset.duration) + guard startMs > 0 || endMs > 0 else { return whole } + + let milliseconds: CMTimeScale = 1000 + let requestedStart = + startMs > 0 ? CMTime(value: CMTimeValue(startMs), timescale: milliseconds) : .zero + let requestedEnd = + endMs > 0 ? CMTime(value: CMTimeValue(endMs), timescale: milliseconds) : asset.duration + + let start = CMTimeMaximum(.zero, CMTimeMinimum(requestedStart, asset.duration)) + let end = CMTimeMinimum(requestedEnd, asset.duration) + let duration = CMTimeSubtract(end, start) + guard duration.isValid, CMTimeGetSeconds(duration) > 0 else { return whole } + return CMTimeRange(start: start, duration: duration) + } + private static func pump( input: AVAssetWriterInput, output: AVAssetReaderOutput, label: String, group: DispatchGroup, + startSeconds: Double, duration: Double?, onProgress: ((Int) -> Void)? ) { @@ -285,8 +322,11 @@ enum NativeVideoCodec { return } if let duration, duration > 0, let onProgress { + // Timestamps are still the recording's, so the trimmed-off head has + // to come off before this is a fraction of the work being done. let seconds = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sample)) - onProgress(Int((seconds / duration * 100).rounded())) + let done = (seconds - startSeconds) / duration + onProgress(Int((min(max(done, 0), 1) * 100).rounded())) } input.append(sample) } @@ -301,6 +341,8 @@ func twonlyRenderVideo( _ overlay: UnsafePointer?, _ output: UnsafePointer?, _ removeAudio: Bool, + _ trimStartMs: Int64, + _ trimEndMs: Int64, _ mediaId: UnsafePointer?, _ progress: @convention(c) (UnsafePointer?, Int32) -> Void ) -> Bool { @@ -311,6 +353,8 @@ func twonlyRenderVideo( overlayPath: overlay.map { String(cString: $0) }, outputPath: String(cString: output), removeAudio: removeAudio, + trimStartMs: trimStartMs, + trimEndMs: trimEndMs, onProgress: { percent in mediaIdString.withCString { progress($0, Int32(percent)) } } diff --git a/ios/Runner/Runner-Bridging-Header.h b/ios/Runner/Runner-Bridging-Header.h index 308a2a56..12417ae4 100644 --- a/ios/Runner/Runner-Bridging-Header.h +++ b/ios/Runner/Runner-Bridging-Header.h @@ -1 +1,2 @@ #import "GeneratedPluginRegistrant.h" +#import diff --git a/lib/app.dart b/lib/app.dart index b6786418..ce837b3d 100644 --- a/lib/app.dart +++ b/lib/app.dart @@ -45,8 +45,6 @@ class App extends StatefulWidget { } class _AppState extends State with WidgetsBindingObserver { - bool _wasPaused = false; - @override void initState() { super.initState(); @@ -58,23 +56,31 @@ class _AppState extends State with WidgetsBindingObserver { void didChangeAppLifecycleState(AppLifecycleState state) { super.didChangeAppLifecycleState(state); if (state == AppLifecycleState.resumed) { - if (_wasPaused) { - AppState.isAppInBackground = false; - twonlyDB.markUpdated(); - unawaited( - rust_api.RustApi.setBackground(inBackground: false), - ); - // The notification service extension wrote to the outbox while the app - // was suspended, and its Rust change broadcast never reached this - // process, so the badge has to be re-read on the way back in. - unawaited(NativeNotificationService.refreshBadgeCount()); - } + AppState.isAppInBackground = false; + twonlyDB.markUpdated(); + // Resuming can follow `inactive` without a `paused` event. Always notify + // Rust so every transition back to a focused app gets an immediate + // WebSocket attempt. + unawaited( + rust_api.RustApi.setBackground(inBackground: false), + ); + // The notification service extension wrote to the outbox while the app + // was suspended, and its Rust change broadcast never reached this + // process, so the badge has to be re-read on the way back in. + unawaited(NativeNotificationService.refreshBadgeCount()); } else if (state == AppLifecycleState.paused) { - _wasPaused = true; AppState.isAppInBackground = true; unawaited( rust_api.RustApi.setBackground(inBackground: true), ); + } else if (state == AppLifecycleState.detached) { + // Last chance before the engine goes away: hand anything still unsent to + // the OS, which delivers it once there is a network again whether or not + // this process is ever started back up. A no-op when `paused` already + // did it. + unawaited( + rust_api.RustApi.handOutboxToOs(), + ); } } diff --git a/lib/core/bridge/api.dart b/lib/core/bridge/api.dart index db4938a0..6ed165da 100644 --- a/lib/core/bridge/api.dart +++ b/lib/core/bridge/api.dart @@ -443,6 +443,11 @@ class RustApi { username: username, ); + /// Hands every queued envelope to an OS-owned transfer. Used when the app + /// is being torn down while messages are still unsent. + static Future handOutboxToOs() => + RustLib.instance.api.crateBridgeApiRustApiHandOutboxToOs(); + /// Creates the media row and its content-encryption material and returns /// the media id the UI addresses every later step by. static Future initializeMediaUpload({ @@ -541,6 +546,13 @@ class RustApi { .api .crateBridgeApiRustApiPerformPasswordlessRecoveryHeartbeat(); + /// Transcodes a captured video while the user is still editing it, so the + /// send itself only has to encrypt and hand over. + static Future prerenderMedia({required String mediaId}) => RustLib + .instance + .api + .crateBridgeApiRustApiPrerenderMedia(mediaId: mediaId); + /// Deletes temporary media whose messages are finished with it. static Future purgeMediaTempFolder() => RustLib.instance.api.crateBridgeApiRustApiPurgeMediaTempFolder(); @@ -736,6 +748,18 @@ class RustApi { requiresAuthentication: requiresAuthentication, ); + /// Stores the cut the editor's video trimmer asked for. Both bounds are + /// milliseconds into the recording, `None` keeps that end of the clip. + static Future setMediaTrim({ + required String mediaId, + PlatformInt64? trimStartMs, + PlatformInt64? trimEndMs, + }) => RustLib.instance.api.crateBridgeApiRustApiSetMediaTrim( + mediaId: mediaId, + trimStartMs: trimStartMs, + trimEndMs: trimEndMs, + ); + static Future setNetworkAvailable({required bool available}) => RustLib .instance .api diff --git a/lib/core/frb_generated.dart b/lib/core/frb_generated.dart index 0a34940a..f3438700 100644 --- a/lib/core/frb_generated.dart +++ b/lib/core/frb_generated.dart @@ -86,7 +86,7 @@ class RustLib extends BaseEntrypoint { String get codegenVersion => '2.12.0'; @override - int get rustContentHash => -1370816897; + int get rustContentHash => 1963960341; static const kDefaultExternalLibraryLoaderConfig = ExternalLibraryLoaderConfig( @@ -285,6 +285,8 @@ abstract class RustLibApi extends BaseApi { required String username, }); + Future crateBridgeApiRustApiHandOutboxToOs(); + Future crateBridgeApiRustApiInitializeMediaUpload({ required String mediaType, PlatformInt64? displayLimitInMilliseconds, @@ -339,6 +341,8 @@ abstract class RustLibApi extends BaseApi { Future crateBridgeApiRustApiPerformPasswordlessRecoveryHeartbeat(); + Future crateBridgeApiRustApiPrerenderMedia({required String mediaId}); + Future crateBridgeApiRustApiPurgeMediaTempFolder(); Future crateBridgeApiRustApiRegister({ @@ -457,6 +461,12 @@ abstract class RustLibApi extends BaseApi { required bool requiresAuthentication, }); + Future crateBridgeApiRustApiSetMediaTrim({ + required String mediaId, + PlatformInt64? trimStartMs, + PlatformInt64? trimEndMs, + }); + Future crateBridgeApiRustApiSetNetworkAvailable({ required bool available, }); @@ -2536,6 +2546,36 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { argNames: ["username"], ); + @override + Future crateBridgeApiRustApiHandOutboxToOs() { + return handler.executeNormal( + NormalTask( + callFfi: (port_) { + final serializer = SseSerializer(generalizedFrbRustBinding); + pdeCallFfi( + generalizedFrbRustBinding, + serializer, + funcId: 57, + port: port_, + ); + }, + codec: SseCodec( + decodeSuccessData: sse_decode_unit, + decodeErrorData: sse_decode_AnyhowException, + ), + constMeta: kCrateBridgeApiRustApiHandOutboxToOsConstMeta, + argValues: [], + apiImpl: this, + ), + ); + } + + TaskConstMeta get kCrateBridgeApiRustApiHandOutboxToOsConstMeta => + const TaskConstMeta( + debugName: "rust_api_hand_outbox_to_os", + argNames: [], + ); + @override Future crateBridgeApiRustApiInitializeMediaUpload({ required String mediaType, @@ -2555,7 +2595,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 57, + funcId: 58, port: port_, ); }, @@ -2592,7 +2632,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 58, + funcId: 59, port: port_, ); }, @@ -2628,7 +2668,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 59, + funcId: 60, port: port_, ); }, @@ -2663,7 +2703,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 60, + funcId: 61, port: port_, ); }, @@ -2700,7 +2740,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 61, + funcId: 62, port: port_, ); }, @@ -2737,7 +2777,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 62, + funcId: 63, port: port_, ); }, @@ -2767,7 +2807,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 63, + funcId: 64, port: port_, ); }, @@ -2800,7 +2840,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 64, + funcId: 65, port: port_, ); }, @@ -2835,7 +2875,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 65, + funcId: 66, port: port_, ); }, @@ -2865,7 +2905,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 66, + funcId: 67, port: port_, ); }, @@ -2900,7 +2940,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 67, + funcId: 68, port: port_, ); }, @@ -2930,7 +2970,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 68, + funcId: 69, port: port_, ); }, @@ -2953,6 +2993,37 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { argNames: [], ); + @override + Future crateBridgeApiRustApiPrerenderMedia({required String mediaId}) { + return handler.executeNormal( + NormalTask( + callFfi: (port_) { + final serializer = SseSerializer(generalizedFrbRustBinding); + sse_encode_String(mediaId, serializer); + pdeCallFfi( + generalizedFrbRustBinding, + serializer, + funcId: 70, + port: port_, + ); + }, + codec: SseCodec( + decodeSuccessData: sse_decode_unit, + decodeErrorData: sse_decode_AnyhowException, + ), + constMeta: kCrateBridgeApiRustApiPrerenderMediaConstMeta, + argValues: [mediaId], + apiImpl: this, + ), + ); + } + + TaskConstMeta get kCrateBridgeApiRustApiPrerenderMediaConstMeta => + const TaskConstMeta( + debugName: "rust_api_prerender_media", + argNames: ["mediaId"], + ); + @override Future crateBridgeApiRustApiPurgeMediaTempFolder() { return handler.executeNormal( @@ -2962,7 +3033,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 69, + funcId: 71, port: port_, ); }, @@ -3001,7 +3072,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 70, + funcId: 72, port: port_, ); }, @@ -3040,7 +3111,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 71, + funcId: 73, port: port_, ); }, @@ -3082,7 +3153,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 72, + funcId: 74, port: port_, ); }, @@ -3113,7 +3184,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 73, + funcId: 75, port: port_, ); }, @@ -3146,7 +3217,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 74, + funcId: 76, port: port_, ); }, @@ -3179,7 +3250,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 75, + funcId: 77, port: port_, ); }, @@ -3214,7 +3285,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 76, + funcId: 78, port: port_, ); }, @@ -3247,7 +3318,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 77, + funcId: 79, port: port_, ); }, @@ -3280,7 +3351,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 78, + funcId: 80, port: port_, ); }, @@ -3313,7 +3384,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 79, + funcId: 81, port: port_, ); }, @@ -3350,7 +3421,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 80, + funcId: 82, port: port_, ); }, @@ -3380,7 +3451,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 81, + funcId: 83, port: port_, ); }, @@ -3410,7 +3481,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 82, + funcId: 84, port: port_, ); }, @@ -3440,7 +3511,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 83, + funcId: 85, port: port_, ); }, @@ -3473,7 +3544,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 84, + funcId: 86, port: port_, ); }, @@ -3504,7 +3575,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 85, + funcId: 87, port: port_, ); }, @@ -3537,7 +3608,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 86, + funcId: 88, port: port_, ); }, @@ -3583,7 +3654,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 87, + funcId: 89, port: port_, ); }, @@ -3639,7 +3710,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 88, + funcId: 90, port: port_, ); }, @@ -3685,7 +3756,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 89, + funcId: 91, port: port_, ); }, @@ -3718,7 +3789,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 90, + funcId: 92, port: port_, ); }, @@ -3753,7 +3824,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 91, + funcId: 93, port: port_, ); }, @@ -3788,7 +3859,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 92, + funcId: 94, port: port_, ); }, @@ -3821,7 +3892,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 93, + funcId: 95, port: port_, ); }, @@ -3852,7 +3923,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 94, + funcId: 96, port: port_, ); }, @@ -3890,7 +3961,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 95, + funcId: 97, port: port_, ); }, @@ -3925,7 +3996,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 96, + funcId: 98, port: port_, ); }, @@ -3948,6 +4019,43 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { argNames: ["mediaId", "requiresAuthentication"], ); + @override + Future crateBridgeApiRustApiSetMediaTrim({ + required String mediaId, + PlatformInt64? trimStartMs, + PlatformInt64? trimEndMs, + }) { + return handler.executeNormal( + NormalTask( + callFfi: (port_) { + final serializer = SseSerializer(generalizedFrbRustBinding); + sse_encode_String(mediaId, serializer); + sse_encode_opt_box_autoadd_i_64(trimStartMs, serializer); + sse_encode_opt_box_autoadd_i_64(trimEndMs, serializer); + pdeCallFfi( + generalizedFrbRustBinding, + serializer, + funcId: 99, + port: port_, + ); + }, + codec: SseCodec( + decodeSuccessData: sse_decode_unit, + decodeErrorData: sse_decode_AnyhowException, + ), + constMeta: kCrateBridgeApiRustApiSetMediaTrimConstMeta, + argValues: [mediaId, trimStartMs, trimEndMs], + apiImpl: this, + ), + ); + } + + TaskConstMeta get kCrateBridgeApiRustApiSetMediaTrimConstMeta => + const TaskConstMeta( + debugName: "rust_api_set_media_trim", + argNames: ["mediaId", "trimStartMs", "trimEndMs"], + ); + @override Future crateBridgeApiRustApiSetNetworkAvailable({ required bool available, @@ -3960,7 +4068,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 97, + funcId: 100, port: port_, ); }, @@ -3991,7 +4099,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 98, + funcId: 101, port: port_, ); }, @@ -4026,7 +4134,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 99, + funcId: 102, port: port_, ); }, @@ -4059,7 +4167,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 100, + funcId: 103, port: port_, ); }, @@ -4094,7 +4202,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 101, + funcId: 104, port: port_, ); }, @@ -4125,7 +4233,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 102, + funcId: 105, port: port_, ); }, @@ -4162,7 +4270,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 103, + funcId: 106, port: port_, ); }, @@ -4214,7 +4322,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 104, + funcId: 107, port: port_, ); }, @@ -4267,7 +4375,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 105, + funcId: 108, port: port_, ); }, @@ -4307,7 +4415,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 106, + funcId: 109, port: port_, ); }, @@ -4340,7 +4448,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 107, + funcId: 110, port: port_, ); }, @@ -4373,7 +4481,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 108, + funcId: 111, port: port_, ); }, @@ -4410,7 +4518,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 109, + funcId: 112, port: port_, ); }, @@ -4442,7 +4550,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 110, + funcId: 113, port: port_, ); }, @@ -4475,7 +4583,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 111, + funcId: 114, port: port_, ); }, @@ -4510,7 +4618,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 112, + funcId: 115, port: port_, ); }, @@ -4542,7 +4650,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 113, + funcId: 116, port: port_, ); }, @@ -4580,7 +4688,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 114, + funcId: 117, port: port_, ); }, @@ -4613,7 +4721,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 115, + funcId: 118, port: port_, ); }, @@ -4651,7 +4759,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 116, + funcId: 119, port: port_, ); }, @@ -4688,7 +4796,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 117, + funcId: 120, port: port_, ); }, @@ -4725,7 +4833,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 118, + funcId: 121, port: port_, ); }, @@ -4763,7 +4871,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 119, + funcId: 122, port: port_, ); }, @@ -4801,7 +4909,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 120, + funcId: 123, port: port_, ); }, @@ -4834,7 +4942,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 121, + funcId: 124, port: port_, ); }, @@ -4866,7 +4974,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 122, + funcId: 125, port: port_, ); }, @@ -4901,7 +5009,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 123, + funcId: 126, port: port_, ); }, @@ -4938,7 +5046,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 124, + funcId: 127, port: port_, ); }, @@ -4971,7 +5079,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 125, + funcId: 128, port: port_, ); }, @@ -5003,7 +5111,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 126, + funcId: 129, port: port_, ); }, @@ -5038,7 +5146,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 127, + funcId: 130, port: port_, ); }, @@ -5077,7 +5185,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 128, + funcId: 131, port: port_, ); }, @@ -5114,7 +5222,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 129, + funcId: 132, port: port_, ); }, @@ -5144,7 +5252,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 130, + funcId: 133, port: port_, ); }, @@ -5176,7 +5284,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 131, + funcId: 134, port: port_, ); }, @@ -5211,7 +5319,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 132, + funcId: 135, port: port_, ); }, @@ -5243,7 +5351,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 133, + funcId: 136, port: port_, ); }, @@ -5281,7 +5389,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 134, + funcId: 137, port: port_, ); }, @@ -5320,7 +5428,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 135, + funcId: 138, port: port_, ); }, @@ -5355,7 +5463,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 136, + funcId: 139, port: port_, ); }, @@ -5390,7 +5498,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 137, + funcId: 140, port: port_, ); }, @@ -5425,7 +5533,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 138, + funcId: 141, port: port_, ); }, @@ -5458,7 +5566,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { return pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 139, + funcId: 142, )!; }, codec: SseCodec( @@ -5498,7 +5606,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 140, + funcId: 143, port: port_, ); }, @@ -5543,7 +5651,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 141, + funcId: 144, port: port_, ); }, @@ -5573,7 +5681,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 142, + funcId: 145, port: port_, ); }, @@ -5606,7 +5714,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 143, + funcId: 146, port: port_, ); }, @@ -5641,7 +5749,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 144, + funcId: 147, port: port_, ); }, @@ -5680,7 +5788,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { return pdeCallFfi( generalizedFrbRustBinding, serializer, - funcId: 145, + funcId: 148, )!; }, codec: SseCodec( diff --git a/lib/globals.dart b/lib/globals.dart index 1b053354..49c7ad7e 100644 --- a/lib/globals.dart +++ b/lib/globals.dart @@ -90,7 +90,6 @@ class AppEnvironment { class AppState { static bool isAppInBackground = true; - static bool isInBackgroundTask = false; static bool allowErrorTrackingViaSentry = false; static bool gotMessageFromServer = false; static int latestAppVersionId = 119; diff --git a/lib/src/database/tables/mediafiles.table.dart b/lib/src/database/tables/mediafiles.table.dart index b1191e7b..f6517813 100644 --- a/lib/src/database/tables/mediafiles.table.dart +++ b/lib/src/database/tables/mediafiles.table.dart @@ -71,6 +71,13 @@ class MediaFiles extends Table { IntColumn get displayLimitInMilliseconds => integer().nullable()(); BoolColumn get removeAudio => boolean().nullable()(); + /// Where the editor's cutter placed the two ends of a video, in milliseconds + /// into the recording. Null on either side means the clip keeps that end. + /// The recording on disk is never cut; the transcode every send performs + /// applies these. + IntColumn get trimStartMs => integer().nullable()(); + IntColumn get trimEndMs => integer().nullable()(); + BlobColumn get downloadToken => blob().nullable()(); BlobColumn get encryptionKey => blob().nullable()(); BlobColumn get encryptionMac => blob().nullable()(); diff --git a/lib/src/database/twonly.db.g.dart b/lib/src/database/twonly.db.g.dart index 86d82cb6..a03fcb28 100644 --- a/lib/src/database/twonly.db.g.dart +++ b/lib/src/database/twonly.db.g.dart @@ -3297,6 +3297,28 @@ class $MediaFilesTable extends MediaFiles 'CHECK ("remove_audio" IN (0, 1))', ), ); + static const VerificationMeta _trimStartMsMeta = const VerificationMeta( + 'trimStartMs', + ); + @override + late final GeneratedColumn trimStartMs = GeneratedColumn( + 'trim_start_ms', + aliasedName, + true, + type: DriftSqlType.int, + requiredDuringInsert: false, + ); + static const VerificationMeta _trimEndMsMeta = const VerificationMeta( + 'trimEndMs', + ); + @override + late final GeneratedColumn trimEndMs = GeneratedColumn( + 'trim_end_ms', + aliasedName, + true, + type: DriftSqlType.int, + requiredDuringInsert: false, + ); static const VerificationMeta _downloadTokenMeta = const VerificationMeta( 'downloadToken', ); @@ -3423,6 +3445,8 @@ class $MediaFilesTable extends MediaFiles reuploadRequestedBy, displayLimitInMilliseconds, removeAudio, + trimStartMs, + trimEndMs, downloadToken, encryptionKey, encryptionMac, @@ -3525,6 +3549,21 @@ class $MediaFilesTable extends MediaFiles ), ); } + if (data.containsKey('trim_start_ms')) { + context.handle( + _trimStartMsMeta, + trimStartMs.isAcceptableOrUnknown( + data['trim_start_ms']!, + _trimStartMsMeta, + ), + ); + } + if (data.containsKey('trim_end_ms')) { + context.handle( + _trimEndMsMeta, + trimEndMs.isAcceptableOrUnknown(data['trim_end_ms']!, _trimEndMsMeta), + ); + } if (data.containsKey('download_token')) { context.handle( _downloadTokenMeta, @@ -3683,6 +3722,14 @@ class $MediaFilesTable extends MediaFiles DriftSqlType.bool, data['${effectivePrefix}remove_audio'], ), + trimStartMs: attachedDatabase.typeMapping.read( + DriftSqlType.int, + data['${effectivePrefix}trim_start_ms'], + ), + trimEndMs: attachedDatabase.typeMapping.read( + DriftSqlType.int, + data['${effectivePrefix}trim_end_ms'], + ), downloadToken: attachedDatabase.typeMapping.read( DriftSqlType.blob, data['${effectivePrefix}download_token'], @@ -3765,6 +3812,13 @@ class MediaFile extends DataClass implements Insertable { final List? reuploadRequestedBy; final int? displayLimitInMilliseconds; final bool? removeAudio; + + /// Where the editor's cutter placed the two ends of a video, in milliseconds + /// into the recording. Null on either side means the clip keeps that end. + /// The recording on disk is never cut; the transcode every send performs + /// applies these. + final int? trimStartMs; + final int? trimEndMs; final Uint8List? downloadToken; final Uint8List? encryptionKey; final Uint8List? encryptionMac; @@ -3790,6 +3844,8 @@ class MediaFile extends DataClass implements Insertable { this.reuploadRequestedBy, this.displayLimitInMilliseconds, this.removeAudio, + this.trimStartMs, + this.trimEndMs, this.downloadToken, this.encryptionKey, this.encryptionMac, @@ -3850,6 +3906,12 @@ class MediaFile extends DataClass implements Insertable { if (!nullToAbsent || removeAudio != null) { map['remove_audio'] = Variable(removeAudio); } + if (!nullToAbsent || trimStartMs != null) { + map['trim_start_ms'] = Variable(trimStartMs); + } + if (!nullToAbsent || trimEndMs != null) { + map['trim_end_ms'] = Variable(trimEndMs); + } if (!nullToAbsent || downloadToken != null) { map['download_token'] = Variable(downloadToken); } @@ -3908,6 +3970,12 @@ class MediaFile extends DataClass implements Insertable { removeAudio: removeAudio == null && nullToAbsent ? const Value.absent() : Value(removeAudio), + trimStartMs: trimStartMs == null && nullToAbsent + ? const Value.absent() + : Value(trimStartMs), + trimEndMs: trimEndMs == null && nullToAbsent + ? const Value.absent() + : Value(trimEndMs), downloadToken: downloadToken == null && nullToAbsent ? const Value.absent() : Value(downloadToken), @@ -3971,6 +4039,8 @@ class MediaFile extends DataClass implements Insertable { json['displayLimitInMilliseconds'], ), removeAudio: serializer.fromJson(json['removeAudio']), + trimStartMs: serializer.fromJson(json['trimStartMs']), + trimEndMs: serializer.fromJson(json['trimEndMs']), downloadToken: serializer.fromJson(json['downloadToken']), encryptionKey: serializer.fromJson(json['encryptionKey']), encryptionMac: serializer.fromJson(json['encryptionMac']), @@ -4011,6 +4081,8 @@ class MediaFile extends DataClass implements Insertable { displayLimitInMilliseconds, ), 'removeAudio': serializer.toJson(removeAudio), + 'trimStartMs': serializer.toJson(trimStartMs), + 'trimEndMs': serializer.toJson(trimEndMs), 'downloadToken': serializer.toJson(downloadToken), 'encryptionKey': serializer.toJson(encryptionKey), 'encryptionMac': serializer.toJson(encryptionMac), @@ -4039,6 +4111,8 @@ class MediaFile extends DataClass implements Insertable { Value?> reuploadRequestedBy = const Value.absent(), Value displayLimitInMilliseconds = const Value.absent(), Value removeAudio = const Value.absent(), + Value trimStartMs = const Value.absent(), + Value trimEndMs = const Value.absent(), Value downloadToken = const Value.absent(), Value encryptionKey = const Value.absent(), Value encryptionMac = const Value.absent(), @@ -4073,6 +4147,8 @@ class MediaFile extends DataClass implements Insertable { ? displayLimitInMilliseconds.value : this.displayLimitInMilliseconds, removeAudio: removeAudio.present ? removeAudio.value : this.removeAudio, + trimStartMs: trimStartMs.present ? trimStartMs.value : this.trimStartMs, + trimEndMs: trimEndMs.present ? trimEndMs.value : this.trimEndMs, downloadToken: downloadToken.present ? downloadToken.value : this.downloadToken, @@ -4134,6 +4210,10 @@ class MediaFile extends DataClass implements Insertable { removeAudio: data.removeAudio.present ? data.removeAudio.value : this.removeAudio, + trimStartMs: data.trimStartMs.present + ? data.trimStartMs.value + : this.trimStartMs, + trimEndMs: data.trimEndMs.present ? data.trimEndMs.value : this.trimEndMs, downloadToken: data.downloadToken.present ? data.downloadToken.value : this.downloadToken, @@ -4180,6 +4260,8 @@ class MediaFile extends DataClass implements Insertable { ..write('reuploadRequestedBy: $reuploadRequestedBy, ') ..write('displayLimitInMilliseconds: $displayLimitInMilliseconds, ') ..write('removeAudio: $removeAudio, ') + ..write('trimStartMs: $trimStartMs, ') + ..write('trimEndMs: $trimEndMs, ') ..write('downloadToken: $downloadToken, ') ..write('encryptionKey: $encryptionKey, ') ..write('encryptionMac: $encryptionMac, ') @@ -4210,6 +4292,8 @@ class MediaFile extends DataClass implements Insertable { reuploadRequestedBy, displayLimitInMilliseconds, removeAudio, + trimStartMs, + trimEndMs, $driftBlobEquality.hash(downloadToken), $driftBlobEquality.hash(encryptionKey), $driftBlobEquality.hash(encryptionMac), @@ -4239,6 +4323,8 @@ class MediaFile extends DataClass implements Insertable { other.reuploadRequestedBy == this.reuploadRequestedBy && other.displayLimitInMilliseconds == this.displayLimitInMilliseconds && other.removeAudio == this.removeAudio && + other.trimStartMs == this.trimStartMs && + other.trimEndMs == this.trimEndMs && $driftBlobEquality.equals(other.downloadToken, this.downloadToken) && $driftBlobEquality.equals(other.encryptionKey, this.encryptionKey) && $driftBlobEquality.equals(other.encryptionMac, this.encryptionMac) && @@ -4272,6 +4358,8 @@ class MediaFilesCompanion extends UpdateCompanion { final Value?> reuploadRequestedBy; final Value displayLimitInMilliseconds; final Value removeAudio; + final Value trimStartMs; + final Value trimEndMs; final Value downloadToken; final Value encryptionKey; final Value encryptionMac; @@ -4298,6 +4386,8 @@ class MediaFilesCompanion extends UpdateCompanion { this.reuploadRequestedBy = const Value.absent(), this.displayLimitInMilliseconds = const Value.absent(), this.removeAudio = const Value.absent(), + this.trimStartMs = const Value.absent(), + this.trimEndMs = const Value.absent(), this.downloadToken = const Value.absent(), this.encryptionKey = const Value.absent(), this.encryptionMac = const Value.absent(), @@ -4325,6 +4415,8 @@ class MediaFilesCompanion extends UpdateCompanion { this.reuploadRequestedBy = const Value.absent(), this.displayLimitInMilliseconds = const Value.absent(), this.removeAudio = const Value.absent(), + this.trimStartMs = const Value.absent(), + this.trimEndMs = const Value.absent(), this.downloadToken = const Value.absent(), this.encryptionKey = const Value.absent(), this.encryptionMac = const Value.absent(), @@ -4353,6 +4445,8 @@ class MediaFilesCompanion extends UpdateCompanion { Expression? reuploadRequestedBy, Expression? displayLimitInMilliseconds, Expression? removeAudio, + Expression? trimStartMs, + Expression? trimEndMs, Expression? downloadToken, Expression? encryptionKey, Expression? encryptionMac, @@ -4384,6 +4478,8 @@ class MediaFilesCompanion extends UpdateCompanion { if (displayLimitInMilliseconds != null) 'display_limit_in_milliseconds': displayLimitInMilliseconds, if (removeAudio != null) 'remove_audio': removeAudio, + if (trimStartMs != null) 'trim_start_ms': trimStartMs, + if (trimEndMs != null) 'trim_end_ms': trimEndMs, if (downloadToken != null) 'download_token': downloadToken, if (encryptionKey != null) 'encryption_key': encryptionKey, if (encryptionMac != null) 'encryption_mac': encryptionMac, @@ -4413,6 +4509,8 @@ class MediaFilesCompanion extends UpdateCompanion { Value?>? reuploadRequestedBy, Value? displayLimitInMilliseconds, Value? removeAudio, + Value? trimStartMs, + Value? trimEndMs, Value? downloadToken, Value? encryptionKey, Value? encryptionMac, @@ -4443,6 +4541,8 @@ class MediaFilesCompanion extends UpdateCompanion { displayLimitInMilliseconds: displayLimitInMilliseconds ?? this.displayLimitInMilliseconds, removeAudio: removeAudio ?? this.removeAudio, + trimStartMs: trimStartMs ?? this.trimStartMs, + trimEndMs: trimEndMs ?? this.trimEndMs, downloadToken: downloadToken ?? this.downloadToken, encryptionKey: encryptionKey ?? this.encryptionKey, encryptionMac: encryptionMac ?? this.encryptionMac, @@ -4522,6 +4622,12 @@ class MediaFilesCompanion extends UpdateCompanion { if (removeAudio.present) { map['remove_audio'] = Variable(removeAudio.value); } + if (trimStartMs.present) { + map['trim_start_ms'] = Variable(trimStartMs.value); + } + if (trimEndMs.present) { + map['trim_end_ms'] = Variable(trimEndMs.value); + } if (downloadToken.present) { map['download_token'] = Variable(downloadToken.value); } @@ -4573,6 +4679,8 @@ class MediaFilesCompanion extends UpdateCompanion { ..write('reuploadRequestedBy: $reuploadRequestedBy, ') ..write('displayLimitInMilliseconds: $displayLimitInMilliseconds, ') ..write('removeAudio: $removeAudio, ') + ..write('trimStartMs: $trimStartMs, ') + ..write('trimEndMs: $trimEndMs, ') ..write('downloadToken: $downloadToken, ') ..write('encryptionKey: $encryptionKey, ') ..write('encryptionMac: $encryptionMac, ') @@ -15510,6 +15618,8 @@ typedef $$MediaFilesTableCreateCompanionBuilder = Value?> reuploadRequestedBy, Value displayLimitInMilliseconds, Value removeAudio, + Value trimStartMs, + Value trimEndMs, Value downloadToken, Value encryptionKey, Value encryptionMac, @@ -15538,6 +15648,8 @@ typedef $$MediaFilesTableUpdateCompanionBuilder = Value?> reuploadRequestedBy, Value displayLimitInMilliseconds, Value removeAudio, + Value trimStartMs, + Value trimEndMs, Value downloadToken, Value encryptionKey, Value encryptionMac, @@ -15662,6 +15774,16 @@ class $$MediaFilesTableFilterComposer builder: (column) => ColumnFilters(column), ); + ColumnFilters get trimStartMs => $composableBuilder( + column: $table.trimStartMs, + builder: (column) => ColumnFilters(column), + ); + + ColumnFilters get trimEndMs => $composableBuilder( + column: $table.trimEndMs, + builder: (column) => ColumnFilters(column), + ); + ColumnFilters get downloadToken => $composableBuilder( column: $table.downloadToken, builder: (column) => ColumnFilters(column), @@ -15817,6 +15939,16 @@ class $$MediaFilesTableOrderingComposer builder: (column) => ColumnOrderings(column), ); + ColumnOrderings get trimStartMs => $composableBuilder( + column: $table.trimStartMs, + builder: (column) => ColumnOrderings(column), + ); + + ColumnOrderings get trimEndMs => $composableBuilder( + column: $table.trimEndMs, + builder: (column) => ColumnOrderings(column), + ); + ColumnOrderings get downloadToken => $composableBuilder( column: $table.downloadToken, builder: (column) => ColumnOrderings(column), @@ -15943,6 +16075,14 @@ class $$MediaFilesTableAnnotationComposer builder: (column) => column, ); + GeneratedColumn get trimStartMs => $composableBuilder( + column: $table.trimStartMs, + builder: (column) => column, + ); + + GeneratedColumn get trimEndMs => + $composableBuilder(column: $table.trimEndMs, builder: (column) => column); + GeneratedColumn get downloadToken => $composableBuilder( column: $table.downloadToken, builder: (column) => column, @@ -16055,6 +16195,8 @@ class $$MediaFilesTableTableManager Value?> reuploadRequestedBy = const Value.absent(), Value displayLimitInMilliseconds = const Value.absent(), Value removeAudio = const Value.absent(), + Value trimStartMs = const Value.absent(), + Value trimEndMs = const Value.absent(), Value downloadToken = const Value.absent(), Value encryptionKey = const Value.absent(), Value encryptionMac = const Value.absent(), @@ -16081,6 +16223,8 @@ class $$MediaFilesTableTableManager reuploadRequestedBy: reuploadRequestedBy, displayLimitInMilliseconds: displayLimitInMilliseconds, removeAudio: removeAudio, + trimStartMs: trimStartMs, + trimEndMs: trimEndMs, downloadToken: downloadToken, encryptionKey: encryptionKey, encryptionMac: encryptionMac, @@ -16109,6 +16253,8 @@ class $$MediaFilesTableTableManager Value?> reuploadRequestedBy = const Value.absent(), Value displayLimitInMilliseconds = const Value.absent(), Value removeAudio = const Value.absent(), + Value trimStartMs = const Value.absent(), + Value trimEndMs = const Value.absent(), Value downloadToken = const Value.absent(), Value encryptionKey = const Value.absent(), Value encryptionMac = const Value.absent(), @@ -16135,6 +16281,8 @@ class $$MediaFilesTableTableManager reuploadRequestedBy: reuploadRequestedBy, displayLimitInMilliseconds: displayLimitInMilliseconds, removeAudio: removeAudio, + trimStartMs: trimStartMs, + trimEndMs: trimEndMs, downloadToken: downloadToken, encryptionKey: encryptionKey, encryptionMac: encryptionMac, diff --git a/lib/src/localization/translations b/lib/src/localization/translations index 17d10180..66a65b77 160000 --- a/lib/src/localization/translations +++ b/lib/src/localization/translations @@ -1 +1 @@ -Subproject commit 17d101801f02cfeaa2a175083111a59fa6d6822c +Subproject commit 66a65b7787ae6c66b8fefbb8d016494de0355ef8 diff --git a/lib/src/providers/connection.provider.dart b/lib/src/providers/connection.provider.dart index c41790b0..5e9a4d6c 100644 --- a/lib/src/providers/connection.provider.dart +++ b/lib/src/providers/connection.provider.dart @@ -9,16 +9,23 @@ class CustomChangeProvider with ChangeNotifier, DiagnosticableTreeMixin { unawaited(_loadConnectionState()); _connSub = apiService.events .where((event) => event.kind == ApiEventKind.connectionStateChanged) - .listen( - (event) => updateConnectionState( - event.state == ApiConnectionState.authenticated, - ), - ); + .listen(_handleConnectionState); } late bool _isConnected; late StreamSubscription _connSub; bool get isConnected => _isConnected; + void _handleConnectionState(ApiEvent event) { + // `suspended` is a lifecycle hint, not a transport disconnect: the socket + // remains alive while Flutter is briefly covered or backgrounded. Keeping + // the last known value prevents the avatar from flashing red during such a + // transition. + if (event.state == ApiConnectionState.suspended) return; + unawaited( + updateConnectionState(event.state == ApiConnectionState.authenticated), + ); + } + Future _loadConnectionState() async { final state = await RustApi.connectionState(); await updateConnectionState(state == ApiConnectionState.authenticated); diff --git a/lib/src/services/api/api.service.dart b/lib/src/services/api/api.service.dart index e9b1e076..ef219f0b 100644 --- a/lib/src/services/api/api.service.dart +++ b/lib/src/services/api/api.service.dart @@ -49,9 +49,7 @@ class ApiService { Future onAuthenticated() async { await FcmNotificationService.initFCMAfterAuthenticated(); - if (AppState.isInBackgroundTask) { - await rust_api.RustApi.reuploadPendingMedia(); - } else if (!AppState.isAppInBackground) { + if (!AppState.isAppInBackground) { unawaitedRustCall( rust_api.RustApi.reuploadPendingMedia(), 'reuploadPendingMedia', diff --git a/lib/src/services/mediafiles/mediafile.service.dart b/lib/src/services/mediafiles/mediafile.service.dart index 2f37a15a..0305f307 100644 --- a/lib/src/services/mediafiles/mediafile.service.dart +++ b/lib/src/services/mediafiles/mediafile.service.dart @@ -59,6 +59,27 @@ class MediaFileService { bool get removeAudio => mediaFile.removeAudio ?? false; + /// Where the editor's cutter placed the two ends of a video. Null on either + /// side means the clip keeps that end. + Duration? get trimStart => mediaFile.trimStartMs == null + ? null + : Duration(milliseconds: mediaFile.trimStartMs!); + Duration? get trimEnd => mediaFile.trimEndMs == null + ? null + : Duration(milliseconds: mediaFile.trimEndMs!); + + /// Rust stores the bounds and applies them in the transcode every send + /// performs, so the recording itself is never rewritten and the cut stays + /// reversible for as long as the editor is open. + Future setTrim(Duration? start, Duration? end) async { + await RustApi.setMediaTrim( + mediaId: mediaFile.mediaId, + trimStartMs: start?.inMilliseconds, + trimEndMs: end?.inMilliseconds, + ); + await updateFromDB(); + } + Future toggleRemoveAudio() async { await RustApi.toggleMediaRemoveAudio(mediaId: mediaFile.mediaId); await updateFromDB(); diff --git a/lib/src/utils/log.dart b/lib/src/utils/log.dart index 78fc8cf1..1b2bb80d 100644 --- a/lib/src/utils/log.dart +++ b/lib/src/utils/log.dart @@ -29,7 +29,7 @@ class Log { record.level >= Level.WARNING) { // ignore: avoid_print print( - '${record.level.name} [${AppState.isInBackgroundTask ? 'b' : 'f'}] [twonly] ${record.loggerName} > ${record.message}', + '${record.level.name} [f] [twonly] ${record.loggerName} > ${record.message}', ); } } @@ -72,7 +72,9 @@ class Log { }, source: record.loggerName, message: record.message, - inBackground: AppState.isInBackgroundTask, + // Background work runs natively now; anything logged from Dart is by + // definition the foreground runtime. + inBackground: false, ); return true; } catch (error) { diff --git a/lib/src/visual/views/camera/camera_preview_components/camera_preview_controller_view.dart b/lib/src/visual/views/camera/camera_preview_components/camera_preview_controller_view.dart index 6c00a352..e4cd1223 100644 --- a/lib/src/visual/views/camera/camera_preview_components/camera_preview_controller_view.dart +++ b/lib/src/visual/views/camera/camera_preview_components/camera_preview_controller_view.dart @@ -16,6 +16,7 @@ import 'package:twonly/locator.dart'; import 'package:twonly/src/database/tables/mediafiles.table.dart'; import 'package:twonly/src/database/twonly.db.dart'; import 'package:twonly/src/services/mediafiles/mediafile.service.dart'; +import 'package:twonly/src/services/subscription.service.dart'; import 'package:twonly/src/services/user.service.dart'; import 'package:twonly/src/utils/log.dart'; import 'package:twonly/src/utils/misc.dart'; @@ -31,13 +32,12 @@ import 'package:twonly/src/visual/views/camera/camera_preview_components/face_fi import 'package:twonly/src/visual/views/camera/camera_preview_components/main_camera_controller.dart'; import 'package:twonly/src/visual/views/camera/camera_preview_components/permissions_view.dart'; import 'package:twonly/src/visual/views/camera/camera_preview_components/send_to.dart'; +import 'package:twonly/src/visual/views/camera/camera_preview_components/video_recording_budget.dart'; import 'package:twonly/src/visual/views/camera/camera_preview_components/video_recording_time.dart'; import 'package:twonly/src/visual/views/camera/share_image_editor.view.dart'; import 'package:twonly/src/visual/views/camera/share_image_editor_components/action_button.dart'; import 'package:twonly/src/visual/views/home.view.dart'; -int maxVideoRecordingTime = 60; - class SelectedCameraDetails { double maxAvailableZoom = 1; double minAvailableZoom = 1; @@ -153,6 +153,7 @@ class _CameraPreviewViewState extends State { DateTime? _videoRecordingStarted; Timer? _videoRecordingTimer; bool _videoRecordingLocked = false; + Duration _currentMaxRecordingTime = Duration.zero; DateTime _currentTime = clock.now(); final GlobalKey keyTriggerButton = GlobalKey(); @@ -166,6 +167,7 @@ class _CameraPreviewViewState extends State { @override void initState() { super.initState(); + unawaited(VideoRecordingBudget.ensureLoaded()); initVolumeControl(); initAsync(); _checkAndInitCamera(); @@ -411,8 +413,14 @@ class _CameraPreviewViewState extends State { ..copySync(mediaFileService.originalPath.path) ..deleteSync(); - // Start with compressing the video, to speed up the process in case the video is not changed. - // unawaited(mediaFileService.compressMedia()); + // Transcode while the user is still editing. A send then only has to + // encrypt and hand over, which is short enough to survive the app being + // closed straight after; the render is discarded if the editor ends up + // drawing something on the clip. + unawaitedRustCall( + RustApi.prerenderMedia(mediaId: mediaId), + 'prerenderMedia', + ); } await _deInitVolumeControl(); @@ -611,6 +619,15 @@ class _CameraPreviewViewState extends State { mc.setFilter(mc.currentFilterType.goRight()); } + /// How long this recording may run before the file stops being uploadable + /// on the user's plan. Frozen when the recording starts so that the ring and + /// the cut-off cannot drift apart if the plan or the camera changes midway. + Duration get _maxVideoRecordingTime => VideoRecordingBudget.maxRecordingTime( + plan: planFromString(userService.currentUser.subscriptionPlan), + recordingSize: mc.cameraController?.value.previewSize, + hasAudio: _hasAudioPermission, + ); + Future startVideoRecording() async { if (mc.cameraController != null && mc.cameraController!.value.isRecordingVideo) { @@ -626,24 +643,31 @@ class _CameraPreviewViewState extends State { try { await mc.cameraController?.startVideoRecording(); - _videoRecordingTimer = Timer.periodic(const Duration(milliseconds: 15), ( + final maxRecordingTime = _maxVideoRecordingTime; + setState(() { + _currentTime = clock.now(); + _videoRecordingStarted = _currentTime; + _currentMaxRecordingTime = maxRecordingTime; + mc.isVideoRecording = true; + }); + _videoRecordingTimer = Timer.periodic(const Duration(milliseconds: 50), ( timer, ) { + final startedAt = _videoRecordingStarted; + if (startedAt == null) { + timer.cancel(); + _videoRecordingTimer = null; + return; + } setState(() { _currentTime = clock.now(); }); - if (_videoRecordingStarted != null && - _currentTime.difference(_videoRecordingStarted!).inSeconds >= - maxVideoRecordingTime) { - timer.cancel(); - _videoRecordingTimer = null; - stopVideoRecording(); + if (_currentTime.difference(startedAt) >= maxRecordingTime) { + // The budget is spent, so this stop is not the user letting go of + // the button: it has to go through even while recording is locked. + unawaited(stopVideoRecording(force: true)); } }); - setState(() { - _videoRecordingStarted = clock.now(); - mc.isVideoRecording = true; - }); } on CameraException catch (e) { setState(() { mc.isVideoRecording = false; @@ -663,6 +687,15 @@ class _CameraPreviewViewState extends State { _videoRecordingTimer = null; } + final startedAt = _videoRecordingStarted; + // Taken here rather than after the recorder has been asked to stop: the + // tail it flushes would otherwise read as recorded time and talk the + // measured write rate down. + final recordedFor = startedAt == null + ? null + : clock.now().difference(startedAt); + final recordingSize = mc.cameraController?.value.previewSize; + await mc.cameraController?.setFlashMode(FlashMode.off); setState(() { @@ -684,7 +717,24 @@ class _CameraPreviewViewState extends State { final videoPath = await mc.cameraController?.stopVideoRecording(); if (videoPath == null) return; await mc.cameraController?.pausePreview(); - if (await pushMediaEditor(null, File(videoPath.path))) { + final videoFile = File(videoPath.path); + // Read before the editor takes the file over, and tell the budget what + // this device really writes so the next recording is measured, not + // estimated. + if (recordedFor != null) { + try { + unawaited( + VideoRecordingBudget.recordMeasurement( + fileSizeInBytes: videoFile.statSync().size, + duration: recordedFor, + recordingSize: recordingSize, + ), + ); + } catch (e) { + Log.warn('Could not measure the recorded video: $e'); + } + } + if (await pushMediaEditor(null, videoFile)) { return; } } on CameraException catch (e) { @@ -838,7 +888,8 @@ class _CameraPreviewViewState extends State { ), VideoRecordingTimer( videoRecordingStarted: _videoRecordingStarted, - maxVideoRecordingTime: maxVideoRecordingTime, + currentTime: _currentTime, + maxRecordingTime: _currentMaxRecordingTime, ), if (!mc.isSharePreviewIsShown && widget.sendToGroup != null || widget.hideControllers) diff --git a/lib/src/visual/views/camera/camera_preview_components/video_recording_budget.dart b/lib/src/visual/views/camera/camera_preview_components/video_recording_budget.dart new file mode 100644 index 00000000..f5f240b0 --- /dev/null +++ b/lib/src/visual/views/camera/camera_preview_components/video_recording_budget.dart @@ -0,0 +1,180 @@ +import 'dart:ui' show Size; + +import 'package:flutter/foundation.dart' show visibleForTesting; +import 'package:twonly/src/services/subscription.service.dart'; +import 'package:twonly/src/utils/keyvalue.dart'; +import 'package:twonly/src/utils/log.dart'; + +/// How long the camera may keep recording before the file outgrows the largest +/// single object the user's plan lets them upload. +/// +/// A recorded video is sent exactly as the camera wrote it: nothing re-encodes +/// it between the recorder and the upload, and the upload encrypts in place, +/// so the recorder's output size *is* the size the server measures against +/// `maximal_upload_size_of_single_media_size`. That makes the recording budget +/// a plain division: plan limit divided by the rate the encoder writes at. +/// +/// The rate is the part that cannot be known up front. Both platforms record +/// `ResolutionPreset.high`, which is 720p, but what an encoder spends on those +/// pixels differs by a factor of several between devices - the same minute +/// that costs 18 MB on one phone costs far more on a phone whose camcorder +/// profile asks for a high bitrate. So the first recording on a device is +/// budgeted from a deliberately pessimistic estimate, and every recording +/// after that is budgeted from what this device actually wrote. +abstract final class VideoRecordingBudget { + /// Mirrors `maximal_upload_size_of_single_media_size` of the server's plans. + /// Every paid plan shares the same per-file limit. + static const int freePlanUploadLimitBytes = 50000000; + static const int paidPlanUploadLimitBytes = 100000000; + + /// Bits the encoder is assumed to spend per pixel per frame before this + /// device has recorded anything. Devices measured so far sit well below + /// this; guessing high only costs a shorter first recording, while guessing + /// low costs a recording the server refuses once it is already made. + static const double _assumedBitsPerPixel = 0.25; + + /// Neither platform reports the recording frame rate, and both record at + /// 30 fps unless the scene is too dark for it. + static const double _assumedFrameRate = 30; + + /// AAC alongside the video, when the microphone is available. + static const double _audioBitsPerSecond = 128000; + + /// Falls back to 720p when the camera has not reported its preview size. + static const Size _assumedRecordingSize = Size(1280, 720); + + /// Leaves the container overhead and a scene busier than the one that was + /// measured somewhere to go. + static const double _headroom = 0.9; + + /// A budget shorter than this makes the camera useless, and one longer than + /// this makes an unwieldy file however well it fits the plan. + static const Duration _shortestBudget = Duration(seconds: 10); + static const Duration _longestBudget = Duration(minutes: 5); + + static const String _storeKey = 'video_recording_bitrate'; + static const String _samplesField = 'bytesPerSecondByResolution'; + + /// Measured write rates in bytes per second, keyed by recording resolution + /// so that a different camera or preset does not inherit a rate that was + /// never measured for it. + static Map _measured = {}; + static bool _loaded = false; + + static int uploadLimitBytesFor(SubscriptionPlan plan) => + plan == SubscriptionPlan.Free + ? freePlanUploadLimitBytes + : paidPlanUploadLimitBytes; + + /// Reads back what earlier recordings measured. Cheap to call repeatedly; + /// the camera view calls it while it is starting up, long before the first + /// recording can be started. + static Future ensureLoaded() async { + if (_loaded) return; + _loaded = true; + try { + final stored = await KeyValueStore.get(_storeKey); + final samples = stored?[_samplesField]; + if (samples is Map) { + _measured = { + for (final entry in samples.entries) + if (entry.value is num) + entry.key.toString(): (entry.value as num).toDouble(), + }; + } + } catch (e) { + Log.warn('Could not read the measured video bitrate: $e'); + } + } + + @visibleForTesting + static void resetForTesting() { + _measured = {}; + _loaded = false; + } + + /// How long the camera may record for [plan] before the file would no longer + /// be uploadable. + static Duration maxRecordingTime({ + required SubscriptionPlan plan, + required Size? recordingSize, + required bool hasAudio, + }) { + final bytesPerSecond = _bytesPerSecond( + recordingSize: recordingSize, + hasAudio: hasAudio, + ); + final seconds = (uploadLimitBytesFor(plan) * _headroom) / bytesPerSecond; + final budget = Duration(milliseconds: (seconds * 1000).round()); + if (budget < _shortestBudget) return _shortestBudget; + if (budget > _longestBudget) return _longestBudget; + return budget; + } + + /// Feeds back what the encoder really wrote, so the next recording on this + /// device is budgeted from a measurement instead of the estimate. + /// + /// A rate above the one on record is adopted at once - it is proof the + /// budget in use was too generous - while a lower one is eased in, because a + /// single still scene encodes far smaller than the device's usual output and + /// should not talk the budget up. + static Future recordMeasurement({ + required int fileSizeInBytes, + required Duration duration, + required Size? recordingSize, + }) async { + // Too short to divide by: the fixed cost of the container and the first + // key frame would swamp the rate. + if (duration.inMilliseconds < 2000 || fileSizeInBytes <= 0) return; + + await ensureLoaded(); + + final key = _resolutionKey(recordingSize); + final observed = fileSizeInBytes / (duration.inMilliseconds / 1000); + final previous = _measured[key]; + final updated = (previous == null || observed > previous) + ? observed + : previous * 0.8 + observed * 0.2; + + _measured[key] = updated; + try { + await KeyValueStore.put(_storeKey, {_samplesField: _measured}); + } catch (e) { + Log.warn('Could not store the measured video bitrate: $e'); + } + } + + static double _bytesPerSecond({ + required Size? recordingSize, + required bool hasAudio, + }) { + final measured = _measured[_resolutionKey(recordingSize)]; + if (measured != null && measured > 0) return measured; + + final size = _sizeOrDefault(recordingSize); + final videoBitsPerSecond = + _assumedBitsPerPixel * size.width * size.height * _assumedFrameRate; + final bitsPerSecond = + videoBitsPerSecond + (hasAudio ? _audioBitsPerSecond : 0); + return bitsPerSecond / 8; + } + + static Size _sizeOrDefault(Size? recordingSize) { + if (recordingSize == null || + recordingSize.width <= 0 || + recordingSize.height <= 0) { + return _assumedRecordingSize; + } + return recordingSize; + } + + /// Orientation is not part of the key: the camera reports the preview size + /// rotated on one platform and not on the other, and only the pixel count + /// matters here. + static String _resolutionKey(Size? recordingSize) { + final size = _sizeOrDefault(recordingSize); + final shorter = size.shortestSide.round(); + final longer = size.longestSide.round(); + return '${longer}x$shorter'; + } +} diff --git a/lib/src/visual/views/camera/camera_preview_components/video_recording_time.dart b/lib/src/visual/views/camera/camera_preview_components/video_recording_time.dart index 5177ecb9..57589293 100644 --- a/lib/src/visual/views/camera/camera_preview_components/video_recording_time.dart +++ b/lib/src/visual/views/camera/camera_preview_components/video_recording_time.dart @@ -1,66 +1,66 @@ -import 'package:clock/clock.dart'; import 'package:flutter/material.dart'; class VideoRecordingTimer extends StatelessWidget { const VideoRecordingTimer({ required this.videoRecordingStarted, - required this.maxVideoRecordingTime, + required this.currentTime, + required this.maxRecordingTime, super.key, }); final DateTime? videoRecordingStarted; - final int maxVideoRecordingTime; + + /// Read once per tick by the camera view, so the ring and the stop that ends + /// the recording agree on how far along it is. + final DateTime currentTime; + final Duration maxRecordingTime; @override Widget build(BuildContext context) { - if (videoRecordingStarted != null) { - final currentTime = clock.now(); - return Positioned( - top: 50, - left: 0, - right: 0, - child: Center( - child: SizedBox( - width: 50, - height: 50, - child: Stack( - children: [ - Center( - child: CircularProgressIndicator( - value: - currentTime - .difference(videoRecordingStarted!) - .inMilliseconds / - (maxVideoRecordingTime * 1000), - strokeWidth: 4, - valueColor: const AlwaysStoppedAnimation(Colors.red), - backgroundColor: Colors.grey[300], - ), - ), - Center( - child: Text( - currentTime - .difference(videoRecordingStarted!) - .inSeconds - .toString(), - textAlign: TextAlign.center, - style: const TextStyle( - fontSize: 17, - shadows: [ - Shadow( - color: Color.fromARGB(122, 0, 0, 0), - blurRadius: 5, - ), - ], - ), - ), - ), - ], - ), - ), - ), - ); - } else { + if (videoRecordingStarted == null) { return const SizedBox.shrink(); } + final elapsed = currentTime.difference(videoRecordingStarted!); + return Positioned( + top: 50, + left: 0, + right: 0, + child: Center( + child: SizedBox( + width: 50, + height: 50, + child: Stack( + children: [ + Center( + child: CircularProgressIndicator( + value: maxRecordingTime.inMilliseconds == 0 + ? 0 + : (elapsed.inMilliseconds / + maxRecordingTime.inMilliseconds) + .clamp(0.0, 1.0), + strokeWidth: 4, + valueColor: const AlwaysStoppedAnimation(Colors.red), + backgroundColor: Colors.grey[300], + ), + ), + Center( + child: Text( + elapsed.inSeconds.toString(), + textAlign: TextAlign.center, + style: const TextStyle( + fontSize: 17, + shadows: [ + Shadow( + color: Color.fromARGB(122, 0, 0, 0), + blurRadius: 5, + ), + ], + ), + ), + ), + ], + ), + ), + ), + ); } } diff --git a/lib/src/visual/views/camera/share_image_editor.view.dart b/lib/src/visual/views/camera/share_image_editor.view.dart index 18c9e8f2..9d5ef0f8 100644 --- a/lib/src/visual/views/camera/share_image_editor.view.dart +++ b/lib/src/visual/views/camera/share_image_editor.view.dart @@ -1,40 +1,40 @@ -// ignore_for_file: inference_failure_on_function_invocation - import 'dart:async'; import 'dart:collection'; -import 'dart:typed_data'; import 'package:drift/drift.dart' show Value; import 'package:flutter/material.dart'; -import 'package:font_awesome_flutter/font_awesome_flutter.dart'; import 'package:twonly/locator.dart'; -import 'package:twonly/src/database/daos/contacts.dao.dart'; import 'package:twonly/src/database/tables/mediafiles.table.dart'; import 'package:twonly/src/database/twonly.db.dart'; import 'package:twonly/src/model/protobuf/client/generated/data.pb.dart'; import 'package:twonly/src/services/mediafiles/mediafile.service.dart'; -import 'package:twonly/src/services/user.service.dart'; import 'package:twonly/src/utils/log.dart'; -import 'package:twonly/src/utils/misc.dart'; -import 'package:twonly/src/visual/components/emoji_picker.bottom.dart'; -import 'package:twonly/src/visual/components/notification_badge.comp.dart'; -import 'package:twonly/src/visual/elements/my_button.element.dart'; import 'package:twonly/src/visual/helpers/media_view_sizing.helper.dart'; import 'package:twonly/src/visual/helpers/screenshot.helper.dart'; import 'package:twonly/src/visual/views/camera/camera_preview_components/main_camera_controller.dart'; -import 'package:twonly/src/visual/views/camera/camera_preview_components/save_to_gallery.dart'; import 'package:twonly/src/visual/views/camera/share_image_contact_selection.view.dart'; -import 'package:twonly/src/visual/views/camera/share_image_contact_selection_components/select_show_time.dart'; -import 'package:twonly/src/visual/views/camera/share_image_editor_components/action_button.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/discard_media_dialog.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/display_time_picker.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/editor_bottom_bar.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/editor_canvas.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/editor_layer_stack.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/editor_media_writer.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/editor_side_toolbar.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/editor_top_toolbar.dart'; import 'package:twonly/src/visual/views/camera/share_image_editor_components/image_item.dart'; -import 'package:twonly/src/visual/views/camera/share_image_editor_components/layer_data.dart'; -import 'package:twonly/src/visual/views/camera/share_image_editor_components/layers_viewer.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/video_trimmer.dart'; import 'package:video_player/video_player.dart'; -List layers = []; -List undoLayers = []; -List removedLayers = []; - +/// Lets the user edit a just taken (or shared) photo/video/gif and send it. +/// +/// The screen is built out of four parts: +/// * [EditorCanvas] shows the media with all edits stacked on top of it +/// * [EditorTopToolbar] holds close/undo/redo +/// * [EditorSideToolbar] holds the editing tools +/// * [EditorBottomBar] holds save to gallery and send +/// +/// The edits themselves live in an [EditorLayerStack], writing them to disk is +/// done by an [EditorMediaWriter]. class ShareImageEditorView extends StatefulWidget { const ShareImageEditorView({ required this.sharedFromGallery, @@ -56,15 +56,29 @@ class ShareImageEditorView extends StatefulWidget { } class _ShareImageEditorView extends State { + final layerStack = EditorLayerStack(); + late final EditorMediaWriter mediaWriter; + double tabDownPosition = 0; bool sendingOrLoadingImage = true; bool loadingImage = true; bool isDisposed = false; HashSet selectedGroupIds = HashSet(); - double widthRatio = 1; - double heightRatio = 1; double pixelRatio = 1; VideoPlayerController? videoController; + + /// The cut the trimmer is showing. Held here as well as in the media row so + /// dragging a handle repaints immediately instead of waiting on a write, and + /// so the send path and the preview always agree on the same bounds. + Duration _trimStart = Duration.zero; + + /// Null until the trimmer is dragged or a stored cut is loaded; it then means + /// "the clip ends here" while null keeps the recording's own end. + Duration? _trimEnd; + + /// The cutter lies over the video, so it can be put away to see the frame + /// underneath it. Open to begin with, otherwise nothing says it is there. + bool _trimmerVisible = true; ImageItem currentImage = ImageItem(); ScreenshotController screenshotController = ScreenshotController(); Timer? _imageLoadingTimer; @@ -76,63 +90,42 @@ class _ShareImageEditorView extends State { void initState() { super.initState(); + mediaWriter = EditorMediaWriter( + mediaService: mediaService, + layerStack: layerStack, + screenshotController: screenshotController, + gifSource: widget.screenshotImage, + requestRebuild: () { + if (mounted) setState(() {}); + }, + ); + if (media.type != MediaType.gif) { - layers.add(FilterLayerData(key: GlobalKey())); + layerStack.addFilterLayer(); } - if (widget.previewLink != null && - widget.previewLink!.shouldGeneratePreview) { - layers.add( - LinkPreviewLayerData(key: GlobalKey(), link: widget.previewLink!.url), - ); + final previewLink = widget.previewLink; + if (previewLink != null && previewLink.shouldGeneratePreview) { + layerStack.addLinkPreviewLayer(previewLink.url); } if (widget.sendToGroup != null) { selectedGroupIds.add(widget.sendToGroup!.groupId); } - if (widget.mediaFileService.mediaFile.type == MediaType.image || - widget.mediaFileService.mediaFile.type == MediaType.gif) { - if (widget.screenshotImage != null) { - loadImage(widget.screenshotImage!); - } else { - if (widget.mediaFileService.tempPath.existsSync()) { - loadImage( - ScreenshotImageHelper(file: widget.mediaFileService.tempPath), - ); - } else if (widget.mediaFileService.originalPath.existsSync()) { - loadImage( - ScreenshotImageHelper(file: widget.mediaFileService.originalPath), - ); - } - } + if (media.type == MediaType.image || media.type == MediaType.gif) { + _loadInitialImage(); } if (media.type == MediaType.video) { - setState(() { - sendingOrLoadingImage = false; - loadingImage = false; - }); - videoController = VideoPlayerController.file( - mediaService.originalPath, - videoPlayerOptions: VideoPlayerOptions(), - ); - videoController?.setLooping(true); - videoController - ?.initialize() - .then((_) async { - await videoController!.play(); - setState(() {}); - }) - // ignore: argument_type_not_assignable_to_error_handler - .catchError(Log.error); + _initVideoController(); } } @override void dispose() { isDisposed = true; - layers.clear(); + layerStack.clear(); videoController?.dispose(); twonlyDB.mediaFilesDao.updateAllMediaFiles( const MediaFilesCompanion( @@ -143,424 +136,74 @@ class _ShareImageEditorView extends State { super.dispose(); } - void updateSelectedGroupIds(String groupId, bool checked) { - if (checked) { - if (media.requiresAuthentication) { - selectedGroupIds.clear(); - } - selectedGroupIds.add(groupId); - } else { - selectedGroupIds.remove(groupId); - } - setState(() {}); - } + // --------------------------------------------------------------------------- + // loading the media + // --------------------------------------------------------------------------- - Future _setMaxShowTime(int? maxShowTime, bool storeAsDefault) async { - await mediaService.setDisplayLimit(maxShowTime); - if (!mounted) return; - setState(() {}); - if (storeAsDefault) { - await UserService.update((user) { - user.defaultShowTime = maxShowTime; - }); - } - } - - Future _setImageDisplayTime() async { - if (media.type == MediaType.video) { - await mediaService.setDisplayLimit( - (media.displayLimitInMilliseconds == null) ? 0 : null, - ); - if (!mounted) return; - setState(() {}); + void _loadInitialImage() { + if (widget.screenshotImage != null) { + loadImage(widget.screenshotImage!); return; } - - final options = [ - 1000, - 2000, - 3000, - 4000, - 5000, - 6000, - 7000, - 8000, - 9000, - 10000, - 15000, - 20000, - null, - ]; - - var initialItem = options.length - 1; - if (media.displayLimitInMilliseconds != null) { - initialItem = options.indexOf(media.displayLimitInMilliseconds); - if (initialItem == -1) { - initialItem = options.length - 1; - } - } - await showModalBottomSheet( - context: context, - backgroundColor: Colors.black, - builder: (context) { - return SelectShowTime( - initialItem: initialItem, - setMaxShowTime: _setMaxShowTime, - options: options, - ); - }, - ); - } - - List get actionsAtTheRight { - if (layers.isNotEmpty && - (layers.first.isEditing || - (layers.last.isEditing && layers.last.hasCustomActionButtons))) { - return []; - } - return [ - if (media.type != MediaType.gif) - ActionButton( - Icons.text_fields_rounded, - tooltipText: context.lang.addTextItem, - onPressed: () async { - layers = layers.where((x) => !x.isDeleted).toList(); - if (layers.any((x) => x.isEditing)) return; - undoLayers.clear(); - removedLayers.clear(); - layers.add( - TextLayerData( - key: GlobalKey(), - textLayersBefore: layers.whereType().length, - ), - ); - setState(() {}); - }, - ), - const SizedBox(height: 8), - if (media.type != MediaType.gif) - ActionButton( - Icons.draw_rounded, - tooltipText: context.lang.addDrawing, - onPressed: () async { - undoLayers.clear(); - removedLayers.clear(); - layers.add(DrawLayerData(key: GlobalKey())); - setState(() {}); - }, - ), - const SizedBox(height: 8), - if (media.type != MediaType.gif) - ActionButton( - Icons.add_reaction_outlined, - tooltipText: context.lang.addEmoji, - onPressed: () async { - final layer = - await showModalBottomSheet( - context: context, - backgroundColor: Colors.black, - builder: (context) { - return const EmojiPickerBottom(); - }, - ) - as Layer?; - if (layer == null) return; - undoLayers.clear(); - removedLayers.clear(); - layers.add(layer); - setState(() {}); - }, - ), - const SizedBox(height: 8), - NotificationBadgeComp( - count: (media.type == MediaType.video) - ? '0' - : media.displayLimitInMilliseconds == null - ? '∞' - : (media.displayLimitInMilliseconds! ~/ 1000).toString(), - child: ActionButton( - (media.type == MediaType.video) - ? media.displayLimitInMilliseconds == null - ? Icons.repeat_rounded - : Icons.repeat_one_rounded - : Icons.timer_outlined, - tooltipText: context.lang.protectAsARealTwonly, - onPressed: _setImageDisplayTime, - ), - ), - if (media.type == MediaType.video) ...[ - const SizedBox(height: 8), - ActionButton( - (mediaService.removeAudio) - ? Icons.volume_off_rounded - : Icons.volume_up_rounded, - tooltipText: 'Enable Audio in Video', - color: (mediaService.removeAudio) - ? Colors.white.withAlpha(160) - : Colors.white, - onPressed: () async { - await mediaService.toggleRemoveAudio(); - if (mediaService.removeAudio) { - await videoController?.setVolume(0); - } else { - await videoController?.setVolume(100); - } - if (mounted) setState(() {}); - }, - ), - ], - if (media.type == MediaType.image) ...[ - const SizedBox(height: 8), - ActionButton( - Icons.crop_rotate_outlined, - tooltipText: 'Crop or rotate image', - color: Colors.white, - onPressed: () async { - final first = layers.first; - if (first is BackgroundLayerData) { - first.isEditing = !first.isEditing; - } - setState(() {}); - // await mediaService.toggleRemoveAudio(); - // if (mediaService.removeAudio) { - // await videoController?.setVolume(0); - // } else { - // await videoController?.setVolume(100); - // } - // if (mounted) setState(() {}); - }, - ), - ], - const SizedBox(height: 8), - ActionButton( - FontAwesomeIcons.shieldHeart, - tooltipText: context.lang.protectAsARealTwonly, - color: media.requiresAuthentication - ? Theme.of(context).colorScheme.primary - : Colors.white, - onPressed: () async { - await mediaService.setRequiresAuth(!media.requiresAuthentication); - selectedGroupIds = HashSet(); - setState(() {}); - }, - ), - ]; - } - - Future _showBackDialog() { - return showDialog( - context: context, - builder: (context) { - return AlertDialog( - title: Text( - context.lang.dialogAskDeleteMediaFilePopTitle, - ), - actions: [ - MyButton( - variant: MyButtonVariant.primaryMiddle, - onPressed: () { - Navigator.pop(context, true); - }, - child: Text(context.lang.dialogAskDeleteMediaFilePopDelete), - ), - TextButton( - child: Text(context.lang.cancel), - onPressed: () { - Navigator.pop(context, false); - }, - ), - ], - ); - }, - ); - } - - Future askToCloseThenClose() async { - final shouldPop = await _showBackDialog() ?? false; - if (mounted && shouldPop) { - Navigator.pop(context); - } - } - - List get actionsAtTheTop { - if (layers.isNotEmpty && - (layers.first.isEditing || - (layers.last.isEditing && layers.last.hasCustomActionButtons))) { - return []; - } - return [ - ActionButton( - FontAwesomeIcons.xmark, - tooltipText: context.lang.close, - onPressed: () async { - final nonImageFilterLayer = layers.where( - (x) => x is! BackgroundLayerData && x is! FilterLayerData, - ); - if (nonImageFilterLayer.isEmpty) { - Navigator.pop(context, false); - } else { - await askToCloseThenClose(); - } - }, - ), - Expanded(child: Container()), - const SizedBox(width: 8), - ActionButton( - FontAwesomeIcons.rotateLeft, - tooltipText: context.lang.undo, - disable: layers.where((x) => !x.isDeleted).length <= 2, - onPressed: () { - if (removedLayers.isNotEmpty) { - final lastLayer = removedLayers.removeLast() - ..isDeleted = false - ..isEditing = false; - layers.add(lastLayer); - setState(() {}); - return; - } - layers = layers.where((x) => !x.isDeleted).toList(); - if (layers.length <= 2) { - // do not remove image layer and filter layer - return; - } - undoLayers.add(layers.removeLast()); - setState(() {}); - }, - ), - const SizedBox(width: 8), - ActionButton( - FontAwesomeIcons.rotateRight, - tooltipText: context.lang.redo, - disable: undoLayers.isEmpty, - onPressed: () { - if (undoLayers.isEmpty) return; - layers.add(undoLayers.removeLast()); - setState(() {}); - }, - ), - const SizedBox(width: 70), - ]; - } - - Future pushShareImageView() async { - final mediaStoreFuture = storeImageAsOriginal(); - - await videoController?.pause(); - if (isDisposed || !mounted) return; - final wasSend = - await Navigator.push( - context, - MaterialPageRoute( - builder: (context) => ShareImageView( - selectedGroupIds: selectedGroupIds, - updateSelectedGroupIds: updateSelectedGroupIds, - mediaStoreFuture: mediaStoreFuture, - mediaFileService: mediaService, - additionalData: getAdditionalData(), - ), - ), - ) - as bool?; - if (wasSend != null && wasSend && mounted) { - widget.mainCameraController?.onImageSend(); - Navigator.pop(context, true); - } else { - await videoController?.play(); - } - } - - Future getEditedImageBytes() async { - if (layers.length == 1) { - if (layers.first is BackgroundLayerData) { - return (layers.first as BackgroundLayerData).image.image; - } - } - if (layers.length == 2) { - final filterLayer = layers[1]; - if (layers.first is BackgroundLayerData && - filterLayer is FilterLayerData) { - if (filterLayer.page == 1) { - return (layers.first as BackgroundLayerData).image.image; - } - } - } - - for (final x in layers) { - x.showCustomButtons = false; - } - setState(() {}); - - // Make a short delay, so the setState does have its effect... - await Future.delayed(const Duration(milliseconds: 80)); - - final image = await screenshotController.capture( - pixelRatio: pixelRatio, - ); - if (image == null) { - Log.warn('screenshotController did not return image bytes'); - return null; - } - - for (final x in layers) { - x.showCustomButtons = true; - } - if (mounted) { - setState(() {}); - } - return image; - } - - Future storeImageAsOriginal() async { - Uint8List? gifBytes; - ScreenshotImageHelper? image; - if (media.type == MediaType.gif) { - gifBytes = await widget.screenshotImage?.getBytes(); - } else { - image = await getEditedImageBytes(); - if (image != null) { - await image.getBytes(); - } - } - - if (mediaService.overlayImagePath.existsSync()) { - mediaService.overlayImagePath.deleteSync(); - } if (mediaService.tempPath.existsSync()) { - mediaService.tempPath.deleteSync(); + loadImage(ScreenshotImageHelper(file: mediaService.tempPath)); + } else if (mediaService.originalPath.existsSync()) { + loadImage(ScreenshotImageHelper(file: mediaService.originalPath)); } - if (mediaService.originalPath.existsSync()) { - if (media.type == MediaType.image) { - mediaService.originalPath.deleteSync(); - } - } - - if (media.type == MediaType.gif) { - if (gifBytes != null) { - mediaService.originalPath.writeAsBytesSync(gifBytes.toList()); - } - } else { - if (image == null) return null; - final bytes = await image.getBytes(); - if (bytes == null) { - Log.warn('imageBytes are empty'); - return null; - } - if (media.type == MediaType.image || media.type == MediaType.gif) { - mediaService.originalPath.writeAsBytesSync(bytes); - } else if (media.type == MediaType.video) { - mediaService.overlayImagePath.writeAsBytesSync(bytes); - } else { - Log.error('MediaType not supported: ${media.type}'); - } - } - return image; } - Future storeIoImageAsDraft( - ScreenshotImageHelper screenshotImage, - ) async { - final imageBytes = await screenshotImage.getBytes(); - mediaService.originalPath.writeAsBytesSync(imageBytes!.toList()); + void _initVideoController() { + setState(() { + sendingOrLoadingImage = false; + loadingImage = false; + }); + videoController = VideoPlayerController.file( + mediaService.originalPath, + videoPlayerOptions: VideoPlayerOptions(), + ); + videoController?.setLooping(true); + videoController + ?.initialize() + .then((_) async { + _loadStoredTrim(); + if (_trimStart > Duration.zero) { + await videoController!.seekTo(_trimStart); + } + await videoController!.play(); + setState(() {}); + }) + // ignore: argument_type_not_assignable_to_error_handler + .catchError(Log.error); + } + + /// Restores a cut made before the editor was closed and reopened on the same + /// draft. Bounds that no longer fit the recording are dropped rather than + /// clamped, because a mismatch means they belong to a different clip. + void _loadStoredTrim() { + final duration = videoController?.value.duration ?? Duration.zero; + final start = mediaService.trimStart; + final end = mediaService.trimEnd; + if (start != null && start > Duration.zero && start < duration) { + _trimStart = start; + } + if (end != null && end > _trimStart && end < duration) { + _trimEnd = end; + } + } + + /// Where the clip ends, with "not cut" resolved to the end of the recording. + Duration get _effectiveTrimEnd => + _trimEnd ?? videoController?.value.duration ?? Duration.zero; + + /// Stores the cut once the finger lifts. An end on the last frame is stored + /// as "not cut" so a clip nobody shortened never carries a bound that a + /// re-encode could round past its own duration. + Future _persistTrim(Duration start, Duration end) async { + final duration = videoController?.value.duration ?? Duration.zero; + await mediaService.setTrim( + start > Duration.zero ? start : null, + end < duration ? end : null, + ); } Future loadImage(ScreenshotImageHelper screenshotImage) async { @@ -568,10 +211,10 @@ class _ShareImageEditorView extends State { screenshotImage.imageBytes == null && screenshotImage.imageBytesFuture != null) { // this ensures that the imageBytes are defined - await storeIoImageAsDraft(screenshotImage); + await mediaWriter.storeAsDraft(screenshotImage); } else { // store this image so it can be used as a draft in case the app is restarted - unawaited(storeIoImageAsDraft(screenshotImage)); + unawaited(mediaWriter.storeAsDraft(screenshotImage)); } if (screenshotImage.image == null) { @@ -598,46 +241,126 @@ class _ShareImageEditorView extends State { }); setState(() { - layers.insert( - 0, - BackgroundLayerData( - key: GlobalKey(), - image: currentImage, - ), - ); + layerStack.insertBackgroundLayer(currentImage); }); - // It is important that the user can sending the image only when the image is fully loaded otherwise if the user - // will click on send before the image is painted the screenshot will be transparent.. + + _waitUntilBackgroundIsPainted(); + } + + /// The user may only send the image once it is fully painted, otherwise the + /// screenshot taken for the export would be transparent. + void _waitUntilBackgroundIsPainted() { _imageLoadingTimer = Timer.periodic(const Duration(milliseconds: 10), ( timer, ) { - final imageLayer = layers.first; - if (imageLayer is BackgroundLayerData) { - if (imageLayer.imageLoaded) { - timer.cancel(); - Future.delayed(const Duration(milliseconds: 50), () { - if (context.mounted) { - setState(() { - sendingOrLoadingImage = false; - loadingImage = false; - }); - } + if (!layerStack.isBackgroundLoaded) return; + timer.cancel(); + Future.delayed(const Duration(milliseconds: 50), () { + if (context.mounted) { + setState(() { + sendingOrLoadingImage = false; + loadingImage = false; }); } - } + }); }); } - AdditionalMessageData? getAdditionalData() { - AdditionalMessageData? additionalData; + // --------------------------------------------------------------------------- + // toolbar actions + // --------------------------------------------------------------------------- - if (widget.previewLink != null) { - additionalData = AdditionalMessageData( - type: AdditionalMessageData_Type.LINK, - link: widget.previewLink!.url.toString(), - ); + Future _toggleAudio() async { + await mediaService.toggleRemoveAudio(); + if (mediaService.removeAudio) { + await videoController?.setVolume(0); + } else { + await videoController?.setVolume(100); + } + if (mounted) setState(() {}); + } + + Future _toggleRequiresAuth() async { + await mediaService.setRequiresAuth(!media.requiresAuthentication); + selectedGroupIds = HashSet(); + if (mounted) setState(() {}); + } + + Future _editDisplayTime() async { + await showDisplayTimePicker( + context, + mediaService: mediaService, + onChanged: () { + if (mounted) setState(() {}); + }, + ); + } + + Future _onClosePressed() async { + if (!layerStack.hasUserAddedLayers) { + Navigator.pop(context, false); + return; + } + await askToCloseThenClose(); + } + + Future askToCloseThenClose() async { + final shouldPop = await askToDiscardMedia(context); + if (mounted && shouldPop) { + Navigator.pop(context); + } + } + + // --------------------------------------------------------------------------- + // sending + // --------------------------------------------------------------------------- + + void updateSelectedGroupIds(String groupId, bool checked) { + if (checked) { + if (media.requiresAuthentication) { + selectedGroupIds.clear(); + } + selectedGroupIds.add(groupId); + } else { + selectedGroupIds.remove(groupId); + } + setState(() {}); + } + + Future storeImageAsOriginal() => + mediaWriter.storeImageAsOriginal(pixelRatio); + + AdditionalMessageData? getAdditionalData() { + if (widget.previewLink == null) return null; + return AdditionalMessageData( + type: AdditionalMessageData_Type.LINK, + link: widget.previewLink!.url.toString(), + ); + } + + Future pushShareImageView() async { + final mediaStoreFuture = storeImageAsOriginal(); + + await videoController?.pause(); + if (isDisposed || !mounted) return; + final wasSend = await Navigator.push( + context, + MaterialPageRoute( + builder: (context) => ShareImageView( + selectedGroupIds: selectedGroupIds, + updateSelectedGroupIds: updateSelectedGroupIds, + mediaStoreFuture: mediaStoreFuture, + mediaFileService: mediaService, + additionalData: getAdditionalData(), + ), + ), + ); + if (wasSend != null && wasSend && mounted) { + widget.mainCameraController?.onImageSend(); + Navigator.pop(context, true); + } else { + await videoController?.play(); } - return additionalData; } Future sendImageToSinglePerson() async { @@ -665,23 +388,22 @@ class _ShareImageEditorView extends State { } } - Widget _buildScreenshotViewer() { - return Screenshot( - controller: screenshotController, - child: LayersViewer( - layers: layers.where((x) => !x.isDeleted).toList(), - onUpdate: () { - for (final layer in layers) { - layer.isEditing = false; - if (layer.isDeleted) { - removedLayers.add(layer); - } - } - layers = layers.where((x) => !x.isDeleted).toList(); - setState(() {}); - }, - ), - ); + // --------------------------------------------------------------------------- + // layout + // --------------------------------------------------------------------------- + + /// The cutter only makes sense once the player knows how long the recording + /// is, and only for a video that is still being edited. + bool get _canTrim => + media.type == MediaType.video && + videoController != null && + videoController!.value.isInitialized && + videoController!.value.duration > Duration.zero; + + /// Tapping anywhere on the media adds a text layer at that position. + void _onCanvasTap() { + layerStack.addTextLayer(offset: Offset(0, tabDownPosition)); + setState(() {}); } @override @@ -710,120 +432,44 @@ class _ShareImageEditorView extends State { tabDownPosition = details.globalPosition.dy; } }, - onTap: () { - if (layers.any((x) => x.isEditing)) { - return; - } - layers = layers.where((x) => !x.isDeleted).toList(); - undoLayers.clear(); - removedLayers.clear(); - layers.add( - TextLayerData( - key: GlobalKey(), - offset: Offset(0, tabDownPosition), - textLayersBefore: layers.whereType().length, - ), - ); - setState(() {}); - }, + onTap: _onCanvasTap, child: MediaViewSizingHelper( requiredHeight: 59, - bottomNavigation: ColoredBox( - color: Theme.of(context).colorScheme.surface, - child: Row( - mainAxisAlignment: MainAxisAlignment.center, - children: [ - SaveToGalleryButton( - storeImageAsOriginal: storeImageAsOriginal, - mediaService: mediaService, - displayButtonLabel: widget.sendToGroup == null, - isLoading: loadingImage, - ), - if (widget.sendToGroup != null) const SizedBox(width: 10), - if (widget.sendToGroup != null) - MyButton( - variant: MyButtonVariant.secondaryMiddle, - onPressed: pushShareImageView, - child: const FaIcon( - FontAwesomeIcons.userPlus, - size: 14, - ), - ), - SizedBox(width: widget.sendToGroup == null ? 20 : 10), - IntrinsicWidth( - child: MyButton( - variant: MyButtonVariant.primaryMiddle, - onPressed: sendingOrLoadingImage - ? null - : () async { - if (widget.sendToGroup == null) { - return pushShareImageView(); - } - await sendImageToSinglePerson(); - }, - child: Row( - mainAxisSize: MainAxisSize.min, - children: [ - if (sendingOrLoadingImage) - const SizedBox( - height: 12, - width: 12, - child: CircularProgressIndicator.adaptive( - strokeWidth: 2, - valueColor: AlwaysStoppedAnimation( - Colors.black87, - ), - ), - ) - else - const FaIcon( - FontAwesomeIcons.solidPaperPlane, - size: 14, - ), - const SizedBox(width: 8), - Text( - (widget.sendToGroup == null) - ? context.lang.shareImagedEditorShareWith - : substringBy( - widget.sendToGroup!.groupName, - 15, - ), - ), - ], - ), - ), - ), - ], - ), + bottomNavigation: EditorBottomBar( + mediaService: mediaService, + sendToGroup: widget.sendToGroup, + isLoadingImage: loadingImage, + isSending: sendingOrLoadingImage, + storeImageAsOriginal: storeImageAsOriginal, + onAddMoreRecipients: pushShareImageView, + onSend: () async { + if (widget.sendToGroup == null) { + return pushShareImageView(); + } + await sendImageToSinglePerson(); + }, ), - child: SizedBox( - height: currentImage.height / pixelRatio, - width: currentImage.width / pixelRatio, - child: Stack( - children: [ - if (videoController != null && - videoController!.value.isInitialized) - Positioned.fill( - child: Center( - child: AspectRatio( - aspectRatio: videoController!.value.aspectRatio, - child: Stack( - children: [ - Positioned.fill( - child: VideoPlayer(videoController!), - ), - Positioned.fill( - child: _buildScreenshotViewer(), - ), - ], - ), - ), - ), + child: EditorCanvas( + layerStack: layerStack, + screenshotController: screenshotController, + image: currentImage, + pixelRatio: pixelRatio, + videoController: videoController, + onLayersUpdated: () => setState(() {}), + bottomOverlay: (_canTrim && _trimmerVisible) + ? VideoTrimmer( + controller: videoController!, + start: _trimStart, + end: _effectiveTrimEnd, + onChanged: (start, end) { + setState(() { + _trimStart = start; + _trimEnd = end; + }); + }, + onChangeEnd: _persistTrim, ) - else - _buildScreenshotViewer(), - ], - ), + : null, ), ), ), @@ -832,8 +478,10 @@ class _ShareImageEditorView extends State { left: 5, right: 0, child: SafeArea( - child: Row( - children: actionsAtTheTop, + child: EditorTopToolbar( + layerStack: layerStack, + onClose: _onClosePressed, + onChanged: () => setState(() {}), ), ), ), @@ -844,9 +492,17 @@ class _ShareImageEditorView extends State { alignment: Alignment.bottomCenter, padding: const EdgeInsets.symmetric(vertical: 16), child: SafeArea( - child: Column( - mainAxisAlignment: MainAxisAlignment.center, - children: actionsAtTheRight, + child: EditorSideToolbar( + layerStack: layerStack, + mediaService: mediaService, + onChanged: () => setState(() {}), + onEditDisplayTime: _editDisplayTime, + onToggleAudio: _toggleAudio, + onToggleRequiresAuth: _toggleRequiresAuth, + canTrim: _canTrim, + trimmerVisible: _trimmerVisible, + onToggleTrimmer: () => + setState(() => _trimmerVisible = !_trimmerVisible), ), ), ), diff --git a/lib/src/visual/views/camera/share_image_editor_components/discard_media_dialog.dart b/lib/src/visual/views/camera/share_image_editor_components/discard_media_dialog.dart new file mode 100644 index 00000000..875f9ded --- /dev/null +++ b/lib/src/visual/views/camera/share_image_editor_components/discard_media_dialog.dart @@ -0,0 +1,31 @@ +import 'package:flutter/material.dart'; +import 'package:twonly/src/utils/misc.dart'; +import 'package:twonly/src/visual/elements/my_button.element.dart'; + +/// Asks the user whether the media and all its edits should be thrown away. +Future askToDiscardMedia(BuildContext context) async { + final shouldDiscard = await showDialog( + context: context, + builder: (context) { + return AlertDialog( + title: Text(context.lang.dialogAskDeleteMediaFilePopTitle), + actions: [ + MyButton( + variant: MyButtonVariant.primaryMiddle, + onPressed: () { + Navigator.pop(context, true); + }, + child: Text(context.lang.dialogAskDeleteMediaFilePopDelete), + ), + TextButton( + child: Text(context.lang.cancel), + onPressed: () { + Navigator.pop(context, false); + }, + ), + ], + ); + }, + ); + return shouldDiscard ?? false; +} diff --git a/lib/src/visual/views/camera/share_image_editor_components/display_time_picker.dart b/lib/src/visual/views/camera/share_image_editor_components/display_time_picker.dart new file mode 100644 index 00000000..7d9211ee --- /dev/null +++ b/lib/src/visual/views/camera/share_image_editor_components/display_time_picker.dart @@ -0,0 +1,74 @@ +import 'package:flutter/material.dart'; +import 'package:twonly/src/database/tables/mediafiles.table.dart'; +import 'package:twonly/src/services/mediafiles/mediafile.service.dart'; +import 'package:twonly/src/services/user.service.dart'; +import 'package:twonly/src/visual/views/camera/share_image_contact_selection_components/select_show_time.dart'; + +const _displayTimeOptionsInMs = [ + 1000, + 2000, + 3000, + 4000, + 5000, + 6000, + 7000, + 8000, + 9000, + 10000, + 15000, + 20000, + null, // unlimited +]; + +/// Lets the user choose how long the receiver may look at the media. +/// +/// Videos have no picker, for them this only toggles between playing once and +/// looping. [onChanged] is called whenever the limit was updated, the picker +/// stays open while the user scrolls through the options. +Future showDisplayTimePicker( + BuildContext context, { + required MediaFileService mediaService, + required VoidCallback onChanged, +}) async { + final media = mediaService.mediaFile; + + if (media.type == MediaType.video) { + await mediaService.setDisplayLimit( + (media.displayLimitInMilliseconds == null) ? 0 : null, + ); + if (!context.mounted) return; + onChanged(); + return; + } + + var initialItem = _displayTimeOptionsInMs.length - 1; + if (media.displayLimitInMilliseconds != null) { + initialItem = _displayTimeOptionsInMs.indexOf( + media.displayLimitInMilliseconds, + ); + if (initialItem == -1) { + initialItem = _displayTimeOptionsInMs.length - 1; + } + } + + await showModalBottomSheet( + context: context, + backgroundColor: Colors.black, + builder: (sheetContext) { + return SelectShowTime( + initialItem: initialItem, + options: _displayTimeOptionsInMs, + setMaxShowTime: (maxShowTime, storeAsDefault) async { + await mediaService.setDisplayLimit(maxShowTime); + if (!context.mounted) return; + onChanged(); + if (storeAsDefault) { + await UserService.update((user) { + user.defaultShowTime = maxShowTime; + }); + } + }, + ); + }, + ); +} diff --git a/lib/src/visual/views/camera/share_image_editor_components/editor_bottom_bar.dart b/lib/src/visual/views/camera/share_image_editor_components/editor_bottom_bar.dart new file mode 100644 index 00000000..f605ce2c --- /dev/null +++ b/lib/src/visual/views/camera/share_image_editor_components/editor_bottom_bar.dart @@ -0,0 +1,92 @@ +import 'package:flutter/material.dart'; +import 'package:font_awesome_flutter/font_awesome_flutter.dart'; +import 'package:twonly/src/database/daos/contacts.dao.dart'; +import 'package:twonly/src/database/twonly.db.dart'; +import 'package:twonly/src/services/mediafiles/mediafile.service.dart'; +import 'package:twonly/src/utils/misc.dart'; +import 'package:twonly/src/visual/elements/my_button.element.dart'; +import 'package:twonly/src/visual/helpers/screenshot.helper.dart'; +import 'package:twonly/src/visual/views/camera/camera_preview_components/save_to_gallery.dart'; + +/// Save to gallery and send, shown below the media. +/// +/// When the editor was opened for a single group ([sendToGroup]) the send +/// button sends directly to that group and an additional button opens the +/// contact selection. +class EditorBottomBar extends StatelessWidget { + const EditorBottomBar({ + required this.mediaService, + required this.isLoadingImage, + required this.isSending, + required this.storeImageAsOriginal, + required this.onAddMoreRecipients, + required this.onSend, + this.sendToGroup, + super.key, + }); + + final MediaFileService mediaService; + final Group? sendToGroup; + final bool isLoadingImage; + final bool isSending; + final Future Function() storeImageAsOriginal; + final VoidCallback onAddMoreRecipients; + final VoidCallback onSend; + + @override + Widget build(BuildContext context) { + final hasFixedReceiver = sendToGroup != null; + + return ColoredBox( + color: Theme.of(context).colorScheme.surface, + child: Row( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + SaveToGalleryButton( + storeImageAsOriginal: storeImageAsOriginal, + mediaService: mediaService, + displayButtonLabel: !hasFixedReceiver, + isLoading: isLoadingImage, + ), + if (hasFixedReceiver) ...[ + const SizedBox(width: 10), + MyButton( + variant: MyButtonVariant.secondaryMiddle, + onPressed: onAddMoreRecipients, + child: const FaIcon(FontAwesomeIcons.userPlus, size: 14), + ), + ], + SizedBox(width: hasFixedReceiver ? 10 : 20), + IntrinsicWidth( + child: MyButton( + variant: MyButtonVariant.primaryMiddle, + onPressed: isSending ? null : onSend, + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + if (isSending) + const SizedBox( + height: 12, + width: 12, + child: CircularProgressIndicator.adaptive( + strokeWidth: 2, + valueColor: AlwaysStoppedAnimation(Colors.black87), + ), + ) + else + const FaIcon(FontAwesomeIcons.solidPaperPlane, size: 14), + const SizedBox(width: 8), + Text( + hasFixedReceiver + ? substringBy(sendToGroup!.groupName, 15) + : context.lang.shareImagedEditorShareWith, + ), + ], + ), + ), + ), + ], + ), + ); + } +} diff --git a/lib/src/visual/views/camera/share_image_editor_components/editor_canvas.dart b/lib/src/visual/views/camera/share_image_editor_components/editor_canvas.dart new file mode 100644 index 00000000..5dd6f806 --- /dev/null +++ b/lib/src/visual/views/camera/share_image_editor_components/editor_canvas.dart @@ -0,0 +1,83 @@ +import 'package:flutter/material.dart'; +import 'package:twonly/src/visual/helpers/screenshot.helper.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/editor_layer_stack.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/image_item.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/layers_viewer.dart'; +import 'package:video_player/video_player.dart'; + +/// The media itself with all editor layers stacked on top of it. +/// +/// For videos the layers are rendered above the running video, for images the +/// background image is just another layer. The whole stack is wrapped in a +/// [Screenshot] so it can be exported as a single image. +class EditorCanvas extends StatelessWidget { + const EditorCanvas({ + required this.layerStack, + required this.screenshotController, + required this.image, + required this.pixelRatio, + required this.onLayersUpdated, + this.videoController, + this.bottomOverlay, + super.key, + }); + + final EditorLayerStack layerStack; + final ScreenshotController screenshotController; + final ImageItem image; + final double pixelRatio; + + /// Called after the user finished interacting with a layer. + final VoidCallback onLayersUpdated; + final VideoPlayerController? videoController; + + /// Editor chrome laid over the bottom of the media, outside the [Screenshot] + /// so it never ends up burnt into what gets sent. Used by the video trimmer. + final Widget? bottomOverlay; + + bool get _hasVideo => + videoController != null && videoController!.value.isInitialized; + + Widget _buildLayers() { + return Screenshot( + controller: screenshotController, + child: LayersViewer( + layers: layerStack.visible, + onUpdate: () { + layerStack.commitPendingEdits(); + onLayersUpdated(); + }, + ), + ); + } + + @override + Widget build(BuildContext context) { + return SizedBox( + height: image.height / pixelRatio, + width: image.width / pixelRatio, + child: Stack( + children: [ + if (_hasVideo) + Positioned.fill( + child: Center( + child: AspectRatio( + aspectRatio: videoController!.value.aspectRatio, + child: Stack( + children: [ + Positioned.fill(child: VideoPlayer(videoController!)), + Positioned.fill(child: _buildLayers()), + ], + ), + ), + ), + ) + else + _buildLayers(), + if (bottomOverlay != null) + Positioned(left: 0, right: 0, bottom: 0, child: bottomOverlay!), + ], + ), + ); + } +} diff --git a/lib/src/visual/views/camera/share_image_editor_components/editor_layer_stack.dart b/lib/src/visual/views/camera/share_image_editor_components/editor_layer_stack.dart new file mode 100644 index 00000000..ace3a603 --- /dev/null +++ b/lib/src/visual/views/camera/share_image_editor_components/editor_layer_stack.dart @@ -0,0 +1,157 @@ +import 'package:flutter/material.dart'; +import 'package:twonly/src/visual/helpers/screenshot.helper.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/image_item.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/layer_data.dart'; + +/// Owns the layer stack of the image editor together with its undo/redo +/// history. +/// +/// The stack always starts with the [BackgroundLayerData] (the photo itself) +/// followed by the [FilterLayerData]; everything the user adds (text, drawings, +/// emojis, link previews) is stacked on top of those two. +/// +/// Layers are not removed immediately when the user deletes them. They are +/// flagged via [Layer.isDeleted] by the layer widgets and only collected later, +/// which is what makes restoring them through [undo] possible. +class EditorLayerStack { + List _layers = []; + final List _undone = []; + final List _removed = []; + + /// All layers including the ones flagged as deleted. + List get all => _layers; + + /// The layers that should actually be painted and exported. + List get visible => _layers.where((x) => !x.isDeleted).toList(); + + bool get isEmpty => _layers.isEmpty; + + bool get anyIsEditing => _layers.any((x) => x.isEditing); + + /// While a layer takes over the whole screen (cropping the background or a + /// layer bringing its own action buttons) the editor toolbars are hidden. + bool get takesOverScreen => + _layers.isNotEmpty && + (_layers.first.isEditing || + (_layers.last.isEditing && _layers.last.hasCustomActionButtons)); + + /// Whether the user added anything on top of the background and the filter. + bool get hasUserAddedLayers => _layers.any( + (x) => x is! BackgroundLayerData && x is! FilterLayerData, + ); + + bool get canUndo => visible.length > 2; + + bool get canRedo => _undone.isNotEmpty; + + bool get isBackgroundLoaded { + final first = _layers.firstOrNull; + return first is BackgroundLayerData && first.imageLoaded; + } + + /// The untouched background image, but only when nothing was drawn on top of + /// it. In that case the editor can skip taking a screenshot and export the + /// original image instead. + ScreenshotImageHelper? get unmodifiedBackgroundImage { + final background = _layers.firstOrNull; + if (background is! BackgroundLayerData) return null; + if (_layers.length == 1) return background.image.image; + if (_layers.length == 2) { + final filter = _layers[1]; + // page == 1 is the "no filter selected" page. + if (filter is FilterLayerData && filter.page == 1) { + return background.image.image; + } + } + return null; + } + + void addFilterLayer() => _layers.add(FilterLayerData(key: GlobalKey())); + + void addLinkPreviewLayer(Uri link) => + _layers.add(LinkPreviewLayerData(key: GlobalKey(), link: link)); + + void insertBackgroundLayer(ImageItem image) => _layers.insert( + 0, + BackgroundLayerData(key: GlobalKey(), image: image), + ); + + /// Adds a new layer on top and drops the redo history. + void add(Layer layer) { + _undone.clear(); + _removed.clear(); + _layers.add(layer); + } + + /// Adds an empty text layer at [offset]. Does nothing while another layer is + /// still being edited. + void addTextLayer({Offset offset = Offset.zero}) { + _layers = visible; + if (anyIsEditing) return; + add( + TextLayerData( + key: GlobalKey(), + offset: offset, + textLayersBefore: _layers.whereType().length, + ), + ); + } + + void addDrawLayer() => add(DrawLayerData(key: GlobalKey())); + + /// Toggles the crop/rotate mode of the background layer. + void toggleBackgroundEditing() { + final background = _layers.firstOrNull; + if (background is BackgroundLayerData) { + background.isEditing = !background.isEditing; + } + } + + /// Restores the last deleted layer, or removes the topmost one. + void undo() { + if (_removed.isNotEmpty) { + _layers.add( + _removed.removeLast() + ..isDeleted = false + ..isEditing = false, + ); + return; + } + _layers = visible; + if (_layers.length <= 2) { + // never remove the background and the filter layer + return; + } + _undone.add(_layers.removeLast()); + } + + void redo() { + if (_undone.isEmpty) return; + _layers.add(_undone.removeLast()); + } + + /// Called after the user finished interacting with a layer: stop editing all + /// layers and move the ones flagged as deleted into the undo history. + void commitPendingEdits() { + for (final layer in _layers) { + layer.isEditing = false; + if (layer.isDeleted) { + _removed.add(layer); + } + } + _layers = visible; + } + + /// The per-layer action buttons must not end up on the exported screenshot. + void setCustomButtonsVisible({required bool visible}) { + for (final layer in _layers) { + layer.showCustomButtons = visible; + } + } + + void clear() { + _layers.clear(); + _undone.clear(); + _removed.clear(); + } +} diff --git a/lib/src/visual/views/camera/share_image_editor_components/editor_media_writer.dart b/lib/src/visual/views/camera/share_image_editor_components/editor_media_writer.dart new file mode 100644 index 00000000..a3b6f708 --- /dev/null +++ b/lib/src/visual/views/camera/share_image_editor_components/editor_media_writer.dart @@ -0,0 +1,118 @@ +import 'dart:typed_data'; + +import 'package:twonly/src/database/tables/mediafiles.table.dart'; +import 'package:twonly/src/database/twonly.db.dart'; +import 'package:twonly/src/services/mediafiles/mediafile.service.dart'; +import 'package:twonly/src/utils/log.dart'; +import 'package:twonly/src/visual/helpers/screenshot.helper.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/editor_layer_stack.dart'; + +/// Turns the edited layer stack into the files the upload/gallery code expects. +/// +/// * images and gifs are written to [MediaFileService.originalPath] +/// * videos keep their original file and get the edits written next to them as +/// [MediaFileService.overlayImagePath] +class EditorMediaWriter { + EditorMediaWriter({ + required this.mediaService, + required this.layerStack, + required this.screenshotController, + required this.gifSource, + required this.requestRebuild, + }); + + final MediaFileService mediaService; + final EditorLayerStack layerStack; + final ScreenshotController screenshotController; + + /// Gifs are not edited, their bytes are taken as they came in. + final ScreenshotImageHelper? gifSource; + + /// Rebuilds the editor and waits until the change is on screen. Needed to + /// hide the per-layer action buttons before taking the screenshot. + final void Function() requestRebuild; + + MediaFile get media => mediaService.mediaFile; + + /// Renders the visible layers into a single image. + Future captureEditedImage(double pixelRatio) async { + final unmodified = layerStack.unmodifiedBackgroundImage; + if (unmodified != null) return unmodified; + + layerStack.setCustomButtonsVisible(visible: false); + requestRebuild(); + + // Make a short delay, so the rebuild does have its effect... + await Future.delayed(const Duration(milliseconds: 80)); + + final image = await screenshotController.capture(pixelRatio: pixelRatio); + if (image == null) { + Log.warn('screenshotController did not return image bytes'); + return null; + } + + layerStack.setCustomButtonsVisible(visible: true); + requestRebuild(); + + return image; + } + + /// Writes the edited media to disk, replacing any previously written + /// temporary/overlay files. + Future storeImageAsOriginal(double pixelRatio) async { + Uint8List? gifBytes; + ScreenshotImageHelper? image; + if (media.type == MediaType.gif) { + gifBytes = await gifSource?.getBytes(); + } else { + image = await captureEditedImage(pixelRatio); + if (image != null) { + await image.getBytes(); + } + } + + _deleteStaleFiles(); + + if (media.type == MediaType.gif) { + if (gifBytes != null) { + mediaService.originalPath.writeAsBytesSync(gifBytes.toList()); + } + return image; + } + + if (image == null) return null; + final bytes = await image.getBytes(); + if (bytes == null) { + Log.warn('imageBytes are empty'); + return null; + } + if (media.type == MediaType.image || media.type == MediaType.gif) { + mediaService.originalPath.writeAsBytesSync(bytes); + } else if (media.type == MediaType.video) { + mediaService.overlayImagePath.writeAsBytesSync(bytes); + } else { + Log.error('MediaType not supported: ${media.type}'); + } + return image; + } + + void _deleteStaleFiles() { + if (mediaService.overlayImagePath.existsSync()) { + mediaService.overlayImagePath.deleteSync(); + } + if (mediaService.tempPath.existsSync()) { + mediaService.tempPath.deleteSync(); + } + if (mediaService.originalPath.existsSync() && + media.type == MediaType.image) { + mediaService.originalPath.deleteSync(); + } + } + + /// Persists the unedited image so it can be restored as a draft in case the + /// app is restarted while the editor is open. + Future storeAsDraft(ScreenshotImageHelper screenshotImage) async { + final imageBytes = await screenshotImage.getBytes(); + mediaService.originalPath.writeAsBytesSync(imageBytes!.toList()); + } +} diff --git a/lib/src/visual/views/camera/share_image_editor_components/editor_side_toolbar.dart b/lib/src/visual/views/camera/share_image_editor_components/editor_side_toolbar.dart new file mode 100644 index 00000000..762da209 --- /dev/null +++ b/lib/src/visual/views/camera/share_image_editor_components/editor_side_toolbar.dart @@ -0,0 +1,163 @@ +import 'package:flutter/material.dart'; +import 'package:font_awesome_flutter/font_awesome_flutter.dart'; +import 'package:twonly/src/database/tables/mediafiles.table.dart'; +import 'package:twonly/src/database/twonly.db.dart'; +import 'package:twonly/src/services/mediafiles/mediafile.service.dart'; +import 'package:twonly/src/utils/misc.dart'; +import 'package:twonly/src/visual/components/emoji_picker.bottom.dart'; +import 'package:twonly/src/visual/components/notification_badge.comp.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/action_button.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/editor_layer_stack.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/layer_data.dart'; + +/// The editing tools next to the media: add text, drawing and emojis, pick the +/// display time and protect the media. +class EditorSideToolbar extends StatelessWidget { + const EditorSideToolbar({ + required this.layerStack, + required this.mediaService, + required this.onChanged, + required this.onEditDisplayTime, + required this.onToggleAudio, + required this.onToggleRequiresAuth, + required this.canTrim, + required this.trimmerVisible, + required this.onToggleTrimmer, + super.key, + }); + + final EditorLayerStack layerStack; + final MediaFileService mediaService; + + /// Called after the layer stack was modified so the editor can rebuild. + final VoidCallback onChanged; + final VoidCallback onEditDisplayTime; + final VoidCallback onToggleAudio; + final VoidCallback onToggleRequiresAuth; + + /// Whether the clip is far enough along for the cutter to be openable at all; + /// it needs a player that knows how long the recording is. + final bool canTrim; + final bool trimmerVisible; + final VoidCallback onToggleTrimmer; + + MediaFile get media => mediaService.mediaFile; + + /// How long the receiver may look at the media, `∞` when unlimited and `0` + /// for videos (they are limited by their own length). + String get _displayTimeLabel { + if (media.type == MediaType.video) return '0'; + final limit = media.displayLimitInMilliseconds; + if (limit == null) return '∞'; + return (limit ~/ 1000).toString(); + } + + IconData get _displayTimeIcon { + if (media.type != MediaType.video) return Icons.timer_outlined; + return (media.displayLimitInMilliseconds == null) + ? Icons.repeat_rounded + : Icons.repeat_one_rounded; + } + + @override + Widget build(BuildContext context) { + if (layerStack.takesOverScreen) return const SizedBox.shrink(); + + final canBeEdited = media.type != MediaType.gif; + + return Column( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + if (canBeEdited) + ActionButton( + Icons.text_fields_rounded, + tooltipText: context.lang.addTextItem, + onPressed: () { + layerStack.addTextLayer(); + onChanged(); + }, + ), + const SizedBox(height: 8), + if (canBeEdited) + ActionButton( + Icons.draw_rounded, + tooltipText: context.lang.addDrawing, + onPressed: () { + layerStack.addDrawLayer(); + onChanged(); + }, + ), + const SizedBox(height: 8), + if (canBeEdited) + ActionButton( + Icons.add_reaction_outlined, + tooltipText: context.lang.addEmoji, + onPressed: () async { + final layer = await showModalBottomSheet( + context: context, + backgroundColor: Colors.black, + builder: (context) => const EmojiPickerBottom(), + ); + if (layer == null) return; + layerStack.add(layer); + onChanged(); + }, + ), + const SizedBox(height: 8), + NotificationBadgeComp( + count: _displayTimeLabel, + child: ActionButton( + _displayTimeIcon, + tooltipText: context.lang.protectAsARealTwonly, + onPressed: onEditDisplayTime, + ), + ), + if (canTrim) ...[ + const SizedBox(height: 8), + ActionButton( + Icons.content_cut_rounded, + tooltipText: 'Trim video', + color: trimmerVisible + ? Theme.of(context).colorScheme.primary + : Colors.white, + onPressed: onToggleTrimmer, + ), + ], + if (media.type == MediaType.video) ...[ + const SizedBox(height: 8), + ActionButton( + (mediaService.removeAudio) + ? Icons.volume_off_rounded + : Icons.volume_up_rounded, + tooltipText: 'Enable Audio in Video', + color: (mediaService.removeAudio) + ? Colors.white.withAlpha(160) + : Colors.white, + onPressed: onToggleAudio, + ), + ], + if (media.type == MediaType.image) ...[ + const SizedBox(height: 8), + ActionButton( + Icons.crop_rotate_outlined, + tooltipText: 'Crop or rotate image', + color: Colors.white, + onPressed: () { + layerStack.toggleBackgroundEditing(); + onChanged(); + }, + ), + ], + const SizedBox(height: 8), + ActionButton( + FontAwesomeIcons.shieldHeart, + tooltipText: context.lang.protectAsARealTwonly, + color: media.requiresAuthentication + ? Theme.of(context).colorScheme.primary + : Colors.white, + onPressed: onToggleRequiresAuth, + ), + ], + ); + } +} diff --git a/lib/src/visual/views/camera/share_image_editor_components/editor_top_toolbar.dart b/lib/src/visual/views/camera/share_image_editor_components/editor_top_toolbar.dart new file mode 100644 index 00000000..0aa13465 --- /dev/null +++ b/lib/src/visual/views/camera/share_image_editor_components/editor_top_toolbar.dart @@ -0,0 +1,58 @@ +import 'package:flutter/material.dart'; +import 'package:font_awesome_flutter/font_awesome_flutter.dart'; +import 'package:twonly/src/utils/misc.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/action_button.dart'; +import 'package:twonly/src/visual/views/camera/share_image_editor_components/editor_layer_stack.dart'; + +/// Close, undo and redo, shown above the media. +class EditorTopToolbar extends StatelessWidget { + const EditorTopToolbar({ + required this.layerStack, + required this.onClose, + required this.onChanged, + super.key, + }); + + final EditorLayerStack layerStack; + final VoidCallback onClose; + + /// Called after the layer stack was modified so the editor can rebuild. + final VoidCallback onChanged; + + @override + Widget build(BuildContext context) { + if (layerStack.takesOverScreen) return const SizedBox.shrink(); + + return Row( + children: [ + ActionButton( + FontAwesomeIcons.xmark, + tooltipText: context.lang.close, + onPressed: onClose, + ), + Expanded(child: Container()), + const SizedBox(width: 8), + ActionButton( + FontAwesomeIcons.rotateLeft, + tooltipText: context.lang.undo, + disable: !layerStack.canUndo, + onPressed: () { + layerStack.undo(); + onChanged(); + }, + ), + const SizedBox(width: 8), + ActionButton( + FontAwesomeIcons.rotateRight, + tooltipText: context.lang.redo, + disable: !layerStack.canRedo, + onPressed: () { + layerStack.redo(); + onChanged(); + }, + ), + const SizedBox(width: 70), + ], + ); + } +} diff --git a/lib/src/visual/views/camera/share_image_editor_components/video_trimmer.dart b/lib/src/visual/views/camera/share_image_editor_components/video_trimmer.dart new file mode 100644 index 00000000..3a8204f0 --- /dev/null +++ b/lib/src/visual/views/camera/share_image_editor_components/video_trimmer.dart @@ -0,0 +1,358 @@ +import 'dart:async'; + +import 'package:flutter/material.dart'; +import 'package:video_player/video_player.dart'; + +/// Cuts a recorded video down to the part the user wants to send. +/// +/// The two handles move the ends of the clip and the line between them is the +/// playhead, which can be dragged to look through the recording. Nothing is +/// written while a handle is moving: [onChanged] keeps the preview in step and +/// [onChangeEnd] fires once the finger lifts, which is the point where the cut +/// is worth storing. +/// +/// Playback is kept inside the selection here rather than by the editor, +/// because this is already the widget listening to the player's position. +class VideoTrimmer extends StatefulWidget { + const VideoTrimmer({ + required this.controller, + required this.start, + required this.end, + required this.onChanged, + required this.onChangeEnd, + super.key, + }); + + final VideoPlayerController controller; + + /// The current cut. Both are real positions in the recording; the editor + /// resolves "not cut on this end" to zero and the full duration before + /// handing them over. + final Duration start; + final Duration end; + + /// Called continuously while a handle is dragged. + final void Function(Duration start, Duration end) onChanged; + + /// Called once, when the finger lifts. + final void Function(Duration start, Duration end) onChangeEnd; + + /// Nothing shorter than this can be selected. A clip the length of a single + /// frame is never what someone was aiming for, and it gives the two handles + /// room to stay apart. + static const Duration minimumSelection = Duration(seconds: 1); + + @override + State createState() => _VideoTrimmerState(); +} + +/// What a drag started on. +enum _Grip { start, end, playhead } + +class _VideoTrimmerState extends State { + static const double _trackHeight = 40; + static const double _handleWidth = 14; + + /// How far from a handle a touch still counts as grabbing it. Fingers are + /// wider than the handles are drawn. + static const double _grabSlop = 22; + + _Grip? _grip; + + /// Whether the clip was playing when the finger went down, so scrubbing can + /// pause it and hand playback back the way it found it. Seeking against a + /// running player fights the drag and makes the preview stutter. + bool _resumeAfterDrag = false; + + /// Where the player is, mirrored so the playhead can be repainted without + /// rebuilding the editor around it. + Duration _position = Duration.zero; + + Duration get _duration => widget.controller.value.duration; + + @override + void initState() { + super.initState(); + _position = widget.controller.value.position; + widget.controller.addListener(_onPlaybackChanged); + } + + @override + void dispose() { + widget.controller.removeListener(_onPlaybackChanged); + super.dispose(); + } + + void _onPlaybackChanged() { + if (!mounted) return; + final position = widget.controller.value.position; + + // The player loops on the whole file, so the cut end has to bring it back + // by hand. Done before the repaint so the playhead never draws past the + // selection. + if (_grip == null && + widget.controller.value.isPlaying && + (position >= widget.end || position < widget.start)) { + unawaited(widget.controller.seekTo(widget.start)); + return; + } + + if (position != _position) { + setState(() => _position = position); + } + } + + Duration _positionAt(double dx, double width) { + if (width <= 0 || _duration == Duration.zero) return Duration.zero; + final fraction = (dx / width).clamp(0.0, 1.0); + return Duration( + milliseconds: (_duration.inMilliseconds * fraction).round(), + ); + } + + double _offsetOf(Duration position, double width) { + if (_duration == Duration.zero) return 0; + final fraction = position.inMilliseconds / _duration.inMilliseconds; + return fraction.clamp(0.0, 1.0) * width; + } + + /// The handle or the playhead nearest to where the finger landed, as long as + /// something is actually within reach. + _Grip _gripFor(double dx, double width) { + final distances = <_Grip, double>{ + _Grip.start: (dx - _offsetOf(widget.start, width)).abs(), + _Grip.end: (dx - _offsetOf(widget.end, width)).abs(), + _Grip.playhead: (dx - _offsetOf(_position, width)).abs(), + }; + final nearest = distances.entries.reduce( + (a, b) => a.value <= b.value ? a : b, + ); + // Anywhere else on the track is a seek: tapping the middle of the clip to + // jump there is more useful than dragging an end that was not aimed at. + return nearest.value <= _grabSlop ? nearest.key : _Grip.playhead; + } + + void _onDrag(double dx, double width) { + final at = _positionAt(dx, width); + switch (_grip) { + case _Grip.start: + final limit = widget.end - VideoTrimmer.minimumSelection; + final start = at > limit ? limit : at; + widget.onChanged(_atLeastZero(start), widget.end); + unawaited(widget.controller.seekTo(_atLeastZero(start))); + case _Grip.end: + final limit = widget.start + VideoTrimmer.minimumSelection; + final end = at < limit ? limit : at; + final capped = end > _duration ? _duration : end; + widget.onChanged(widget.start, capped); + // Seeking to the very last frame tends to land on a black one, so the + // preview shows the frame just before the new end instead. + unawaited( + widget.controller.seekTo(capped - const Duration(milliseconds: 40)), + ); + case _Grip.playhead: + final clamped = at < widget.start + ? widget.start + : (at > widget.end ? widget.end : at); + setState(() => _position = clamped); + unawaited(widget.controller.seekTo(clamped)); + case null: + break; + } + } + + static Duration _atLeastZero(Duration value) => + value < Duration.zero ? Duration.zero : value; + + static String _format(Duration value) { + final seconds = value.inSeconds; + return '${seconds ~/ 60}:${(seconds % 60).toString().padLeft(2, '0')}'; + } + + @override + Widget build(BuildContext context) { + if (_duration == Duration.zero) return const SizedBox.shrink(); + final selection = widget.end - widget.start; + final isTrimmed = + widget.start > Duration.zero || + widget.end < _duration - const Duration(milliseconds: 100); + + return Container( + margin: const EdgeInsets.fromLTRB(12, 0, 12, 10), + padding: const EdgeInsets.fromLTRB(12, 6, 12, 6), + decoration: BoxDecoration( + // Translucent rather than opaque: the frames being cut away stay + // visible underneath, which is most of what makes a cut readable. + color: Colors.black.withAlpha(90), + borderRadius: BorderRadius.circular(16), + ), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + LayoutBuilder( + builder: (context, constraints) { + // The handles sit inside the ends of the track, so the positions + // they can be dragged to are one handle narrower than the box. + final width = constraints.maxWidth - _handleWidth * 2; + return GestureDetector( + behavior: HitTestBehavior.opaque, + onHorizontalDragStart: (details) { + _grip = _gripFor( + details.localPosition.dx - _handleWidth, + width, + ); + _resumeAfterDrag = widget.controller.value.isPlaying; + if (_resumeAfterDrag) { + unawaited(widget.controller.pause()); + } + _onDrag(details.localPosition.dx - _handleWidth, width); + }, + onHorizontalDragUpdate: (details) => + _onDrag(details.localPosition.dx - _handleWidth, width), + onHorizontalDragEnd: (_) { + _grip = null; + widget.onChangeEnd(widget.start, widget.end); + if (_resumeAfterDrag) { + _resumeAfterDrag = false; + unawaited(widget.controller.play()); + } + }, + onTapUp: (details) { + _grip = _Grip.playhead; + _onDrag(details.localPosition.dx - _handleWidth, width); + _grip = null; + }, + child: SizedBox( + height: _trackHeight, + child: _TrimTrack( + trackWidth: width, + handleWidth: _handleWidth, + startOffset: _offsetOf(widget.start, width), + endOffset: _offsetOf(widget.end, width), + playheadOffset: _offsetOf(_position, width), + accent: Theme.of(context).colorScheme.primary, + ), + ), + ); + }, + ), + const SizedBox(height: 4), + Text( + isTrimmed + ? '${_format(widget.start)} – ${_format(widget.end)} · ${_format(selection)}' + : _format(_duration), + style: TextStyle( + fontSize: 11, + color: Colors.white.withAlpha(isTrimmed ? 235 : 150), + fontFeatures: const [FontFeature.tabularFigures()], + shadows: const [ + Shadow(color: Color.fromARGB(122, 0, 0, 0), blurRadius: 4), + ], + ), + ), + ], + ), + ); + } +} + +/// The bar itself: the whole recording, the selected span, and the playhead. +class _TrimTrack extends StatelessWidget { + const _TrimTrack({ + required this.trackWidth, + required this.handleWidth, + required this.startOffset, + required this.endOffset, + required this.playheadOffset, + required this.accent, + }); + + final double trackWidth; + final double handleWidth; + final double startOffset; + final double endOffset; + final double playheadOffset; + final Color accent; + + @override + Widget build(BuildContext context) { + final selectionWidth = (endOffset - startOffset).clamp(0.0, trackWidth); + + return Stack( + children: [ + // The recording in full, so the cut-off parts stay visible as the + // context the selection was taken out of. + Positioned.fill( + left: handleWidth, + right: handleWidth, + child: DecoratedBox( + decoration: BoxDecoration( + color: Colors.white.withAlpha(46), + borderRadius: BorderRadius.circular(6), + ), + ), + ), + Positioned( + left: startOffset, + width: selectionWidth + handleWidth * 2, + top: 0, + bottom: 0, + child: DecoratedBox( + decoration: BoxDecoration( + color: accent.withAlpha(40), + border: Border.symmetric( + horizontal: BorderSide(color: accent, width: 2), + ), + borderRadius: BorderRadius.circular(8), + ), + ), + ), + _handle(left: startOffset), + _handle(left: endOffset + handleWidth), + Positioned( + left: playheadOffset + handleWidth - 1, + top: 4, + bottom: 4, + width: 3, + child: DecoratedBox( + decoration: BoxDecoration( + color: Colors.white, + borderRadius: BorderRadius.circular(2), + boxShadow: const [ + BoxShadow(color: Color.fromARGB(122, 0, 0, 0), blurRadius: 4), + ], + ), + ), + ), + ], + ); + } + + Widget _handle({required double left}) { + return Positioned( + left: left, + top: 0, + bottom: 0, + width: handleWidth, + child: Center( + child: Container( + width: handleWidth, + decoration: BoxDecoration( + color: accent, + borderRadius: BorderRadius.circular(6), + ), + child: Center( + child: Container( + width: 2, + height: 14, + decoration: BoxDecoration( + color: Colors.white.withAlpha(220), + borderRadius: BorderRadius.circular(1), + ), + ), + ), + ), + ), + ); + } +} diff --git a/rust/src/api/proto/api/http/http_requests.proto b/rust/src/api/proto/api/http/http_requests.proto index 9e5e7011..2dd00075 100644 --- a/rust/src/api/proto/api/http/http_requests.proto +++ b/rust/src/api/proto/api/http/http_requests.proto @@ -31,6 +31,29 @@ message AttachmentDispatch { bytes download_token = 4; } +// A message envelope handed to a background transfer instead of the websocket. +// +// The socket only exists while the app runs, so an envelope queued offline +// would otherwise wait for the next app launch rather than the next network. +message OutboxDispatch { + string dispatch_id = 1; + int64 recipient_user_id = 2; + bytes encrypted_body = 3; + bool wake_receiver = 4; +} + +message OutboxDispatchBatch { + repeated OutboxDispatch dispatches = 1; +} + +message OutboxDispatchResult { + // Envelopes the server has accepted, whether just now or on an earlier + // delivery of the same request. + int32 accepted = 1; + // Envelopes it will never accept, so the transfer must not be retried. + int32 rejected = 2; +} + message AttachmentManifest { int64 encrypted_object_size = 1; repeated AttachmentDispatch dispatches = 2; diff --git a/rust/src/api/runtime/client.rs b/rust/src/api/runtime/client.rs index aa2f618a..510954c8 100644 --- a/rust/src/api/runtime/client.rs +++ b/rust/src/api/runtime/client.rs @@ -50,6 +50,10 @@ pub(crate) struct ApiClient { pub(crate) config: ApiConfig, pub(crate) state: RwLock, pub(crate) ws_client: Mutex>>, + /// Serializes replacing/closing the client slot. The WebSocket runner emits + /// events asynchronously, so overlapping replacements must not clear each + /// other's pending requests or publish stale state. + connection_change: Mutex<()>, pub(crate) pending: PendingRequests, pub(crate) next_sequence: Mutex, pub(crate) events: broadcast::Sender, @@ -77,6 +81,7 @@ impl ApiClient { config, state: RwLock::const_new(ApiConnectionState::Stopped), ws_client: Mutex::const_new(None), + connection_change: Mutex::const_new(()), pending: Arc::new(Mutex::const_new(HashMap::new())), next_sequence: Mutex::const_new(1), events: API_EVENTS.clone(), @@ -200,15 +205,15 @@ impl ApiClient { } async fn reconnect(self: &Arc, stale: &Arc, reason: &str) -> Result<()> { + let connection_change = self.connection_change.lock().await; { - let mut guard = self.ws_client.lock().await; + let guard = self.ws_client.lock().await; if !guard .as_ref() .is_some_and(|current| Arc::ptr_eq(current, stale)) { return Ok(()); } - guard.take(); } // Counted only for a socket that was still the live one, so a stale @@ -226,14 +231,8 @@ impl ApiClient { "API WebSocket is dead, reconnecting" ); - self.is_authenticated.store(false, Ordering::Release); - // Dropping the sender ends the catch-up loop bound to the dead socket. - *self.catch_up_tx.lock().await = None; - if let Err(error) = stale.shutdown_graceful(Duration::from_secs(5)).await { - tracing::warn!("dead WebSocket did not shut down cleanly: {error}"); - } - self.fail_pending().await; - self.set_state(ApiConnectionState::Stopped).await; + self.discard_current_connection(reason).await; + drop(connection_change); tokio::time::sleep(delay).await; @@ -259,6 +258,7 @@ impl ApiClient { if *guard != state { *guard = state; drop(guard); + tracing::info!(?state, "API connection state changed"); let _ = self.events.send(ApiEvent { kind: ApiEventKind::ConnectionStateChanged, state: Some(state), @@ -267,7 +267,60 @@ impl ApiClient { } } + /// Returns whether `connection` still owns the live client slot. + /// + /// A replaced socket can emit its final disconnect/shutdown events after + /// the new socket has started. Those events must not overwrite the new + /// connection's state or schedule another reconnect. + async fn is_current_connection(&self, connection: &Arc) -> bool { + self.ws_client + .lock() + .await + .as_ref() + .is_some_and(|current| Arc::ptr_eq(current, connection)) + } + + /// Drops the current transport without waiting for its graceful-shutdown + /// timeout. Used for external reachability changes, where waiting would + /// defeat the purpose of an immediate reconnect. + async fn discard_current_connection(&self, reason: &str) { + let current = self.ws_client.lock().await.take(); + self.is_authenticated.store(false, Ordering::Release); + // Dropping the sender ends the catch-up loop bound to the old socket. + *self.catch_up_tx.lock().await = None; + + if let Some(current) = current { + tracing::info!(reason, "discarding API WebSocket"); + current.shutdown(); + } + + self.fail_pending().await; + self.set_state(ApiConnectionState::Stopped).await; + } + + /// Starts a fresh connection immediately, bypassing any retry sleep owned + /// by the previous transport. The replacement client still uses the normal + /// exponential backoff if this first attempt fails. + async fn reconnect_now(self: &Arc, reason: &str) -> Result<()> { + let _connection_change = self.connection_change.lock().await; + if API_PERMANENTLY_REJECTED.load(Ordering::Acquire) + || self.deliberately_closed.load(Ordering::Acquire) + || self.in_background.load(Ordering::Acquire) + || !self.network_available.load(Ordering::Acquire) + { + return Ok(()); + } + + self.discard_current_connection(reason).await; + self.connect_inner().await + } + pub async fn connect(self: &Arc) -> Result<()> { + let _connection_change = self.connection_change.lock().await; + self.connect_inner().await + } + + async fn connect_inner(self: &Arc) -> Result<()> { if API_PERMANENTLY_REJECTED.load(Ordering::Acquire) { self.set_state(ApiConnectionState::PermanentlyRejected) .await; @@ -339,6 +392,12 @@ impl ApiClient { msg = receiver.recv() => { match msg { Ok(msg) => { + let Some(connection) = connection.upgrade() else { + break; + }; + if !self_clone.is_current_connection(&connection).await { + break; + } // Tungstenite message if let stream_tungstenite::tokio_tungstenite::tungstenite::Message::Binary(bytes) = &*msg { self_clone.handle_incoming(bytes).await; @@ -349,6 +408,12 @@ impl ApiClient { } ev = events.recv() => { use stream_tungstenite::ConnectionEvent; + let Some(connection) = connection.upgrade() else { + break; + }; + if !self_clone.is_current_connection(&connection).await { + break; + } match ev { Ok(ConnectionEvent::Connected { .. }) => { let is_auth = self_clone.is_authenticated.load(Ordering::Acquire); @@ -377,12 +442,10 @@ impl ApiClient { // send. Replace it instead of sitting on it. Ok(ConnectionEvent::FatalError { .. } | ConnectionEvent::Shutdown) => { self_clone.is_authenticated.store(false, Ordering::Release); - if let Some(connection) = connection.upgrade() { - self_clone.schedule_reconnect( - &connection, - "supervisor stopped reconnecting", - ); - } + self_clone.schedule_reconnect( + &connection, + "supervisor stopped reconnecting", + ); } Ok(_) => {} Err(_) => break, @@ -397,6 +460,7 @@ impl ApiClient { pub async fn close(&self) { self.deliberately_closed.store(true, Ordering::Release); + let _connection_change = self.connection_change.lock().await; self.is_authenticated.store(false, Ordering::Release); // Dropping the sender ends the catch-up loop for this connection. *self.catch_up_tx.lock().await = None; @@ -422,24 +486,31 @@ impl ApiClient { self.in_background.store(in_background, Ordering::Release); if in_background { self.set_state(ApiConnectionState::Suspended).await; - } else if self.ws_client.lock().await.is_some() { - self.set_state(ApiConnectionState::Authenticated).await; - // Returning to the foreground: pick up whatever arrived while the - // socket was suspended. - self.request_catch_up().await; - } else if self.network_available.load(Ordering::Acquire) { - self.connect().await?; + } else { + // Mobile platforms can freeze the process while keeping the socket + // object alive. Its TCP connection may be dead even though neither + // the OS nor tungstenite has reported that yet, so foregrounding + // always starts a fresh attempt instead of waiting for a timeout. + self.reconnect_now("application entered the foreground") + .await?; } Ok(()) } pub(crate) async fn set_network_available(self: &Arc, available: bool) -> Result<()> { self.network_available.store(available, Ordering::Release); - if available - & !self.in_background.load(Ordering::Acquire) - & self.ws_client.lock().await.is_none() - { - self.connect().await?; + if available { + // A Wi-Fi/mobile/route change can leave an apparently connected + // socket bound to the old network. Replace it now, including when + // its own supervisor is currently sleeping in reconnect backoff. + self.reconnect_now("network became available or changed") + .await?; + } else { + // Reflect loss of reachability immediately instead of showing an + // authenticated state until the receive timeout expires. + let _connection_change = self.connection_change.lock().await; + self.discard_current_connection("network became unavailable") + .await; } Ok(()) } diff --git a/rust/src/api/runtime/helpers.rs b/rust/src/api/runtime/helpers.rs index 594a775b..15ac59ed 100644 --- a/rust/src/api/runtime/helpers.rs +++ b/rust/src/api/runtime/helpers.rs @@ -9,10 +9,11 @@ use crate::api::runtime::ApiRuntime; use crate::api::Server; use crate::bridge::api::ServerResult; -use crate::context::{Context, RuntimeMode}; +use crate::context::Context; use crate::error::{Result, TwonlyError}; use crate::services::direct_media_upload::DirectMediaUploadService; use crate::services::groups::GroupService; +use crate::services::media_upload::MediaUploadService; use crate::services::mediafiles::MediaFileService; use prost::Message as ProstMessage; use std::future::Future; @@ -43,7 +44,7 @@ pub(crate) fn response_error_code(bytes: &[u8]) -> Result> { pub(crate) fn schedule_post_authentication(ctx: &Arc, in_background: bool) { // Notification workers only drain and commit the mailbox. Media downloads, // maintenance, and outbox replay belong to the main application runtime. - if ctx.runtime_mode == RuntimeMode::Notification { + if ctx.is_notification_runtime() { return; } let ctx = ctx.clone(); @@ -77,6 +78,12 @@ pub(crate) fn schedule_post_authentication(ctx: &Arc, in_background: bo if let Err(error) = messages::retransmit_queued_receipts(&ctx).await { tracing::warn!("failed to retransmit queued receipts: {error}"); } + // A preparation that a terminated process left half-finished is only + // resumed by a sweep like this one; a mid-session reconnect is just as + // good a moment for it as a cold start, and far more frequent. + if let Err(error) = MediaUploadService::new(&ctx).finish_started_uploads().await { + tracing::warn!("failed to finish started media uploads: {error}"); + } if let Err(error) = MediaFileService::new(&ctx).download_pending().await { tracing::warn!("failed to download pending media: {error}"); } diff --git a/rust/src/bridge/api.rs b/rust/src/bridge/api.rs index c015adc4..9e9d4ede 100644 --- a/rust/src/bridge/api.rs +++ b/rust/src/bridge/api.rs @@ -7,13 +7,13 @@ use crate::api::ApiRuntime; pub use crate::api::PqcPreKeyInput; use crate::api::Server; use crate::context::Context; -use crate::context::RuntimeMode; use crate::error::{Result, TwonlyError}; use crate::frb_generated::StreamSink; use crate::services::avatars; use crate::services::contacts::ContactService; use crate::services::media_upload::{MediaSizeReport, MediaUploadService}; use crate::services::messages::MessageService; +use crate::services::outbox_dispatch::OutboxDispatchService; use crate::user_config::UserConfig; use flutter_rust_bridge::frb; use std::collections::HashMap; @@ -54,7 +54,7 @@ impl ApiConfig { Ok(Self { websocket_url: format!("{}client", RustApi::api_base_url("wss".to_owned())), legacy_user_app_version: user.as_ref().map_or(0, |value| value.app_version), - in_background: context.runtime_mode == RuntimeMode::Notification, + in_background: context.is_notification_runtime(), can_use_login_token_for_auth: user .as_ref() .is_some_and(|value| value.can_use_login_token_for_auth), @@ -302,6 +302,19 @@ impl RustApi { .await } + /// Stores the cut the editor's video trimmer asked for. Both bounds are + /// milliseconds into the recording, `None` keeps that end of the clip. + pub async fn set_media_trim( + media_id: String, + trim_start_ms: Option, + trim_end_ms: Option, + ) -> Result<()> { + let ctx = Context::get_static()?; + MediaUploadService::new(ctx) + .set_trim(&media_id, trim_start_ms, trim_end_ms) + .await + } + pub async fn toggle_media_remove_audio(media_id: String) -> Result<()> { let ctx = Context::get_static()?; MediaUploadService::new(ctx) @@ -390,14 +403,47 @@ impl RustApi { pub async fn set_background(in_background: bool) -> Result<()> { let ctx = Context::get_static()?; - if !in_background { + if in_background { + // The socket is about to stop being an option, so anything still + // waiting on it is handed to the OS, which delivers it whether or + // not this process survives. + let ctx = ctx.clone(); + tokio::spawn(async move { + if let Err(error) = OutboxDispatchService::new(&ctx).hand_pending_to_os().await { + tracing::warn!(%error, "could not hand queued envelopes to the OS"); + } + }); + } else { // Coming back to the foreground is the first chance to notice that // the OS finished a media transfer while this process was idle. crate::services::direct_media_upload::watch_pending_uploads(ctx); + let ctx = ctx.clone(); + tokio::spawn(async move { + // Envelopes the OS has been carrying are cleaned up here; the + // socket takes over for anything still unacknowledged. + if let Err(error) = OutboxDispatchService::new(&ctx).purge_expired().await { + tracing::warn!(%error, "could not purge finished outbox dispatches"); + } + }); } ApiRuntime::set_background(ctx, in_background).await } + /// Transcodes a captured video while the user is still editing it, so the + /// send itself only has to encrypt and hand over. + pub async fn prerender_media(media_id: String) -> Result<()> { + let ctx = Context::get_static()?; + MediaUploadService::new(ctx).prerender(&media_id).await + } + + /// Hands every queued envelope to an OS-owned transfer. Used when the app + /// is being torn down while messages are still unsent. + pub async fn hand_outbox_to_os() -> Result<()> { + let ctx = Context::get_static()?; + OutboxDispatchService::new(ctx).hand_pending_to_os().await?; + Ok(()) + } + pub async fn set_network_available(available: bool) -> Result<()> { let ctx = Context::get_static()?; ApiRuntime::set_network_available(ctx, available).await diff --git a/rust/src/context.rs b/rust/src/context.rs index ae801893..5af7fc55 100644 --- a/rust/src/context.rs +++ b/rust/src/context.rs @@ -19,7 +19,7 @@ use crate::utils::Shared; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::{ path::{Path, PathBuf}, - sync::Arc, + sync::{Arc, OnceLock as StdOnceLock}, }; use tokio::sync::{Mutex, Notify, OnceCell, RwLock}; use zeroize::Zeroize; @@ -36,6 +36,14 @@ pub enum RuntimeMode { pub struct Context { pub config: InitConfig, pub(crate) runtime_mode: RuntimeMode, + /// A native worker can initialize this process before Flutter does. Once + /// Flutter claims it, background jobs must neither open nor close a second + /// API connection through the shared client. + flutter_claimed: AtomicBool, + /// Runtime owned by flutter_rust_bridge. Native workers may call into the + /// same library from a temporary runtime; UI-facing pollers must be spawned + /// here so they survive that native call returning. + foreground_runtime: StdOnceLock, pub rust_db: Arc>>, pub app_db: Arc>>, pub(crate) secure_storage: SecureStorage, @@ -116,6 +124,8 @@ impl Context { let ctx = Arc::new(Context { config, runtime_mode: RuntimeMode::Standalone, + flutter_claimed: AtomicBool::new(false), + foreground_runtime: StdOnceLock::new(), rust_db, app_db, secure_storage, @@ -191,13 +201,20 @@ impl Context { // Logging is process-wide and owns app.log directly. Initialize it // before the context check so notification and Flutter runtimes both // have a sink even when the main context already exists. + let foreground_already_owns_process = GLOBAL_CONTEXT + .get() + .is_some_and(|context| context.is_flutter_runtime()); init_tracing( Path::new(&config.data_dir), - runtime_mode != RuntimeMode::Flutter, + runtime_mode != RuntimeMode::Flutter && !foreground_already_owns_process, ); if GLOBAL_CONTEXT.initialized() { tracing::info!("twonly already initialized. Ensuring storage directories exist."); + if runtime_mode == RuntimeMode::Flutter { + let context = GLOBAL_CONTEXT.get().ok_or(TwonlyError::Initialization)?; + Self::claim_for_flutter(context).await?; + } return Ok(()); } @@ -278,6 +295,8 @@ impl Context { let ctx = Arc::new(Context { config, runtime_mode, + flutter_claimed: AtomicBool::new(true), + foreground_runtime: StdOnceLock::new(), secure_storage, rust_db: rust_db_handle, app_db, @@ -319,6 +338,8 @@ impl Context { let ctx = Arc::new(Context { config, runtime_mode, + flutter_claimed: AtomicBool::new(false), + foreground_runtime: StdOnceLock::new(), rust_db: rust_db_handle, app_db, key_manager, @@ -341,12 +362,45 @@ impl Context { }) .await; let ctx = res?; - if runtime_mode != RuntimeMode::Notification { + if runtime_mode == RuntimeMode::Flutter { + Self::claim_for_flutter(ctx).await?; + } else if runtime_mode != RuntimeMode::Notification { ApiRuntime::connect(ctx).await?; } Ok(()) } + async fn claim_for_flutter(context: &Arc) -> Result<()> { + if let Ok(runtime) = tokio::runtime::Handle::try_current() { + let _ = context.foreground_runtime.set(runtime); + } + let newly_claimed = !context.flutter_claimed.swap(true, Ordering::AcqRel); + if newly_claimed && context.runtime_mode != RuntimeMode::Flutter { + tracing::info!("promoting native background runtime to Flutter ownership"); + // Rebuild the client so the next authentication is explicitly + // foreground. The old background client is closed before Flutter + // starts observing connection state. + ApiRuntime::reload_configuration(context).await + } else { + ApiRuntime::connect(context).await + } + } + + pub(crate) fn is_flutter_runtime(&self) -> bool { + self.runtime_mode == RuntimeMode::Flutter || self.flutter_claimed.load(Ordering::Acquire) + } + + pub(crate) fn is_notification_runtime(&self) -> bool { + self.runtime_mode == RuntimeMode::Notification + && !self.flutter_claimed.load(Ordering::Acquire) + } + + pub(crate) fn foreground_runtime(&self) -> Option { + self.is_flutter_runtime() + .then(|| self.foreground_runtime.get().cloned()) + .flatten() + } + pub(super) fn get_static() -> Result<&'static Arc> { GLOBAL_CONTEXT.get().ok_or(TwonlyError::Initialization) } diff --git a/rust/src/database/app/migrations/0009_outbox_dispatch.sql b/rust/src/database/app/migrations/0009_outbox_dispatch.sql new file mode 100644 index 00000000..eaef76ee --- /dev/null +++ b/rust/src/database/app/migrations/0009_outbox_dispatch.sql @@ -0,0 +1,21 @@ +-- Envelopes handed to an OS-owned transfer instead of the websocket. +-- +-- The socket only exists while the app runs, so a message queued while offline +-- waits for the next app launch rather than the next network. An envelope +-- prepared here is POSTed by WorkManager or a background URLSession, which both +-- wait for connectivity themselves and survive the app being closed. +-- +-- The receipt itself stays in `receipts` and keeps being retried over the +-- socket: this is an accelerator, not a replacement. A recipient that receives +-- both copies deduplicates them by receipt id, exactly as it already does for a +-- socket retransmission. +CREATE TABLE outbox_dispatch_jobs ( + receipt_id TEXT PRIMARY KEY, + dispatch_id TEXT NOT NULL UNIQUE, + contact_id INTEGER NOT NULL, + body_path TEXT NOT NULL, + expires_at INTEGER NOT NULL, + created_at INTEGER NOT NULL +); + +CREATE INDEX outbox_dispatch_jobs_expiry_idx ON outbox_dispatch_jobs(expires_at); diff --git a/rust/src/database/app/migrations/0010_media_trim.sql b/rust/src/database/app/migrations/0010_media_trim.sql new file mode 100644 index 00000000..53775195 --- /dev/null +++ b/rust/src/database/app/migrations/0010_media_trim.sql @@ -0,0 +1,11 @@ +-- Where the editor's cutter placed the two ends of a recorded video. +-- +-- Only the bounds are stored, never a cut copy of the clip: the send already +-- transcodes every video once, so the trim is applied by that pass and the +-- original recording stays intact until the media is retired. That also keeps +-- the cut reversible while the editor is open. +-- +-- NULL on either side means "not cut there", which is what an untouched clip +-- and every non-video media file carries. +ALTER TABLE media_files ADD COLUMN trim_start_ms INTEGER; +ALTER TABLE media_files ADD COLUMN trim_end_ms INTEGER; diff --git a/rust/src/database/app/mod.rs b/rust/src/database/app/mod.rs index 6962b396..beebbb20 100644 --- a/rust/src/database/app/mod.rs +++ b/rust/src/database/app/mod.rs @@ -16,7 +16,7 @@ mod legacy_import; pub mod tables; pub const APP_DATABASE_FILE: &str = "app_db.sqlite"; -pub const APP_SCHEMA_VERSION: i64 = 5; +pub const APP_SCHEMA_VERSION: i64 = 6; /// Tables imported from the legacy Drift database. Every entry must exist in /// Drift schema 25, because a missing table aborts the whole import. Rust-only diff --git a/rust/src/database/app/tables/mediafiles.rs b/rust/src/database/app/tables/mediafiles.rs index 679b90ff..90e747b8 100644 --- a/rust/src/database/app/tables/mediafiles.rs +++ b/rust/src/database/app/tables/mediafiles.rs @@ -32,20 +32,24 @@ impl MediaFile { /// Single owner of the "media reached the server" transition. The state /// change, the per-recipient message actions, and the receipt bookkeeping /// have to become visible together, so they share one transaction. + /// + /// Every statement below is written to touch only the rows that still need + /// it, because this runs more than once for the same media: the upload + /// settles it, and a receiver's response or reaction can arrive later and + /// settle it again. It must not short-circuit on the media row already + /// being `uploaded` - a message added to a media that was uploaded earlier + /// would then never leave the "sending" state, however long ago the + /// recipient received it. pub async fn mark_uploaded( transaction: &mut Transaction<'_, Sqlite>, media_id: &str, ) -> Result<()> { - let updated = sqlx::query!( + sqlx::query!( "UPDATE media_files SET upload_state = 'uploaded' WHERE media_id = ? AND upload_state IS NOT 'uploaded'", media_id, ) .execute(&mut **transaction) - .await? - .rows_affected(); - if updated == 0 { - return Ok(()); - } + .await?; let now = chrono::Utc::now().timestamp(); sqlx::query!( @@ -56,8 +60,10 @@ impl MediaFile { JOIN group_members ON group_members.group_id = messages.group_id WHERE messages.media_id = ? AND (group_members.member_state IS NULL OR group_members.member_state != 'leftGroup') + -- The first acknowledgement is the true one; a later settle of the + -- same media must not move a timestamp that already stands. ON CONFLICT(message_id, contact_id, type) - DO UPDATE SET action_at = excluded.action_at + DO NOTHING "#, now, media_id, @@ -65,19 +71,21 @@ impl MediaFile { .execute(&mut **transaction) .await?; - sqlx::query!( - "UPDATE messages SET ack_by_server = ? WHERE media_id = ?", + let acknowledged = sqlx::query!( + "UPDATE messages SET ack_by_server = ? WHERE media_id = ? AND ack_by_server IS NULL", now, media_id, ) .execute(&mut **transaction) - .await?; + .await? + .rows_affected(); sqlx::query!( r#" UPDATE receipts SET ack_by_server_at = ?, retry_count = 1, last_retry = ?, mark_for_retry = NULL - WHERE EXISTS( + WHERE ack_by_server_at IS NULL + AND EXISTS( SELECT 1 FROM messages JOIN group_members ON group_members.group_id = messages.group_id @@ -94,10 +102,13 @@ impl MediaFile { .execute(&mut **transaction) .await?; - tracing::info!( - media_id, - "marked media upload as successful after receiver response" - ); + if acknowledged > 0 { + tracing::info!( + media_id, + acknowledged, + "marked media upload as successful after receiver response" + ); + } Ok(()) } } diff --git a/rust/src/frb_generated.rs b/rust/src/frb_generated.rs index 4ad89108..baf69c6b 100644 --- a/rust/src/frb_generated.rs +++ b/rust/src/frb_generated.rs @@ -38,7 +38,7 @@ flutter_rust_bridge::frb_generated_boilerplate!( default_rust_auto_opaque = RustAutoOpaqueMoi, ); pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_VERSION: &str = "2.12.0"; -pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = -1370816897; +pub(crate) const FLUTTER_RUST_BRIDGE_CODEGEN_CONTENT_HASH: i32 = 1963960341; // Section: executor @@ -2078,6 +2078,41 @@ fn wire__crate__bridge__api__rust_api_get_user_id_from_username_impl( }, ) } +fn wire__crate__bridge__api__rust_api_hand_outbox_to_os_impl( + port_: flutter_rust_bridge::for_generated::MessagePort, + ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr, + rust_vec_len_: i32, + data_len_: i32, +) { + FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( + flutter_rust_bridge::for_generated::TaskInfo { + debug_name: "rust_api_hand_outbox_to_os", + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + }, + move || { + let message = unsafe { + flutter_rust_bridge::for_generated::Dart2RustMessageSse::from_wire( + ptr_, + rust_vec_len_, + data_len_, + ) + }; + let mut deserializer = + flutter_rust_bridge::for_generated::SseDeserializer::new(message); + deserializer.end(); + move |context| async move { + transform_result_sse::<_, flutter_rust_bridge::for_generated::anyhow::Error>( + (move || async move { + let output_ok = crate::bridge::api::RustApi::hand_outbox_to_os().await?; + Ok(output_ok) + })() + .await, + ) + } + }, + ) +} fn wire__crate__bridge__api__rust_api_initialize_media_upload_impl( port_: flutter_rust_bridge::for_generated::MessagePort, ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr, @@ -2562,6 +2597,43 @@ fn wire__crate__bridge__api__rust_api_perform_passwordless_recovery_heartbeat_im }, ) } +fn wire__crate__bridge__api__rust_api_prerender_media_impl( + port_: flutter_rust_bridge::for_generated::MessagePort, + ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr, + rust_vec_len_: i32, + data_len_: i32, +) { + FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( + flutter_rust_bridge::for_generated::TaskInfo { + debug_name: "rust_api_prerender_media", + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + }, + move || { + let message = unsafe { + flutter_rust_bridge::for_generated::Dart2RustMessageSse::from_wire( + ptr_, + rust_vec_len_, + data_len_, + ) + }; + let mut deserializer = + flutter_rust_bridge::for_generated::SseDeserializer::new(message); + let api_media_id = ::sse_decode(&mut deserializer); + deserializer.end(); + move |context| async move { + transform_result_sse::<_, flutter_rust_bridge::for_generated::anyhow::Error>( + (move || async move { + let output_ok = + crate::bridge::api::RustApi::prerender_media(api_media_id).await?; + Ok(output_ok) + })() + .await, + ) + } + }, + ) +} fn wire__crate__bridge__api__rust_api_purge_media_temp_folder_impl( port_: flutter_rust_bridge::for_generated::MessagePort, ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr, @@ -3669,6 +3741,49 @@ fn wire__crate__bridge__api__rust_api_set_media_requires_authentication_impl( }, ) } +fn wire__crate__bridge__api__rust_api_set_media_trim_impl( + port_: flutter_rust_bridge::for_generated::MessagePort, + ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr, + rust_vec_len_: i32, + data_len_: i32, +) { + FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::( + flutter_rust_bridge::for_generated::TaskInfo { + debug_name: "rust_api_set_media_trim", + port: Some(port_), + mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal, + }, + move || { + let message = unsafe { + flutter_rust_bridge::for_generated::Dart2RustMessageSse::from_wire( + ptr_, + rust_vec_len_, + data_len_, + ) + }; + let mut deserializer = + flutter_rust_bridge::for_generated::SseDeserializer::new(message); + let api_media_id = ::sse_decode(&mut deserializer); + let api_trim_start_ms = >::sse_decode(&mut deserializer); + let api_trim_end_ms = >::sse_decode(&mut deserializer); + deserializer.end(); + move |context| async move { + transform_result_sse::<_, flutter_rust_bridge::for_generated::anyhow::Error>( + (move || async move { + let output_ok = crate::bridge::api::RustApi::set_media_trim( + api_media_id, + api_trim_start_ms, + api_trim_end_ms, + ) + .await?; + Ok(output_ok) + })() + .await, + ) + } + }, + ) +} fn wire__crate__bridge__api__rust_api_set_network_available_impl( port_: flutter_rust_bridge::for_generated::MessagePort, ptr_: flutter_rust_bridge::for_generated::PlatformGeneralizedUint8ListPtr, @@ -4469,7 +4584,7 @@ fn wire__crate__bridge__wrapper__key_manager__rust_key_manager_remove_local_cred rust_vec_len_: i32, data_len_: i32, ) { - FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::(flutter_rust_bridge::for_generated::TaskInfo{ debug_name: "rust_key_manager_remove_local_credentials", port: Some(port_), mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal }, move || { + FLUTTER_RUST_BRIDGE_HANDLER.wrap_async::(flutter_rust_bridge::for_generated::TaskInfo{ debug_name: "rust_key_manager_remove_local_credentials", port: Some(port_), mode: flutter_rust_bridge::for_generated::FfiCallMode::Normal }, move || { let message = unsafe { flutter_rust_bridge::for_generated::Dart2RustMessageSse::from_wire(ptr_, rust_vec_len_, data_len_) }; let mut deserializer = flutter_rust_bridge::for_generated::SseDeserializer::new(message); deserializer.end(); move |context| async move { @@ -6545,93 +6660,96 @@ fn pde_ffi_dispatcher_primary_impl( 54 => wire__crate__bridge__api__rust_api_get_user_by_id_impl(port, ptr, rust_vec_len, data_len), 55 => wire__crate__bridge__api__rust_api_get_user_data_impl(port, ptr, rust_vec_len, data_len), 56 => wire__crate__bridge__api__rust_api_get_user_id_from_username_impl(port, ptr, rust_vec_len, data_len), -57 => wire__crate__bridge__api__rust_api_initialize_media_upload_impl(port, ptr, rust_vec_len, data_len), -58 => wire__crate__bridge__api__rust_api_insert_and_send_additional_data_impl(port, ptr, rust_vec_len, data_len), -59 => wire__crate__bridge__api__rust_api_insert_and_send_ask_about_user_impl(port, ptr, rust_vec_len, data_len), -60 => wire__crate__bridge__api__rust_api_insert_and_send_contact_share_impl(port, ptr, rust_vec_len, data_len), -61 => wire__crate__bridge__api__rust_api_insert_and_send_text_impl(port, ptr, rust_vec_len, data_len), -62 => wire__crate__bridge__api__rust_api_ipa_purchase_impl(port, ptr, rust_vec_len, data_len), -63 => wire__crate__bridge__api__rust_api_load_plan_balance_impl(port, ptr, rust_vec_len, data_len), -64 => wire__crate__bridge__api__rust_api_media_size_limit_report_impl(port, ptr, rust_vec_len, data_len), -65 => wire__crate__bridge__api__rust_api_media_step_finished_impl(port, ptr, rust_vec_len, data_len), -66 => wire__crate__bridge__api__rust_api_notification_badge_count_impl(port, ptr, rust_vec_len, data_len), -67 => wire__crate__bridge__api__rust_api_notify_messages_opened_impl(port, ptr, rust_vec_len, data_len), -68 => wire__crate__bridge__api__rust_api_perform_passwordless_recovery_heartbeat_impl(port, ptr, rust_vec_len, data_len), -69 => wire__crate__bridge__api__rust_api_purge_media_temp_folder_impl(port, ptr, rust_vec_len, data_len), -70 => wire__crate__bridge__api__rust_api_register_impl(port, ptr, rust_vec_len, data_len), -71 => wire__crate__bridge__api__rust_api_register_passwordless_notification_impl(port, ptr, rust_vec_len, data_len), -72 => wire__crate__bridge__api__rust_api_register_passwordless_recovery_impl(port, ptr, rust_vec_len, data_len), -73 => wire__crate__bridge__api__rust_api_reload_configuration_impl(port, ptr, rust_vec_len, data_len), -74 => wire__crate__bridge__api__rust_api_remove_additional_user_impl(port, ptr, rust_vec_len, data_len), -75 => wire__crate__bridge__api__rust_api_remove_media_files_impl(port, ptr, rust_vec_len, data_len), -76 => wire__crate__bridge__api__rust_api_report_user_impl(port, ptr, rust_vec_len, data_len), -77 => wire__crate__bridge__api__rust_api_request_binary_impl(port, ptr, rust_vec_len, data_len), -78 => wire__crate__bridge__api__rust_api_request_contact_by_username_impl(port, ptr, rust_vec_len, data_len), -79 => wire__crate__bridge__api__rust_api_request_media_reupload_impl(port, ptr, rust_vec_len, data_len), -80 => wire__crate__bridge__api__rust_api_request_memories_upload_impl(port, ptr, rust_vec_len, data_len), -81 => wire__crate__bridge__api__rust_api_retransmit_all_messages_impl(port, ptr, rust_vec_len, data_len), -82 => wire__crate__bridge__api__rust_api_retry_pending_media_reuploads_impl(port, ptr, rust_vec_len, data_len), -83 => wire__crate__bridge__api__rust_api_reupload_pending_media_impl(port, ptr, rust_vec_len, data_len), -84 => wire__crate__bridge__api__rust_api_save_media_to_gallery_impl(port, ptr, rust_vec_len, data_len), -85 => wire__crate__bridge__api__rust_api_send_binary_impl(port, ptr, rust_vec_len, data_len), -86 => wire__crate__bridge__api__rust_api_send_contact_profile_impl(port, ptr, rust_vec_len, data_len), -87 => wire__crate__bridge__api__rust_api_send_encrypted_content_impl(port, ptr, rust_vec_len, data_len), -88 => wire__crate__bridge__api__rust_api_send_encrypted_content_to_group_impl(port, ptr, rust_vec_len, data_len), -89 => wire__crate__bridge__api__rust_api_send_media_to_groups_impl(port, ptr, rust_vec_len, data_len), -90 => wire__crate__bridge__api__rust_api_send_queued_message_impl(port, ptr, rust_vec_len, data_len), -91 => wire__crate__bridge__api__rust_api_send_text_message_impl(port, ptr, rust_vec_len, data_len), -92 => wire__crate__bridge__api__rust_api_send_typing_impl(port, ptr, rust_vec_len, data_len), -93 => wire__crate__bridge__api__rust_api_set_background_impl(port, ptr, rust_vec_len, data_len), -94 => wire__crate__bridge__api__rust_api_set_login_token_impl(port, ptr, rust_vec_len, data_len), -95 => wire__crate__bridge__api__rust_api_set_media_display_limit_impl(port, ptr, rust_vec_len, data_len), -96 => wire__crate__bridge__api__rust_api_set_media_requires_authentication_impl(port, ptr, rust_vec_len, data_len), -97 => wire__crate__bridge__api__rust_api_set_network_available_impl(port, ptr, rust_vec_len, data_len), -98 => wire__crate__bridge__api__rust_api_store_media_impl(port, ptr, rust_vec_len, data_len), -99 => wire__crate__bridge__api__rust_api_submit_recovery_share_impl(port, ptr, rust_vec_len, data_len), -100 => wire__crate__bridge__api__rust_api_toggle_media_remove_audio_impl(port, ptr, rust_vec_len, data_len), -101 => wire__crate__bridge__api__rust_api_try_request_contact_by_id_impl(port, ptr, rust_vec_len, data_len), -102 => wire__crate__bridge__api__rust_api_update_fcm_token_impl(port, ptr, rust_vec_len, data_len), -103 => wire__crate__bridge__api__rust_api_update_signed_pre_key_impl(port, ptr, rust_vec_len, data_len), -104 => wire__crate__bridge__api__rust_api_upload_pqc_pre_keys_impl(port, ptr, rust_vec_len, data_len), -105 => wire__crate__bridge__wrapper__app_database__rust_app_database_changes_impl(port, ptr, rust_vec_len, data_len), -106 => wire__crate__bridge__wrapper__app_database__rust_app_database_execute_impl(port, ptr, rust_vec_len, data_len), -107 => wire__crate__bridge__wrapper__app_database__rust_app_database_legacy_import_complete_impl(port, ptr, rust_vec_len, data_len), -108 => wire__crate__bridge__wrapper__app_database__rust_app_database_migrate_legacy_database_impl(port, ptr, rust_vec_len, data_len), -109 => wire__crate__bridge__wrapper__app_database__rust_app_database_select_impl(port, ptr, rust_vec_len, data_len), -110 => wire__crate__bridge__wrapper__backup__rust_backup_archive_create_backup_archive_impl(port, ptr, rust_vec_len, data_len), -111 => wire__crate__bridge__wrapper__backup__rust_backup_archive_get_backup_download_token_impl(port, ptr, rust_vec_len, data_len), -112 => wire__crate__bridge__wrapper__backup__rust_backup_archive_restore_backup_archive_impl(port, ptr, rust_vec_len, data_len), -113 => wire__crate__bridge__wrapper__backup__rust_backup_identity_get_backup_id_impl(port, ptr, rust_vec_len, data_len), -114 => wire__crate__bridge__wrapper__backup__rust_backup_identity_get_backup_password_keys_impl(port, ptr, rust_vec_len, data_len), -115 => wire__crate__bridge__wrapper__backup__rust_backup_identity_get_identity_backup_bytes_impl(port, ptr, rust_vec_len, data_len), -116 => wire__crate__bridge__wrapper__backup__rust_backup_identity_import_backup_password_keys_impl(port, ptr, rust_vec_len, data_len), -117 => wire__crate__bridge__wrapper__backup__rust_backup_identity_restore_identity_backup_impl(port, ptr, rust_vec_len, data_len), -118 => wire__crate__bridge__wrapper__backup__rust_backup_identity_set_backup_password_keys_impl(port, ptr, rust_vec_len, data_len), -119 => wire__crate__bridge__wrapper__key_manager__rust_key_manager_decrypt_cloud_media_key_impl(port, ptr, rust_vec_len, data_len), -120 => wire__crate__bridge__wrapper__key_manager__rust_key_manager_encrypt_cloud_media_key_impl(port, ptr, rust_vec_len, data_len), -121 => wire__crate__bridge__wrapper__key_manager__rust_key_manager_get_signal_identity_impl(port, ptr, rust_vec_len, data_len), -122 => wire__crate__bridge__wrapper__key_manager__rust_key_manager_get_user_id_impl(port, ptr, rust_vec_len, data_len), -123 => wire__crate__bridge__wrapper__key_manager__rust_key_manager_import_serialized_impl(port, ptr, rust_vec_len, data_len), -124 => wire__crate__bridge__wrapper__key_manager__rust_key_manager_import_signal_identity_impl(port, ptr, rust_vec_len, data_len), -125 => wire__crate__bridge__wrapper__key_manager__rust_key_manager_remove_local_credentials_impl(port, ptr, rust_vec_len, data_len), -126 => wire__crate__bridge__wrapper__key_manager__rust_key_manager_serialize_impl(port, ptr, rust_vec_len, data_len), -127 => wire__crate__bridge__wrapper__key_manager__rust_key_manager_set_user_id_impl(port, ptr, rust_vec_len, data_len), -128 => wire__crate__bridge__wrapper__signal__rust_signal_decrypt_impl(port, ptr, rust_vec_len, data_len), -129 => wire__crate__bridge__wrapper__signal__rust_signal_encrypt_impl(port, ptr, rust_vec_len, data_len), -130 => wire__crate__bridge__wrapper__signal__rust_signal_generate_bundle_impl(port, ptr, rust_vec_len, data_len), -131 => wire__crate__bridge__wrapper__signal__rust_signal_generate_pqc_prekeys_impl(port, ptr, rust_vec_len, data_len), -132 => wire__crate__bridge__wrapper__signal__rust_signal_get_contact_public_key_impl(port, ptr, rust_vec_len, data_len), -133 => wire__crate__bridge__wrapper__signal__rust_signal_get_user_public_key_impl(port, ptr, rust_vec_len, data_len), -134 => wire__crate__bridge__wrapper__signal__rust_signal_process_prekey_bundle_impl(port, ptr, rust_vec_len, data_len), -135 => wire__crate__bridge__wrapper__rust_utils_generate_shares_impl(port, ptr, rust_vec_len, data_len), -136 => wire__crate__bridge__wrapper__rust_utils_recover_secret_impl(port, ptr, rust_vec_len, data_len), -137 => wire__crate__bridge__groups__update_chat_deletion_time_impl(port, ptr, rust_vec_len, data_len), -138 => wire__crate__bridge__groups__update_group_name_impl(port, ptr, rust_vec_len, data_len), -140 => wire__crate__bridge__user_config__user_config_api_create_impl(port, ptr, rust_vec_len, data_len), -141 => wire__crate__bridge__user_config__user_config_api_import_json_impl(port, ptr, rust_vec_len, data_len), -142 => wire__crate__bridge__user_config__user_config_api_load_impl(port, ptr, rust_vec_len, data_len), -143 => wire__crate__bridge__user_config__user_config_api_save_impl(port, ptr, rust_vec_len, data_len), -144 => wire__crate__bridge__user_config__user_config_api_update_impl(port, ptr, rust_vec_len, data_len), +57 => wire__crate__bridge__api__rust_api_hand_outbox_to_os_impl(port, ptr, rust_vec_len, data_len), +58 => wire__crate__bridge__api__rust_api_initialize_media_upload_impl(port, ptr, rust_vec_len, data_len), +59 => wire__crate__bridge__api__rust_api_insert_and_send_additional_data_impl(port, ptr, rust_vec_len, data_len), +60 => wire__crate__bridge__api__rust_api_insert_and_send_ask_about_user_impl(port, ptr, rust_vec_len, data_len), +61 => wire__crate__bridge__api__rust_api_insert_and_send_contact_share_impl(port, ptr, rust_vec_len, data_len), +62 => wire__crate__bridge__api__rust_api_insert_and_send_text_impl(port, ptr, rust_vec_len, data_len), +63 => wire__crate__bridge__api__rust_api_ipa_purchase_impl(port, ptr, rust_vec_len, data_len), +64 => wire__crate__bridge__api__rust_api_load_plan_balance_impl(port, ptr, rust_vec_len, data_len), +65 => wire__crate__bridge__api__rust_api_media_size_limit_report_impl(port, ptr, rust_vec_len, data_len), +66 => wire__crate__bridge__api__rust_api_media_step_finished_impl(port, ptr, rust_vec_len, data_len), +67 => wire__crate__bridge__api__rust_api_notification_badge_count_impl(port, ptr, rust_vec_len, data_len), +68 => wire__crate__bridge__api__rust_api_notify_messages_opened_impl(port, ptr, rust_vec_len, data_len), +69 => wire__crate__bridge__api__rust_api_perform_passwordless_recovery_heartbeat_impl(port, ptr, rust_vec_len, data_len), +70 => wire__crate__bridge__api__rust_api_prerender_media_impl(port, ptr, rust_vec_len, data_len), +71 => wire__crate__bridge__api__rust_api_purge_media_temp_folder_impl(port, ptr, rust_vec_len, data_len), +72 => wire__crate__bridge__api__rust_api_register_impl(port, ptr, rust_vec_len, data_len), +73 => wire__crate__bridge__api__rust_api_register_passwordless_notification_impl(port, ptr, rust_vec_len, data_len), +74 => wire__crate__bridge__api__rust_api_register_passwordless_recovery_impl(port, ptr, rust_vec_len, data_len), +75 => wire__crate__bridge__api__rust_api_reload_configuration_impl(port, ptr, rust_vec_len, data_len), +76 => wire__crate__bridge__api__rust_api_remove_additional_user_impl(port, ptr, rust_vec_len, data_len), +77 => wire__crate__bridge__api__rust_api_remove_media_files_impl(port, ptr, rust_vec_len, data_len), +78 => wire__crate__bridge__api__rust_api_report_user_impl(port, ptr, rust_vec_len, data_len), +79 => wire__crate__bridge__api__rust_api_request_binary_impl(port, ptr, rust_vec_len, data_len), +80 => wire__crate__bridge__api__rust_api_request_contact_by_username_impl(port, ptr, rust_vec_len, data_len), +81 => wire__crate__bridge__api__rust_api_request_media_reupload_impl(port, ptr, rust_vec_len, data_len), +82 => wire__crate__bridge__api__rust_api_request_memories_upload_impl(port, ptr, rust_vec_len, data_len), +83 => wire__crate__bridge__api__rust_api_retransmit_all_messages_impl(port, ptr, rust_vec_len, data_len), +84 => wire__crate__bridge__api__rust_api_retry_pending_media_reuploads_impl(port, ptr, rust_vec_len, data_len), +85 => wire__crate__bridge__api__rust_api_reupload_pending_media_impl(port, ptr, rust_vec_len, data_len), +86 => wire__crate__bridge__api__rust_api_save_media_to_gallery_impl(port, ptr, rust_vec_len, data_len), +87 => wire__crate__bridge__api__rust_api_send_binary_impl(port, ptr, rust_vec_len, data_len), +88 => wire__crate__bridge__api__rust_api_send_contact_profile_impl(port, ptr, rust_vec_len, data_len), +89 => wire__crate__bridge__api__rust_api_send_encrypted_content_impl(port, ptr, rust_vec_len, data_len), +90 => wire__crate__bridge__api__rust_api_send_encrypted_content_to_group_impl(port, ptr, rust_vec_len, data_len), +91 => wire__crate__bridge__api__rust_api_send_media_to_groups_impl(port, ptr, rust_vec_len, data_len), +92 => wire__crate__bridge__api__rust_api_send_queued_message_impl(port, ptr, rust_vec_len, data_len), +93 => wire__crate__bridge__api__rust_api_send_text_message_impl(port, ptr, rust_vec_len, data_len), +94 => wire__crate__bridge__api__rust_api_send_typing_impl(port, ptr, rust_vec_len, data_len), +95 => wire__crate__bridge__api__rust_api_set_background_impl(port, ptr, rust_vec_len, data_len), +96 => wire__crate__bridge__api__rust_api_set_login_token_impl(port, ptr, rust_vec_len, data_len), +97 => wire__crate__bridge__api__rust_api_set_media_display_limit_impl(port, ptr, rust_vec_len, data_len), +98 => wire__crate__bridge__api__rust_api_set_media_requires_authentication_impl(port, ptr, rust_vec_len, data_len), +99 => wire__crate__bridge__api__rust_api_set_media_trim_impl(port, ptr, rust_vec_len, data_len), +100 => wire__crate__bridge__api__rust_api_set_network_available_impl(port, ptr, rust_vec_len, data_len), +101 => wire__crate__bridge__api__rust_api_store_media_impl(port, ptr, rust_vec_len, data_len), +102 => wire__crate__bridge__api__rust_api_submit_recovery_share_impl(port, ptr, rust_vec_len, data_len), +103 => wire__crate__bridge__api__rust_api_toggle_media_remove_audio_impl(port, ptr, rust_vec_len, data_len), +104 => wire__crate__bridge__api__rust_api_try_request_contact_by_id_impl(port, ptr, rust_vec_len, data_len), +105 => wire__crate__bridge__api__rust_api_update_fcm_token_impl(port, ptr, rust_vec_len, data_len), +106 => wire__crate__bridge__api__rust_api_update_signed_pre_key_impl(port, ptr, rust_vec_len, data_len), +107 => wire__crate__bridge__api__rust_api_upload_pqc_pre_keys_impl(port, ptr, rust_vec_len, data_len), +108 => wire__crate__bridge__wrapper__app_database__rust_app_database_changes_impl(port, ptr, rust_vec_len, data_len), +109 => wire__crate__bridge__wrapper__app_database__rust_app_database_execute_impl(port, ptr, rust_vec_len, data_len), +110 => wire__crate__bridge__wrapper__app_database__rust_app_database_legacy_import_complete_impl(port, ptr, rust_vec_len, data_len), +111 => wire__crate__bridge__wrapper__app_database__rust_app_database_migrate_legacy_database_impl(port, ptr, rust_vec_len, data_len), +112 => wire__crate__bridge__wrapper__app_database__rust_app_database_select_impl(port, ptr, rust_vec_len, data_len), +113 => wire__crate__bridge__wrapper__backup__rust_backup_archive_create_backup_archive_impl(port, ptr, rust_vec_len, data_len), +114 => wire__crate__bridge__wrapper__backup__rust_backup_archive_get_backup_download_token_impl(port, ptr, rust_vec_len, data_len), +115 => wire__crate__bridge__wrapper__backup__rust_backup_archive_restore_backup_archive_impl(port, ptr, rust_vec_len, data_len), +116 => wire__crate__bridge__wrapper__backup__rust_backup_identity_get_backup_id_impl(port, ptr, rust_vec_len, data_len), +117 => wire__crate__bridge__wrapper__backup__rust_backup_identity_get_backup_password_keys_impl(port, ptr, rust_vec_len, data_len), +118 => wire__crate__bridge__wrapper__backup__rust_backup_identity_get_identity_backup_bytes_impl(port, ptr, rust_vec_len, data_len), +119 => wire__crate__bridge__wrapper__backup__rust_backup_identity_import_backup_password_keys_impl(port, ptr, rust_vec_len, data_len), +120 => wire__crate__bridge__wrapper__backup__rust_backup_identity_restore_identity_backup_impl(port, ptr, rust_vec_len, data_len), +121 => wire__crate__bridge__wrapper__backup__rust_backup_identity_set_backup_password_keys_impl(port, ptr, rust_vec_len, data_len), +122 => wire__crate__bridge__wrapper__key_manager__rust_key_manager_decrypt_cloud_media_key_impl(port, ptr, rust_vec_len, data_len), +123 => wire__crate__bridge__wrapper__key_manager__rust_key_manager_encrypt_cloud_media_key_impl(port, ptr, rust_vec_len, data_len), +124 => wire__crate__bridge__wrapper__key_manager__rust_key_manager_get_signal_identity_impl(port, ptr, rust_vec_len, data_len), +125 => wire__crate__bridge__wrapper__key_manager__rust_key_manager_get_user_id_impl(port, ptr, rust_vec_len, data_len), +126 => wire__crate__bridge__wrapper__key_manager__rust_key_manager_import_serialized_impl(port, ptr, rust_vec_len, data_len), +127 => wire__crate__bridge__wrapper__key_manager__rust_key_manager_import_signal_identity_impl(port, ptr, rust_vec_len, data_len), +128 => wire__crate__bridge__wrapper__key_manager__rust_key_manager_remove_local_credentials_impl(port, ptr, rust_vec_len, data_len), +129 => wire__crate__bridge__wrapper__key_manager__rust_key_manager_serialize_impl(port, ptr, rust_vec_len, data_len), +130 => wire__crate__bridge__wrapper__key_manager__rust_key_manager_set_user_id_impl(port, ptr, rust_vec_len, data_len), +131 => wire__crate__bridge__wrapper__signal__rust_signal_decrypt_impl(port, ptr, rust_vec_len, data_len), +132 => wire__crate__bridge__wrapper__signal__rust_signal_encrypt_impl(port, ptr, rust_vec_len, data_len), +133 => wire__crate__bridge__wrapper__signal__rust_signal_generate_bundle_impl(port, ptr, rust_vec_len, data_len), +134 => wire__crate__bridge__wrapper__signal__rust_signal_generate_pqc_prekeys_impl(port, ptr, rust_vec_len, data_len), +135 => wire__crate__bridge__wrapper__signal__rust_signal_get_contact_public_key_impl(port, ptr, rust_vec_len, data_len), +136 => wire__crate__bridge__wrapper__signal__rust_signal_get_user_public_key_impl(port, ptr, rust_vec_len, data_len), +137 => wire__crate__bridge__wrapper__signal__rust_signal_process_prekey_bundle_impl(port, ptr, rust_vec_len, data_len), +138 => wire__crate__bridge__wrapper__rust_utils_generate_shares_impl(port, ptr, rust_vec_len, data_len), +139 => wire__crate__bridge__wrapper__rust_utils_recover_secret_impl(port, ptr, rust_vec_len, data_len), +140 => wire__crate__bridge__groups__update_chat_deletion_time_impl(port, ptr, rust_vec_len, data_len), +141 => wire__crate__bridge__groups__update_group_name_impl(port, ptr, rust_vec_len, data_len), +143 => wire__crate__bridge__user_config__user_config_api_create_impl(port, ptr, rust_vec_len, data_len), +144 => wire__crate__bridge__user_config__user_config_api_import_json_impl(port, ptr, rust_vec_len, data_len), +145 => wire__crate__bridge__user_config__user_config_api_load_impl(port, ptr, rust_vec_len, data_len), +146 => wire__crate__bridge__user_config__user_config_api_save_impl(port, ptr, rust_vec_len, data_len), +147 => wire__crate__bridge__user_config__user_config_api_update_impl(port, ptr, rust_vec_len, data_len), _ => unreachable!(), } } @@ -6649,12 +6767,12 @@ fn pde_ffi_dispatcher_sync_impl( 37 => { wire__crate__bridge__api__rust_api_decode_avatar_svg_impl(ptr, rust_vec_len, data_len) } - 139 => wire__crate__bridge__user_config__user_config_api_clone_impl( + 142 => wire__crate__bridge__user_config__user_config_api_clone_impl( ptr, rust_vec_len, data_len, ), - 145 => wire__crate__bridge__logging__write_log_impl(ptr, rust_vec_len, data_len), + 148 => wire__crate__bridge__logging__write_log_impl(ptr, rust_vec_len, data_len), _ => unreachable!(), } } diff --git a/rust/src/log.rs b/rust/src/log.rs index 8fd09dfa..25cae3bb 100644 --- a/rust/src/log.rs +++ b/rust/src/log.rs @@ -283,9 +283,13 @@ where tracing::Level::WARN => "\x1b[33m", tracing::Level::ERROR => "\x1b[31m", }; - write!(writer, "{level_color}{:<5}\x1b[0m ", metadata.level())?; + write!( + writer, + "{level_color}{:<5}\x1b[0m [twonly] ", + metadata.level() + )?; } else { - write!(writer, "{time} {:<5} ", metadata.level())?; + write!(writer, "{time} {:<5} [twonly] ", metadata.level())?; } if ansi { diff --git a/rust/src/native/background.rs b/rust/src/native/background.rs new file mode 100644 index 00000000..9003687c --- /dev/null +++ b/rust/src/native/background.rs @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2026, Tobias Müller git@tsmr.eu + * + */ + +//! Stable synchronous entry points for the platform's background schedulers. +//! +//! `WorkManager` and `BGTaskScheduler` may start the process with no Flutter +//! engine in it, so these mirror the notification ABI in +//! [`crate::native::notifications`]: they own their Tokio runtime and take the +//! storage directories rather than assuming an initialized context. + +use crate::bridge::InitConfig; +use crate::services::background::{self, Job, RunOutcome}; +use std::ffi::{c_char, CStr, CString}; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::{LazyLock, Mutex, MutexGuard}; + +/// One maintenance run at a time per process. Two concurrent runs would only +/// contend on the same per-media locks and the same socket. +static MAINTENANCE_WORKER: LazyLock> = LazyLock::new(|| Mutex::new(())); + +fn worker_lock() -> MutexGuard<'static, ()> { + MAINTENANCE_WORKER + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +fn runtime() -> Result { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .map_err(|error| format!("could not create maintenance runtime: {error}")) +} + +unsafe fn required_string(pointer: *const c_char, name: &str) -> Result { + if pointer.is_null() { + return Err(format!("{name} is null")); + } + // SAFETY: Native callers promise a valid NUL-terminated string for the + // duration of this synchronous function call. + unsafe { CStr::from_ptr(pointer) } + .to_str() + .map(str::to_owned) + .map_err(|error| format!("{name} is not UTF-8: {error}")) +} + +unsafe fn optional_string(pointer: *const c_char) -> Option { + if pointer.is_null() { + return None; + } + unsafe { CStr::from_ptr(pointer) } + .to_str() + .ok() + .map(str::to_owned) + .filter(|value| !value.is_empty()) +} + +fn response_json(outcome: Option, error: Option) -> *mut c_char { + let json = match (outcome, error) { + (Some(outcome), None) => format!( + r#"{{"ok":true,"pending_uploads":{}}}"#, + outcome.pending_uploads + ), + (_, Some(error)) => format!( + r#"{{"ok":false,"error":{}}}"#, + serde_json::to_string(&error).unwrap_or_else(|_| "null".into()) + ), + (None, None) => r#"{"ok":false,"error":"missing background outcome"}"#.to_owned(), + }; + CString::new(json) + .expect("JSON serializers must escape interior NUL bytes") + .into_raw() +} + +/// Runs one maintenance job and returns a JSON `{"ok":bool,"error":string?}`. +/// +/// A null or empty `media_id` runs the full flush; otherwise only that media +/// file is prepared. The returned pointer must be released with +/// `twonly_background_string_free`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn twonly_background_run( + database_dir: *const c_char, + data_dir: *const c_char, + media_id: *const c_char, +) -> *mut c_char { + let result = catch_unwind(AssertUnwindSafe(|| { + let _worker = worker_lock(); + let database_dir = unsafe { required_string(database_dir, "database_dir") }?; + let data_dir = unsafe { required_string(data_dir, "data_dir") }?; + let job = match unsafe { optional_string(media_id) } { + Some(media_id) => Job::PrepareMedia(media_id), + None => Job::Flush, + }; + runtime()? + .block_on(background::run( + InitConfig { + database_dir, + data_dir, + }, + job, + )) + .map_err(|error| error.to_string()) + })); + + match result { + Ok(Ok(outcome)) => response_json(Some(outcome), None), + Ok(Err(error)) => response_json(None, Some(error)), + Err(_) => response_json(None, Some("background maintenance panicked".into())), + } +} + +/// Releases a string returned by a Twonly background entry point. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn twonly_background_string_free(pointer: *mut c_char) { + if !pointer.is_null() { + // SAFETY: The pointer was allocated by `CString::into_raw` above and + // ownership is transferred back exactly once by the native caller. + drop(unsafe { CString::from_raw(pointer) }); + } +} + +#[cfg(target_os = "android")] +mod android_jni { + use super::*; + use jni::objects::{JClass, JString}; + use jni::sys::jstring; + use jni::JNIEnv; + + #[unsafe(no_mangle)] + pub extern "system" fn Java_eu_twonly_directmedia_NativeMediaPrepareBridge_run( + mut env: JNIEnv<'_>, + _class: JClass<'_>, + database_dir: JString<'_>, + data_dir: JString<'_>, + media_id: JString<'_>, + ) -> jstring { + let result = (|| -> Result { + let java_string = + |env: &mut JNIEnv<'_>, value: JString<'_>| -> Result { + env.get_string(&value) + .map(Into::into) + .map_err(|error| format!("invalid Java string: {error}")) + }; + let database_dir = CString::new(java_string(&mut env, database_dir)?) + .map_err(|error| error.to_string())?; + let data_dir = CString::new(java_string(&mut env, data_dir)?) + .map_err(|error| error.to_string())?; + // A null Java string is the flush job, so it is not required here. + let media_id = if media_id.is_null() { + None + } else { + Some( + CString::new(java_string(&mut env, media_id)?) + .map_err(|error| error.to_string())?, + ) + }; + // SAFETY: Each CString remains alive for the synchronous ABI call. + let pointer = unsafe { + twonly_background_run( + database_dir.as_ptr(), + data_dir.as_ptr(), + media_id + .as_ref() + .map_or(std::ptr::null(), |value| value.as_ptr()), + ) + }; + if pointer.is_null() { + return Err("Rust maintenance worker returned null".into()); + } + // SAFETY: The C ABI returns a valid owned CString. + let json = unsafe { CStr::from_ptr(pointer) } + .to_string_lossy() + .into_owned(); + unsafe { twonly_background_string_free(pointer) }; + Ok(json) + })(); + let json = result.unwrap_or_else(|error| { + format!( + r#"{{"ok":false,"error":{}}}"#, + serde_json::to_string(&error).unwrap_or_else(|_| "null".into()) + ) + }); + env.new_string(json) + .map(JString::into_raw) + .unwrap_or(std::ptr::null_mut()) + } +} diff --git a/rust/src/native/mod.rs b/rust/src/native/mod.rs index e8ef7467..e89578b5 100644 --- a/rust/src/native/mod.rs +++ b/rust/src/native/mod.rs @@ -10,8 +10,10 @@ //! decoder, durable background transfer, the photo library and notifications. //! None of them require the Flutter engine to be running. +pub(crate) mod background; pub(crate) mod gallery; pub(crate) mod image; pub(crate) mod notifications; +pub(crate) mod prepare; pub(crate) mod transfer; pub(crate) mod video; diff --git a/rust/src/native/prepare.rs b/rust/src/native/prepare.rs new file mode 100644 index 00000000..a01e4021 --- /dev/null +++ b/rust/src/native/prepare.rs @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2026, Tobias Müller git@tsmr.eu + * + */ + +//! Keeping media preparation alive when the app is not. +//! +//! Everything between the send button and [`crate::native::transfer::schedule`] +//! — transcoding, encryption, and one Signal ratchet step per recipient — runs +//! in this process. Once the transfer is scheduled the OS owns it, but until +//! then a process death strands the send until the next launch. Each platform +//! protects that window differently: +//! +//! * Android runs the preparation inside a `WorkManager` job, which survives +//! the task being swiped away and is re-run after a reboot. +//! * iOS cannot outlive a force quit at all, so the preparation stays in this +//! process under a `UIApplication` background task assertion, which buys the +//! roughly thirty seconds that a backgrounded app is still allowed to run. + +/// How this platform is keeping a preparation alive. +pub(crate) enum Preparation { + /// The OS owns a job that runs the preparation itself; the caller must not + /// start one. + #[cfg_attr(not(target_os = "android"), allow(dead_code))] + Scheduled, + /// The preparation belongs to this process. The caller runs it and holds + /// the guard for as long as it does. + InProcess(Guard), +} + +/// Releases the platform's background execution assertion when dropped. +pub(crate) struct Guard { + #[cfg_attr(not(target_os = "ios"), allow(dead_code))] + assertion: Option, +} + +impl Guard { + /// A guard for a platform that offers no assertion to take. + #[cfg_attr(target_os = "ios", allow(dead_code))] + pub(crate) fn unprotected() -> Self { + Self { assertion: None } + } +} + +impl Drop for Guard { + fn drop(&mut self) { + #[cfg(target_os = "ios")] + if let Some(assertion) = self.assertion.take() { + ios::end_background_task(assertion); + } + } +} + +/// Asks the platform to protect the preparation of `media_id`. +/// +/// Never fails: a platform that cannot protect the work still has to prepare +/// it, and an interrupted preparation is picked up by the next maintenance +/// pass. The reason is logged so a missing native hook is visible. +pub(crate) fn begin(media_id: &str) -> Preparation { + #[cfg(target_os = "android")] + { + match android::schedule(media_id) { + Ok(()) => return Preparation::Scheduled, + Err(error) => { + tracing::warn!(media_id, %error, "no WorkManager preparation job; preparing in process"); + } + } + } + #[cfg(target_os = "ios")] + { + let _ = media_id; + return Preparation::InProcess(Guard { + assertion: ios::begin_background_task(), + }); + } + #[cfg(not(target_os = "ios"))] + { + let _ = media_id; + Preparation::InProcess(Guard::unprotected()) + } +} + +#[cfg(target_os = "android")] +mod android { + use crate::error::{Result, TwonlyError}; + use crate::native::transfer::android::jni_env; + use jni::objects::{JObject, JValue}; + + pub(super) fn schedule(media_id: &str) -> Result<()> { + let prepare = crate::native::transfer::android::prepare_class()?; + let mut env = jni_env()?; + let media_id = env + .new_string(media_id) + .map_err(|error| TwonlyError::Generic(error.to_string()))?; + let media_id = JObject::from(media_id); + let call = env + .call_static_method( + prepare, + "schedule", + "(Ljava/lang/String;)Z", + &[JValue::Object(&media_id)], + ) + .and_then(|value| value.z()); + // A pending Java exception would be raised as a fatal error when this + // thread next enters the VM, taking the process with it. + if env.exception_check().unwrap_or(false) { + let _ = env.exception_describe(); + let _ = env.exception_clear(); + } + if call.map_err(|error| TwonlyError::Generic(error.to_string()))? { + Ok(()) + } else { + Err(TwonlyError::Generic( + "Android rejected the media preparation job".into(), + )) + } + } +} + +#[cfg(target_os = "ios")] +mod ios { + use std::os::raw::c_char; + + type BeginBackgroundTask = unsafe extern "C" fn() -> u64; + type EndBackgroundTask = unsafe extern "C" fn(u64); + + fn symbol(name: &std::ffi::CStr) -> Option<*mut std::ffi::c_void> { + // The implementation lives in the app executable, so it is resolved at + // runtime rather than being required when the cdylib is linked. + let pointer = unsafe { libc::dlsym(libc::RTLD_DEFAULT, name.as_ptr().cast::()) }; + (!pointer.is_null()).then_some(pointer) + } + + pub(super) fn begin_background_task() -> Option { + let callback = symbol(c"twonly_begin_background_task")?; + // SAFETY: the Swift @_cdecl declaration has this exact C ABI signature. + let callback: BeginBackgroundTask = unsafe { std::mem::transmute(callback) }; + let identifier = unsafe { callback() }; + // UIBackgroundTaskInvalid is reported as zero. + (identifier != 0).then_some(identifier) + } + + pub(super) fn end_background_task(identifier: u64) { + let Some(callback) = symbol(c"twonly_end_background_task") else { + return; + }; + // SAFETY: the Swift @_cdecl declaration has this exact C ABI signature. + let callback: EndBackgroundTask = unsafe { std::mem::transmute(callback) }; + unsafe { callback(identifier) }; + } +} diff --git a/rust/src/native/transfer.rs b/rust/src/native/transfer.rs index 30773586..fb1624c2 100644 --- a/rust/src/native/transfer.rs +++ b/rust/src/native/transfer.rs @@ -40,12 +40,14 @@ pub(crate) mod android { use std::sync::OnceLock; const TRANSFER_CLASS: &str = "eu/twonly/directmedia/DirectMediaTransfer"; + const PREPARE_CLASS: &str = "eu/twonly/directmedia/DirectMediaPrepare"; const MEDIA_CODEC_CLASS: &str = "eu/twonly/media/NativeImageCodec"; const VIDEO_CODEC_CLASS: &str = "eu/twonly/media/NativeVideoCodec"; const GALLERY_CLASS: &str = "eu/twonly/media/NativeGallery"; static JAVA_VM: OnceLock = OnceLock::new(); static TRANSFER: OnceLock = OnceLock::new(); + static PREPARE: OnceLock = OnceLock::new(); static MEDIA_CODEC: OnceLock = OnceLock::new(); static VIDEO_CODEC: OnceLock = OnceLock::new(); static GALLERY: OnceLock = OnceLock::new(); @@ -65,6 +67,7 @@ pub(crate) mod android { Ok(mut env) => { for (name, cache) in [ (TRANSFER_CLASS, &TRANSFER), + (PREPARE_CLASS, &PREPARE), (MEDIA_CODEC_CLASS, &MEDIA_CODEC), (VIDEO_CODEC_CLASS, &VIDEO_CODEC), (GALLERY_CLASS, &GALLERY), @@ -104,6 +107,10 @@ pub(crate) mod android { }) } + pub(crate) fn prepare_class() -> Result<&'static GlobalRef> { + cached_class(&PREPARE, PREPARE_CLASS) + } + pub(crate) fn media_codec_class() -> Result<&'static GlobalRef> { cached_class(&MEDIA_CODEC, MEDIA_CODEC_CLASS) } diff --git a/rust/src/native/video.rs b/rust/src/native/video.rs index 856a1b95..f718c21a 100644 --- a/rust/src/native/video.rs +++ b/rust/src/native/video.rs @@ -11,8 +11,9 @@ //! background or was relaunched without its UI. use crate::error::{Result, TwonlyError}; +use std::collections::HashMap; use std::path::{Path, PathBuf}; -use std::sync::OnceLock; +use std::sync::{LazyLock, Mutex}; /// One render request. The overlay is a pre-rasterised PNG the size of the /// video frame; the editor draws it once when the user hits send. Owned so the @@ -25,22 +26,62 @@ pub(crate) struct RenderRequest { pub overlay: Option, pub output: PathBuf, pub remove_audio: bool, + /// Where the editor's cutter placed the two ends, in milliseconds into the + /// recording. `None` keeps that end. Both platforms take the cut as part of + /// the same pass that composites and encodes, so trimming costs nothing on + /// top of a render that was going to happen anyway. + pub trim_start_ms: Option, + pub trim_end_ms: Option, +} + +/// The bounds as the platform entry points take them: milliseconds, with a +/// negative value standing for "not cut on this end". Keeping the C and JNI +/// signatures to plain integers avoids an optional across either boundary. +#[cfg_attr(not(any(target_os = "android", target_os = "ios")), allow(dead_code))] +impl RenderRequest { + fn trim_start_or_negative(&self) -> i64 { + self.trim_start_ms.unwrap_or(-1) + } + + fn trim_end_or_negative(&self) -> i64 { + self.trim_end_ms.unwrap_or(-1) + } } /// The platform reports progress from its own thread — Android's main looper, -/// a GCD timer on iOS — which is not inside the async runtime. Spawning from -/// there would panic, and a panic unwinding back through the C ABI aborts the -/// process, so the runtime is captured while still on a Rust thread. +/// a GCD timer on iOS — which is not inside the async runtime. Keep the runtime +/// for each active render only. A process-wide `OnceLock` used to retain +/// the first WorkManager runtime forever, even after that runtime was destroyed. #[cfg_attr(not(any(target_os = "android", target_os = "ios")), allow(dead_code))] -static RUNTIME: OnceLock = OnceLock::new(); +static PROGRESS_RUNTIMES: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); #[cfg_attr(not(any(target_os = "android", target_os = "ios")), allow(dead_code))] -fn remember_runtime() { - if let Ok(handle) = tokio::runtime::Handle::try_current() { - let _ = RUNTIME.set(handle); +struct ProgressRuntimeGuard { + media_id: String, +} + +#[cfg_attr(not(any(target_os = "android", target_os = "ios")), allow(dead_code))] +impl Drop for ProgressRuntimeGuard { + fn drop(&mut self) { + if let Ok(mut runtimes) = PROGRESS_RUNTIMES.lock() { + runtimes.remove(&self.media_id); + } } } +#[cfg_attr(not(any(target_os = "android", target_os = "ios")), allow(dead_code))] +fn remember_runtime(media_id: &str) -> Option { + let handle = tokio::runtime::Handle::try_current().ok()?; + PROGRESS_RUNTIMES + .lock() + .ok()? + .insert(media_id.to_owned(), handle); + Some(ProgressRuntimeGuard { + media_id: media_id.to_owned(), + }) +} + /// Reports transcoding progress back into the media row so the send state in /// chat can show it. Called by the platform while a render is running. #[cfg_attr(not(any(target_os = "android", target_os = "ios")), allow(dead_code))] @@ -48,25 +89,32 @@ fn report_progress(media_id: &str, percent: i64) { let Ok(ctx) = crate::context::Context::get_static() else { return; }; - let Some(runtime) = RUNTIME.get() else { + let Some(runtime) = PROGRESS_RUNTIMES + .lock() + .ok() + .and_then(|runtimes| runtimes.get(media_id).cloned()) + else { return; }; let ctx = ctx.clone(); let media_id = media_id.to_owned(); runtime.spawn(async move { let database = ctx.app_db.read().await.clone(); - let _ = + if let Err(error) = sqlx::query("UPDATE media_files SET pre_progressing_process = ? WHERE media_id = ?") .bind(percent.clamp(0, 100)) .bind(&media_id) .execute(&database.pool) - .await; + .await + { + tracing::warn!(media_id, %error, "could not store video render progress"); + } }); } #[cfg(target_os = "ios")] pub(crate) fn render(request: &RenderRequest) -> Result<()> { - remember_runtime(); + let _progress_runtime = remember_runtime(&request.media_id); use std::ffi::{CStr, CString}; use std::os::raw::{c_char, c_int}; @@ -76,6 +124,8 @@ pub(crate) fn render(request: &RenderRequest) -> Result<()> { *const c_char, *const c_char, bool, + i64, + i64, *const c_char, ProgressCallback, ) -> bool; @@ -126,6 +176,8 @@ pub(crate) fn render(request: &RenderRequest) -> Result<()> { .map_or(std::ptr::null(), |path| path.as_ptr()), output.as_ptr(), request.remove_audio, + request.trim_start_or_negative(), + request.trim_end_or_negative(), media_id.as_ptr(), progress, ) @@ -141,7 +193,7 @@ pub(crate) fn render(request: &RenderRequest) -> Result<()> { #[cfg(target_os = "android")] pub(crate) fn render(request: &RenderRequest) -> Result<()> { - remember_runtime(); + let _progress_runtime = remember_runtime(&request.media_id); use crate::native::transfer::android::{jni_env, video_codec_class}; use jni::objects::{JObject, JValue}; @@ -163,12 +215,14 @@ pub(crate) fn render(request: &RenderRequest) -> Result<()> { .call_static_method( class, "render", - "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZLjava/lang/String;)Z", + "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;ZJJLjava/lang/String;)Z", &[ JValue::Object(&input), JValue::Object(&overlay), JValue::Object(&output), JValue::Bool(u8::from(request.remove_audio)), + JValue::Long(request.trim_start_or_negative()), + JValue::Long(request.trim_end_or_negative()), JValue::Object(&media_id), ], ) diff --git a/rust/src/services/background.rs b/rust/src/services/background.rs new file mode 100644 index 00000000..75f6594c --- /dev/null +++ b/rust/src/services/background.rs @@ -0,0 +1,163 @@ +/* + * Copyright (c) 2026, Tobias Müller git@tsmr.eu + * + */ + +//! Work the app does when it is not running. +//! +//! Everything a send needs is durable — the receipt outbox, the media state +//! machine, the direct-media job rows — but until now only a Flutter launch +//! resumed any of it. A user who sends and closes the app, or who composes +//! offline and never reopens, is exactly the case that leaves. These entry +//! points are driven by the platform's own schedulers (`WorkManager` on +//! Android, `BGTaskScheduler` on iOS) and need neither Flutter nor a visible +//! app. + +use crate::api::messages::incoming::messages; +use crate::api::runtime::ApiRuntime; +use crate::bridge::api::ApiConnectionState; +use crate::bridge::InitConfig; +use crate::context::Context; +use crate::error::Result; +use crate::services::direct_media_upload::DirectMediaUploadService; +use crate::services::media_upload::MediaUploadService; +use crate::services::outbox_dispatch::OutboxDispatchService; +use std::sync::Arc; +use std::time::Duration; + +/// How long to wait for the socket to authenticate before giving up on the +/// parts of the flush that need it. A background job has a hard budget of its +/// own, so this stays well inside it. +const AUTHENTICATION_TIMEOUT: Duration = Duration::from_secs(20); +const AUTHENTICATION_POLL: Duration = Duration::from_millis(250); + +/// What a maintenance run should do. +pub enum Job { + /// Prepare one media file and hand it to the OS uploader. This is the job + /// scheduled the moment the user hits send. + PrepareMedia(String), + /// Resume everything that was left in flight: interrupted preparations, + /// transfers the OS has since settled, and the message outbox. + Flush, +} + +/// State the native scheduler needs in order to decide whether this job is +/// finished or must be retried later by the OS. +pub struct RunOutcome { + pub pending_uploads: bool, +} + +/// Runs a maintenance job in a process that may have no Flutter engine. +/// +/// Safe to call concurrently with a running app: every step it performs is the +/// same idempotent one the app runs itself, guarded by the same locks. +pub async fn run(config: InitConfig, job: Job) -> Result { + Context::init_notification(config).await?; + let ctx = Context::get_static()?.clone(); + + // A send that has not been prepared yet needs an upload slot, and one for a + // contact with no Signal session yet needs a prekey bundle. Both want the + // network, so the connection is opened before the work rather than after. + // + // Except when the app is up in this same process: this job owns a Tokio + // runtime that is dropped the moment it returns, so connecting here would + // put the socket's reader on a runtime that is about to go away and kill a + // connection the app is relying on. The app's own post-authentication sweep + // already does all of this, so the job sticks to the media work. + let owns_connection = !ctx.is_flutter_runtime(); + let connected = if owns_connection { + connect_and_authenticate(&ctx).await + } else { + matches!( + ApiRuntime::connection_state(&ctx).await, + Ok(ApiConnectionState::Authenticated) + ) + }; + if !connected { + tracing::info!("background maintenance is running without a server connection"); + } + + match &job { + Job::PrepareMedia(media_id) => { + MediaUploadService::new(&ctx).start_upload(media_id).await?; + } + Job::Flush => { + if let Err(error) = MediaUploadService::new(&ctx).finish_started_uploads().await { + tracing::warn!(%error, "could not finish started media uploads"); + } + if let Err(error) = MediaUploadService::new(&ctx).reupload_pending().await { + tracing::warn!(%error, "could not retry pending media reuploads"); + } + } + } + + // A watcher spawned from this function would be cancelled when the native + // entry point drops its temporary Tokio runtime. Perform one status pass + // synchronously instead and tell WorkManager to retry durably while a job + // is still waiting for the server. + let direct_uploads = DirectMediaUploadService::new(&ctx); + if let Err(error) = direct_uploads.reconcile().await { + tracing::warn!(%error, "could not reconcile direct-media uploads"); + } + let pending_uploads = direct_uploads + .pending_job_count() + .await + .map(|count| count > 0) + .unwrap_or_else(|error| { + tracing::warn!(%error, "could not count pending direct-media uploads"); + // A failed count must not let the OS discard the only durable retry. + true + }); + + if connected && owns_connection { + // The Flutter runtime does this from its post-authentication sweep, + // which deliberately does not run in a background runtime. + if let Err(error) = ApiRuntime::replay_outbox(&ctx).await { + tracing::warn!(%error, "could not replay the API outbox"); + } + if let Err(error) = messages::retransmit_queued_receipts(&ctx).await { + tracing::warn!(%error, "could not retransmit queued receipts"); + } + if let Err(error) = DirectMediaUploadService::new(&ctx).preload_slots().await { + tracing::warn!(%error, "could not preload direct-media upload slots"); + } + } else if owns_connection { + // Nothing can be sent right now, so leave the envelopes with the OS, + // which waits for connectivity by itself. + if let Err(error) = OutboxDispatchService::new(&ctx).hand_pending_to_os().await { + tracing::warn!(%error, "could not hand queued envelopes to the OS"); + } + } + + // Flutter may have claimed a process that WorkManager started while this + // run was in flight. Never close the replacement foreground client. + if owns_connection && ctx.is_notification_runtime() { + ApiRuntime::close(&ctx).await?; + } + Ok(RunOutcome { pending_uploads }) +} + +/// Opens the socket and waits for the handshake, bounded. A background job that +/// cannot reach the server still has offline work worth doing, so a failure +/// here is reported rather than raised. +async fn connect_and_authenticate(ctx: &Arc) -> bool { + if matches!( + ApiRuntime::connection_state(ctx).await, + Ok(ApiConnectionState::Authenticated) + ) { + return true; + } + if let Err(error) = ApiRuntime::connect(ctx).await { + tracing::info!(%error, "background maintenance could not connect"); + return false; + } + let deadline = tokio::time::Instant::now() + AUTHENTICATION_TIMEOUT; + while tokio::time::Instant::now() < deadline { + match ApiRuntime::connection_state(ctx).await { + Ok(ApiConnectionState::Authenticated) => return true, + Ok(ApiConnectionState::PermanentlyRejected) => return false, + _ => tokio::time::sleep(AUTHENTICATION_POLL).await, + } + } + false +} diff --git a/rust/src/services/direct_media_upload.rs b/rust/src/services/direct_media_upload.rs index b060ac17..5e664e46 100644 --- a/rust/src/services/direct_media_upload.rs +++ b/rust/src/services/direct_media_upload.rs @@ -18,38 +18,75 @@ use sqlx::FromRow; use std::collections::HashMap; use std::io::Write; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; use std::time::Duration; const REFRESH_BEFORE_SECONDS: i64 = 24 * 60 * 60; -/// How long a running app keeps asking the server about transfers the OS is -/// carrying for it. Anything still unfinished after this is picked up by the -/// reconciliation pass on the next launch. -const WATCH_BUDGET: Duration = Duration::from_secs(10 * 60); +/// When to top the slot cache up, until the server has said otherwise. +/// +/// A slot is the one part of preparing a send that needs the network, so the +/// cache decides whether a message composed offline can still be handed to the +/// OS — which then waits for connectivity by itself — or has to sit in the +/// database until the app is opened again. The server caps how many a client +/// may hold and advertises where it refills; this is only the value used before +/// the first answer has arrived. +const DEFAULT_SLOT_REFILL_THRESHOLD: i64 = 5; +/// Where the server's advertised refill threshold is remembered between runs. +const REFILL_THRESHOLD_KEY: &str = "direct_media_refill_threshold"; const WATCH_FIRST_DELAY: Duration = Duration::from_secs(2); const WATCH_MAX_DELAY: Duration = Duration::from_secs(120); static WATCHING: AtomicBool = AtomicBool::new(false); +/// Bumped every time something asks to be watched. A watcher already in flight +/// reads this as "new work arrived" and drops back to the short poll interval. +/// Without it a send made while an earlier watch had already backed off would +/// wait out that watch's two-minute cadence before its first look, long after +/// the recipient has the file. +static WATCH_REQUESTS: AtomicU64 = AtomicU64::new(0); + +/// Clears the process-wide watcher claim even when the runtime carrying the +/// task is shut down. Background platform entry points own short-lived Tokio +/// runtimes, so ordinary code after an `.await` is not guaranteed to run. +struct WatchingGuard; + +impl Drop for WatchingGuard { + fn drop(&mut self) { + WATCHING.store(false, Ordering::SeqCst); + } +} + /// The OS finishes a transfer without telling this process, so a send would keep /// showing as "sending" until the app is restarted. While the app is alive, poll /// the server for the attachments it is still waiting on, backing off as the /// wait grows, and stop as soon as everything has settled. pub fn watch_pending_uploads(ctx: &Arc) { + let Some(runtime) = ctx.foreground_runtime() else { + // A killed-app worker has no long-lived runtime. Its native scheduler + // uses the `pending_uploads` result from background::run instead. + return; + }; + let requested = WATCH_REQUESTS.fetch_add(1, Ordering::SeqCst) + 1; if WATCHING.swap(true, Ordering::SeqCst) { + // A watcher is already running and will pick the request up on its next + // pass, so this does not start a second poller against the same jobs. return; } let ctx = ctx.clone(); - tokio::spawn(async move { + runtime.spawn(async move { + let guard = WatchingGuard; let service = DirectMediaUploadService::new(&ctx); - let deadline = tokio::time::Instant::now() + WATCH_BUDGET; + let mut seen = requested; let mut delay = WATCH_FIRST_DELAY; - while tokio::time::Instant::now() < deadline { + loop { tokio::time::sleep(delay).await; if let Err(error) = service.reconcile().await { tracing::warn!(%error, "could not reconcile direct-media uploads"); } + // The jobs themselves bound this loop: `reconcile` settles one whose + // slot has expired, so a server that never answers still ends the + // watch rather than leaving it running for the life of the app. match service.pending_job_count().await { Ok(0) => break, Ok(_) => {} @@ -58,9 +95,26 @@ pub fn watch_pending_uploads(ctx: &Arc) { break; } } - delay = (delay * 2).min(WATCH_MAX_DELAY); + let requests = WATCH_REQUESTS.load(Ordering::SeqCst); + if requests == seen { + delay = (delay * 2).min(WATCH_MAX_DELAY); + } else { + // Something was handed to the OS since the last pass, so look + // again soon instead of on the interval an older wait grew to. + seen = requests; + delay = WATCH_FIRST_DELAY; + } + } + // Release the claim before checking for a request that raced the last + // pass. `WatchingGuard` also performs this release if this future is + // cancelled because its runtime is being destroyed. + drop(guard); + // A request that arrived between the loop stopping and the flag being + // cleared found `WATCHING` still set and returned without starting a + // watcher. Nothing else would poll for it, so pick it up here. + if WATCH_REQUESTS.load(Ordering::SeqCst) != seen { + watch_pending_uploads(&ctx); } - WATCHING.store(false, Ordering::SeqCst); }); } @@ -157,6 +211,24 @@ impl DirectMediaUploadService { .header("x-twonly-login-token", hex::encode(login_token))) } + /// Tops the slot cache up without blocking the caller. Failures are + /// expected — this runs exactly when the network may be gone — and the next + /// reconnect preloads again. + fn spawn_slot_refill(&self) { + let ctx = self.ctx.clone(); + tokio::spawn(async move { + let service = DirectMediaUploadService::new(&ctx); + if let Ok(usable) = service.usable_slot_count().await { + if usable > service.refill_threshold().await { + return; + } + } + if let Err(error) = service.preload_slots().await { + tracing::info!(%error, "could not top up the direct-media slot cache"); + } + }); + } + pub async fn preload_slots(&self) -> Result { let database = self.ctx.app_db.read().await.clone(); let known = sqlx::query_scalar::<_, String>( @@ -197,6 +269,18 @@ impl DirectMediaUploadService { )?; let count = slots.slots.len(); let mut transaction = database.pool.begin().await?; + // The server decides how deep this cache may be. Remembering the number + // keeps an offline launch from refilling against a stale guess. + if slots.refill_threshold > 0 { + sqlx::query( + r#"INSERT INTO app_metadata(key, value) VALUES(?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value"#, + ) + .bind(REFILL_THRESHOLD_KEY) + .bind(slots.refill_threshold.to_string()) + .execute(&mut *transaction) + .await?; + } for slot in slots.slots { let upload = slot.media_upload.ok_or_else(|| { TwonlyError::Generic("server returned an upload slot without a POST".into()) @@ -232,9 +316,24 @@ impl DirectMediaUploadService { .await?) } + /// The refill point the server last advertised. Asking for slots the server + /// will not issue costs a request per send, so its number wins over ours. + async fn refill_threshold(&self) -> i64 { + let database = self.ctx.app_db.read().await.clone(); + sqlx::query_scalar::<_, String>("SELECT value FROM app_metadata WHERE key = ?") + .bind(REFILL_THRESHOLD_KEY) + .fetch_optional(&database.pool) + .await + .ok() + .flatten() + .and_then(|value| value.parse::().ok()) + .filter(|threshold| *threshold > 0) + .unwrap_or(DEFAULT_SLOT_REFILL_THRESHOLD) + } + async fn ensure_slots(&self) -> Result<()> { let usable = self.usable_slot_count().await?; - if usable > 5 { + if usable > self.refill_threshold().await { return Ok(()); } match self.preload_slots().await { @@ -279,9 +378,18 @@ impl DirectMediaUploadService { .await?) } - /// Native background transfers cannot report back into a process that may - /// not be running, so the server's attachment state is the authority. Every - /// launch settles the jobs the device believes are still in flight. + /// The server's attachment state is the authority on what happened to a + /// transfer the OS carried. + /// + /// Not because the native side cannot say - both uploaders finish in this + /// process holding the HTTP status - but because that word can be lost: iOS + /// does not relaunch after a force quit, `WorkManager` records its `Result` + /// nowhere durable, and either can complete while this library is loaded + /// but uninitialised. A status the server still holds survives all three. + /// + /// A 2xx would not settle it anyway: the upload is accepted with a 202 + /// while the manifest is still being reconciled, so only the attachment's + /// own state says whether the recipients were dispatched. pub async fn reconcile(&self) -> Result<()> { let database = self.ctx.app_db.read().await.clone(); let jobs = sqlx::query_as::<_, PendingJob>( @@ -310,6 +418,12 @@ impl DirectMediaUploadService { continue; } }; + tracing::info!( + attachment_id = job.attachment_id, + media_id = job.media_id, + ?state, + "read direct-media attachment status" + ); match state { AttachmentState::Ready => self.settle(&job, Outcome::Uploaded).await?, AttachmentState::Rejected => self.settle(&job, Outcome::Rejected).await?, @@ -383,6 +497,9 @@ impl DirectMediaUploadService { } else { "abandoned" }; + // The cache has just lost one; refill in the background so the next + // send does not have to be online to reserve one. + self.spawn_slot_refill(); sqlx::query("UPDATE direct_media_upload_slots SET state = ? WHERE attachment_id = ?") .bind(slot_state) .bind(&job.attachment_id) @@ -561,6 +678,9 @@ impl DirectMediaUploadService { .bind(&slot.attachment_id) .execute(&database.pool) .await?; + // If Flutter is alive this lands on its long-lived Rust runtime even + // when preparation itself was called by WorkManager. Otherwise it is a + // no-op and the durable native retry owns reconciliation. watch_pending_uploads(&self.ctx); Ok(()) } @@ -746,6 +866,13 @@ fn write_multipart_body( mod tests { use super::*; + #[test] + fn cancelled_watcher_releases_process_claim() { + WATCHING.store(true, Ordering::SeqCst); + drop(WatchingGuard); + assert!(!WATCHING.load(Ordering::SeqCst)); + } + #[test] fn multipart_body_is_deterministic_and_keeps_exact_media_bytes() { let directory = tempfile::tempdir().unwrap(); diff --git a/rust/src/services/media_codec.rs b/rust/src/services/media_codec.rs index b2258314..1ead9166 100644 --- a/rust/src/services/media_codec.rs +++ b/rust/src/services/media_codec.rs @@ -5,16 +5,17 @@ //! Image codec work, previously done by `flutter_image_compress`. //! -//! Both platforms encode WebP with libwebp — iOS has no system WebP encoder at -//! all and Android's `Bitmap.compress` routes through Skia to the same library -//! — so linking libwebp here is the identical code path without the Dart hop. -//! Only formats this crate cannot decode (HEIC/HEIF/AVIF) are handed to the -//! platform, which decodes them to PNG for us. +//! Rust links libwebp directly. The previous Flutter plugin used Android's +//! system `Bitmap` pipeline and SDWebImageWebPCoder on iOS; those may ultimately +//! use libwebp too, but their resize, pixel format, build, and encoder settings +//! are not identical to this path. Only formats this crate cannot decode +//! (HEIC/HEIF/AVIF) are handed to the platform, which decodes them to PNG for us. use crate::error::{Result, TwonlyError}; use crate::native::image as native_image; use image::{DynamicImage, ImageReader}; use std::path::Path; +use std::time::Instant; /// Quality the sender's media is encoded with, and the lower quality retried /// when the first attempt is too large to be worth sending. @@ -35,73 +36,207 @@ const MIN_CROPPED_EDGE: u32 = 10; /// Encodes with libwebp. RGB input is encoded without an alpha channel so /// opaque photos do not pay for one. fn encode_webp(image: &DynamicImage, quality: f32) -> Result> { + let total_started = Instant::now(); let has_alpha = image.color().has_alpha(); - let encoder = if has_alpha { + let conversion_started = Instant::now(); + let (encoded, input_bytes, conversion_ms, encode_ms) = if has_alpha { let rgba = image.to_rgba8(); - webp::Encoder::from_rgba(rgba.as_raw(), rgba.width(), rgba.height()) - .encode_simple(false, quality) + let conversion_ms = elapsed_ms(conversion_started); + let input_bytes = rgba.len(); + let encode_started = Instant::now(); + let encoded = webp::Encoder::from_rgba(rgba.as_raw(), rgba.width(), rgba.height()) + .encode_simple(false, quality); + ( + encoded, + input_bytes, + conversion_ms, + elapsed_ms(encode_started), + ) } else { let rgb = image.to_rgb8(); - webp::Encoder::from_rgb(rgb.as_raw(), rgb.width(), rgb.height()) - .encode_simple(false, quality) + let conversion_ms = elapsed_ms(conversion_started); + let input_bytes = rgb.len(); + let encode_started = Instant::now(); + let encoded = webp::Encoder::from_rgb(rgb.as_raw(), rgb.width(), rgb.height()) + .encode_simple(false, quality); + ( + encoded, + input_bytes, + conversion_ms, + elapsed_ms(encode_started), + ) }; - let memory = encoder + let memory = encoded .map_err(|error| TwonlyError::Generic(format!("webp encoding failed: {error:?}")))?; - Ok(memory.to_vec()) + let copy_started = Instant::now(); + let output = memory.to_vec(); + let copy_ms = elapsed_ms(copy_started); + tracing::info!( + width = image.width(), + height = image.height(), + ?has_alpha, + quality, + input_bytes, + output_bytes = output.len(), + conversion_ms, + encode_ms, + copy_ms, + total_ms = elapsed_ms(total_started), + debug_build = cfg!(debug_assertions), + "WebP encode timing" + ); + Ok(output) } /// Decodes any still image. The platform is only consulted for the formats this /// crate has no decoder for, which in practice means HEIC/HEIF from an iPhone. pub(crate) fn decode(path: &Path) -> Result { + let total_started = Instant::now(); + let source_bytes = file_size(path); let reader = ImageReader::open(path)? .with_guessed_format() .map_err(|error| TwonlyError::Generic(error.to_string()))?; + let format = reader.format(); + let rust_decode_started = Instant::now(); match reader.decode() { - Ok(image) => Ok(image), + Ok(image) => { + tracing::info!( + decoder = "rust", + ?format, + source_bytes, + width = image.width(), + height = image.height(), + color = ?image.color(), + rust_decode_ms = elapsed_ms(rust_decode_started), + total_ms = elapsed_ms(total_started), + "image decode timing" + ); + Ok(image) + } Err(error) => { + let rust_decode_ms = elapsed_ms(rust_decode_started); tracing::info!( path = %path.display(), %error, "asking the platform to decode an unsupported image format" ); let decoded = path.with_extension("decoded.png"); + let platform_started = Instant::now(); native_image::decode_to_png(path, &decoded)?; + let platform_decode_and_png_write_ms = elapsed_ms(platform_started); + let decoded_png_bytes = file_size(&decoded); + let png_decode_started = Instant::now(); let image = ImageReader::open(&decoded)? .with_guessed_format() .map_err(|error| TwonlyError::Generic(error.to_string()))? .decode() .map_err(|error| TwonlyError::Generic(error.to_string()))?; + let png_decode_ms = elapsed_ms(png_decode_started); let _ = std::fs::remove_file(&decoded); + tracing::info!( + decoder = "platform-via-png", + ?format, + source_bytes, + decoded_png_bytes, + width = image.width(), + height = image.height(), + color = ?image.color(), + rust_decode_ms, + platform_decode_and_png_write_ms, + png_decode_ms, + total_ms = elapsed_ms(total_started), + "image decode timing" + ); Ok(image) } } } -/// Produces the file that actually gets encrypted and uploaded. A first pass at -/// high quality is re-encoded lower only when the result is big enough that the +/// Produces the file that actually gets encrypted and uploaded. Unlike the old +/// plugin call, this currently keeps the source dimensions. A first pass at high +/// quality is re-encoded lower only when the result is big enough that the /// quality loss is worth the transfer. pub(crate) fn compress_for_send(source: &Path, destination: &Path) -> Result<()> { + let total_started = Instant::now(); + let source_bytes = file_size(source); + let decode_started = Instant::now(); let image = decode(source)?; + let decode_ms = elapsed_ms(decode_started); + let (width, height) = (image.width(), image.height()); + let high_quality_started = Instant::now(); let mut encoded = encode_webp(&image, SEND_QUALITY)?; + let high_quality_encode_ms = elapsed_ms(high_quality_started); + let high_quality_bytes = encoded.len(); + let mut low_quality_encode_ms = None; if encoded.len() >= LARGE_IMAGE_BYTES { + let low_quality_started = Instant::now(); match encode_webp(&image, SEND_QUALITY_LARGE) { - Ok(smaller) => encoded = smaller, + Ok(smaller) => { + low_quality_encode_ms = Some(elapsed_ms(low_quality_started)); + encoded = smaller; + } Err(error) => { + low_quality_encode_ms = Some(elapsed_ms(low_quality_started)); tracing::warn!(%error, "keeping the high quality image after a failed re-encode"); } } } - write_atomically(destination, &encoded) + let write_started = Instant::now(); + write_atomically(destination, &encoded)?; + let write_ms = elapsed_ms(write_started); + tracing::info!( + source_bytes, + width, + height, + high_quality_bytes, + output_bytes = encoded.len(), + retried_at_lower_quality = low_quality_encode_ms.is_some(), + decode_ms, + high_quality_encode_ms, + low_quality_encode_ms, + write_ms, + total_ms = elapsed_ms(total_started), + debug_build = cfg!(debug_assertions), + "send image compression timing" + ); + Ok(()) } /// Scales down so the shorter edge lands on `THUMBNAIL_MIN_EDGE`, never scaling /// an already small image up. A GIF decodes to its first frame, which is /// exactly what its thumbnail should show. pub(crate) fn create_image_thumbnail(source: &Path, destination: &Path) -> Result<()> { + let total_started = Instant::now(); + let source_bytes = file_size(source); + let decode_started = Instant::now(); let image = decode(source)?; + let decode_ms = elapsed_ms(decode_started); + let (source_width, source_height) = (image.width(), image.height()); + let resize_started = Instant::now(); let thumbnail = downscale(&image, THUMBNAIL_MIN_EDGE); + let resize_ms = elapsed_ms(resize_started); + let (output_width, output_height) = (thumbnail.width(), thumbnail.height()); + let encode_started = Instant::now(); let encoded = encode_webp(&thumbnail, THUMBNAIL_QUALITY)?; - write_atomically(destination, &encoded) + let encode_ms = elapsed_ms(encode_started); + let write_started = Instant::now(); + write_atomically(destination, &encoded)?; + let write_ms = elapsed_ms(write_started); + tracing::info!( + source_bytes, + source_width, + source_height, + output_width, + output_height, + output_bytes = encoded.len(), + decode_ms, + resize_ms, + encode_ms, + write_ms, + total_ms = elapsed_ms(total_started), + "image thumbnail timing" + ); + Ok(()) } fn downscale(image: &DynamicImage, min_edge: u32) -> DynamicImage { @@ -122,6 +257,19 @@ fn downscale(image: &DynamicImage, min_edge: u32) -> DynamicImage { ) } +/// Whether an editor overlay would actually change the frames underneath it. +/// +/// The editor writes an overlay for every video, including one the user drew +/// nothing on. Compositing a fully transparent layer produces the same frames +/// at the cost of a whole transcode, so the send path asks this first. +pub(crate) fn has_visible_content(path: &Path) -> Result { + let image = decode(path)?; + if !image.color().has_alpha() { + return Ok(true); + } + Ok(opaque_bounds(&image.to_rgba8()).is_some()) +} + /// Trims fully transparent borders left by the editor. Returns whether the /// image was rewritten. pub(crate) fn crop_transparent_borders(path: &Path) -> Result { @@ -178,6 +326,16 @@ fn write_atomically(destination: &Path, bytes: &[u8]) -> Result<()> { Ok(()) } +fn elapsed_ms(started: Instant) -> u64 { + started.elapsed().as_millis() as u64 +} + +fn file_size(path: &Path) -> u64 { + std::fs::metadata(path) + .map(|metadata| metadata.len()) + .unwrap_or(0) +} + #[cfg(test)] mod tests { use super::*; diff --git a/rust/src/services/media_upload.rs b/rust/src/services/media_upload.rs index d41be700..bc3de5e8 100644 --- a/rust/src/services/media_upload.rs +++ b/rust/src/services/media_upload.rs @@ -15,6 +15,7 @@ use crate::context::Context; use crate::database::app::tables::{Group, MediaFile}; use crate::error::{Result, TwonlyError}; use crate::native::gallery; +use crate::native::prepare; use crate::native::video; use crate::services::direct_media_upload::DirectMediaUploadService; use crate::services::media_codec; @@ -66,6 +67,8 @@ struct MediaRow { display_limit_in_milliseconds: Option, reupload_requested_by: Option, remove_audio: Option, + trim_start_ms: Option, + trim_end_ms: Option, created_at: i64, } @@ -202,12 +205,100 @@ impl MediaUploadService { // Preparation compresses and encrypts, which is far too slow to keep the // send button blocked; the media row is already durable at this point. - let ctx = self.ctx.clone(); - tokio::spawn(async move { - if let Err(error) = MediaUploadService::new(&ctx).start_upload(&media_id).await { - tracing::warn!(media_id, %error, "starting the media upload failed"); + // It is also the only part of a send that no OS transfer is carrying + // yet, so it is handed to the platform rather than left in a bare task + // that dies with the process. + self.spawn_preparation(media_id); + Ok(()) + } + + /// Runs `start_upload` under whatever protection the platform offers. On + /// Android the work itself moves into a `WorkManager` job and nothing is + /// started here. + fn spawn_preparation(&self, media_id: String) { + match prepare::begin(&media_id) { + prepare::Preparation::Scheduled => { + tracing::info!(media_id, "handed media preparation to the OS"); } - }); + prepare::Preparation::InProcess(guard) => { + let ctx = self.ctx.clone(); + tokio::spawn(async move { + // Held for the whole preparation: dropping it tells the + // platform this process no longer needs to keep running. + let _guard = guard; + match MediaUploadService::new(&ctx).start_upload(&media_id).await { + Ok(()) => { + // This branch runs on the application's long-lived + // runtime (iOS and desktop). Android preparation is + // scheduled through WorkManager and reconciles + // durably there instead. + crate::services::direct_media_upload::watch_pending_uploads(&ctx); + } + Err(error) => { + tracing::warn!(media_id, %error, "starting the media upload failed"); + } + } + }); + } + } + } + + /// Transcodes a captured video before the user has chosen recipients. + /// + /// A hardware transcode is by far the longest thing between the send button + /// and the point where the OS owns the transfer, and it does not depend on + /// anything the editor produces. Doing it up front leaves the send with + /// nothing but an encryption pass, which is short enough to finish inside + /// the grace period a closing app gets. + /// + /// The result is only used when the editor's final state agrees with it, + /// so a user who does draw on the clip pays for a wasted render rather than + /// getting a second encoding pass over an already encoded file. + pub async fn prerender(&self, media_id: &str) -> Result<()> { + let lock = media_lock(media_id); + let _guard = lock.lock().await; + + let Some(media) = self.load(media_id).await? else { + return Ok(()); + }; + // Anything past `initialized` is either being sent or already sent, and + // a pre-render would race the send for the same output file. + if media.media_type != "video" || media.upload_state.as_deref() != Some("initialized") { + return Ok(()); + } + let files = MediaFileService::new(&self.ctx); + let fingerprint = render_fingerprint(&media); + if files.prerender_matches(media_id, &fingerprint) { + return Ok(()); + } + let original = files.original_path(media_id, &media.media_type); + if !original.exists() { + return Ok(()); + } + // A marker from earlier settings would otherwise outlive its clip. + files.discard_prerender(media_id); + + let output = files.prerendered_path(media_id); + MediaFileService::ensure_parent(&output)?; + let (trim_start_ms, trim_end_ms) = trim_bounds(&media); + let request = video::RenderRequest { + media_id: media_id.to_owned(), + input: original, + overlay: None, + output: output.clone(), + remove_audio: media.remove_audio.unwrap_or(0) != 0, + trim_start_ms, + trim_end_ms, + }; + if let Err(error) = blocking(move || video::render(&request)).await { + // The send path renders from the original, so this costs nothing + // beyond the time already spent. + tracing::info!(media_id, %error, "pre-rendering the video failed"); + remove_file(&output); + return Ok(()); + } + files.write_prerender_marker(media_id, &fingerprint)?; + tracing::info!(media_id, "pre-rendered the captured video"); Ok(()) } @@ -318,7 +409,7 @@ impl MediaUploadService { let pending = sqlx::query_as::<_, MediaRow>( r#"SELECT media_id, type AS media_type, upload_state, requires_authentication, is_draft_media, display_limit_in_milliseconds, reupload_requested_by, - remove_audio, created_at + remove_audio, trim_start_ms, trim_end_ms, created_at FROM media_files WHERE upload_state IN ('initialized', 'preprocessing', 'uploading', 'uploadLimitReached')"#, @@ -861,6 +952,28 @@ impl MediaUploadService { Ok(()) } + /// Stores where the editor's cutter placed the two ends of a video. + /// + /// Both bounds are milliseconds into the recording; `None` means the clip + /// keeps that end. The recording itself is never rewritten - the transcode + /// every send performs applies the cut - so this stays reversible for as + /// long as the editor is open. + pub async fn set_trim( + &self, + media_id: &str, + trim_start_ms: Option, + trim_end_ms: Option, + ) -> Result<()> { + let database = self.ctx.app_db.read().await.clone(); + sqlx::query("UPDATE media_files SET trim_start_ms = ?, trim_end_ms = ? WHERE media_id = ?") + .bind(trim_start_ms) + .bind(trim_end_ms) + .bind(media_id) + .execute(&database.pool) + .await?; + Ok(()) + } + pub async fn toggle_remove_audio(&self, media_id: &str) -> Result<()> { let database = self.ctx.app_db.read().await.clone(); sqlx::query( @@ -869,6 +982,11 @@ impl MediaUploadService { .bind(media_id) .execute(&database.pool) .await?; + drop(database); + // The pre-rendered clip carries the old audio decision, and + // `prerender_matches` would reject it anyway; dropping it now frees the + // space and lets a fresh pre-render start. + MediaFileService::new(&self.ctx).discard_prerender(media_id); Ok(()) } @@ -956,7 +1074,7 @@ impl MediaUploadService { Ok(sqlx::query_as::<_, MediaRow>( r#"SELECT media_id, type AS media_type, upload_state, requires_authentication, is_draft_media, display_limit_in_milliseconds, reupload_requested_by, - remove_audio, created_at + remove_audio, trim_start_ms, trim_end_ms, created_at FROM media_files WHERE media_id = ?"#, ) .bind(media_id) @@ -989,15 +1107,28 @@ impl MediaUploadService { "video" => self.render_video(media, &original, &temp).await, "image" => { let (source, destination) = (original.clone(), temp.clone()); - if let Err(error) = - blocking(move || media_codec::compress_for_send(&source, &destination)).await + let started = std::time::Instant::now(); + let mut used_uncompressed_fallback = false; + match blocking(move || media_codec::compress_for_send(&source, &destination)).await { - // Sending the original beats not sending at all, which is - // what the Flutter implementation did on a codec failure. - tracing::warn!(media_id = media.media_id, %error, "sending the uncompressed image"); - MediaFileService::ensure_parent(&temp)?; - std::fs::copy(&original, &temp)?; + Ok(()) => {} + Err(error) => { + // Sending the original beats not sending at all, which is + // what the Flutter implementation did on a codec failure. + tracing::warn!(media_id = media.media_id, %error, "sending the uncompressed image"); + MediaFileService::ensure_parent(&temp)?; + std::fs::copy(&original, &temp)?; + used_uncompressed_fallback = true; + } } + tracing::info!( + media_id = media.media_id, + source_bytes = file_size_or_zero(&original), + output_bytes = file_size_or_zero(&temp), + used_uncompressed_fallback, + total_ms = started.elapsed().as_millis() as u64, + "image compression phase finished" + ); Ok(()) } // GIF keeps its animation and audio is already in its delivery @@ -1016,16 +1147,54 @@ impl MediaUploadService { async fn render_video(&self, media: &MediaRow, original: &Path, temp: &Path) -> Result<()> { let files = MediaFileService::new(&self.ctx); let overlay = files.overlay_image_path(&media.media_id); - let overlay = overlay.exists().then_some(overlay); + // The editor writes an overlay for every video, including one nobody + // drew on. Compositing a fully transparent layer produces the same + // frames, and recognising that is what lets a pre-rendered clip be sent + // without touching an encoder. A file that cannot be read is treated as + // meaningful, so an unreadable overlay never silently drops artwork. + let overlay = if overlay.exists() { + let path = overlay.clone(); + let visible = blocking(move || media_codec::has_visible_content(&path)) + .await + .unwrap_or_else(|error| { + tracing::warn!(media_id = media.media_id, %error, "could not inspect the overlay"); + true + }); + visible.then_some(overlay) + } else { + None + }; + + let remove_audio = media.remove_audio.unwrap_or(0) != 0; + if overlay.is_none() && files.prerender_matches(&media.media_id, &render_fingerprint(media)) + { + // The clip the editor was opened on is the clip that gets sent. + MediaFileService::ensure_parent(temp)?; + match std::fs::rename(files.prerendered_path(&media.media_id), temp) { + Ok(()) => { + tracing::info!(media_id = media.media_id, "sending the pre-rendered video"); + files.discard_prerender(&media.media_id); + return Ok(()); + } + Err(error) => { + tracing::warn!(media_id = media.media_id, %error, "could not use the pre-rendered video"); + } + } + } + files.discard_prerender(&media.media_id); + // Every clip is rendered, including small ones with no overlay. What a // camera produces varies by device and is not guaranteed to play on the // other platform; the render is what makes the output predictable. + let (trim_start_ms, trim_end_ms) = trim_bounds(media); let request = video::RenderRequest { media_id: media.media_id.clone(), input: original.to_path_buf(), overlay, output: temp.to_path_buf(), - remove_audio: media.remove_audio.unwrap_or(0) != 0, + remove_audio, + trim_start_ms, + trim_end_ms, }; let media_id = media.media_id.clone(); if let Err(error) = blocking(move || video::render(&request)).await { @@ -1141,6 +1310,39 @@ impl MediaUploadService { } } +/// Everything the video transcode reads off the media row. +/// +/// A clip pre-rendered while the user was still editing may only be sent when +/// the editor's final state asks for exactly the same render, so this is the +/// one description both sides compare. **Any new input the render starts to +/// honour — a trim, a speed change, a filter — has to be added here in the same +/// change, or a pre-rendered clip will be sent ignoring it.** +fn render_fingerprint(media: &MediaRow) -> String { + // A plain string rather than a struct: it is only ever compared, never + // read back, and a stable textual form makes a stale marker obvious in a + // temp directory listing. + let (start, end) = trim_bounds(media); + format!( + "v2;remove_audio={};trim={}..{}", + media.remove_audio.unwrap_or(0) != 0, + start.map_or_else(|| "-".to_owned(), |value| value.to_string()), + end.map_or_else(|| "-".to_owned(), |value| value.to_string()), + ) +} + +/// The cut the editor asked for, as milliseconds into the recording. +/// +/// A bound is dropped when it cannot describe a cut: a negative offset, or an +/// end at or before the start. The renderers are handed the raw numbers, so a +/// nonsensical pair here would produce an empty clip rather than a whole one. +fn trim_bounds(media: &MediaRow) -> (Option, Option) { + let start = media.trim_start_ms.filter(|value| *value > 0); + let end = media + .trim_end_ms + .filter(|value| *value > start.unwrap_or(0)); + (start, end) +} + /// Image codecs are CPU-bound: a full-resolution photo takes long enough to /// encode that it must not run on an async worker. async fn blocking(task: F) -> Result @@ -1161,6 +1363,12 @@ fn remove_file(path: &Path) { } } +fn file_size_or_zero(path: &Path) -> u64 { + std::fs::metadata(path) + .map(|metadata| metadata.len()) + .unwrap_or(0) +} + /// Drift persists this column as a JSON array of contact ids. Tolerate the /// single-scalar form that an earlier Rust write produced. fn parse_reupload_requested_by(value: Option<&str>) -> Vec { diff --git a/rust/src/services/mediafiles.rs b/rust/src/services/mediafiles.rs index 48b9e8e0..b5bb634a 100644 --- a/rust/src/services/mediafiles.rs +++ b/rust/src/services/mediafiles.rs @@ -378,6 +378,9 @@ impl MediaFileService { base.join("tmp") .join(format!("{media_id}.ffmpeg.{extension}")), base.join("tmp").join(format!("{media_id}.overlay.png")), + base.join("tmp").join(format!("{media_id}.prerendered.mp4")), + base.join("tmp") + .join(format!("{media_id}.prerendered.json")), base.join("stored").join(format!("{media_id}.{extension}")), base.join("stored") .join(format!("{media_id}.thumbnail.webp")), @@ -420,6 +423,42 @@ impl MediaFileService { self.media_path("tmp", media_id, ".upload", Self::extension(media_type)) } + /// The transcode produced before the user picked recipients. It is only a + /// candidate: `prerender_matches` decides whether the editor's final + /// settings are still the ones it was produced with. + pub(crate) fn prerendered_path(&self, media_id: &str) -> PathBuf { + self.media_path("tmp", media_id, ".prerendered", "mp4") + } + + fn prerender_marker_path(&self, media_id: &str) -> PathBuf { + self.media_path("tmp", media_id, ".prerendered", "json") + } + + /// Records the render inputs the pre-rendered clip was produced with. + /// Written only after the render finished, so a marker on disk also proves + /// the clip is whole. + pub(crate) fn write_prerender_marker(&self, media_id: &str, fingerprint: &str) -> Result<()> { + let marker = self.prerender_marker_path(media_id); + Self::ensure_parent(&marker)?; + std::fs::write(&marker, fingerprint.as_bytes())?; + Ok(()) + } + + /// Whether the pre-rendered clip can be sent as it stands, which is true + /// only while the editor still asks for exactly what it was produced with. + pub(crate) fn prerender_matches(&self, media_id: &str, fingerprint: &str) -> bool { + if !self.prerendered_path(media_id).exists() { + return false; + } + std::fs::read_to_string(self.prerender_marker_path(media_id)) + .is_ok_and(|marker| marker == fingerprint) + } + + pub(crate) fn discard_prerender(&self, media_id: &str) { + let _ = std::fs::remove_file(self.prerendered_path(media_id)); + let _ = std::fs::remove_file(self.prerender_marker_path(media_id)); + } + pub(crate) fn temp_path(&self, media_id: &str, media_type: &str) -> PathBuf { PathBuf::from(&self.ctx.config.data_dir) .join("mediafiles/tmp") diff --git a/rust/src/services/mod.rs b/rust/src/services/mod.rs index 6b7af4f9..01670689 100644 --- a/rust/src/services/mod.rs +++ b/rust/src/services/mod.rs @@ -4,6 +4,7 @@ */ pub mod avatars; +pub mod background; pub mod contacts; pub mod direct_media_upload; pub mod groups; @@ -13,3 +14,4 @@ pub mod media_upload; pub mod mediafiles; pub mod messages; pub mod notifications; +pub mod outbox_dispatch; diff --git a/rust/src/services/notifications.rs b/rust/src/services/notifications.rs index b1fbafad..2c2384b0 100644 --- a/rust/src/services/notifications.rs +++ b/rust/src/services/notifications.rs @@ -6,7 +6,7 @@ use crate::api::proto::client::{self as proto, encrypted_content}; use crate::api::runtime::ApiRuntime; use crate::bridge::InitConfig; -use crate::context::{Context, RuntimeMode}; +use crate::context::Context; use crate::database::app::AppDatabase; use crate::error::Result; use crate::user_config::UserConfig; @@ -415,7 +415,8 @@ pub async fn process_wakeup( let deadline = std::time::Duration::from_millis(deadline_ms.clamp(1_000, 28_000)); - let completed = if ctx.runtime_mode == RuntimeMode::Notification { + let owns_connection = ctx.is_notification_runtime(); + let completed = if owns_connection { let generation = ctx.mailbox_generation(); ApiRuntime::connect(&ctx).await?; tokio::time::timeout(deadline, ctx.wait_for_mailbox_after(generation)) @@ -443,7 +444,7 @@ pub async fn process_wakeup( let mut batch = pending_batch(&ctx, locale).await?; batch.completed = completed; - if ctx.runtime_mode == RuntimeMode::Notification { + if owns_connection && ctx.is_notification_runtime() { ApiRuntime::close(&ctx).await?; } Ok(batch) diff --git a/rust/src/services/outbox_dispatch.rs b/rust/src/services/outbox_dispatch.rs new file mode 100644 index 00000000..1a02eb7d --- /dev/null +++ b/rust/src/services/outbox_dispatch.rs @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2026, Tobias Müller git@tsmr.eu + * + */ + +//! Handing queued envelopes to an OS-owned transfer. +//! +//! A message only reaches the server over the websocket, which exists only +//! while the app is running. A message composed offline therefore waits for the +//! next app launch rather than the next network — the opposite of what a +//! messenger promises. Media already avoids this: its envelopes ride inside the +//! attachment manifest, which `WorkManager` and a background `URLSession` both +//! deliver on their own once connectivity returns. +//! +//! This gives plain messages the same route. The envelope is encrypted here, +//! written to a request file, and POSTed to `v2/messages/dispatch` by the same +//! native transfer that carries media. +//! +//! It is an accelerator, not a replacement. The receipt stays in the outbox and +//! is still retransmitted over the socket, because the outcome of an OS +//! transfer can be lost before it reaches this process - iOS does not relaunch +//! after a force quit, and `WorkManager` records its `Result` nowhere durable. +//! A recipient that receives both copies discards the second by receipt id, +//! exactly as it already does for a socket retransmission. + +use crate::api::messages::incoming::messages; +use crate::api::proto::http_requests::{OutboxDispatch, OutboxDispatchBatch}; +use crate::bridge::api::RustApi; +use crate::context::Context; +use crate::error::Result; +use crate::native::transfer; +use crate::utils::new_uuid_v7; +use prost::Message as _; +use serde::Serialize; +use sqlx::FromRow; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; + +/// How long the native transfer keeps retrying an envelope. Past this the +/// recipient's session has usually moved on far enough that the socket +/// retransmission — which re-encrypts — is the better copy anyway. +const DISPATCH_VALIDITY_SECONDS: i64 = 24 * 60 * 60; +/// Handing over a very large backlog at once would write one request file per +/// receipt; beyond this the socket is the more sensible route. +const MAXIMUM_HANDOVERS: usize = 20; + +#[derive(FromRow)] +struct ExpiredJob { + body_path: String, +} + +/// Mirrors the single-request form of the descriptor the native uploaders read. +#[derive(Serialize)] +struct NativeRequest { + role: String, + url: String, + method: String, + headers: HashMap, + body_path: String, +} + +#[derive(Serialize)] +struct NativeDescriptor { + attachment_id: String, + expires_at: i64, + media: NativeRequest, +} + +pub struct OutboxDispatchService { + ctx: Arc, +} + +impl OutboxDispatchService { + pub fn new(ctx: &Arc) -> Self { + Self { ctx: ctx.clone() } + } + + fn job_dir(&self) -> PathBuf { + PathBuf::from(&self.ctx.config.data_dir).join("outbox-dispatch") + } + + /// Hands every receipt that has not reached the server to the OS. + /// + /// Called when the app is going away — backgrounded, or shutting down — + /// which is exactly when the socket is about to stop being an option. + pub async fn hand_pending_to_os(&self) -> Result { + self.purge_expired().await?; + + let database = self.ctx.app_db.read().await.clone(); + let receipt_ids = sqlx::query_scalar::<_, String>( + r#"SELECT receipt_id FROM receipts + WHERE will_be_retried_by_media_upload = 0 + AND deferred_until_session IS NULL + AND ack_by_server_at IS NULL + AND receipt_id NOT IN (SELECT receipt_id FROM outbox_dispatch_jobs) + AND (mark_for_retry_after_accepted IS NULL OR EXISTS( + SELECT 1 FROM contacts + WHERE contacts.user_id = receipts.contact_id AND contacts.accepted = 1 + )) + ORDER BY created_at + LIMIT ?"#, + ) + .bind(MAXIMUM_HANDOVERS as i64) + .fetch_all(&database.pool) + .await?; + drop(database); + + let mut handed = 0; + for receipt_id in receipt_ids { + match self.hand_to_os(&receipt_id).await { + Ok(true) => handed += 1, + Ok(false) => {} + // A peer with no session yet needs a prekey bundle from the + // server, which is the one thing this path cannot do offline. + Err(error) => { + tracing::info!(receipt_id, %error, "could not hand the envelope to the OS"); + } + } + } + if handed > 0 { + tracing::info!(handed, "handed queued envelopes to the OS transfer"); + } + Ok(handed) + } + + /// Encrypts one queued receipt and schedules its delivery with the OS. + /// + /// Returns whether anything was scheduled: a receipt that has meanwhile + /// been sent, or one whose contact deleted their account, is not an error. + pub async fn hand_to_os(&self, receipt_id: &str) -> Result { + let Some(prepared) = + messages::prepare_queued_receipt_details(&self.ctx, receipt_id).await? + else { + return Ok(false); + }; + + let dispatch_id = new_uuid_v7(); + let batch = OutboxDispatchBatch { + dispatches: vec![OutboxDispatch { + dispatch_id: dispatch_id.clone(), + recipient_user_id: prepared.contact_id, + encrypted_body: prepared.message.encode_to_vec(), + wake_receiver: prepared.wake_receiver, + }], + }; + + let directory = self.job_dir().join(&dispatch_id); + std::fs::create_dir_all(&directory)?; + let body_path = directory.join("dispatch.pb"); + std::fs::write(&body_path, batch.encode_to_vec())?; + + let expires_at = chrono::Utc::now().timestamp() + DISPATCH_VALIDITY_SECONDS; + let user_id = self.ctx.user_id().await?; + let login_token = self.ctx.key_manager.lock().await.main_key.get_login_token(); + let descriptor = NativeDescriptor { + attachment_id: format!("outbox-{dispatch_id}"), + expires_at, + media: NativeRequest { + role: "media".into(), + url: format!( + "{}v2/messages/dispatch", + RustApi::api_base_url("https".into()) + ), + method: "POST".into(), + headers: HashMap::from([ + ("content-type".into(), "application/x-protobuf".into()), + ( + "x-twonly-user-id".into(), + hex::encode(user_id.to_be_bytes()), + ), + ("x-twonly-login-token".into(), hex::encode(login_token)), + ]), + body_path: body_path.to_string_lossy().into_owned(), + }, + }; + + let database = self.ctx.app_db.read().await.clone(); + // The row is written before the hand-off so a process death between the + // two leaves a job the expiry sweep can clean up, rather than a file + // nothing knows about. + sqlx::query( + r#"INSERT INTO outbox_dispatch_jobs + (receipt_id, dispatch_id, contact_id, body_path, expires_at, created_at) + VALUES (?, ?, ?, ?, ?, ?)"#, + ) + .bind(receipt_id) + .bind(&dispatch_id) + .bind(prepared.contact_id) + .bind(body_path.to_string_lossy().as_ref()) + .bind(expires_at) + .bind(chrono::Utc::now().timestamp()) + .execute(&database.pool) + .await?; + drop(database); + + let descriptor_json = serde_json::to_string(&descriptor)?; + if let Err(error) = transfer::schedule(&descriptor_json) { + self.forget(receipt_id).await; + return Err(error); + } + Ok(true) + } + + /// Drops a job and its request file. The receipt is untouched: the socket + /// path remains responsible for it either way. + async fn forget(&self, receipt_id: &str) { + let database = self.ctx.app_db.read().await.clone(); + let removed = sqlx::query_as::<_, ExpiredJob>( + "DELETE FROM outbox_dispatch_jobs WHERE receipt_id = ? RETURNING body_path", + ) + .bind(receipt_id) + .fetch_optional(&database.pool) + .await; + if let Ok(Some(job)) = removed { + Self::remove_job_files(&job.body_path); + } + } + + /// Clears out jobs the native transfer has given up on. Nothing reports a + /// finished transfer back, so age is the only signal there is. + pub async fn purge_expired(&self) -> Result<()> { + let database = self.ctx.app_db.read().await.clone(); + let expired = sqlx::query_as::<_, ExpiredJob>( + r#"DELETE FROM outbox_dispatch_jobs + WHERE expires_at <= CAST(strftime('%s','now') AS INTEGER) + OR receipt_id NOT IN (SELECT receipt_id FROM receipts) + RETURNING body_path"#, + ) + .fetch_all(&database.pool) + .await?; + for job in expired { + Self::remove_job_files(&job.body_path); + } + Ok(()) + } + + fn remove_job_files(body_path: &str) { + let path = PathBuf::from(body_path); + match std::fs::remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + tracing::warn!(path = %path.display(), %error, "could not remove a dispatch body"); + } + } + if let Some(parent) = path.parent() { + let _ = std::fs::remove_dir(parent); + } + } +} diff --git a/rust_builder/ios/Classes/TwonlyNotifications.h b/rust_builder/ios/Classes/TwonlyNotifications.h index c40a441f..fc84b33e 100644 --- a/rust_builder/ios/Classes/TwonlyNotifications.h +++ b/rust_builder/ios/Classes/TwonlyNotifications.h @@ -18,6 +18,18 @@ char *twonly_notification_acknowledge(const char *event_ids_json); void twonly_notification_string_free(char *pointer); +/* Runs one maintenance job: preparing a single media file when `media_id` is + given, or resuming everything left in flight when it is NULL. Returns a JSON + `{"ok":bool,"error":string?}` that must be released with + `twonly_background_string_free`. */ +char *twonly_background_run( + const char *database_dir, + const char *data_dir, + const char *media_id +); + +void twonly_background_string_free(char *pointer); + #ifdef __cplusplus } #endif diff --git a/test/visual/video_recording_budget_test.dart b/test/visual/video_recording_budget_test.dart new file mode 100644 index 00000000..996ef774 --- /dev/null +++ b/test/visual/video_recording_budget_test.dart @@ -0,0 +1,180 @@ +import 'dart:io'; +import 'dart:ui' show Size; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:twonly/globals.dart'; +import 'package:twonly/src/services/subscription.service.dart'; +import 'package:twonly/src/visual/views/camera/camera_preview_components/video_recording_budget.dart'; + +void main() { + late Directory tempDir; + + const hd = Size(1280, 720); + + setUp(() { + tempDir = Directory.systemTemp.createTempSync('twonly_video_budget_test_'); + AppEnvironment.initTesting( + customCacheDir: tempDir.path, + customSupportDir: tempDir.path, + ); + VideoRecordingBudget.resetForTesting(); + }); + + tearDown(() { + if (tempDir.existsSync()) { + try { + tempDir.deleteSync(recursive: true); + } catch (_) {} + } + }); + + Duration budgetFor(SubscriptionPlan plan, {Size? size = hd}) => + VideoRecordingBudget.maxRecordingTime( + plan: plan, + recordingSize: size, + hasAudio: true, + ); + + group('VideoRecordingBudget', () { + test('a paid plan may record twice as long as the free plan', () { + final free = budgetFor(SubscriptionPlan.Free); + final pro = budgetFor(SubscriptionPlan.Pro); + + expect(pro.inMilliseconds, greaterThan(free.inMilliseconds)); + expect(pro.inMilliseconds, closeTo(2 * free.inMilliseconds, 1)); + // Every paid plan shares the same per-file limit. + expect(budgetFor(SubscriptionPlan.Family), pro); + expect(budgetFor(SubscriptionPlan.Tester), pro); + }); + + test('without a measurement the estimate stays under the plan limit', () { + final free = budgetFor(SubscriptionPlan.Free); + // 720p30 at the assumed 0.25 bits per pixel plus AAC is 880 kB/s, so + // 50 MB is reached just after 51 seconds even before the headroom. + expect(free.inSeconds, inInclusiveRange(45, 60)); + }); + + test('an unknown recording size falls back to 720p', () { + expect( + budgetFor(SubscriptionPlan.Free, size: null), + budgetFor(SubscriptionPlan.Free), + ); + expect( + budgetFor(SubscriptionPlan.Free, size: Size.zero), + budgetFor(SubscriptionPlan.Free), + ); + }); + + test('a bigger recording size shortens the budget', () { + final hdBudget = budgetFor(SubscriptionPlan.Free); + final fullHdBudget = budgetFor( + SubscriptionPlan.Free, + size: const Size(1920, 1080), + ); + expect(fullHdBudget.inMilliseconds, lessThan(hdBudget.inMilliseconds)); + }); + + test('a measured recording replaces the estimate', () async { + // The 18 MB per minute a real device wrote. + await VideoRecordingBudget.recordMeasurement( + fileSizeInBytes: 18000000, + duration: const Duration(seconds: 60), + recordingSize: hd, + ); + + // 50 MB minus headroom at 300 kB/s. + expect(budgetFor(SubscriptionPlan.Free).inSeconds, 150); + // 100 MB would allow 300s, which the ceiling caps at five minutes. + expect(budgetFor(SubscriptionPlan.Pro), const Duration(minutes: 5)); + }); + + test('the measurement is reused after a restart', () async { + await VideoRecordingBudget.recordMeasurement( + fileSizeInBytes: 18000000, + duration: const Duration(seconds: 60), + recordingSize: hd, + ); + final beforeRestart = budgetFor(SubscriptionPlan.Free); + + VideoRecordingBudget.resetForTesting(); + // Nothing loaded yet, so the pessimistic estimate is what is on offer. + expect(budgetFor(SubscriptionPlan.Free), lessThan(beforeRestart)); + + await VideoRecordingBudget.ensureLoaded(); + expect(budgetFor(SubscriptionPlan.Free), beforeRestart); + }); + + test('the measurement is kept per resolution', () async { + await VideoRecordingBudget.recordMeasurement( + fileSizeInBytes: 18000000, + duration: const Duration(seconds: 60), + recordingSize: hd, + ); + + // A rotated preview size is the same recording. + expect( + budgetFor(SubscriptionPlan.Free, size: const Size(720, 1280)), + budgetFor(SubscriptionPlan.Free), + ); + // A different recording size has not been measured. + expect( + budgetFor(SubscriptionPlan.Free, size: const Size(1920, 1080)), + lessThan(budgetFor(SubscriptionPlan.Free)), + ); + }); + + test('a higher rate is adopted at once, a lower one eases in', () async { + await VideoRecordingBudget.recordMeasurement( + fileSizeInBytes: 18000000, + duration: const Duration(seconds: 60), + recordingSize: hd, + ); + final atThreeHundredKilobytes = budgetFor(SubscriptionPlan.Free); + + // A busier scene writing 600 kB/s halves the budget immediately. + await VideoRecordingBudget.recordMeasurement( + fileSizeInBytes: 36000000, + duration: const Duration(seconds: 60), + recordingSize: hd, + ); + expect(budgetFor(SubscriptionPlan.Free).inSeconds, 75); + + // A still scene writing 300 kB/s again does not hand the budget straight + // back: 600 kB/s eased towards 300 kB/s is 540 kB/s. + await VideoRecordingBudget.recordMeasurement( + fileSizeInBytes: 18000000, + duration: const Duration(seconds: 60), + recordingSize: hd, + ); + final eased = budgetFor(SubscriptionPlan.Free); + expect(eased.inSeconds, 83); + expect(eased, lessThan(atThreeHundredKilobytes)); + }); + + test('a recording too short to divide by is ignored', () async { + final estimated = budgetFor(SubscriptionPlan.Free); + + await VideoRecordingBudget.recordMeasurement( + fileSizeInBytes: 400000, + duration: const Duration(milliseconds: 300), + recordingSize: hd, + ); + await VideoRecordingBudget.recordMeasurement( + fileSizeInBytes: 0, + duration: const Duration(seconds: 30), + recordingSize: hd, + ); + + expect(budgetFor(SubscriptionPlan.Free), estimated); + }); + + test('an implausible rate is still left usable', () async { + await VideoRecordingBudget.recordMeasurement( + fileSizeInBytes: 500000000, + duration: const Duration(seconds: 5), + recordingSize: hd, + ); + expect(budgetFor(SubscriptionPlan.Free), const Duration(seconds: 10)); + }); + }); +}