From 9667ea21b6bbb702646c0c022824836d064a9a7c Mon Sep 17 00:00:00 2001 From: otsmr Date: Sun, 7 Dec 2025 03:23:54 +0100 Subject: [PATCH] starting with #327 --- .gitmodules | 0 .../camera_preview.dart | 8 +- .../camera_preview_controller_view.dart | 2 + lib/src/views/camera/camera_send_to_view.dart | 1 + .../painters/barcode_detector_painter.dart | 124 +++++++++++++++++ .../painters/coordinates_translator.dart | 52 +++++++ .../views/contact/contact_verify.view.dart | 57 ++++---- .../contact/contact_verify_qr_scan.view.dart | 41 +++--- lib/src/views/home.view.dart | 127 ++++++++++++++++++ pubspec.lock | 29 ++-- pubspec.yaml | 5 +- 11 files changed, 380 insertions(+), 66 deletions(-) delete mode 100644 .gitmodules create mode 100644 lib/src/views/camera/painters/barcode_detector_painter.dart create mode 100644 lib/src/views/camera/painters/coordinates_translator.dart diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e69de29..0000000 diff --git a/lib/src/views/camera/camera_preview_components/camera_preview.dart b/lib/src/views/camera/camera_preview_components/camera_preview.dart index 9911479..5b117da 100644 --- a/lib/src/views/camera/camera_preview_components/camera_preview.dart +++ b/lib/src/views/camera/camera_preview_components/camera_preview.dart @@ -7,10 +7,12 @@ class HomeViewCameraPreview extends StatelessWidget { const HomeViewCameraPreview({ required this.controller, required this.screenshotController, + required this.customPaint, super.key, }); final CameraController? controller; + final CustomPaint? customPaint; final ScreenshotController screenshotController; @override @@ -33,7 +35,7 @@ class HomeViewCameraPreview extends StatelessWidget { child: SizedBox( width: controller!.value.previewSize!.height, height: controller!.value.previewSize!.width, - child: CameraPreview(controller!), + child: CameraPreview(controller!, child: customPaint), ), ), ), @@ -48,11 +50,13 @@ class SendToCameraPreview extends StatelessWidget { const SendToCameraPreview({ required this.cameraController, required this.screenshotController, + required this.customPaint, super.key, }); final CameraController? cameraController; final ScreenshotController screenshotController; + final CustomPaint? customPaint; @override Widget build(BuildContext context) { @@ -73,7 +77,7 @@ class SendToCameraPreview extends StatelessWidget { child: SizedBox( width: cameraController!.value.previewSize!.height, height: cameraController!.value.previewSize!.width, - child: CameraPreview(cameraController!), + child: CameraPreview(cameraController!, child: customPaint), ), ), ), diff --git a/lib/src/views/camera/camera_preview_controller_view.dart b/lib/src/views/camera/camera_preview_controller_view.dart index bd4fa5c..71201a8 100644 --- a/lib/src/views/camera/camera_preview_controller_view.dart +++ b/lib/src/views/camera/camera_preview_controller_view.dart @@ -52,6 +52,8 @@ Future<(SelectedCameraDetails, CameraController)?> initializeCameraController( gCameras[cameraId], ResolutionPreset.high, enableAudio: await Permission.microphone.isGranted, + imageFormatGroup: + Platform.isAndroid ? ImageFormatGroup.nv21 : ImageFormatGroup.bgra8888, ); await cameraController.initialize().then((_) async { diff --git a/lib/src/views/camera/camera_send_to_view.dart b/lib/src/views/camera/camera_send_to_view.dart index 578c955..2b7b656 100644 --- a/lib/src/views/camera/camera_send_to_view.dart +++ b/lib/src/views/camera/camera_send_to_view.dart @@ -72,6 +72,7 @@ class CameraSendToViewState extends State { SendToCameraPreview( cameraController: cameraController, screenshotController: screenshotController, + customPaint: null, ), CameraPreviewControllerView( selectCamera: selectCamera, diff --git a/lib/src/views/camera/painters/barcode_detector_painter.dart b/lib/src/views/camera/painters/barcode_detector_painter.dart new file mode 100644 index 0000000..7cc1e51 --- /dev/null +++ b/lib/src/views/camera/painters/barcode_detector_painter.dart @@ -0,0 +1,124 @@ +import 'dart:io'; +import 'dart:ui'; +import 'dart:ui' as ui; + +import 'package:camera/camera.dart'; +import 'package:flutter/material.dart'; +import 'package:google_mlkit_barcode_scanning/google_mlkit_barcode_scanning.dart'; + +import 'coordinates_translator.dart'; + +class BarcodeDetectorPainter extends CustomPainter { + BarcodeDetectorPainter( + this.barcodes, + this.imageSize, + this.rotation, + this.cameraLensDirection, + ); + + final List barcodes; + final Size imageSize; + final InputImageRotation rotation; + final CameraLensDirection cameraLensDirection; + + @override + void paint(Canvas canvas, Size size) { + final Paint paint = Paint() + ..style = PaintingStyle.stroke + ..strokeWidth = 3.0 + ..color = Colors.lightGreenAccent; + + final Paint background = Paint()..color = Color(0x99000000); + + for (final Barcode barcode in barcodes) { + final ParagraphBuilder builder = ParagraphBuilder( + ParagraphStyle( + textAlign: TextAlign.left, + fontSize: 16, + textDirection: TextDirection.ltr), + ); + builder.pushStyle( + ui.TextStyle(color: Colors.lightGreenAccent, background: background)); + builder.addText('${barcode.displayValue}'); + builder.pop(); + + final left = translateX( + barcode.boundingBox.left, + size, + imageSize, + rotation, + cameraLensDirection, + ); + final top = translateY( + barcode.boundingBox.top, + size, + imageSize, + rotation, + cameraLensDirection, + ); + final right = translateX( + barcode.boundingBox.right, + size, + imageSize, + rotation, + cameraLensDirection, + ); + // final bottom = translateY( + // barcode.boundingBox.bottom, + // size, + // imageSize, + // rotation, + // cameraLensDirection, + // ); + // + // // Draw a bounding rectangle around the barcode + // canvas.drawRect( + // Rect.fromLTRB(left, top, right, bottom), + // paint, + // ); + + final List cornerPoints = []; + for (final point in barcode.cornerPoints) { + final double x = translateX( + point.x.toDouble(), + size, + imageSize, + rotation, + cameraLensDirection, + ); + final double y = translateY( + point.y.toDouble(), + size, + imageSize, + rotation, + cameraLensDirection, + ); + + cornerPoints.add(Offset(x, y)); + } + + // Add the first point to close the polygon + cornerPoints.add(cornerPoints.first); + canvas.drawPoints(PointMode.polygon, cornerPoints, paint); + + canvas.drawParagraph( + builder.build() + ..layout(ParagraphConstraints( + width: (right - left).abs(), + )), + Offset( + Platform.isAndroid && + cameraLensDirection == CameraLensDirection.front + ? right + : left, + top), + ); + } + } + + @override + bool shouldRepaint(BarcodeDetectorPainter oldDelegate) { + return oldDelegate.imageSize != imageSize || + oldDelegate.barcodes != barcodes; + } +} diff --git a/lib/src/views/camera/painters/coordinates_translator.dart b/lib/src/views/camera/painters/coordinates_translator.dart new file mode 100644 index 0000000..36feed1 --- /dev/null +++ b/lib/src/views/camera/painters/coordinates_translator.dart @@ -0,0 +1,52 @@ +import 'dart:io'; +import 'dart:ui'; + +import 'package:camera/camera.dart'; +import 'package:google_mlkit_barcode_scanning/google_mlkit_barcode_scanning.dart'; + +double translateX( + double x, + Size canvasSize, + Size imageSize, + InputImageRotation rotation, + CameraLensDirection cameraLensDirection, +) { + switch (rotation) { + case InputImageRotation.rotation90deg: + return x * + canvasSize.width / + (Platform.isIOS ? imageSize.width : imageSize.height); + case InputImageRotation.rotation270deg: + return canvasSize.width - + x * + canvasSize.width / + (Platform.isIOS ? imageSize.width : imageSize.height); + case InputImageRotation.rotation0deg: + case InputImageRotation.rotation180deg: + switch (cameraLensDirection) { + case CameraLensDirection.back: + return x * canvasSize.width / imageSize.width; + default: + return canvasSize.width - x * canvasSize.width / imageSize.width; + } + } +} + +double translateY( + double y, + Size canvasSize, + Size imageSize, + InputImageRotation rotation, + CameraLensDirection cameraLensDirection, +) { + switch (rotation) { + case InputImageRotation.rotation90deg: + case InputImageRotation.rotation270deg: + return y * + canvasSize.height / + (Platform.isIOS ? imageSize.height : imageSize.width); + case InputImageRotation.rotation0deg: + case InputImageRotation.rotation180deg: + return y * canvasSize.height / imageSize.height; + } +} diff --git a/lib/src/views/contact/contact_verify.view.dart b/lib/src/views/contact/contact_verify.view.dart index 2b37dd7..de226c4 100644 --- a/lib/src/views/contact/contact_verify.view.dart +++ b/lib/src/views/contact/contact_verify.view.dart @@ -3,7 +3,6 @@ import 'dart:convert'; import 'package:drift/drift.dart' hide Column; import 'package:flutter/material.dart'; -import 'package:flutter_zxing/flutter_zxing.dart'; import 'package:font_awesome_flutter/font_awesome_flutter.dart'; import 'package:image/image.dart' as imglib; import 'package:libsignal_protocol_dart/libsignal_protocol_dart.dart'; @@ -52,36 +51,36 @@ class _ContactVerifyViewState extends State { _fingerprint = await generateSessionFingerPrint(widget.contact.userId); if (_fingerprint != null) { - final result = zx.encodeBarcode( - contents: base64Encode( - _fingerprint!.scannableFingerprint.fingerprints, - ), - params: EncodeParams( - width: 150, - height: 150, - ), - ); - if (result.isValid && result.data != null) { - final img = imglib.Image.fromBytes( - width: 150, - height: 150, - bytes: result.data!.buffer, - numChannels: 1, - ); - _qrCodeImageBytes = imglib.encodePng(img); - } + // final result = zx.encodeBarcode( + // contents: base64Encode( + // _fingerprint!.scannableFingerprint.fingerprints, + // ), + // params: EncodeParams( + // width: 150, + // height: 150, + // ), + // ); + // if (result.isValid && result.data != null) { + // final img = imglib.Image.fromBytes( + // width: 150, + // height: 150, + // bytes: result.data!.buffer, + // numChannels: 1, + // ); + // _qrCodeImageBytes = imglib.encodePng(img); + // } } - final contact = twonlyDB.contactsDao - .getContactByUserId(widget.contact.userId) - .watchSingleOrNull(); - _contactSub = contact.listen((contact) { - if (contact == null) return; - setState(() { - _contact = contact; - }); - }); - setState(() {}); + // final contact = twonlyDB.contactsDao + // .getContactByUserId(widget.contact.userId) + // .watchSingleOrNull(); + // _contactSub = contact.listen((contact) { + // if (contact == null) return; + // setState(() { + // _contact = contact; + // }); + // }); + // setState(() {}); } Future openQrScanner() async { diff --git a/lib/src/views/contact/contact_verify_qr_scan.view.dart b/lib/src/views/contact/contact_verify_qr_scan.view.dart index 5328b42..1998f51 100644 --- a/lib/src/views/contact/contact_verify_qr_scan.view.dart +++ b/lib/src/views/contact/contact_verify_qr_scan.view.dart @@ -1,10 +1,6 @@ -import 'dart:convert'; - import 'package:flutter/material.dart'; -import 'package:flutter_zxing/flutter_zxing.dart'; import 'package:libsignal_protocol_dart/libsignal_protocol_dart.dart'; import 'package:twonly/src/database/twonly.db.dart'; -import 'package:twonly/src/utils/log.dart'; class ContactVerifyQrScanView extends StatefulWidget { const ContactVerifyQrScanView( @@ -23,23 +19,24 @@ class ContactVerifyQrScanView extends StatefulWidget { class _ContactVerifyQrScanViewState extends State { @override Widget build(BuildContext context) { - return Scaffold( - body: ReaderWidget( - onScan: (result) async { - var isValid = false; - try { - if (result.text != null) { - final otherFingerPrint = base64Decode(result.text!); - isValid = widget.fingerprint.scannableFingerprint.compareTo( - otherFingerPrint, - ); - } - } catch (e) { - Log.error('$e'); - } - return Navigator.pop(context, isValid); - }, - ), - ); + return Text('Not yet implemented.'); + // return Scaffold( + // body: ReaderWidget( + // onScan: (result) async { + // var isValid = false; + // try { + // if (result.text != null) { + // final otherFingerPrint = base64Decode(result.text!); + // isValid = widget.fingerprint.scannableFingerprint.compareTo( + // otherFingerPrint, + // ); + // } + // } catch (e) { + // Log.error('$e'); + // } + // return Navigator.pop(context, isValid); + // }, + // ), + // ); } } diff --git a/lib/src/views/home.view.dart b/lib/src/views/home.view.dart index 05b3320..7c7c893 100644 --- a/lib/src/views/home.view.dart +++ b/lib/src/views/home.view.dart @@ -1,8 +1,11 @@ import 'dart:async'; +import 'dart:io'; import 'package:camera/camera.dart'; import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; import 'package:flutter_local_notifications/flutter_local_notifications.dart'; import 'package:font_awesome_flutter/font_awesome_flutter.dart'; +import 'package:google_mlkit_barcode_scanning/google_mlkit_barcode_scanning.dart'; import 'package:screenshot/screenshot.dart'; import 'package:twonly/globals.dart'; import 'package:twonly/src/services/mediafiles/mediafile.service.dart'; @@ -10,6 +13,7 @@ import 'package:twonly/src/services/notifications/setup.notifications.dart'; import 'package:twonly/src/utils/misc.dart'; import 'package:twonly/src/views/camera/camera_preview_components/camera_preview.dart'; import 'package:twonly/src/views/camera/camera_preview_controller_view.dart'; +import 'package:twonly/src/views/camera/painters/barcode_detector_painter.dart'; import 'package:twonly/src/views/camera/share_image_editor_view.dart'; import 'package:twonly/src/views/chats/chat_list.view.dart'; import 'package:twonly/src/views/memories/memories.view.dart'; @@ -108,7 +112,9 @@ class HomeViewState extends State { void dispose() { unawaited(selectNotificationStream.close()); disableCameraTimer?.cancel(); + cameraController?.stopImageStream(); cameraController?.dispose(); + cameraController = null; super.dispose(); } @@ -123,6 +129,10 @@ class HomeViewState extends State { cameraController = opts.$2; initCameraStarted = false; } + if (cameraController?.description.lensDirection == + CameraLensDirection.back) { + await cameraController?.startImageStream(_processCameraImage); + } setState(() {}); return cameraController; } @@ -134,11 +144,127 @@ class HomeViewState extends State { if (cameraController!.value.isRecordingVideo) { return; } + await cameraController!.stopImageStream(); await cameraController!.dispose(); cameraController = null; await selectCamera((selectedCameraDetails.cameraId + 1) % 2, false); } + final BarcodeScanner _barcodeScanner = BarcodeScanner(); + bool _canProcess = true; + bool _isBusy = false; + CustomPaint? _customPaint; + String? _text; + + final _orientations = { + DeviceOrientation.portraitUp: 0, + DeviceOrientation.landscapeLeft: 90, + DeviceOrientation.portraitDown: 180, + DeviceOrientation.landscapeRight: 270, + }; + + void _processCameraImage(CameraImage image) { + final inputImage = _inputImageFromCameraImage(image); + if (inputImage == null) return; + _processImage(inputImage); + } + + InputImage? _inputImageFromCameraImage(CameraImage image) { + if (cameraController == null) return null; + + // get image rotation + // it is used in android to convert the InputImage from Dart to Java: https://github.com/flutter-ml/google_ml_kit_flutter/blob/master/packages/google_mlkit_commons/android/src/main/java/com/google_mlkit_commons/InputImageConverter.java + // `rotation` is not used in iOS to convert the InputImage from Dart to Obj-C: https://github.com/flutter-ml/google_ml_kit_flutter/blob/master/packages/google_mlkit_commons/ios/Classes/MLKVisionImage%2BFlutterPlugin.m + // in both platforms `rotation` and `camera.lensDirection` can be used to compensate `x` and `y` coordinates on a canvas: https://github.com/flutter-ml/google_ml_kit_flutter/blob/master/packages/example/lib/vision_detector_views/painters/coordinates_translator.dart + final camera = cameraController!.description; + final sensorOrientation = camera.sensorOrientation; + // print( + // 'lensDirection: ${camera.lensDirection}, sensorOrientation: $sensorOrientation, ${_controller?.value.deviceOrientation} ${_controller?.value.lockedCaptureOrientation} ${_controller?.value.isCaptureOrientationLocked}'); + InputImageRotation? rotation; + if (Platform.isIOS) { + rotation = InputImageRotationValue.fromRawValue(sensorOrientation); + } else if (Platform.isAndroid) { + var rotationCompensation = + _orientations[cameraController!.value.deviceOrientation]; + if (rotationCompensation == null) return null; + if (camera.lensDirection == CameraLensDirection.front) { + // front-facing + rotationCompensation = (sensorOrientation + rotationCompensation) % 360; + } else { + // back-facing + rotationCompensation = + (sensorOrientation - rotationCompensation + 360) % 360; + } + rotation = InputImageRotationValue.fromRawValue(rotationCompensation); + // print('rotationCompensation: $rotationCompensation'); + } + if (rotation == null) return null; + // print('final rotation: $rotation'); + + // get image format + var format = InputImageFormatValue.fromRawValue(image.format.raw as int); + // validate format depending on platform + // only supported formats: + // * nv21 for Android + // * bgra8888 for iOS + if (Platform.isAndroid && format == InputImageFormat.yuv420) { + // https://developer.android.com/reference/kotlin/androidx/camera/core/ImageAnalysis#OUTPUT_IMAGE_FORMAT_NV21() + format = InputImageFormat.nv21; + } + if (format == null || + (Platform.isAndroid && format != InputImageFormat.nv21) || + (Platform.isIOS && format != InputImageFormat.bgra8888)) { + return null; + } + + // since format is constraint to nv21 or bgra8888, both only have one plane + if (image.planes.length != 1) return null; + final plane = image.planes.first; + + // compose InputImage using bytes + return InputImage.fromBytes( + bytes: plane.bytes, + metadata: InputImageMetadata( + size: Size(image.width.toDouble(), image.height.toDouble()), + rotation: rotation, // used only in Android + format: format, // used only in iOS + bytesPerRow: plane.bytesPerRow, // used only in iOS + ), + ); + } + + Future _processImage(InputImage inputImage) async { + if (!_canProcess) return; + if (_isBusy) return; + _isBusy = true; + setState(() { + _text = ''; + }); + final barcodes = await _barcodeScanner.processImage(inputImage); + if (inputImage.metadata?.size != null && + inputImage.metadata?.rotation != null && + cameraController != null) { + final painter = BarcodeDetectorPainter( + barcodes, + inputImage.metadata!.size, + inputImage.metadata!.rotation, + cameraController!.description.lensDirection); + _customPaint = CustomPaint(painter: painter); + } else { + String text = 'Barcodes found: ${barcodes.length}\n\n'; + for (final barcode in barcodes) { + text += 'Barcode: ${barcode.rawValue}\n\n'; + } + _text = text; + // TODO: set _customPaint to draw boundingRect on top of image + _customPaint = null; + } + _isBusy = false; + if (mounted) { + setState(() {}); + } + } + Future initAsync() async { final notificationAppLaunchDetails = await flutterLocalNotificationsPlugin.getNotificationAppLaunchDetails(); @@ -174,6 +300,7 @@ class HomeViewState extends State { HomeViewCameraPreview( controller: cameraController, screenshotController: screenshotController, + customPaint: _customPaint, ), Shade( opacity: offsetRatio, diff --git a/pubspec.lock b/pubspec.lock index 82b8e83..a591354 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -173,11 +173,11 @@ packages: dependency: "direct overridden" description: path: "packages/camera/camera_android_camerax" - ref: aef58af205a5f3ce6588a5c59bb2e734aab943f0 - resolved-ref: aef58af205a5f3ce6588a5c59bb2e734aab943f0 + ref: "43b87faec960306f98d767253b9bf2cee61be630" + resolved-ref: "43b87faec960306f98d767253b9bf2cee61be630" url: "https://github.com/otsmr/flutter-packages.git" source: git - version: "0.6.23+2" + version: "0.6.25+1" camera_avfoundation: dependency: transitive description: @@ -786,13 +786,6 @@ packages: description: flutter source: sdk version: "0.0.0" - flutter_zxing: - dependency: "direct main" - description: - path: "dependencies/flutter_zxing" - relative: true - source: path - version: "2.1.0" font_awesome_flutter: dependency: "direct main" description: @@ -825,6 +818,22 @@ packages: url: "https://pub.dev" source: hosted version: "2.1.3" + google_mlkit_barcode_scanning: + dependency: "direct main" + description: + name: google_mlkit_barcode_scanning + sha256: b38505df2d3fdf7830979d60fee55039c2f442d189b2e06fcb2fe494ba65d0db + url: "https://pub.dev" + source: hosted + version: "0.14.1" + google_mlkit_commons: + dependency: transitive + description: + name: google_mlkit_commons + sha256: "8f40fbac10685cad4715d11e6a0d86837d9ad7168684dfcad29610282a88e67a" + url: "https://pub.dev" + source: hosted + version: "0.11.0" graphs: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 1fcfc73..827233f 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -42,11 +42,10 @@ dependencies: path: flutter_secure_storage/ flutter_svg: ^2.0.17 flutter_volume_controller: ^1.3.4 - flutter_zxing: - path: ./dependencies/flutter_zxing font_awesome_flutter: ^10.10.0 gal: ^2.3.1 get: ^4.7.2 + google_mlkit_barcode_scanning: ^0.14.1 hand_signature: ^3.0.3 hashlib: ^2.0.0 http: ^1.3.0 @@ -85,7 +84,7 @@ dependency_overrides: git: url: https://github.com/otsmr/flutter-packages.git path: packages/camera/camera_android_camerax - ref: aef58af205a5f3ce6588a5c59bb2e734aab943f0 + ref: 43b87faec960306f98d767253b9bf2cee61be630 emoji_picker_flutter: # Fixes the issue with recent emojis (solved by https://github.com/Fintasys/emoji_picker_flutter/pull/238) # Using override until this gets merged.