starting with #327

lib/src/views/camera/camera_preview_components/camera_preview.dart

@@ -7,10 +7,12 @@ class HomeViewCameraPreview extends StatelessWidget {
   const HomeViewCameraPreview({
     required this.controller,
     required this.screenshotController,
+    required this.customPaint,
     super.key,
   });
 
   final CameraController? controller;
+  final CustomPaint? customPaint;
   final ScreenshotController screenshotController;
 
   @override
@@ -33,7 +35,7 @@ class HomeViewCameraPreview extends StatelessWidget {
                 child: SizedBox(
                   width: controller!.value.previewSize!.height,
                   height: controller!.value.previewSize!.width,
-                  child: CameraPreview(controller!),
+                  child: CameraPreview(controller!, child: customPaint),
                 ),
               ),
             ),
@@ -48,11 +50,13 @@ class SendToCameraPreview extends StatelessWidget {
   const SendToCameraPreview({
     required this.cameraController,
     required this.screenshotController,
+    required this.customPaint,
     super.key,
   });
 
   final CameraController? cameraController;
   final ScreenshotController screenshotController;
+  final CustomPaint? customPaint;
 
   @override
   Widget build(BuildContext context) {
@@ -73,7 +77,7 @@ class SendToCameraPreview extends StatelessWidget {
                 child: SizedBox(
                   width: cameraController!.value.previewSize!.height,
                   height: cameraController!.value.previewSize!.width,
-                  child: CameraPreview(cameraController!),
+                  child: CameraPreview(cameraController!, child: customPaint),
                 ),
               ),
             ),

lib/src/views/camera/camera_preview_controller_view.dart

@@ -52,6 +52,8 @@ Future<(SelectedCameraDetails, CameraController)?> initializeCameraController(
     gCameras[cameraId],
     ResolutionPreset.high,
     enableAudio: await Permission.microphone.isGranted,
+    imageFormatGroup:
+        Platform.isAndroid ? ImageFormatGroup.nv21 : ImageFormatGroup.bgra8888,
   );
 
   await cameraController.initialize().then((_) async {

lib/src/views/camera/camera_send_to_view.dart

@@ -72,6 +72,7 @@ class CameraSendToViewState extends State<CameraSendToView> {
             SendToCameraPreview(
               cameraController: cameraController,
               screenshotController: screenshotController,
+              customPaint: null,
             ),
             CameraPreviewControllerView(
               selectCamera: selectCamera,

lib/src/views/camera/painters/barcode_detector_painter.dart

@@ -0,0 +1,124 @@
+import 'dart:io';
+import 'dart:ui';
+import 'dart:ui' as ui;
+
+import 'package:camera/camera.dart';
+import 'package:flutter/material.dart';
+import 'package:google_mlkit_barcode_scanning/google_mlkit_barcode_scanning.dart';
+
+import 'coordinates_translator.dart';
+
+class BarcodeDetectorPainter extends CustomPainter {
+  BarcodeDetectorPainter(
+    this.barcodes,
+    this.imageSize,
+    this.rotation,
+    this.cameraLensDirection,
+  );
+
+  final List<Barcode> barcodes;
+  final Size imageSize;
+  final InputImageRotation rotation;
+  final CameraLensDirection cameraLensDirection;
+
+  @override
+  void paint(Canvas canvas, Size size) {
+    final Paint paint = Paint()
+      ..style = PaintingStyle.stroke
+      ..strokeWidth = 3.0
+      ..color = Colors.lightGreenAccent;
+
+    final Paint background = Paint()..color = Color(0x99000000);
+
+    for (final Barcode barcode in barcodes) {
+      final ParagraphBuilder builder = ParagraphBuilder(
+        ParagraphStyle(
+            textAlign: TextAlign.left,
+            fontSize: 16,
+            textDirection: TextDirection.ltr),
+      );
+      builder.pushStyle(
+          ui.TextStyle(color: Colors.lightGreenAccent, background: background));
+      builder.addText('${barcode.displayValue}');
+      builder.pop();
+
+      final left = translateX(
+        barcode.boundingBox.left,
+        size,
+        imageSize,
+        rotation,
+        cameraLensDirection,
+      );
+      final top = translateY(
+        barcode.boundingBox.top,
+        size,
+        imageSize,
+        rotation,
+        cameraLensDirection,
+      );
+      final right = translateX(
+        barcode.boundingBox.right,
+        size,
+        imageSize,
+        rotation,
+        cameraLensDirection,
+      );
+      // final bottom = translateY(
+      //   barcode.boundingBox.bottom,
+      //   size,
+      //   imageSize,
+      //   rotation,
+      //   cameraLensDirection,
+      // );
+      //
+      // // Draw a bounding rectangle around the barcode
+      // canvas.drawRect(
+      //   Rect.fromLTRB(left, top, right, bottom),
+      //   paint,
+      // );
+
+      final List<Offset> cornerPoints = <Offset>[];
+      for (final point in barcode.cornerPoints) {
+        final double x = translateX(
+          point.x.toDouble(),
+          size,
+          imageSize,
+          rotation,
+          cameraLensDirection,
+        );
+        final double y = translateY(
+          point.y.toDouble(),
+          size,
+          imageSize,
+          rotation,
+          cameraLensDirection,
+        );
+
+        cornerPoints.add(Offset(x, y));
+      }
+
+      // Add the first point to close the polygon
+      cornerPoints.add(cornerPoints.first);
+      canvas.drawPoints(PointMode.polygon, cornerPoints, paint);
+
+      canvas.drawParagraph(
+        builder.build()
+          ..layout(ParagraphConstraints(
+            width: (right - left).abs(),
+          )),
+        Offset(
+            Platform.isAndroid &&
+                    cameraLensDirection == CameraLensDirection.front
+                ? right
+                : left,
+            top),
+      );
+    }
+  }
+
+  @override
+  bool shouldRepaint(BarcodeDetectorPainter oldDelegate) {
+    return oldDelegate.imageSize != imageSize ||
+        oldDelegate.barcodes != barcodes;
+  }
+}

lib/src/views/camera/painters/coordinates_translator.dart

@@ -0,0 +1,52 @@
+import 'dart:io';
+import 'dart:ui';
+
+import 'package:camera/camera.dart';
+import 'package:google_mlkit_barcode_scanning/google_mlkit_barcode_scanning.dart';
+
+double translateX(
+  double x,
+  Size canvasSize,
+  Size imageSize,
+  InputImageRotation rotation,
+  CameraLensDirection cameraLensDirection,
+) {
+  switch (rotation) {
+    case InputImageRotation.rotation90deg:
+      return x *
+          canvasSize.width /
+          (Platform.isIOS ? imageSize.width : imageSize.height);
+    case InputImageRotation.rotation270deg:
+      return canvasSize.width -
+          x *
+              canvasSize.width /
+              (Platform.isIOS ? imageSize.width : imageSize.height);
+    case InputImageRotation.rotation0deg:
+    case InputImageRotation.rotation180deg:
+      switch (cameraLensDirection) {
+        case CameraLensDirection.back:
+          return x * canvasSize.width / imageSize.width;
+        default:
+          return canvasSize.width - x * canvasSize.width / imageSize.width;
+      }
+  }
+}
+
+double translateY(
+  double y,
+  Size canvasSize,
+  Size imageSize,
+  InputImageRotation rotation,
+  CameraLensDirection cameraLensDirection,
+) {
+  switch (rotation) {
+    case InputImageRotation.rotation90deg:
+    case InputImageRotation.rotation270deg:
+      return y *
+          canvasSize.height /
+          (Platform.isIOS ? imageSize.height : imageSize.width);
+    case InputImageRotation.rotation0deg:
+    case InputImageRotation.rotation180deg:
+      return y * canvasSize.height / imageSize.height;
+  }
+}

lib/src/views/contact/contact_verify.view.dart

@@ -3,7 +3,6 @@ import 'dart:convert';
 
 import 'package:drift/drift.dart' hide Column;
 import 'package:flutter/material.dart';
-import 'package:flutter_zxing/flutter_zxing.dart';
 import 'package:font_awesome_flutter/font_awesome_flutter.dart';
 import 'package:image/image.dart' as imglib;
 import 'package:libsignal_protocol_dart/libsignal_protocol_dart.dart';
@@ -52,36 +51,36 @@ class _ContactVerifyViewState extends State<ContactVerifyView> {
     _fingerprint = await generateSessionFingerPrint(widget.contact.userId);
 
     if (_fingerprint != null) {
-      final result = zx.encodeBarcode(
-        contents: base64Encode(
-          _fingerprint!.scannableFingerprint.fingerprints,
-        ),
-        params: EncodeParams(
-          width: 150,
-          height: 150,
-        ),
-      );
-      if (result.isValid && result.data != null) {
-        final img = imglib.Image.fromBytes(
-          width: 150,
-          height: 150,
-          bytes: result.data!.buffer,
-          numChannels: 1,
-        );
-        _qrCodeImageBytes = imglib.encodePng(img);
-      }
+      // final result = zx.encodeBarcode(
+      //   contents: base64Encode(
+      //     _fingerprint!.scannableFingerprint.fingerprints,
+      //   ),
+      //   params: EncodeParams(
+      //     width: 150,
+      //     height: 150,
+      //   ),
+      // );
+      // if (result.isValid && result.data != null) {
+      //   final img = imglib.Image.fromBytes(
+      //     width: 150,
+      //     height: 150,
+      //     bytes: result.data!.buffer,
+      //     numChannels: 1,
+      //   );
+      //   _qrCodeImageBytes = imglib.encodePng(img);
+      // }
     }
 
-    final contact = twonlyDB.contactsDao
-        .getContactByUserId(widget.contact.userId)
-        .watchSingleOrNull();
-    _contactSub = contact.listen((contact) {
-      if (contact == null) return;
-      setState(() {
-        _contact = contact;
-      });
-    });
-    setState(() {});
+    // final contact = twonlyDB.contactsDao
+    //     .getContactByUserId(widget.contact.userId)
+    //     .watchSingleOrNull();
+    // _contactSub = contact.listen((contact) {
+    //   if (contact == null) return;
+    //   setState(() {
+    //     _contact = contact;
+    //   });
+    // });
+    // setState(() {});
   }
 
   Future<void> openQrScanner() async {

lib/src/views/contact/contact_verify_qr_scan.view.dart

@@ -1,10 +1,6 @@
-import 'dart:convert';
-
 import 'package:flutter/material.dart';
-import 'package:flutter_zxing/flutter_zxing.dart';
 import 'package:libsignal_protocol_dart/libsignal_protocol_dart.dart';
 import 'package:twonly/src/database/twonly.db.dart';
-import 'package:twonly/src/utils/log.dart';
 
 class ContactVerifyQrScanView extends StatefulWidget {
   const ContactVerifyQrScanView(
@@ -23,23 +19,24 @@ class ContactVerifyQrScanView extends StatefulWidget {
 class _ContactVerifyQrScanViewState extends State<ContactVerifyQrScanView> {
   @override
   Widget build(BuildContext context) {
-    return Scaffold(
-      body: ReaderWidget(
-        onScan: (result) async {
-          var isValid = false;
-          try {
-            if (result.text != null) {
-              final otherFingerPrint = base64Decode(result.text!);
-              isValid = widget.fingerprint.scannableFingerprint.compareTo(
-                otherFingerPrint,
-              );
-            }
-          } catch (e) {
-            Log.error('$e');
-          }
-          return Navigator.pop(context, isValid);
-        },
-      ),
-    );
+    return Text('Not yet implemented.');
+    // return Scaffold(
+    //   body: ReaderWidget(
+    //     onScan: (result) async {
+    //       var isValid = false;
+    //       try {
+    //         if (result.text != null) {
+    //           final otherFingerPrint = base64Decode(result.text!);
+    //           isValid = widget.fingerprint.scannableFingerprint.compareTo(
+    //             otherFingerPrint,
+    //           );
+    //         }
+    //       } catch (e) {
+    //         Log.error('$e');
+    //       }
+    //       return Navigator.pop(context, isValid);
+    //     },
+    //   ),
+    // );
   }
 }

lib/src/views/home.view.dart

@@ -1,8 +1,11 @@
 import 'dart:async';
+import 'dart:io';
 import 'package:camera/camera.dart';
 import 'package:flutter/material.dart';
+import 'package:flutter/services.dart';
 import 'package:flutter_local_notifications/flutter_local_notifications.dart';
 import 'package:font_awesome_flutter/font_awesome_flutter.dart';
+import 'package:google_mlkit_barcode_scanning/google_mlkit_barcode_scanning.dart';
 import 'package:screenshot/screenshot.dart';
 import 'package:twonly/globals.dart';
 import 'package:twonly/src/services/mediafiles/mediafile.service.dart';
@@ -10,6 +13,7 @@ import 'package:twonly/src/services/notifications/setup.notifications.dart';
 import 'package:twonly/src/utils/misc.dart';
 import 'package:twonly/src/views/camera/camera_preview_components/camera_preview.dart';
 import 'package:twonly/src/views/camera/camera_preview_controller_view.dart';
+import 'package:twonly/src/views/camera/painters/barcode_detector_painter.dart';
 import 'package:twonly/src/views/camera/share_image_editor_view.dart';
 import 'package:twonly/src/views/chats/chat_list.view.dart';
 import 'package:twonly/src/views/memories/memories.view.dart';
@@ -108,7 +112,9 @@ class HomeViewState extends State<HomeView> {
   void dispose() {
     unawaited(selectNotificationStream.close());
     disableCameraTimer?.cancel();
+    cameraController?.stopImageStream();
     cameraController?.dispose();
+    cameraController = null;
     super.dispose();
   }
 
@@ -123,6 +129,10 @@ class HomeViewState extends State<HomeView> {
       cameraController = opts.$2;
       initCameraStarted = false;
     }
+    if (cameraController?.description.lensDirection ==
+        CameraLensDirection.back) {
+      await cameraController?.startImageStream(_processCameraImage);
+    }
     setState(() {});
     return cameraController;
   }
@@ -134,11 +144,127 @@ class HomeViewState extends State<HomeView> {
     if (cameraController!.value.isRecordingVideo) {
       return;
     }
+    await cameraController!.stopImageStream();
     await cameraController!.dispose();
     cameraController = null;
     await selectCamera((selectedCameraDetails.cameraId + 1) % 2, false);
   }
 
+  final BarcodeScanner _barcodeScanner = BarcodeScanner();
+  bool _canProcess = true;
+  bool _isBusy = false;
+  CustomPaint? _customPaint;
+  String? _text;
+
+  final _orientations = {
+    DeviceOrientation.portraitUp: 0,
+    DeviceOrientation.landscapeLeft: 90,
+    DeviceOrientation.portraitDown: 180,
+    DeviceOrientation.landscapeRight: 270,
+  };
+
+  void _processCameraImage(CameraImage image) {
+    final inputImage = _inputImageFromCameraImage(image);
+    if (inputImage == null) return;
+    _processImage(inputImage);
+  }
+
+  InputImage? _inputImageFromCameraImage(CameraImage image) {
+    if (cameraController == null) return null;
+
+    // get image rotation
+    // it is used in android to convert the InputImage from Dart to Java: https://github.com/flutter-ml/google_ml_kit_flutter/blob/master/packages/google_mlkit_commons/android/src/main/java/com/google_mlkit_commons/InputImageConverter.java
+    // `rotation` is not used in iOS to convert the InputImage from Dart to Obj-C: https://github.com/flutter-ml/google_ml_kit_flutter/blob/master/packages/google_mlkit_commons/ios/Classes/MLKVisionImage%2BFlutterPlugin.m
+    // in both platforms `rotation` and `camera.lensDirection` can be used to compensate `x` and `y` coordinates on a canvas: https://github.com/flutter-ml/google_ml_kit_flutter/blob/master/packages/example/lib/vision_detector_views/painters/coordinates_translator.dart
+    final camera = cameraController!.description;
+    final sensorOrientation = camera.sensorOrientation;
+    // print(
+    //     'lensDirection: ${camera.lensDirection}, sensorOrientation: $sensorOrientation, ${_controller?.value.deviceOrientation} ${_controller?.value.lockedCaptureOrientation} ${_controller?.value.isCaptureOrientationLocked}');
+    InputImageRotation? rotation;
+    if (Platform.isIOS) {
+      rotation = InputImageRotationValue.fromRawValue(sensorOrientation);
+    } else if (Platform.isAndroid) {
+      var rotationCompensation =
+          _orientations[cameraController!.value.deviceOrientation];
+      if (rotationCompensation == null) return null;
+      if (camera.lensDirection == CameraLensDirection.front) {
+        // front-facing
+        rotationCompensation = (sensorOrientation + rotationCompensation) % 360;
+      } else {
+        // back-facing
+        rotationCompensation =
+            (sensorOrientation - rotationCompensation + 360) % 360;
+      }
+      rotation = InputImageRotationValue.fromRawValue(rotationCompensation);
+      // print('rotationCompensation: $rotationCompensation');
+    }
+    if (rotation == null) return null;
+    // print('final rotation: $rotation');
+
+    // get image format
+    var format = InputImageFormatValue.fromRawValue(image.format.raw as int);
+    // validate format depending on platform
+    // only supported formats:
+    // * nv21 for Android
+    // * bgra8888 for iOS
+    if (Platform.isAndroid && format == InputImageFormat.yuv420) {
+      // https://developer.android.com/reference/kotlin/androidx/camera/core/ImageAnalysis#OUTPUT_IMAGE_FORMAT_NV21()
+      format = InputImageFormat.nv21;
+    }
+    if (format == null ||
+        (Platform.isAndroid && format != InputImageFormat.nv21) ||
+        (Platform.isIOS && format != InputImageFormat.bgra8888)) {
+      return null;
+    }
+
+    // since format is constraint to nv21 or bgra8888, both only have one plane
+    if (image.planes.length != 1) return null;
+    final plane = image.planes.first;
+
+    // compose InputImage using bytes
+    return InputImage.fromBytes(
+      bytes: plane.bytes,
+      metadata: InputImageMetadata(
+        size: Size(image.width.toDouble(), image.height.toDouble()),
+        rotation: rotation, // used only in Android
+        format: format, // used only in iOS
+        bytesPerRow: plane.bytesPerRow, // used only in iOS
+      ),
+    );
+  }
+
+  Future<void> _processImage(InputImage inputImage) async {
+    if (!_canProcess) return;
+    if (_isBusy) return;
+    _isBusy = true;
+    setState(() {
+      _text = '';
+    });
+    final barcodes = await _barcodeScanner.processImage(inputImage);
+    if (inputImage.metadata?.size != null &&
+        inputImage.metadata?.rotation != null &&
+        cameraController != null) {
+      final painter = BarcodeDetectorPainter(
+          barcodes,
+          inputImage.metadata!.size,
+          inputImage.metadata!.rotation,
+          cameraController!.description.lensDirection);
+      _customPaint = CustomPaint(painter: painter);
+    } else {
+      String text = 'Barcodes found: ${barcodes.length}\n\n';
+      for (final barcode in barcodes) {
+        text += 'Barcode: ${barcode.rawValue}\n\n';
+      }
+      _text = text;
+      // TODO: set _customPaint to draw boundingRect on top of image
+      _customPaint = null;
+    }
+    _isBusy = false;
+    if (mounted) {
+      setState(() {});
+    }
+  }
+
   Future<void> initAsync() async {
     final notificationAppLaunchDetails =
         await flutterLocalNotificationsPlugin.getNotificationAppLaunchDetails();
@@ -174,6 +300,7 @@ class HomeViewState extends State<HomeView> {
             HomeViewCameraPreview(
               controller: cameraController,
               screenshotController: screenshotController,
+              customPaint: _customPaint,
             ),
             Shade(
               opacity: offsetRatio,

pubspec.lock

@@ -173,11 +173,11 @@ packages:
     dependency: "direct overridden"
     description:
       path: "packages/camera/camera_android_camerax"
-      ref: aef58af205a5f3ce6588a5c59bb2e734aab943f0
-      resolved-ref: aef58af205a5f3ce6588a5c59bb2e734aab943f0
+      ref: "43b87faec960306f98d767253b9bf2cee61be630"
+      resolved-ref: "43b87faec960306f98d767253b9bf2cee61be630"
       url: "https://github.com/otsmr/flutter-packages.git"
     source: git
-    version: "0.6.23+2"
+    version: "0.6.25+1"
   camera_avfoundation:
     dependency: transitive
     description:
@@ -786,13 +786,6 @@ packages:
     description: flutter
     source: sdk
     version: "0.0.0"
-  flutter_zxing:
-    dependency: "direct main"
-    description:
-      path: "dependencies/flutter_zxing"
-      relative: true
-    source: path
-    version: "2.1.0"
   font_awesome_flutter:
     dependency: "direct main"
     description:
@@ -825,6 +818,22 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "2.1.3"
+  google_mlkit_barcode_scanning:
+    dependency: "direct main"
+    description:
+      name: google_mlkit_barcode_scanning
+      sha256: b38505df2d3fdf7830979d60fee55039c2f442d189b2e06fcb2fe494ba65d0db
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.14.1"
+  google_mlkit_commons:
+    dependency: transitive
+    description:
+      name: google_mlkit_commons
+      sha256: "8f40fbac10685cad4715d11e6a0d86837d9ad7168684dfcad29610282a88e67a"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.11.0"
   graphs:
     dependency: transitive
     description:

pubspec.yaml

@@ -42,11 +42,10 @@ dependencies:
       path: flutter_secure_storage/
   flutter_svg: ^2.0.17
   flutter_volume_controller: ^1.3.4
-  flutter_zxing:
-    path: ./dependencies/flutter_zxing
   font_awesome_flutter: ^10.10.0
   gal: ^2.3.1
   get: ^4.7.2
+  google_mlkit_barcode_scanning: ^0.14.1
   hand_signature: ^3.0.3
   hashlib: ^2.0.0
   http: ^1.3.0
@@ -85,7 +84,7 @@ dependency_overrides:
     git:
       url: https://github.com/otsmr/flutter-packages.git
       path: packages/camera/camera_android_camerax 
-      ref: aef58af205a5f3ce6588a5c59bb2e734aab943f0
+      ref: 43b87faec960306f98d767253b9bf2cee61be630
   emoji_picker_flutter:
     # Fixes the issue with recent emojis (solved by https://github.com/Fintasys/emoji_picker_flutter/pull/238)
     # Using override until this gets merged.