opencv: Having java.lang.Exception: unknown exception with text detection algorithm

We can close this issue 😃 I’ll check out on how to do a (good) pull request ! Thanks again for everything !

@Zappyford, @berak, try to vary input’s width and height:

./bin/example_dnn_text_detection -m=frozen_east_text_detection.pb  -i=48361260-6e553080-e6a1-11e8-8f2b-ae6c2bea2231.jpg --width=320 --height=320

./bin/example_dnn_text_detection -m=frozen_east_text_detection.pb  -i=48361260-6e553080-e6a1-11e8-8f2b-ae6c2bea2231.jpg --width=640 --height=640

It also make sense to check which input size is used in TensorFlow’s demo.

@dkurt

@dkurt i’ve tried in the original sample in C++ with width=320 and height=320 and it’s the same. @berak here is my code :

import org.opencv.core.*;
import org.opencv.dnn.Dnn;
import org.opencv.dnn.Net;
import org.opencv.imgcodecs.Imgcodecs;
import org.opencv.imgproc.Imgproc;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;


public class TextDetection {
    public static void main(String[] args) {
        System.loadLibrary(Core.NATIVE_LIBRARY_NAME);

        // Model from https://github.com/argman/EAST
        // You can find it here : https://github.com/opencv/opencv_extra/blob/master/testdata/dnn/download_models.py#L309
        Net net = Dnn.readNetFromTensorflow("path/to/model");
        // input image
        Mat frame = Imgcodecs.imread("path/to/jpg");
        Imgproc.cvtColor(frame, frame, Imgproc.COLOR_RGBA2RGB);
        Mat blob = Dnn.blobFromImage(frame, 1.0, new Size(320, 320), new Scalar(123.68, 116.78, 103.94), true, false);
        net.setInput(blob);
        List<Mat> outs = new ArrayList<>(2);
        List<String> outNames = Arrays.asList("feature_fusion/Conv_7/Sigmoid", "feature_fusion/concat_3");
        net.forward(outs, outNames);
        // scores.shape : (1,1,80,80)
        Mat scores = outs.get(0).reshape(1, 80);
        // geometry.shape : (1,5,80,80)
        // My lord and savior : http://answers.opencv.org/question/175676/javaandroid-access-4-dim-mat-planes/
        Mat geometry = outs.get(1).reshape(1, 5 * 80);
        List<Float> confidencesList = new ArrayList<>();
        // Decode predicted bounding boxes.
        List<RotatedRect> boxesList = decode(scores, geometry, confidencesList);

        float[] confidencesArray = new float[confidencesList.size()];
        for (int i = 0; i < confidencesList.size(); ++i) {
            confidencesArray[i] = confidencesList.get(i) != null ? confidencesList.get(i) : Float.NaN;
        }

        MatOfFloat confidences = new MatOfFloat(confidencesArray);
        RotatedRect[] boxesArray = boxesList.toArray(new RotatedRect[0]);
        MatOfRotatedRect boxes = new MatOfRotatedRect(boxesArray);
        MatOfInt indices = new MatOfInt();
        // Apply non-maximum suppression procedure.
        Dnn.NMSBoxesRotated(boxes, confidences, 0.5f,0.4f, indices);

        // Render detections
        Point ratio = new Point((float)frame.cols()/320, (float)frame.rows()/320);
        int[] indexes = indices.toArray();
        for(int i = 0; i<indexes.length;++i) {
            RotatedRect rot = boxesArray[indexes[i]];
            Point[] vertices = new Point[4];
            rot.points(vertices);
            for (int j = 0; j < 4; ++j) {
                vertices[j].x *= ratio.x;
                vertices[j].y *= ratio.y;
            }
            for (int j = 0; j < 4; ++j) {
                Imgproc.line(frame, vertices[j], vertices[(j + 1) % 4], new Scalar(0, 255, 0), 1);
            }
        }
        Imgcodecs.imwrite("path/to/output/image", frame);
    }

    private static List<RotatedRect> decode(Mat scores, Mat geometry, List<Float> confidences) {
        if (scores.dims() != 2 && geometry.dims() != 2 && scores.height() != 80 &&
                scores.width() != 80 && geometry.height() != 400 && geometry.width() != 80) {
            throw new RuntimeException("That sucks mate");
        }
        List<RotatedRect> detections = new ArrayList<>();
        float scoreThresh = 0.5f;
        for (int y = 0; y < 80; ++y) {
            Mat scoresData = scores.row(y);
            //1st plane
            Mat x0Data = geometry.submat(0, 80, 0, 80).row(y);
            //2nd plane
            Mat x1Data = geometry.submat(80, 2 * 80, 0, 80).row(y);
            Mat x2Data = geometry.submat(2 * 80, 3 * 80, 0, 80).row(y);
            Mat x3Data = geometry.submat(3 * 80, 4 * 80, 0, 80).row(y);
            Mat anglesData = geometry.submat(4 * 80, 5 * 80, 0, 80).row(y);

            for (int x = 0; x < 80; ++x) {
                double score = scoresData.get(0, x)[0];
                if (score >= scoreThresh) {
                    double offsetX = x * 4.0;
                    double offsetY = y * 4.0;
                    double angle = anglesData.get(0, x)[0];
                    double cosA = Math.cos(angle);
                    double sinA = Math.sin(angle);
                    double x0 = x0Data.get(0, x)[0];
                    double x1 = x1Data.get(0, x)[0];
                    double x2 = x2Data.get(0, x)[0];
                    double x3 = x3Data.get(0, x)[0];
                    double h = x0 + x2;
                    double w = x1 + x3;
                    Point offset = new Point(offsetX + cosA * x1 + sinA * x2, offsetY - sinA * x1 + cosA * x2);
                    Point p1 = new Point(-1 * sinA * h + offset.x, -1 * cosA * h + offset.y);
                    Point p3 = new Point(-1 * sinA * w + offset.x, -1 * cosA * w + offset.y);
                    RotatedRect r = new RotatedRect(new Point(0.5 * (p1.x + p3.x), 0.5 * (p1.y + p3.y)), new Size(w, h), -1 * angle * 180 / Math.PI);
                    detections.add(r);
                    confidences.add((float) score);
                }
            }
        }
        return detections;
    }
}


`

Wow ! Ok I’ll try it ! Thank you !

i think, it’s this: https://github.com/opencv/opencv/issues/12324

(the fix didn’t make it into the 3.4.3 release, unfortunately)

@Zappyford , maybe you can take a look, if you have this fix ?

but then, there will be another problem: there is no working java version for NMSBoxesRotated() yet 😭