tfjs: Some androids failure to compile fragment shader

TensorFlow.js version

0.13.5

Browser version

  • Android: 6.0.1
  • Chrome: 70.0.3536.110
  • Error Devices:
    • Xperia Z3 SO-01G
    • GALAXY S5 SC-04F
    • AQUOS EVER SH-04G

Describe the problem or feature request

Tested tfjs-examples on mobile and some android devices failure to compile fragment shader. if the gpu spec is low, it tend to failure. All failure devices threw following error. Do you have a minimum requirement?

Link QR iPhone7 Plus Xperia XZ2 Xperia Z3
Posenet Sample image OK OK Error
Custom Layer image OK OK Error

Error log

1    
2      precision highp float;
3      precision highp int;
4      varying vec2 resultUV;
5      const vec2 halfCR = vec2(0.5, 0.5);
6    
7      struct ivec5
8      {
9        int x;
10       int y;
11       int z;
Fragment shader compilation failed.
12       int w;                                                                   
13       int u;
14     };
15   
16     struct ivec6
17     {
18       int x;
19       int y;
20       int z;
21       int w;
22       int u;
23       int v;
24     };
25   
26     bool isNaN(float val) {
27       return (val < 0.0 || 0.0 < val || val == 0.0) ? false : true;
28     }
29   
30     bool hasNaN(vec4 values) {
31       vec4 v1 = values * values;
32       vec4 v2 = values * values;
33       return any(notEqual(v1, v2));
34     }
35   
36     float getNaN(vec4 values) {
37       return dot(vec4(1), values);
38     }
39   
40     int round(float value) {
41       return int(floor(value + 0.5));
42     }
43   
44     int imod(int x, int y) {
45       return x - y * (x / y);
46     }
47   
48     //Based on the work of Dave Hoskins
49     //https://www.shadertoy.com/view/4djSRW
50     #define HASHSCALE1 443.8975
51     float random(float seed){
52       vec2 p = resultUV * seed;
53       vec3 p3  = fract(vec3(p.xyx) * HASHSCALE1);
54       p3 += dot(p3, p3.yzx + 19.19);
55       return fract((p3.x + p3.y) * p3.z);
56     }
57   
58     
59   vec2 UVfrom1D(int texNumR, int texNumC, int index) {
60     int texR = index / texNumC;
61     int texC = index - texR * texNumC;
62     return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
63   }
64   
65     
66   vec2 UVfrom2D(int texNumR, int texNumC, int numC, int row, int col) {
67     int index = row * numC + col;
68     int texR = index / texNumC;
69     int texC = index - texR * texNumC;
70     return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
71   }
72   
73     
74   vec2 UVfrom3D(int texNumR, int texNumC, int stride0,
75       int stride1, int row, int col, int depth) {
76     // Explicitly use integer operations as dot() only works on floats.
77     int index = row * stride0 + col * stride1 + depth;
78     int texR = index / texNumC;
79     int texC = index - texR * texNumC;
80     return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
81   }
82   
83     
84   vec2 UVfrom4D(int texNumR, int texNumC, int stride0,
85       int stride1, int stride2, int row, int col, int depth,
86       int depth2) {
87     // Explicitly use integer operations as dot() only works on floats.
88     int index = row * stride0 + col * stride1 + depth * stride2 + depth2;
89     int texR = index / texNumC;
90     int texC = index - texR * texNumC;
91     return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
92   }
93   
94     
95   vec2 UVfrom5D(int texNumR, int texNumC, int stride0,
96       int stride1, int stride2, int stride3, int row, int col, int depth,
97       int depth2, int depth3) {
98     // Explicitly use integer operations as dot() only works on floats.
99     int index = row * stride0 + col * stride1 +
100                depth * stride2 + depth2 * stride3 + depth3;
101    int texR = index / texNumC;
102    int texC = index - texR * texNumC;
103    return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
104  }
105  
106    
107  vec2 UVfrom6D(int texNumR, int texNumC, int stride0,
108      int stride1, int stride2, int stride3, int stride4,
109      int row, int col, int depth, int depth2, int depth3, int depth4) {
110    // Explicitly use integer operations as dot() only works on floats.
111    int index = row * stride0 + col * stride1 + depth * stride2 + depth2 *
112      stride3 + depth3 * stride4 + depth4;
113    int texR = index / texNumC;
114    int texC = index - texR * texNumC;
115    return (vec2(texC, texR) + halfCR) / vec2(texNumC, texNumR);
116  }
117  
118  
119  
120    float sampleTexture(sampler2D textureSampler, vec2 uv) {
121      return texture2D(textureSampler, uv).r;
122    }
123  
124  
125    void setOutput(float val) {
126      gl_FragColor = vec4(val, 0, 0, 0);
127    }
128  
129  uniform sampler2D A;
130  
131      ivec4 getOutputCoords() {
132        ivec2 resTexRC = ivec2(resultUV.yx *
133          vec2(241, 867));
134        int index = resTexRC.x * 867 + resTexRC.y;
135  
136        int r = index / 208947;
137        index -= r * 208947;
138  
139        int c = index / 867;
140        index -= c * 867;
141  
142        int d = index / 3;
143        int d2 = index - d * 3;
144  
145        return ivec4(r, c, d, d2);
146      }
147    
148  
149      float getAFlat(int index) {
150        vec2 uv = UVfrom1D(500, 1800, index);
151        return sampleTexture(A, uv);
152      }
153    
154        
155          float getA(int row, int col, int depth) {
156            int texR = row;
157            int texC = col * 3 + depth;
158            vec2 uv = (vec2(texC, texR) + halfCR) /
159                       vec2(1800.0, 500.0);
160            return sampleTexture(A, uv);
161          }
162        
163        float getA(int row, int col, int depth, int depth2) {
164          return getA(col, depth, depth2);
165        }
166      
167  
168        const vec2 effectiveInputOverOutputRatioRC = vec2(
169            2.074688796680498,
170            2.0761245674740483);
171        const vec2 inputShapeRC = vec2(500.0, 600.0);
172  
173        void main() {
174          ivec4 coords = getOutputCoords();
175          int b = coords[0];
176          int d = coords[3];
177          ivec2 yRC = coords.yz;
178  
179          // Fractional source index.
180          vec2 sourceFracIndexRC = vec2(yRC) * effectiveInputOverOutputRatioRC;
181  
182          // Compute the four integer indices.
183          ivec2 sourceFloorRC = ivec2(sourceFracIndexRC);
184          ivec2 sourceCeilRC = ivec2(
185            min(inputShapeRC - 1.0, ceil(sourceFracIndexRC)));
186  
187          float topLeft = getA(b, sourceFloorRC.x, sourceFloorRC.y, d);
188          float bottomLeft = getA(b, sourceCeilRC.x, sourceFloorRC.y, d);
189          float topRight = getA(b, sourceFloorRC.x, sourceCeilRC.y, d);
190          float bottomRight = getA(b, sourceCeilRC.x, sourceCeilRC.y, d);
191  
192          vec2 fracRC = sourceFracIndexRC - vec2(sourceFloorRC);
193  
194          float top = topLeft + (topRight - topLeft) * fracRC.y;
195          float bottom = bottomLeft + (bottomRight - bottomLeft) * fracRC.y;
196          float newValue = top + (bottom - top) * fracRC.x;
197  
198          setOutput(newValue);
199        }
200 
tf-core.esm.js:17 Uncaught Error: Failed to compile fragment shader.
    at createFragmentShader (tf-core.esm.js:17)
    at e.createProgram (tf-core.esm.js:17)
    at compileProgram (tf-core.esm.js:17)
    at tf-core.esm.js:17
    at e.getAndSaveBinary (tf-core.esm.js:17)
    at e.compileAndRun (tf-core.esm.js:17)
    at e.slice (tf-core.esm.js:17)
    at ENV.engine.runKernel.$x (tf-core.esm.js:17)
    at tf-core.esm.js:17
    at e.scopedRun (tf-core.esm.js:17)
createFragmentShader @ tf-core.esm.js:17
e.createProgram @ tf-core.esm.js:17
compileProgram @ tf-core.esm.js:17
(anonymous) @ tf-core.esm.js:17
e.getAndSaveBinary @ tf-core.esm.js:17
e.compileAndRun @ tf-core.esm.js:17
e.slice @ tf-core.esm.js:17
ENV.engine.runKernel.$x @ tf-core.esm.js:17
(anonymous) @ tf-core.esm.js:17
e.scopedRun @ tf-core.esm.js:17
e.runKernel @ tf-core.esm.js:17
slice_ @ tf-core.esm.js:17
slice @ tf-core.esm.js:17
e.slice @ tf-core.esm.js:17
(anonymous) @ tf-core.esm.js:17
split @ tf-core.esm.js:17
e.split @ tf-core.esm.js:17
ENV.engine.runKernel.$x @ tf-core.esm.js:17
(anonymous) @ tf-core.esm.js:17
e.scopedRun @ tf-core.esm.js:17
e.runKernel @ tf-core.esm.js:17
split_ @ tf-core.esm.js:17
slice @ tf-core.esm.js:17
customLayerDemo @ index.js:27
parcelRequire.3.babel-runtime/helpers/slicedToArray @ index.js:56
newRequire @ custom-layer.32c31cce.js:48
(anonymous) @ custom-layer.32c31cce.js:75
(anonymous) @ custom-layer.32c31cce.js:101

About this issue

  • Original URL
  • State: closed
  • Created 6 years ago
  • Reactions: 3
  • Comments: 21 (3 by maintainers)

Most upvoted comments

Is there any reason why this patch hasn’t become a pull request upstream? Wondering if there are next steps for tfjs as a whole rather than locally patching.

Here is a reduced test case, using Galaxy S5 running Chrome. Can anyone determine if this is either a device limitation or a bug in TFJS?

This code succeeds:

import * as tf from '@tensorflow/tfjs'

tf.split(tf.zeros([1, 65535, 1, 2]), 2, 3)

This code fails:

import * as tf from '@tensorflow/tfjs'

tf.split(tf.zeros([1, 65536, 1, 2]), 2, 3)

Full console output: tfjs-error

In my case, the hack I posted above seems to actually work. I replaced the “getLogicalCoordinatesFromFlatIndex” function directly in tf-core.esm.js (I’m not working from source, so I’ve been directly editing from node_modules package). Here is what I replaced it with:

function getLogicalCoordinatesFromFlatIndex(coords, shape, index = 'index') {
  var strides = computeStrides(shape);
  var returnVal = `int aa = 0; ${strides.map((stride, i) => {
    var quantity = Math.floor(stride / 65535);
    var remainder = stride % 65535;
    var line0 = `aa = ${quantity}; aa = aa * 65535; aa = aa + ${remainder}`;
    var line1 = `int ${coords[i]} = ${index} / aa`;
    var line2 = i === strides.length - 1 ?
        `int ${coords[i + 1]} = ${index} - ${coords[i]} * aa` :
        `index -= ${coords[i]} * aa`;
    return `${line0}; ${line1}; ${line2};`;
  })
  .join('')}`;
  return returnVal;
}

It’s definitely a hack, and I’m not sure of any side-effects of doing it this way, but for my use-case it seems to work. My Galaxy S5 can now compile the shader and run predictions on the model with no issues (aside from performance).

I’m done looking into this issue, so hopefully someone can take over where I left off and make a more meaningful contribution.

I’ve done some additional testing of tfjs-examples on my Galaxy S5 (Chrome). I could not test some examples because they were broken for other reasons (in most cases the XHR loading data files failed because the resource no longer exists).

Hopefully this helps to better isolate the issue. I will try to build a reduced test case from it.

Here are the results:

Example Working? Error Message Notes
addition-rnn
boston-housing All three regressors work
cart-pole
custom-layer Error: Failed to compile fragment shader. Error shows immediately upon visiting the page
iris-fitDataset
iris
lstm-text-generation
mnist-core
mobilenet Error: Failed to compile fragment shader. Error shows immediately upon visiting the page
polynomial-regression-core
polynomial-regression
sentiment
translation
tsne-mnist-canvas gl_util.ts:99 WebGL: INVALID_ENUM: texImage2D: invalid format
gl_util.ts:99 WebGL: INVALID_ENUM: texImage2D: invalid format
webgl_util.ts:91 ERROR: unsupported shader version
index.html:1 [.WebGL-0x9d2b8a00]GL ERROR :GL_INVALID_VALUE : glTexImage2D: invalid internal_format GL_CLOSE_PATH_NV
webgl_util.ts:92 Uncaught (in promise) Error: Failed to compile vertex shader.
Error shows when clicking “Start T-SNE”
webcam-transfer-learning Error: Failed to compile fragment shader. Error shows upon allowing access to webcam

$(document).ready(function(){ run() }) async function run(){ await faceapi.loadMtcnnModel(‘./models’); await faceapi.loadFaceRecognitionModel(‘./models’); const videoEl=document.getElementById(‘inputVideo’) navigator.getUserMedia( {video:{}}, stream=>videoEl.srcObject=stream, err=>console.err(err) ) } async function onPlay(videoEl){

setInterval( async ()=>{console.log('it is running');

const mtcnnForwardParams = {
    maxNumScales: 10,
    scaleFactor: 0.709,
    scoreThresholds: [0.6, 0.7, 0.7],
    minFaceSize: 200
  }
  const canvas=document.getElementById('overlay')
const mtcnnResults=await faceapi.mtcnn(videoEl,mtcnnForwardParams)
canvas.getContext('2d').clearRect(0, 0, canvas.width, canvas.height)
faceapi.drawDetection(canvas, mtcnnResults.map(res => res.faceDetection), { withScore: false })
  faceapi.drawLandmarks(canvas, mtcnnResults.map(res => res.faceLandmarks), { lineWidth: 4, color: 'red' })
},1000)

}

this is my javascript code,i got this error,please help to sort out

script2.js:27 Uncaught (in promise) TypeError: faceapi.drawDetection is not a function at script2.js:27

@bryan-14four I’ve tried locally here, and it worked. I’ll try now with a builded version. But thanks in advance.

@bryan-14four I’ve updated the method directly in the file that you mentioned, but no success. Do we have to run some command to apply changes or something?

Here’s what I did for my project. At the time I used face-api.js version ^0.18.0 which uses tfjs-core v0.14.2, so you’ll have to set package.json deps to {“face-api.js”: “^0.18.0”} (unless newer versions use the same version of tfjs). This solution uses patch-package to patch tfjs-core v0.14.2.

  1. npm i patch-package
  2. In package.json, add to “scripts” section: "postinstall": "patch-package"
  3. Create a folder called patches in the root dir of the app (same place as package.json).
  4. Extract the patch file I’ve included below to the patches folder.
  5. Run npm i again and it should install the modules and apply the patch.

Patch file: @tensorflow+tfjs-core+0.14.2.patch.zip

Good luck!

I have an update on this. The two code snippets above produce different shader code, where the test with the 65536 int fails on shader compilation inside of the getOutputCoords() function.


Success: tf.split(tf.zeros([1, 65535, 1, 2]), 2, 3) ->

ivec4 getOutputCoords() {
  ...
  int r = index / 65535;
  index -= r * 65535;
  ...
}

Failure: tf.split(tf.zeros([1, 65536, 1, 2]), 2, 3) ->

ivec4 getOutputCoords() {
  ...
  int r = index / 65536;
  index -= r * 65536;
  ...
}

I pulled these shader sources out manually and wrote some webGL code to run them. In the failing test, the shader compilation fails both on the int r = ... and index -= ... lines due to an “integer constant overflow”. Interestingly, I’ve found a way to make the shader compile by replacing the failing function with this logically-equivalent one:

ivec4 getOutputCoords() {
  ...
  int aa = 256;
  int bb = 256;
  int r = index / (aa * bb);
  index -= r * (aa * bb);
  ...
}

I’m not sure if this code actually works as a fix, or if the integer primitive itself is limited to 16 bits on this device (a possibility is that the “aa * bb” portion is overflowing silently, whereas the constant “65536” fails with an error). I will do more testing and report back.