cordova-plugin-iosrtc: republishing Track fail using Twillio (Was Error: removeTrack() must be called with a RTCRtpSender instance as argument)

I have used Google with the error message or bug in association with the library and Cordova words to make sure the issue I’m reporting is only related to iOSRTC.
I have provided steps to reproduce (e.g. sample code or updated extra/renderer-and-libwebrtc-tests.js file).
I have provided third party library name and version, ios, Xcode and plugin version and adapter.js version if used.

Versions affected

cordova-plugin-iosrtc version 6.0.13
iOS 13.5.1
Twilio 2.6.0

Description

Trying to switch between cameras by unpublishing the previous track and then publishing a new track.

Unpublishing track returns error: removeTrack() must be called with a RTCRtpSender instance as argument

Steps to reproduce

I have adjusted the Twilio sample to implement switching between cameras:

<!DOCTYPE HTML>
<html>
<head>
    <title>
        Twilio Video Room
    </title>
    <meta name="viewport" content="viewport-fit=cover, width=device-width, initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, user-scalable=no">
    <meta http-equiv="Content-Security-Policy"
          content="default-src 'self' data: gap: wss://* https://* 'unsafe-eval' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; media-src *">
    <script type="text/javascript" src="cordova.js"></script>
</head>
<body>
<br><br><br>
<div id="local-media" style="border: red 1px solid;"></div>
<div id="remote-media"></div>
<button onclick="toggleVideo()">Toggle video</button>
<button onclick="leaveMeeting()">Leave meeting</button>
<button onclick="startMeeting()">Start meeting</button>
<button onclick="switchCamera()">Switch camera</button>
<script type="text/javascript">

const token = '';
const roomName = 'test';
const scriptUrls = [];
let videoTrack = null;
let audioTrack = null;
let room = null;
let facing = true;

function loadScript(scriptUrl) {

  if (scriptUrls[scriptUrl]) {
    return Promise.resolve(scriptUrl);
  }

  return new Promise(function (resolve, reject) {
    // load adapter.js
    var script = document.createElement("script");
    script.type = "text/javascript";
    script.src = scriptUrl;
    script.async = false;
    document.getElementsByTagName("head")[0].appendChild(script);
    script.onload = function () {
      scriptUrls[scriptUrl] = true;
      console.debug('loadScript.loaded', script.src);
      resolve(scriptUrl);
    };
  });
}

async function startMeeting() {

  videoTrack = await getVideoTrack();
  audioTrack = await Twilio.Video.createLocalAudioTrack();
  const localMediaContainer = document.getElementById('local-media');
  localMediaContainer.appendChild(videoTrack.attach());
  localMediaContainer.appendChild(audioTrack.attach());
  console.log(videoTrack, audioTrack);

  Twilio.Video.connect(token, {
    tracks: [videoTrack, audioTrack],
    name: roomName,
    sdpSemantics: 'plan-b',
    bundlePolicy: 'max-compat'
  }).then(_room => {
    room = _room;
    console.log(`Successfully joined a Room: ${room}`);

    // Attach the Tracks of the Room's Participants.
    room.participants.forEach(function (participant) {
      console.log("Already in Room: '" + participant.identity + "'");
      participantConnected(participant);
    });

    room.on('participantConnected', participant => {
      console.log(`A remote Participant connected: ${participant}`);
      participantConnected(participant);
    });

    room.on('participantDisconnected', participant => {
      console.log(`A remote Participant connected: ${participant}`);
      participantDisconnected(participant);
    });

  }, error => {
    console.error(`Unable to connect to Room: ${error.message}`);
  });

  function participantConnected(participant) {
    console.log('Participant "%s" connected', participant.identity);
    const div = document.createElement('div');
    div.id = participant.sid;
    participant.tracks.forEach((publication) => {
      console.log('subbing to existing publication', publication);
      trackSubscribed(div, publication);
    });

    participant.on('trackPublished', (publication) => {
      trackSubscribed(div, publication)
    });
    participant.on('trackUnpublished', trackUnsubscribed);

    document.getElementById('remote-media').appendChild(div);
  }

  function participantDisconnected(participant) {
    console.log('Participant "%s" disconnected', participant.identity);

    var div = document.getElementById(participant.sid);
    if (div) {
      div.remove();
    }
  }

  function trackSubscribed(div, publication) {
    console.log('sub publication', publication);
    if(publication.track){
      attachTrack(div, publication.track);
    }
    publication.on('subscribed', track => attachTrack(div, track));
    publication.on('unsubscribed', track => detachTrack(track));
  }

  function attachTrack(div, track){
    console.log('attachTrack', track);
    div.appendChild(track.attach());
  }



  function trackUnsubscribed(publication) {
    console.log('unsub publication', publication);
    if(publication.track){
      detachTrack(publication.track);
    }
  }
}

function detachTrack(track) {
  console.log('detachTrack', track);
  track.detach().forEach(element => element.remove());
}

function toggleVideo() {
  console.log(videoTrack, room);
  if (videoTrack.isEnabled) {
    videoTrack.disable();
    // room.localParticipant.unpublishTrack(videoTrack);
    console.log('disable');
  } else {
    videoTrack.enable();
    // room.localParticipant.publishTrack(videoTrack);
    console.log('enable');
  }
}

async function switchCamera(){
  facing = !facing;
  // Get new track
  const newTrack = await getVideoTrack();

  // Stop old track
  videoTrack.stop();
  detachTrack(videoTrack);

  // Unpublish previous track
  room.localParticipant.unpublishTrack(videoTrack);
  
  // Publish new track
  room.localParticipant.publishTrack(newTrack);
}

function getVideoTrack(){
  return Twilio.Video.createLocalVideoTrack({facingMode: facing ? 'user' : {exact: 'environment'}});
}

function leaveMeeting() {
  room.disconnect();
  videoTrack.stop();
  audioTrack.stop();
  detachTrack(videoTrack);
  detachTrack(audioTrack);
}

async function ready() {
  // Note: This allow this sample to run on any Browser
  var cordova = window.cordova;
  if (cordova && cordova.plugins && cordova.plugins.iosrtc) {

    // Expose WebRTC and GetUserMedia SHIM as Globals (Optional)
    // Alternatively WebRTC API will be inside cordova.plugins.iosrtc namespace
    cordova.plugins.iosrtc.registerGlobals();

    // Enable iosrtc debug (Optional)
    cordova.plugins.iosrtc.debug.enable('*', true);
  }
  console.log('loading Twilio');

  await loadScript('https://media.twiliocdn.com/sdk/js/video/releases/2.6.0/twilio-video.js');
  console.log('loaded');
}

if(!window.cordova){
  ready();
}

document.addEventListener('deviceready', ready, false);
</script>
</body>
</html>

Expected results

Should switch to other camera.

Actual results

Error: removeTrack() must be called with a RTCRtpSender instance as argument

About this issue

Original URL
State: closed
Created 4 years ago
Comments: 42 (29 by maintainers)

Most upvoted comments

We are testing cordova with both mediasoup and the new Azure Communication Services (ACS) and both clients run into issues because clone and addTransceiver aren’t implemented. With the number of new streaming platforms being released lately (ACS, Zoom, NVIDIA, Twilio, etc.) it’s going to be critical that these modern interfaces are supported.

cah-dgreif on Oct 16, 2020

@hthetiot Thanks, i’ll give it a try.

Also something i came upon. Might be unrelated, but maybe something worth taking note of:

I was trying to track down why unified plan doesn’t work with Twilio. Twilio sets a isUnifiedPlan flag to implement unified-plan specific track management: https://github.com/twilio/twilio-video.js/blob/c86e0001d3706e2c13b1914cffb2f08faae2285c/lib/signaling/v2/peerconnection.js

const sdpFormat = getSdpFormat(configuration.sdpSemantics);
const isUnifiedPlan = sdpFormat === 'unified';

The problem here is that getSdpFormat depends on browser detection and in WKWebView it will always return null, thus isUnifiedPlan will always be false. In this case RTCPeerConnection still initializes with unified plan (default), but Twilio will use plan-b specific track management. Thus we get errors like: setRemoteDescription() failed: Failed to set remote answer sdp: The order of m-lines in answer doesn't match order in offer

I tried hardcoding Twilio to use unified plan, but it seems in that case Twilio is using RTCPeerConnection.addTransceiver which is not yet supported in iosrtc. See the following lines: It sets localMediaStream in case of plan-b.

const localMediaStream = isUnifiedPlan ? null : new options.MediaStream();

Which is later referenced to whether to use addTransceiver (in case of unified-plan) or addTrack (in case of plan-b).

 addMediaTrackSender(mediaTrackSender) {
    if (this._peerConnection.signalingState === 'closed' || this._rtpSenders.has(mediaTrackSender)) {
      return;
    }
    let sender;
    if (this._localMediaStream) {
      this._localMediaStream.addTrack(mediaTrackSender.track);
      sender = this._peerConnection.addTrack(mediaTrackSender.track, this._localMediaStream);
    } else {
      const transceiver = this._addOrUpdateTransceiver(mediaTrackSender.track);
      sender = transceiver.sender;
    }
    mediaTrackSender.addSender(sender);
    this._rtpSenders.set(mediaTrackSender, sender);
  }

So it seems to get unified-plan working with Twilio we would need Twilio to implement a way of setting isUnifiedPlan from configuration and in iosrtc we would need addTransceiver support.

henritoivar on Sep 23, 2020

@hthetiot I’m still having the same issue with unpublishing and republishing ( video not reappearing on the other side / or black)

But it seems i can now work around and avoid using unpublish/publish as i can now use replaceTrack (available in twilio using track.restart) to switch between front/rear camera. Also it seems i can implement muting the video/audio track by using enable/disable track without unpublishing/publishing.

@sboudouk I did try using twilio 2.4 and had the same issues. Actually i’m using Capacitor. I will also test this on cordova and get back with the results.

henritoivar on Sep 16, 2020

The following issue with black remote video is also present: #541

arf, can you confirm @sboudouk ?

I did not get into this issue, probably because I am using twilio 2.4.

What version of cordova ios are you running @henritoivar ? Can you try downgrading your twilio version to 2.4.X to see if it fix the issues ?

I have no access to my workstation for now, will test it asap.

sboudouk on Sep 16, 2020