feat(captions): write in-band captions from DASH fmp4 segments to the textTrack API (#108)

Author: ldayananda

src/master-playlist-controller.js

@@ -106,6 +106,7 @@ export class MasterPlaylistController extends videojs.EventTarget {
     }, false).track;
 
     this.decrypter_ = new Decrypter();
+    this.inbandTextTracks_ = {};
 
     const segmentLoaderSettings = {
       hls: this.hls_,
@@ -119,7 +120,8 @@ export class MasterPlaylistController extends videojs.EventTarget {
       bandwidth,
       syncController: this.syncController_,
       decrypter: this.decrypter_,
-      sourceType: this.sourceType_
+      sourceType: this.sourceType_,
+      inbandTextTracks: this.inbandTextTracks_
     };
 
     this.masterPlaylistLoader_ = this.sourceType_ === 'dash' ?

src/media-segment-request.js

@@ -1,5 +1,6 @@
 import videojs from 'video.js';
 import { createTransferableMessage } from './bin-utils';
+import mp4probe from 'mux.js/lib/mp4/probe';
 
 export const REQUEST_ERRORS = {
   FAILURE: 2,
@@ -171,7 +172,7 @@ const handleKeyResponse = (segment, finishProcessingFn) => (error, request) => {
  * @param {Function} finishProcessingFn - a callback to execute to continue processing
  *                                        this request
  */
-const handleInitSegmentResponse = (segment, finishProcessingFn) => (error, request) => {
+const handleInitSegmentResponse = (segment, captionParser, finishProcessingFn) => (error, request) => {
   const response = request.response;
   const errorObj = handleErrors(error, request);
 
@@ -190,6 +191,15 @@ const handleInitSegmentResponse = (segment, finishProcessingFn) => (error, reque
   }
 
   segment.map.bytes = new Uint8Array(request.response);
+
+  // Initialize CaptionParser if it hasn't been yet
+  if (!captionParser.isInitialized()) {
+    captionParser.init();
+  }
+
+  segment.map.timescales = mp4probe.timescale(segment.map.bytes);
+  segment.map.videoTrackIds = mp4probe.videoTrackIds(segment.map.bytes);
+
   return finishProcessingFn(null, segment);
 };
 
@@ -203,9 +213,10 @@ const handleInitSegmentResponse = (segment, finishProcessingFn) => (error, reque
  * @param {Function} finishProcessingFn - a callback to execute to continue processing
  *                                        this request
  */
-const handleSegmentResponse = (segment, finishProcessingFn) => (error, request) => {
+const handleSegmentResponse = (segment, captionParser, finishProcessingFn) => (error, request) => {
   const response = request.response;
   const errorObj = handleErrors(error, request);
+  let parsed;
 
   if (errorObj) {
     return finishProcessingFn(errorObj, segment);
@@ -229,6 +240,25 @@ const handleSegmentResponse = (segment, finishProcessingFn) => (error, request)
     segment.bytes = new Uint8Array(request.response);
   }
 
+  // This is likely an FMP4 and has the init segment.
+  // Run through the CaptionParser in case there are captions.
+  if (segment.map && segment.map.bytes) {
+    // Initialize CaptionParser if it hasn't been yet
+    if (!captionParser.isInitialized()) {
+      captionParser.init();
+    }
+
+    parsed = captionParser.parse(
+      segment.bytes,
+      segment.map.videoTrackIds,
+      segment.map.timescales);
+
+    if (parsed && parsed.captions) {
+      segment.captionStreams = parsed.captionStreams;
+      segment.fmp4Captions = parsed.captions;
+    }
+  }
+
   return finishProcessingFn(null, segment);
 };
 
@@ -393,6 +423,7 @@ const handleProgress = (segment, progressFn) => (event) => {
 export const mediaSegmentRequest = (xhr,
                                     xhrOptions,
                                     decryptionWorker,
+                                    captionParser,
                                     segment,
                                     progressFn,
                                     doneFn) => {
@@ -420,6 +451,7 @@ export const mediaSegmentRequest = (xhr,
       headers: segmentXhrHeaders(segment.map)
     });
     const initSegmentRequestCallback = handleInitSegmentResponse(segment,
+                                                                 captionParser,
                                                                  finishProcessingFn);
     const initSegmentXhr = xhr(initSegmentOptions, initSegmentRequestCallback);
 
@@ -431,7 +463,9 @@ export const mediaSegmentRequest = (xhr,
     responseType: 'arraybuffer',
     headers: segmentXhrHeaders(segment)
   });
-  const segmentRequestCallback = handleSegmentResponse(segment, finishProcessingFn);
+  const segmentRequestCallback = handleSegmentResponse(segment,
+                                                       captionParser,
+                                                       finishProcessingFn);
   const segmentXhr = xhr(segmentRequestOptions, segmentRequestCallback);
 
   segmentXhr.addEventListener('progress', handleProgress(segment, progressFn));

src/segment-loader.js

@@ -11,6 +11,8 @@ import { initSegmentId } from './bin-utils';
 import { mediaSegmentRequest, REQUEST_ERRORS } from './media-segment-request';
 import { TIME_FUDGE_FACTOR, timeUntilRebuffer as timeUntilRebuffer_ } from './ranges';
 import { minRebufferMaxBandwidthSelector } from './playlist-selectors';
+import { addCaptionData, createCaptionsTrackIfNotExists } from './util/text-tracks';
+import { CaptionParser } from 'mux.js/lib/mp4';
 import logger from './util/logger';
 
 // in ms
@@ -166,6 +168,7 @@ export default class SegmentLoader extends videojs.EventTarget {
     this.segmentMetadataTrack_ = settings.segmentMetadataTrack;
     this.goalBufferLength_ = settings.goalBufferLength;
     this.sourceType_ = settings.sourceType;
+    this.inbandTextTracks_ = settings.inbandTextTracks;
     this.state_ = 'INIT';
 
     // private instance variables
@@ -180,6 +183,8 @@ export default class SegmentLoader extends videojs.EventTarget {
     // Fragmented mp4 playback
     this.activeInitSegmentId_ = null;
     this.initSegments_ = {};
+    // Fmp4 CaptionParser
+    this.captionParser_ = new CaptionParser();
 
     this.decrypter_ = settings.decrypter;
 
@@ -240,6 +245,7 @@ export default class SegmentLoader extends videojs.EventTarget {
       this.sourceUpdater_.dispose();
     }
     this.resetStats_();
+    this.captionParser_.reset();
   }
 
   /**
@@ -340,7 +346,9 @@ export default class SegmentLoader extends videojs.EventTarget {
       this.initSegments_[id] = storedMap = {
         resolvedUri: map.resolvedUri,
         byterange: map.byterange,
-        bytes: map.bytes
+        bytes: map.bytes,
+        timescales: map.timescales,
+        videoTrackIds: map.videoTrackIds
       };
     }
 
@@ -544,6 +552,8 @@ export default class SegmentLoader extends videojs.EventTarget {
     this.ended_ = false;
     this.resetLoader();
     this.remove(0, this.duration_());
+    // clears fmp4 captions
+    this.captionParser_.clearAllCaptions();
     this.trigger('reseteverything');
   }
 
@@ -578,6 +588,12 @@ export default class SegmentLoader extends videojs.EventTarget {
       this.sourceUpdater_.remove(start, end);
     }
     removeCuesFromTrack(start, end, this.segmentMetadataTrack_);
+
+    if (this.inbandTextTracks_) {
+      for (let id in this.inbandTextTracks_) {
+        removeCuesFromTrack(start, end, this.inbandTextTracks_[id]);
+      }
+    }
   }
 
   /**
@@ -666,11 +682,13 @@ export default class SegmentLoader extends videojs.EventTarget {
     //   (we are crossing a discontinuity somehow)
     // - The "timestampOffset" for the start of this segment is less than
     //   the currently set timestampOffset
+    // Also, clear captions if we are crossing a discontinuity boundary
     if (segmentInfo.timeline !== this.currentTimeline_ ||
         ((segmentInfo.startOfSegment !== null) &&
         segmentInfo.startOfSegment < this.sourceUpdater_.timestampOffset())) {
       this.syncController_.reset();
       segmentInfo.timestampOffset = segmentInfo.startOfSegment;
+      this.captionParser_.clearAllCaptions();
     }
 
     this.loadSegment_(segmentInfo);
@@ -952,6 +970,7 @@ export default class SegmentLoader extends videojs.EventTarget {
     segmentInfo.abortRequests = mediaSegmentRequest(this.hls_.xhr,
       this.xhrOptions_,
       this.decrypter_,
+      this.captionParser_,
       this.createSimplifiedSegmentObj_(segmentInfo),
       // progress callback
       this.handleProgress_.bind(this),
@@ -1112,6 +1131,24 @@ export default class SegmentLoader extends videojs.EventTarget {
     }
 
     segmentInfo.endOfAllRequests = simpleSegment.endOfAllRequests;
+
+    // This has fmp4 captions, add them to text tracks
+    if (simpleSegment.fmp4Captions) {
+      createCaptionsTrackIfNotExists(
+        this.inbandTextTracks_,
+        this.hls_.tech_,
+        simpleSegment.captionStreams);
+      addCaptionData({
+        inbandTextTracks: this.inbandTextTracks_,
+        captionArray: simpleSegment.fmp4Captions,
+        // fmp4s will not have a timestamp offset
+        timestampOffset: 0
+      });
+      // Reset stored captions since we added parsed
+      // captions to a text track at this point
+      this.captionParser_.clearParsedCaptions();
+    }
+
     this.handleSegment_();
   }
 

src/util/text-tracks.js

@@ -0,0 +1,63 @@
+/**
+ * Create captions text tracks on video.js if they do not exist
+ *
+ * @param {Object} inbandTextTracks a reference to current inbandTextTracks
+ * @param {Object} tech the video.js tech
+ * @param {Object} captionStreams the caption streams to create
+ * @private
+ */
+export const createCaptionsTrackIfNotExists = function(inbandTextTracks, tech, captionStreams) {
+  for (let trackId in captionStreams) {
+    if (!inbandTextTracks[trackId]) {
+      tech.trigger({type: 'usage', name: 'hls-608'});
+      let track = tech.textTracks().getTrackById(trackId);
+
+      if (track) {
+        // Resuse an existing track with a CC# id because this was
+        // very likely created by videojs-contrib-hls from information
+        // in the m3u8 for us to use
+        inbandTextTracks[trackId] = track;
+      } else {
+        // Otherwise, create a track with the default `CC#` label and
+        // without a language
+        inbandTextTracks[trackId] = tech.addRemoteTextTrack({
+          kind: 'captions',
+          id: trackId,
+          label: trackId
+        }, false).track;
+      }
+    }
+  }
+};
+
+export const addCaptionData = function({
+    inbandTextTracks,
+    captionArray,
+    timestampOffset
+  }) {
+  if (!captionArray) {
+    return;
+  }
+
+  const Cue = window.WebKitDataCue || window.VTTCue;
+
+  captionArray.forEach((caption) => {
+    const track = caption.stream;
+    let startTime = caption.startTime;
+    let endTime = caption.endTime;
+
+    if (!inbandTextTracks[track]) {
+      return;
+    }
+
+    startTime += timestampOffset;
+    endTime += timestampOffset;
+
+    inbandTextTracks[track].addCue(
+      new Cue(
+        startTime,
+        endTime,
+        caption.text
+      ));
+  });
+};