Source: lib/text/text_engine.js

  1. /*! @license
  2.  * Shaka Player
  3.  * Copyright 2016 Google LLC
  4.  * SPDX-License-Identifier: Apache-2.0
  5.  */
  7. goog.provide('shaka.text.TextEngine');
  9. goog.require('goog.asserts');
  10. goog.require('shaka.log');
  11. goog.require('shaka.media.ClosedCaptionParser');
  12. goog.require('shaka.text.Cue');
  13. goog.require('shaka.util.BufferUtils');
  14. goog.require('shaka.util.IDestroyable');
  15. goog.require('shaka.util.MimeUtils');
  18. // TODO: revisit this when Closure Compiler supports partially-exported classes.
  19. /**
  20.  * @summary Manages text parsers and cues.
  21.  * @implements {shaka.util.IDestroyable}
  22.  * @export
  23.  */
  24. shaka.text.TextEngine = class {
  25.   /** @param {shaka.extern.TextDisplayer} displayer */
  26.   constructor(displayer) {
  27.     /** @private {?shaka.extern.TextParser} */
  28.     this.parser_ = null;
  30.     /** @private {shaka.extern.TextDisplayer} */
  31.     this.displayer_ = displayer;
  33.     /** @private {boolean} */
  34.     this.segmentRelativeVttTiming_ = false;
  36.     /** @private {number} */
  37.     this.timestampOffset_ = 0;
  39.     /** @private {number} */
  40.     this.appendWindowStart_ = 0;
  42.     /** @private {number} */
  43.     this.appendWindowEnd_ = Infinity;
  45.     /** @private {?number} */
  46.     this.bufferStart_ = null;
  48.     /** @private {?number} */
  49.     this.bufferEnd_ = null;
  51.     /** @private {string} */
  52.     this.selectedClosedCaptionId_ = '';
  54.     /**
  55.      * The closed captions map stores the CEA closed captions by closed captions
  56.      * id and start and end time.
  57.      * It's used as the buffer of closed caption text streams, to show captions
  58.      * when we start displaying captions or switch caption tracks, we need to be
  59.      * able to get the cues for the other language and display them without
  60.      * re-fetching the video segments they were embedded in.
  61.      * Structure of closed caption map:
  62.      * closed caption id -> {start and end time -> cues}
  63.      * @private {!Map.<string, !Map.<string, !Array.<shaka.text.Cue>>>} */
  64.     this.closedCaptionsMap_ = new Map();
  65.   }
  67.   /**
  68.    * @param {string} mimeType
  69.    * @param {!shaka.extern.TextParserPlugin} plugin
  70.    * @export
  71.    */
  72.   static registerParser(mimeType, plugin) {
  73.     shaka.text.TextEngine.parserMap_[mimeType] = plugin;
  74.   }
  76.   /**
  77.    * @param {string} mimeType
  78.    * @export
  79.    */
  80.   static unregisterParser(mimeType) {
  81.     delete shaka.text.TextEngine.parserMap_[mimeType];
  82.   }
  84.   /**
  85.    * @return {?shaka.extern.TextParserPlugin}
  86.    * @export
  87.    */
  88.   static findParser(mimeType) {
  89.     return shaka.text.TextEngine.parserMap_[mimeType];
  90.   }
  92.   /**
  93.    * @param {string} mimeType
  94.    * @return {boolean}
  95.    */
  96.   static isTypeSupported(mimeType) {
  97.     if (shaka.text.TextEngine.parserMap_[mimeType]) {
  98.       // An actual parser is available.
  99.       return true;
  100.     }
  101.     if (mimeType == shaka.util.MimeUtils.CEA608_CLOSED_CAPTION_MIMETYPE ||
  102.         mimeType == shaka.util.MimeUtils.CEA708_CLOSED_CAPTION_MIMETYPE ) {
  103.       return !!shaka.media.ClosedCaptionParser.findDecoder();
  104.     }
  105.     return false;
  106.   }
  108.   // TODO: revisit this when the compiler supports partially-exported classes.
  109.   /**
  110.    * @override
  111.    * @export
  112.    */
  113.   destroy() {
  114.     this.parser_ = null;
  115.     this.displayer_ = null;
  116.     this.closedCaptionsMap_.clear();
  118.     return Promise.resolve();
  119.   }
  121.   /**
  122.    * @param {!shaka.extern.TextDisplayer} displayer
  123.    */
  124.   setDisplayer(displayer) {
  125.     this.displayer_ = displayer;
  126.   }
  128.   /**
  129.    * Initialize the parser.  This can be called multiple times, but must be
  130.    * called at least once before appendBuffer.
  131.    *
  132.    * @param {string} mimeType
  133.    * @param {boolean} sequenceMode
  134.    * @param {boolean} segmentRelativeVttTiming
  135.    * @param {string} manifestType
  136.    */
  137.   initParser(mimeType, sequenceMode, segmentRelativeVttTiming, manifestType) {
  138.     // No parser for CEA, which is extracted from video and side-loaded
  139.     // into TextEngine and TextDisplayer.
  140.     if (mimeType == shaka.util.MimeUtils.CEA608_CLOSED_CAPTION_MIMETYPE ||
  141.         mimeType == shaka.util.MimeUtils.CEA708_CLOSED_CAPTION_MIMETYPE) {
  142.       this.parser_ = null;
  143.       return;
  144.     }
  146.     const factory = shaka.text.TextEngine.parserMap_[mimeType];
  147.     goog.asserts.assert(
  148.         factory, 'Text type negotiation should have happened already');
  149.     this.parser_ = factory();
  150.     if (this.parser_.setSequenceMode) {
  151.       this.parser_.setSequenceMode(sequenceMode);
  152.     } else {
  153.       shaka.log.alwaysWarn(
  154.           'Text parsers should have a "setSequenceMode" method!');
  155.     }
  156.     if (this.parser_.setManifestType) {
  157.       this.parser_.setManifestType(manifestType);
  158.     } else {
  159.       shaka.log.alwaysWarn(
  160.           'Text parsers should have a "setManifestType" method!');
  161.     }
  162.     this.segmentRelativeVttTiming_ = segmentRelativeVttTiming;
  163.   }
  165.   /**
  166.    * @param {BufferSource} buffer
  167.    * @param {?number} startTime relative to the start of the presentation
  168.    * @param {?number} endTime relative to the start of the presentation
  169.    * @param {?string=} uri
  170.    * @return {!Promise}
  171.    */
  172.   async appendBuffer(buffer, startTime, endTime, uri) {
  173.     goog.asserts.assert(
  174.         this.parser_, 'The parser should already be initialized');
  176.     // Start the operation asynchronously to avoid blocking the caller.
  177.     await Promise.resolve();
  179.     // Check that TextEngine hasn't been destroyed.
  180.     if (!this.parser_ || !this.displayer_) {
  181.       return;
  182.     }
  184.     if (startTime == null || endTime == null) {
  185.       this.parser_.parseInit(shaka.util.BufferUtils.toUint8(buffer));
  186.       return;
  187.     }
  189.     const vttOffset = this.segmentRelativeVttTiming_ ?
  190.         startTime : this.timestampOffset_;
  192.     /** @type {shaka.extern.TextParser.TimeContext} **/
  193.     const time = {
  194.       periodStart: this.timestampOffset_,
  195.       segmentStart: startTime,
  196.       segmentEnd: endTime,
  197.       vttOffset: vttOffset,
  198.     };
  200.     // Parse the buffer and add the new cues.
  201.     const allCues = this.parser_.parseMedia(
  202.         shaka.util.BufferUtils.toUint8(buffer), time, uri);
  203.     const cuesToAppend = allCues.filter((cue) => {
  204.       return cue.startTime >= this.appendWindowStart_ &&
  205.           cue.startTime < this.appendWindowEnd_;
  206.     });
  208.     this.displayer_.append(cuesToAppend);
  210.     // NOTE: We update the buffered range from the start and end times
  211.     // passed down from the segment reference, not with the start and end
  212.     // times of the parsed cues.  This is important because some segments
  213.     // may contain no cues, but we must still consider those ranges
  214.     // buffered.
  215.     if (this.bufferStart_ == null) {
  216.       this.bufferStart_ = Math.max(startTime, this.appendWindowStart_);
  217.     } else {
  218.       // We already had something in buffer, and we assume we are extending
  219.       // the range from the end.
  220.       goog.asserts.assert(
  221.           this.bufferEnd_ != null,
  222.           'There should already be a buffered range end.');
  223.       goog.asserts.assert(
  224.           (startTime - this.bufferEnd_) <= 1,
  225.           'There should not be a gap in text references >1s');
  226.     }
  227.     this.bufferEnd_ = Math.min(endTime, this.appendWindowEnd_);
  228.   }
  230.   /**
  231.    * @param {number} startTime relative to the start of the presentation
  232.    * @param {number} endTime relative to the start of the presentation
  233.    * @return {!Promise}
  234.    */
  235.   async remove(startTime, endTime) {
  236.     // Start the operation asynchronously to avoid blocking the caller.
  237.     await Promise.resolve();
  239.     if (this.displayer_ && this.displayer_.remove(startTime, endTime)) {
  240.       if (this.bufferStart_ == null) {
  241.         goog.asserts.assert(
  242.             this.bufferEnd_ == null, 'end must be null if startTime is null');
  243.       } else {
  244.         goog.asserts.assert(
  245.             this.bufferEnd_ != null,
  246.             'end must be non-null if startTime is non-null');
  248.         // Update buffered range.
  249.         if (endTime <= this.bufferStart_ || startTime >= this.bufferEnd_) {
  250.           // No intersection.  Nothing was removed.
  251.         } else if (startTime <= this.bufferStart_ &&
  252.                    endTime >= this.bufferEnd_) {
  253.           // We wiped out everything.
  254.           this.bufferStart_ = this.bufferEnd_ = null;
  255.         } else if (startTime <= this.bufferStart_ &&
  256.                    endTime < this.bufferEnd_) {
  257.           // We removed from the beginning of the range.
  258.           this.bufferStart_ = endTime;
  259.         } else if (startTime > this.bufferStart_ &&
  260.                    endTime >= this.bufferEnd_) {
  261.           // We removed from the end of the range.
  262.           this.bufferEnd_ = startTime;
  263.         } else {
  264.           // We removed from the middle?  StreamingEngine isn't supposed to.
  265.           goog.asserts.assert(
  266.               false, 'removal from the middle is not supported by TextEngine');
  267.         }
  268.       }
  269.     }
  270.   }
  272.   /** @param {number} timestampOffset */
  273.   setTimestampOffset(timestampOffset) {
  274.     this.timestampOffset_ = timestampOffset;
  275.   }
  277.   /**
  278.    * @param {number} appendWindowStart
  279.    * @param {number} appendWindowEnd
  280.    */
  281.   setAppendWindow(appendWindowStart, appendWindowEnd) {
  282.     this.appendWindowStart_ = appendWindowStart;
  283.     this.appendWindowEnd_ = appendWindowEnd;
  284.   }
  286.   /**
  287.    * @return {?number} Time in seconds of the beginning of the buffered range,
  288.    *   or null if nothing is buffered.
  289.    */
  290.   bufferStart() {
  291.     return this.bufferStart_;
  292.   }
  294.   /**
  295.    * @return {?number} Time in seconds of the end of the buffered range,
  296.    *   or null if nothing is buffered.
  297.    */
  298.   bufferEnd() {
  299.     return this.bufferEnd_;
  300.   }
  302.   /**
  303.    * @param {number} t A timestamp
  304.    * @return {boolean}
  305.    */
  306.   isBuffered(t) {
  307.     if (this.bufferStart_ == null || this.bufferEnd_ == null) {
  308.       return false;
  309.     }
  310.     return t >= this.bufferStart_ && t < this.bufferEnd_;
  311.   }
  313.   /**
  314.    * @param {number} t A timestamp
  315.    * @return {number} Number of seconds ahead of 't' we have buffered
  316.    */
  317.   bufferedAheadOf(t) {
  318.     if (this.bufferEnd_ == null || this.bufferEnd_ < t) {
  319.       return 0;
  320.     }
  322.     goog.asserts.assert(
  323.         this.bufferStart_ != null,
  324.         'start should not be null if end is not null');
  326.     return this.bufferEnd_ - Math.max(t, this.bufferStart_);
  327.   }
  329.   /**
  330.    * Set the selected closed captions id.
  331.    * Append the cues stored in the closed captions map until buffer end time.
  332.    * This is to fill the gap between buffered and unbuffered captions, and to
  333.    * avoid duplicates that would be caused by any future video segments parsed
  334.    * for captions.
  335.    *
  336.    * @param {string} id
  337.    * @param {number} bufferEndTime Load any stored cues up to this time.
  338.    */
  339.   setSelectedClosedCaptionId(id, bufferEndTime) {
  340.     this.selectedClosedCaptionId_ = id;
  342.     const captionsMap = this.closedCaptionsMap_.get(id);
  343.     if (captionsMap) {
  344.       for (const startAndEndTime of captionsMap.keys()) {
  345.         /** @type {Array.<!shaka.text.Cue>} */
  346.         const cues = captionsMap.get(startAndEndTime)
  347.             .filter((c) => c.endTime <= bufferEndTime);
  348.         if (cues) {
  349.           this.displayer_.append(cues);
  350.         }
  351.       }
  352.     }
  353.   }
  355.   /**
  356.    * @param {!shaka.text.Cue} cue the cue to apply the timestamp to recursively
  357.    * @param {number} videoTimestampOffset the timestamp offset of the video
  358.    * @private
  359.    */
  360.   applyVideoTimestampOffsetRecursive_(cue, videoTimestampOffset) {
  361.     cue.startTime += videoTimestampOffset;
  362.     cue.endTime += videoTimestampOffset;
  363.     for (const nested of cue.nestedCues) {
  364.       this.applyVideoTimestampOffsetRecursive_(nested, videoTimestampOffset);
  365.     }
  366.   }
  368.   /**
  369.    * Store the closed captions in the text engine, and append the cues to the
  370.    * text displayer.  This is a side-channel used for embedded text only.
  371.    *
  372.    * @param {!Array<!shaka.extern.ICaptionDecoder.ClosedCaption>} closedCaptions
  373.    * @param {?number} startTime relative to the start of the presentation
  374.    * @param {?number} endTime relative to the start of the presentation
  375.    * @param {number} videoTimestampOffset the timestamp offset of the video
  376.    *   stream in which these captions were embedded
  377.    */
  378.   storeAndAppendClosedCaptions(
  379.       closedCaptions, startTime, endTime, videoTimestampOffset) {
  380.     const startAndEndTime = startTime + ' ' + endTime;
  381.     /** @type {!Map.<string, !Map.<string, !Array.<!shaka.text.Cue>>>} */
  382.     const captionsMap = new Map();
  384.     for (const caption of closedCaptions) {
  385.       const id = caption.stream;
  386.       const cue = caption.cue;
  387.       if (!captionsMap.has(id)) {
  388.         captionsMap.set(id, new Map());
  389.       }
  390.       if (!captionsMap.get(id).has(startAndEndTime)) {
  391.         captionsMap.get(id).set(startAndEndTime, []);
  392.       }
  394.       // Adjust CEA captions with respect to the timestamp offset of the video
  395.       // stream in which they were embedded.
  396.       this.applyVideoTimestampOffsetRecursive_(cue, videoTimestampOffset);
  398.       const keepThisCue =
  399.           cue.startTime >= this.appendWindowStart_ &&
  400.           cue.startTime < this.appendWindowEnd_;
  401.       if (!keepThisCue) {
  402.         continue;
  403.       }
  405.       captionsMap.get(id).get(startAndEndTime).push(cue);
  406.       if (id == this.selectedClosedCaptionId_) {
  407.         this.displayer_.append([cue]);
  408.       }
  409.     }
  411.     for (const id of captionsMap.keys()) {
  412.       if (!this.closedCaptionsMap_.has(id)) {
  413.         this.closedCaptionsMap_.set(id, new Map());
  414.       }
  415.       for (const startAndEndTime of captionsMap.get(id).keys()) {
  416.         const cues = captionsMap.get(id).get(startAndEndTime);
  417.         this.closedCaptionsMap_.get(id).set(startAndEndTime, cues);
  418.       }
  419.     }
  421.     if (this.bufferStart_ == null) {
  422.       this.bufferStart_ = Math.max(startTime, this.appendWindowStart_);
  423.     } else {
  424.       this.bufferStart_ = Math.min(
  425.           this.bufferStart_, Math.max(startTime, this.appendWindowStart_));
  426.     }
  428.     this.bufferEnd_ = Math.max(
  429.         this.bufferEnd_, Math.min(endTime, this.appendWindowEnd_));
  430.   }
  432.   /**
  433.    * Get the number of closed caption channels.
  434.    *
  435.    * This function is for TESTING ONLY. DO NOT USE in the library.
  436.    *
  437.    * @return {number}
  438.    */
  439.   getNumberOfClosedCaptionChannels() {
  440.     return this.closedCaptionsMap_.size;
  441.   }
  443.   /**
  444.    * Get the number of closed caption cues for a given channel. If there is
  445.    * no channel for the given channel id, this will return 0.
  446.    *
  447.    * This function is for TESTING ONLY. DO NOT USE in the library.
  448.    *
  449.    * @param {string} channelId
  450.    * @return {number}
  451.    */
  452.   getNumberOfClosedCaptionsInChannel(channelId) {
  453.     const channel = this.closedCaptionsMap_.get(channelId);
  454.     return channel ? channel.size : 0;
  455.   }
  456. };
  458. /** @private {!Object.<string, !shaka.extern.TextParserPlugin>} */
  459. shaka.text.TextEngine.parserMap_ = {};