Parse frame payloads for WXXX and TXXX tags

Decode the fields of simple ID3 tags. Adjust the starting PTS value for the metadata stream so that it aligns with video and audio streams.

Parse frame payloads for WXXX and TXXX tags
Decode the fields of simple ID3 tags. Adjust the starting PTS value for the metadata stream so that it aligns with video and audio streams.
David LaPalomento
Showing 6 changed files with 262 additions and 23 deletions
example.html
src/metadata-stream.js
src/segment-parser.js
test/metadata-stream_test.js
test/muxer/index.html
test/segment-parser.js
--- a/example.html
View file @9a3c335
+++ b/example.html
View file @9a3c335
@@ -18,13 +18,14 @@
  <!-- segment handling -->
  <script src="src/xhr.js"></script>
  <script src="src/flv-tag.js"></script>
+  <script src="src/stream.js"></script>
  <script src="src/exp-golomb.js"></script>
  <script src="src/h264-stream.js"></script>
  <script src="src/aac-stream.js"></script>
+  <script src="src/metadata-stream.js"></script>
  <script src="src/segment-parser.js"></script>
  <!-- m3u8 handling -->
-  <script src="src/stream.js"></script>
  <script src="src/m3u8/m3u8-parser.js"></script>
  <script src="src/playlist-loader.js"></script>
--- a/src/metadata-stream.js
View file @9a3c335
+++ b/src/metadata-stream.js
View file @9a3c335
@@ -5,16 +5,61 @@
 */
 (function(window, videojs, undefined) {
  'use strict';
-  var defaults = {
+  var
+    defaults = {
      debug: false
-  }, MetadataStream;
+    },
+    parseString = function(bytes, start, end) {
+      var i, result = '';
+      for (i = start; i < end; i++) {
+        result += '%' + ('00' + bytes[i].toString(16)).slice(-2);
+      }
+      return window.decodeURIComponent(result);
+    },
+    tagParsers = {
+      'TXXX': function(tag) {
+        var i;
+        if (tag.data[0] !== 3) {
+          // ignore frames with unrecognized character encodings
+          return;
+        }
+        for (i = 1; i < tag.data.length; i++) {
+          if (tag.data[i] === 0) {
+            // parse the text fields
+            tag.description = parseString(tag.data, 1, i);
+            tag.value = parseString(tag.data, i + 1, tag.data.length);
+            break;
+          }
+        }
+      },
+      'WXXX': function(tag) {
+        var i;
+        if (tag.data[0] !== 3) {
+          // ignore frames with unrecognized character encodings
+          return;
+        }
+        for (i = 1; i < tag.data.length; i++) {
+          if (tag.data[i] === 0) {
+            // parse the description and URL fields
+            tag.description = parseString(tag.data, 1, i);
+            tag.url = parseString(tag.data, i + 1, tag.data.length);
+            break;
+          }
+        }
+      }
+    },
+    MetadataStream;
  MetadataStream = function(options) {
-    var settings = videojs.util.mergeOptions(defaults, options);
+    var settings = {
+      debug: !!(options && options.debug)
+    };
    MetadataStream.prototype.init.call(this);
    this.push = function(chunk) {
-      var tagSize, frameStart, frameSize;
+      var tagSize, frameStart, frameSize, frame;
      // ignore events that don't look like ID3 data
      if (chunk.data.length < 10 ||
@@ -45,6 +90,13 @@
                   (chunk.data[19]);
      }
+      // adjust the PTS values to align with the video and audio
+      // streams
+      if (this.timestampOffset) {
+        chunk.pts -= this.timestampOffset;
+        chunk.dts -= this.timestampOffset;
+      }
      // parse one or more ID3 frames
      // http://id3.org/id3v2.3.0#ID3v2_frame_overview
      chunk.frames = [];
@@ -58,13 +110,17 @@
          return videojs.log('Malformed ID3 frame encountered. Skipping metadata parsing.');
        }
-        chunk.frames.push({
+        frame = {
          id: String.fromCharCode(chunk.data[frameStart]) +
            String.fromCharCode(chunk.data[frameStart + 1]) +
            String.fromCharCode(chunk.data[frameStart + 2]) +
            String.fromCharCode(chunk.data[frameStart + 3]),
          data: chunk.data.subarray(frameStart + 10, frameStart + frameSize + 10)
-        });
+        };
+        if (tagParsers[frame.id]) {
+          tagParsers[frame.id](frame);
+        }
+        chunk.frames.push(frame);
        frameStart += 10; // advance past the frame header
        frameStart += frameSize; // advance past the frame body
--- a/src/segment-parser.js
View file @9a3c335
+++ b/src/segment-parser.js
View file @9a3c335
@@ -4,6 +4,7 @@
    FlvTag = videojs.Hls.FlvTag,
    H264Stream = videojs.Hls.H264Stream,
    AacStream = videojs.Hls.AacStream,
+    MetadataStream = videojs.Hls.MetadataStream,
    MP2T_PACKET_LENGTH,
    STREAM_TYPES;
@@ -27,6 +28,9 @@
      programMapTable: {}
    };
+    // allow in-band metadata to be observed
+    self.metadataStream = new MetadataStream();
    // For information on the FLV format, see
    // http://download.macromedia.com/f4v/video_file_format_spec_v10_1.pdf.
    // Technically, this function returns the header and a metadata FLV tag
@@ -287,7 +291,8 @@
          self.stream.pmtPid = (data[offset + 2] & 0x1F) << 8 | data[offset + 3];
        }
      } else if (pid === self.stream.programMapTable[STREAM_TYPES.h264] ||
-                 pid === self.stream.programMapTable[STREAM_TYPES.adts]) {
+                 pid === self.stream.programMapTable[STREAM_TYPES.adts] ||
+                 pid === self.stream.programMapTable[STREAM_TYPES.metadata]) {
        if (pusi) {
          // comment out for speed
          if (0x00 !== data[offset + 0] || 0x00 !== data[offset + 1] || 0x01 !== data[offset + 2]) {
@@ -328,9 +333,16 @@
              dts /= 45;
            }
          }
          // Skip past "optional" portion of PTS header
          offset += pesHeaderLength;
+          // align the metadata stream PTS values with the start of
+          // the other elementary streams
+          if (!self.metadataStream.timestampOffset) {
+            self.metadataStream.timestampOffset = pts;
+          }
          if (pid === self.stream.programMapTable[STREAM_TYPES.h264]) {
            h264Stream.setNextTimeStamp(pts,
                                        dts,
@@ -339,6 +351,12 @@
            aacStream.setNextTimeStamp(pts,
                                       pesPacketSize,
                                       dataAlignmentIndicator);
+          } else {
+            self.metadataStream.push({
+              pts: pts,
+              dts: dts,
+              data: data.subarray(offset)
+            });
          }
        }
@@ -383,19 +401,18 @@
            // the PID for this entry
            elementaryPID = (data[offset + 1] & 0x1F) << 8 | data[offset + 2];
-            if (streamType === STREAM_TYPES.h264) {
+            if (streamType === STREAM_TYPES.h264 &&
-              if (self.stream.programMapTable[streamType] &&
+                self.stream.programMapTable[streamType] &&
                self.stream.programMapTable[streamType] !== elementaryPID) {
              throw new Error("Program has more than 1 video stream");
-              }
+            } else if (streamType === STREAM_TYPES.adts &&
-              self.stream.programMapTable[streamType] = elementaryPID;
+                       self.stream.programMapTable[streamType] &&
-            } else if (streamType === STREAM_TYPES.adts) {
-              if (self.stream.programMapTable[streamType] &&
                       self.stream.programMapTable[streamType] !== elementaryPID) {
              throw new Error("Program has more than 1 audio Stream");
            }
+            // add the stream type entry to the map
            self.stream.programMapTable[streamType] = elementaryPID;
-            }
            // TODO add support for MP3 audio
            // the length of the entry descriptor
@@ -435,7 +452,8 @@
  videojs.Hls.SegmentParser.MP2T_PACKET_LENGTH = MP2T_PACKET_LENGTH = 188;
  videojs.Hls.SegmentParser.STREAM_TYPES = STREAM_TYPES = {
    h264: 0x1b,
-    adts: 0x0f
+    adts: 0x0f,
+    metadata: 0x15
  };
 })(window);
--- a/test/metadata-stream_test.js
View file @9a3c335
+++ b/test/metadata-stream_test.js
View file @9a3c335
@@ -21,7 +21,7 @@
    throws(block, [expected], [message])
  */
-  var metadataStream, stringToInts, stringToCString, id3Frame;
+  var metadataStream, stringToInts, stringToCString, id3Tag, id3Frame;
  module('MetadataStream', {
    setup: function() {
@@ -45,6 +45,30 @@
    return stringToInts(string).concat([0x00]);
  };
+  id3Tag = function() {
+    var
+      frames = Array.prototype.concat.apply([], Array.prototype.slice.call(arguments)),
+      result = stringToInts('ID3').concat([
+        0x03, 0x00,            // version 3.0 of ID3v2 (aka ID3v.2.3.0)
+        0x40,                  // flags. include an extended header
+        0x00, 0x00, 0x00, 0x00, // size. set later
+        // extended header
+        0x00, 0x00, 0x00, 0x06, // extended header size. no CRC
+        0x00, 0x00,             // extended flags
+        0x00, 0x00, 0x00, 0x02  // size of padding
+      ], frames),
+      size;
+    size = result.length - 10;
+    result[6] = (size >>> 24) & 0xff;
+    result[7] = (size >>> 16) & 0xff;
+    result[8] = (size >>>  8) & 0xff;
+    result[9] = (size)        & 0xff
+    return result;
+  }
  id3Frame = function(type) {
    var result = stringToInts(type).concat([
      0x00, 0x00, 0x00, 0x00, // size
@@ -128,6 +152,8 @@
    deepEqual(new Uint8Array(events[0].frames[1].data),
              new Uint8Array([0x04, 0x03, 0x02, 0x01]),
              'attached the frame payload');
+    equal(events[0].pts, 1000, 'did not modify the PTS');
+    equal(events[0].dts, 1000, 'did not modify the PTS');
  });
  test('skips non-ID3 metadata events', function() {
@@ -158,4 +184,100 @@
  // too large/small tag size values
  // too large/small frame size values
+  test('translates PTS and DTS values based on the timestamp offset', function() {
+    var events = [];
+    metadataStream.on('data', function(event) {
+      events.push(event);
+    });
+    metadataStream.timestampOffset = 800;
+    metadataStream.push({
+      trackId: 7,
+      pts: 1000,
+      dts: 900,
+      // header
+      data: new Uint8Array(id3Tag(id3Frame('XFFF', [0]), [0x00, 0x00]))
+    });
+    equal(events.length, 1, 'emitted an event');
+    equal(events[0].pts, 200, 'translated pts');
+    equal(events[0].dts, 100, 'translated dts');
+  });
+  test('parses TXXX tags', function() {
+    var events = [];
+    metadataStream.on('data', function(event) {
+      events.push(event);
+    });
+    metadataStream.push({
+      trackId: 7,
+      pts: 1000,
+      dts: 900,
+      // header
+      data: new Uint8Array(id3Tag(id3Frame('TXXX',
+                                           0x00,
+                                           stringToCString('get done'),
+                                           stringToInts('{ "key": "value" }')),
+                                  [0x00, 0x00]))
+    });
+    equal(events.length, 1, 'parsed one tag');
+    equal(events[0].frames.length, 1, 'parsed one frame');
+    equal(events[0].frames[0].description, 'get done', 'parsed the description');
+    equal(events[0].frames[0].value, '{ "key": "value" }', 'parsed the value');
+  });
+  test('parses WXXX tags', function() {
+    var events = [], url = 'http://example.com/path/file?abc=7&d=4#ty';
+    metadataStream.on('data', function(event) {
+      events.push(event);
+    });
+    metadataStream.push({
+      trackId: 7,
+      pts: 1000,
+      dts: 900,
+      // header
+      data: new Uint8Array(id3Tag(id3Frame('WXXX',
+                                           0x00,
+                                           stringToCString(''),
+                                           stringToInts(url)),
+                                  [0x00, 0x00]))
+    });
+    equal(events.length, 1, 'parsed one tag');
+    equal(events[0].frames.length, 1, 'parsed one frame');
+    equal(events[0].frames[0].description, '', 'parsed the description');
+    equal(events[0].frames[0].url, url, 'parsed the value');
+  });
+  test('parses TXXX tags with characters that have a single-digit hexadecimal representation', function() {
+    var events = [], value = String.fromCharCode(7);
+    metadataStream.on('data', function(event) {
+      events.push(event);
+    });
+    metadataStream.push({
+      trackId: 7,
+      pts: 1000,
+      dts: 900,
+      // header
+      data: new Uint8Array(id3Tag(id3Frame('TXXX',
+                                           0x00,
+                                           stringToCString(''),
+                                           stringToInts(value)),
+                                  [0x00, 0x00]))
+    });
+    equal(events[0].frames[0].value,
+          value,
+          'parsed the single-digit character');
+  });
 })(window, window.videojs);
--- a/test/muxer/index.html
View file @9a3c335
+++ b/test/muxer/index.html
View file @9a3c335
@@ -109,11 +109,18 @@
  </div>
+  <script>
+    window.videojs = {
+      Hls: {}
+    };
+  </script>
  <!-- transmuxing -->
+  <script src="../../src/stream.js"></script>
  <script src="../../src/flv-tag.js"></script>
  <script src="../../src/exp-golomb.js"></script>
  <script src="../../src/h264-stream.js"></script>
  <script src="../../src/aac-stream.js"></script>
+  <script src="../../src/metadata-stream.js"></script>
  <script src="../../src/segment-parser.js"></script>
  <script src="../../node_modules/pkcs7/dist/pkcs7.unpad.js"></script>
  <script src="../../src/decrypter.js"></script>
--- a/test/segment-parser.js
View file @9a3c335
+++ b/test/segment-parser.js
View file @9a3c335
@@ -29,6 +29,11 @@
    extend = window.videojs.util.mergeOptions,
+    makePat,
+    makePsi,
+    makePmt,
+    makePacket,
    testAudioTag,
    testVideoTag,
    testScriptTag,
@@ -54,14 +59,15 @@
    deepEqual(expectedHeader, header, 'the rest of the header is correct');
  });
-  test('parses PMTs with program descriptors', function() {
+  // Create a PMT packet
-    var
+  // @return {Array} bytes
  makePmt = function(options) {
    var
      result = [],
      entryCount = 0,
      k,
      sectionLength;
    for (k in options.pids) {
      entryCount++;
    }
@@ -108,11 +114,15 @@
    // CRC_32
    result.push([0x00, 0x00, 0x00, 0x00]); // invalid CRC but we don't check it
    return result;
-      },
+  };
+  // Create a PAT packet
+  // @return {Array} bytes
  makePat = function(options) {
    var
      result = [],
      k;
    // table_id
    result.push(0x00);
    // section_syntax_indicator '0' reserved section_length
@@ -136,7 +146,10 @@
      result.push(options.programs[k] & 0xff);
    }
    return result;
-      },
+  };
+  // Create a PAT or PMT packet based on the specified options
+  // @return {Array} bytes
  makePsi = function(options) {
    var result = [];
@@ -148,7 +161,10 @@
      return result.concat(makePat(options));
    }
    return result.concat(makePmt(options));
-      },
+  };
+  // Construct an M2TS packet
+  // @return {Array} bytes
  makePacket = function(options) {
    var
      result = [],
@@ -170,7 +186,10 @@
    // ensure the resulting packet is the correct size
    result.length = window.videojs.Hls.SegmentParser.MP2T_PACKET_LENGTH;
    return result;
-      },
+  };
+  test('parses PMTs with program descriptors', function() {
+    var
      h264Type = window.videojs.Hls.SegmentParser.STREAM_TYPES.h264,
      adtsType = window.videojs.Hls.SegmentParser.STREAM_TYPES.adts;
@@ -191,6 +210,22 @@
    strictEqual(parser.stream.programMapTable[adtsType], 0x03, 'audio is PID 3');
  });
+  test('recognizes metadata streams', function() {
+    parser.parseSegmentBinaryData(new Uint8Array(makePacket({
+      programs: {
+        0x01: [0x01]
+      }
+    }).concat(makePacket({
+      pid: 0x01,
+      pids: {
+        // Rec. ITU-T H.222.0 (06/2012), Table 2-34
+        0x02: 0x15 // Metadata carried in PES packets
+      }
+    }))));
+    equal(parser.stream.programMapTable[0x15], 0x02, 'metadata is PID 2');
+  });
  test('parses the first bipbop segment', function() {
    parser.parseSegmentBinaryData(window.bcSegment);