Parse frame payloads for WXXX and TXXX tags

Decode the fields of simple ID3 tags. Adjust the starting PTS value for the metadata stream so that it aligns with video and audio streams.

@@ -18,13 +18,14 @@
   <!-- segment handling -->
   <script src="src/xhr.js"></script>
   <script src="src/flv-tag.js"></script>
+  <script src="src/stream.js"></script>
   <script src="src/exp-golomb.js"></script>
   <script src="src/h264-stream.js"></script>
   <script src="src/aac-stream.js"></script>
+  <script src="src/metadata-stream.js"></script>
   <script src="src/segment-parser.js"></script>
 
   <!-- m3u8 handling -->
-  <script src="src/stream.js"></script>
   <script src="src/m3u8/m3u8-parser.js"></script>
   <script src="src/playlist-loader.js"></script>
 

@@ -5,16 +5,61 @@
  */
 (function(window, videojs, undefined) {
   'use strict';
-  var defaults = {
-    debug: false
-  }, MetadataStream;
+  var
+    defaults = {
+      debug: false
+    },
+    parseString = function(bytes, start, end) {
+      var i, result = '';
+      for (i = start; i < end; i++) {
+        result += '%' + ('00' + bytes[i].toString(16)).slice(-2);
+      }
+      return window.decodeURIComponent(result);
+    },
+    tagParsers = {
+      'TXXX': function(tag) {
+        var i;
+        if (tag.data[0] !== 3) {
+          // ignore frames with unrecognized character encodings
+          return;
+        }
+
+        for (i = 1; i < tag.data.length; i++) {
+          if (tag.data[i] === 0) {
+            // parse the text fields
+            tag.description = parseString(tag.data, 1, i);
+            tag.value = parseString(tag.data, i + 1, tag.data.length);
+            break;
+          }
+        }
+      },
+      'WXXX': function(tag) {
+        var i;
+        if (tag.data[0] !== 3) {
+          // ignore frames with unrecognized character encodings
+          return;
+        }
+
+        for (i = 1; i < tag.data.length; i++) {
+          if (tag.data[i] === 0) {
+            // parse the description and URL fields
+            tag.description = parseString(tag.data, 1, i);
+            tag.url = parseString(tag.data, i + 1, tag.data.length);
+            break;
+          }
+        }
+      }
+    },
+    MetadataStream;
 
   MetadataStream = function(options) {
-    var settings = videojs.util.mergeOptions(defaults, options);
+    var settings = {
+      debug: !!(options && options.debug)
+    };
     MetadataStream.prototype.init.call(this);
 
     this.push = function(chunk) {
-      var tagSize, frameStart, frameSize;
+      var tagSize, frameStart, frameSize, frame;
 
       // ignore events that don't look like ID3 data
       if (chunk.data.length < 10 ||
@@ -45,6 +90,13 @@
                    (chunk.data[19]);
       }
 
+      // adjust the PTS values to align with the video and audio
+      // streams
+      if (this.timestampOffset) {
+        chunk.pts -= this.timestampOffset;
+        chunk.dts -= this.timestampOffset;
+      }
+
       // parse one or more ID3 frames
       // http://id3.org/id3v2.3.0#ID3v2_frame_overview
       chunk.frames = [];
@@ -58,13 +110,17 @@
           return videojs.log('Malformed ID3 frame encountered. Skipping metadata parsing.');
         }
 
-        chunk.frames.push({
+        frame = {
           id: String.fromCharCode(chunk.data[frameStart]) +
             String.fromCharCode(chunk.data[frameStart + 1]) +
             String.fromCharCode(chunk.data[frameStart + 2]) +
             String.fromCharCode(chunk.data[frameStart + 3]),
           data: chunk.data.subarray(frameStart + 10, frameStart + frameSize + 10)
-        });
+        };
+        if (tagParsers[frame.id]) {
+          tagParsers[frame.id](frame);
+        }
+        chunk.frames.push(frame);
 
         frameStart += 10; // advance past the frame header
         frameStart += frameSize; // advance past the frame body

@@ -4,6 +4,7 @@
     FlvTag = videojs.Hls.FlvTag,
     H264Stream = videojs.Hls.H264Stream,
     AacStream = videojs.Hls.AacStream,
+    MetadataStream = videojs.Hls.MetadataStream,
     MP2T_PACKET_LENGTH,
     STREAM_TYPES;
 
@@ -27,6 +28,9 @@
       programMapTable: {}
     };
 
+    // allow in-band metadata to be observed
+    self.metadataStream = new MetadataStream();
+
     // For information on the FLV format, see
     // http://download.macromedia.com/f4v/video_file_format_spec_v10_1.pdf.
     // Technically, this function returns the header and a metadata FLV tag
@@ -287,7 +291,8 @@
           self.stream.pmtPid = (data[offset + 2] & 0x1F) << 8 | data[offset + 3];
         }
       } else if (pid === self.stream.programMapTable[STREAM_TYPES.h264] ||
-                 pid === self.stream.programMapTable[STREAM_TYPES.adts]) {
+                 pid === self.stream.programMapTable[STREAM_TYPES.adts] ||
+                 pid === self.stream.programMapTable[STREAM_TYPES.metadata]) {
         if (pusi) {
           // comment out for speed
           if (0x00 !== data[offset + 0] || 0x00 !== data[offset + 1] || 0x01 !== data[offset + 2]) {
@@ -328,9 +333,16 @@
               dts /= 45;
             }
           }
+
           // Skip past "optional" portion of PTS header
           offset += pesHeaderLength;
 
+          // align the metadata stream PTS values with the start of
+          // the other elementary streams
+          if (!self.metadataStream.timestampOffset) {
+            self.metadataStream.timestampOffset = pts;
+          }
+
           if (pid === self.stream.programMapTable[STREAM_TYPES.h264]) {
             h264Stream.setNextTimeStamp(pts,
                                         dts,
@@ -339,6 +351,12 @@
             aacStream.setNextTimeStamp(pts,
                                        pesPacketSize,
                                        dataAlignmentIndicator);
+          } else {
+            self.metadataStream.push({
+              pts: pts,
+              dts: dts,
+              data: data.subarray(offset)
+            });
           }
         }
 
@@ -383,19 +401,18 @@
             // the PID for this entry
             elementaryPID = (data[offset + 1] & 0x1F) << 8 | data[offset + 2];
 
-            if (streamType === STREAM_TYPES.h264) {
-              if (self.stream.programMapTable[streamType] &&
-                  self.stream.programMapTable[streamType] !== elementaryPID) {
-                throw new Error("Program has more than 1 video stream");
-              }
-              self.stream.programMapTable[streamType] = elementaryPID;
-            } else if (streamType === STREAM_TYPES.adts) {
-              if (self.stream.programMapTable[streamType] &&
-                  self.stream.programMapTable[streamType] !== elementaryPID) {
-                throw new Error("Program has more than 1 audio Stream");
-              }
-              self.stream.programMapTable[streamType] = elementaryPID;
+            if (streamType === STREAM_TYPES.h264 &&
+                self.stream.programMapTable[streamType] &&
+                self.stream.programMapTable[streamType] !== elementaryPID) {
+              throw new Error("Program has more than 1 video stream");
+            } else if (streamType === STREAM_TYPES.adts &&
+                       self.stream.programMapTable[streamType] &&
+                       self.stream.programMapTable[streamType] !== elementaryPID) {
+              throw new Error("Program has more than 1 audio Stream");
             }
+            // add the stream type entry to the map
+            self.stream.programMapTable[streamType] = elementaryPID;
+
             // TODO add support for MP3 audio
 
             // the length of the entry descriptor
@@ -435,7 +452,8 @@
   videojs.Hls.SegmentParser.MP2T_PACKET_LENGTH = MP2T_PACKET_LENGTH = 188;
   videojs.Hls.SegmentParser.STREAM_TYPES = STREAM_TYPES = {
     h264: 0x1b,
-    adts: 0x0f
+    adts: 0x0f,
+    metadata: 0x15
   };
 
 })(window);

@@ -21,7 +21,7 @@
     throws(block, [expected], [message])
   */
 
-  var metadataStream, stringToInts, stringToCString, id3Frame;
+  var metadataStream, stringToInts, stringToCString, id3Tag, id3Frame;
 
   module('MetadataStream', {
     setup: function() {
@@ -45,6 +45,30 @@
     return stringToInts(string).concat([0x00]);
   };
 
+  id3Tag = function() {
+    var
+      frames = Array.prototype.concat.apply([], Array.prototype.slice.call(arguments)),
+      result = stringToInts('ID3').concat([
+        0x03, 0x00,            // version 3.0 of ID3v2 (aka ID3v.2.3.0)
+        0x40,                  // flags. include an extended header
+        0x00, 0x00, 0x00, 0x00, // size. set later
+
+        // extended header
+        0x00, 0x00, 0x00, 0x06, // extended header size. no CRC
+        0x00, 0x00,             // extended flags
+        0x00, 0x00, 0x00, 0x02  // size of padding
+      ], frames),
+      size;
+
+    size = result.length - 10;
+    result[6] = (size >>> 24) & 0xff;
+    result[7] = (size >>> 16) & 0xff;
+    result[8] = (size >>>  8) & 0xff;
+    result[9] = (size)        & 0xff
+
+    return result;
+  }
+
   id3Frame = function(type) {
     var result = stringToInts(type).concat([
       0x00, 0x00, 0x00, 0x00, // size
@@ -128,6 +152,8 @@
     deepEqual(new Uint8Array(events[0].frames[1].data),
               new Uint8Array([0x04, 0x03, 0x02, 0x01]),
               'attached the frame payload');
+    equal(events[0].pts, 1000, 'did not modify the PTS');
+    equal(events[0].dts, 1000, 'did not modify the PTS');
   });
 
   test('skips non-ID3 metadata events', function() {
@@ -158,4 +184,100 @@
   // too large/small tag size values
   // too large/small frame size values
 
+  test('translates PTS and DTS values based on the timestamp offset', function() {
+    var events = [];
+    metadataStream.on('data', function(event) {
+      events.push(event);
+    });
+
+    metadataStream.timestampOffset = 800;
+
+    metadataStream.push({
+      trackId: 7,
+      pts: 1000,
+      dts: 900,
+
+      // header
+      data: new Uint8Array(id3Tag(id3Frame('XFFF', [0]), [0x00, 0x00]))
+    });
+
+    equal(events.length, 1, 'emitted an event');
+    equal(events[0].pts, 200, 'translated pts');
+    equal(events[0].dts, 100, 'translated dts');
+  });
+
+  test('parses TXXX tags', function() {
+    var events = [];
+    metadataStream.on('data', function(event) {
+      events.push(event);
+    });
+
+    metadataStream.push({
+      trackId: 7,
+      pts: 1000,
+      dts: 900,
+
+      // header
+      data: new Uint8Array(id3Tag(id3Frame('TXXX',
+                                           0x00,
+                                           stringToCString('get done'),
+                                           stringToInts('{ "key": "value" }')),
+                                  [0x00, 0x00]))
+    });
+
+    equal(events.length, 1, 'parsed one tag');
+    equal(events[0].frames.length, 1, 'parsed one frame');
+    equal(events[0].frames[0].description, 'get done', 'parsed the description');
+    equal(events[0].frames[0].value, '{ "key": "value" }', 'parsed the value');
+  });
+
+  test('parses WXXX tags', function() {
+    var events = [], url = 'http://example.com/path/file?abc=7&d=4#ty';
+    metadataStream.on('data', function(event) {
+      events.push(event);
+    });
+
+    metadataStream.push({
+      trackId: 7,
+      pts: 1000,
+      dts: 900,
+
+      // header
+      data: new Uint8Array(id3Tag(id3Frame('WXXX',
+                                           0x00,
+                                           stringToCString(''),
+                                           stringToInts(url)),
+                                  [0x00, 0x00]))
+    });
+
+    equal(events.length, 1, 'parsed one tag');
+    equal(events[0].frames.length, 1, 'parsed one frame');
+    equal(events[0].frames[0].description, '', 'parsed the description');
+    equal(events[0].frames[0].url, url, 'parsed the value');
+  });
+
+  test('parses TXXX tags with characters that have a single-digit hexadecimal representation', function() {
+    var events = [], value = String.fromCharCode(7);
+    metadataStream.on('data', function(event) {
+      events.push(event);
+    });
+
+    metadataStream.push({
+      trackId: 7,
+      pts: 1000,
+      dts: 900,
+
+      // header
+      data: new Uint8Array(id3Tag(id3Frame('TXXX',
+                                           0x00,
+                                           stringToCString(''),
+                                           stringToInts(value)),
+                                  [0x00, 0x00]))
+    });
+
+    equal(events[0].frames[0].value,
+          value,
+          'parsed the single-digit character');
+  });
+
 })(window, window.videojs);

@@ -109,11 +109,18 @@
   </div>
 
 
+  <script>
+    window.videojs = {
+      Hls: {}
+    };
+  </script>
   <!-- transmuxing -->
+  <script src="../../src/stream.js"></script>
   <script src="../../src/flv-tag.js"></script>
   <script src="../../src/exp-golomb.js"></script>
   <script src="../../src/h264-stream.js"></script>
   <script src="../../src/aac-stream.js"></script>
+  <script src="../../src/metadata-stream.js"></script>
   <script src="../../src/segment-parser.js"></script>
   <script src="../../node_modules/pkcs7/dist/pkcs7.unpad.js"></script>
   <script src="../../src/decrypter.js"></script>

@@ -29,6 +29,11 @@
 
     extend = window.videojs.util.mergeOptions,
 
+    makePat,
+    makePsi,
+    makePmt,
+    makePacket,
+
     testAudioTag,
     testVideoTag,
     testScriptTag,
@@ -54,123 +59,137 @@
     deepEqual(expectedHeader, header, 'the rest of the header is correct');
   });
 
+  // Create a PMT packet
+  // @return {Array} bytes
+  makePmt = function(options) {
+    var
+      result = [],
+      entryCount = 0,
+      k,
+      sectionLength;
+
+    for (k in options.pids) {
+      entryCount++;
+    }
+    // table_id
+    result.push(0x02);
+    // section_syntax_indicator '0' reserved section_length
+    // 13 + (program_info_length) + (n * 5 + ES_info_length[n])
+    sectionLength = 13 + (5 * entryCount) + 17;
+    result.push(0x80 | (0xF00 & sectionLength >>> 8));
+    result.push(sectionLength & 0xFF);
+    // program_number
+    result.push(0x00);
+    result.push(0x01);
+    // reserved version_number current_next_indicator
+    result.push(0x01);
+    // section_number
+    result.push(0x00);
+    // last_section_number
+    result.push(0x00);
+    // reserved PCR_PID
+    result.push(0xe1);
+    result.push(0x00);
+    // reserved program_info_length
+    result.push(0xf0);
+    result.push(0x11); // hard-coded 17 byte descriptor
+    // program descriptors
+    result = result.concat([
+      0x25, 0x0f, 0xff, 0xff,
+      0x49, 0x44, 0x33, 0x20,
+      0xff, 0x49, 0x44, 0x33,
+      0x20, 0x00, 0x1f, 0x00,
+      0x01
+    ]);
+    for (k in options.pids) {
+      // stream_type
+      result.push(options.pids[k]);
+      // reserved elementary_PID
+      result.push(0xe0 | (k & 0x1f00) >>> 8);
+      result.push(k & 0xff);
+      // reserved ES_info_length
+      result.push(0xf0);
+      result.push(0x00); // ES_info_length = 0
+    }
+    // CRC_32
+    result.push([0x00, 0x00, 0x00, 0x00]); // invalid CRC but we don't check it
+    return result;
+  };
+
+  // Create a PAT packet
+  // @return {Array} bytes
+  makePat = function(options) {
+    var
+      result = [],
+      k;
+
+    // table_id
+    result.push(0x00);
+    // section_syntax_indicator '0' reserved section_length
+    result.push(0x80);
+    result.push(0x0d); // section_length for one program
+    // transport_stream_id
+    result.push(0x00);
+    result.push(0x00);
+    // reserved version_number current_next_indicator
+    result.push(0x01); // current_next_indicator is 1
+    // section_number
+    result.push(0x00);
+    // last_section_number
+    result.push(0x00);
+    for (k in options.programs) {
+      // program_number
+      result.push((k & 0xFF00) >>> 8);
+      result.push(k & 0x00FF);
+      // reserved program_map_pid
+      result.push((options.programs[k] & 0x1f00) >>> 8);
+      result.push(options.programs[k] & 0xff);
+    }
+    return result;
+  };
+
+  // Create a PAT or PMT packet based on the specified options
+  // @return {Array} bytes
+  makePsi = function(options) {
+    var result = [];
+
+    // pointer_field
+    if (options.payloadUnitStartIndicator) {
+      result.push(0x00);
+    }
+    if (options.programs) {
+      return result.concat(makePat(options));
+    }
+    return result.concat(makePmt(options));
+  };
+
+  // Construct an M2TS packet
+  // @return {Array} bytes
+  makePacket = function(options) {
+    var
+      result = [],
+      settings = extend({
+        payloadUnitStartIndicator: true,
+        pid: 0x00
+      }, options);
+
+    // header
+    // sync_byte
+    result.push(0x47);
+    // transport_error_indicator payload_unit_start_indicator transport_priority PID
+    result.push((settings.pid & 0x1f) << 8 | 0x40);
+    result.push(settings.pid & 0xff);
+    // transport_scrambling_control adaptation_field_control continuity_counter
+    result.push(0x10);
+    result = result.concat(makePsi(settings));
+
+    // ensure the resulting packet is the correct size
+    result.length = window.videojs.Hls.SegmentParser.MP2T_PACKET_LENGTH;
+    return result;
+  };
+
   test('parses PMTs with program descriptors', function() {
     var
-      makePmt = function(options) {
-        var
-          result = [],
-          entryCount = 0,
-          k,
-          sectionLength;
-        for (k in options.pids) {
-          entryCount++;
-        }
-        // table_id
-        result.push(0x02);
-        // section_syntax_indicator '0' reserved section_length
-        // 13 + (program_info_length) + (n * 5 + ES_info_length[n])
-        sectionLength = 13 + (5 * entryCount) + 17;
-        result.push(0x80 | (0xF00 & sectionLength >>> 8));
-        result.push(sectionLength & 0xFF);
-        // program_number
-        result.push(0x00);
-        result.push(0x01);
-        // reserved version_number current_next_indicator
-        result.push(0x01);
-        // section_number
-        result.push(0x00);
-        // last_section_number
-        result.push(0x00);
-        // reserved PCR_PID
-        result.push(0xe1);
-        result.push(0x00);
-        // reserved program_info_length
-        result.push(0xf0);
-        result.push(0x11); // hard-coded 17 byte descriptor
-        // program descriptors
-        result = result.concat([
-          0x25, 0x0f, 0xff, 0xff,
-          0x49, 0x44, 0x33, 0x20,
-          0xff, 0x49, 0x44, 0x33,
-          0x20, 0x00, 0x1f, 0x00,
-          0x01
-        ]);
-        for (k in options.pids) {
-          // stream_type
-          result.push(options.pids[k]);
-          // reserved elementary_PID
-          result.push(0xe0 | (k & 0x1f00) >>> 8);
-          result.push(k & 0xff);
-          // reserved ES_info_length
-          result.push(0xf0);
-          result.push(0x00); // ES_info_length = 0
-        }
-        // CRC_32
-        result.push([0x00, 0x00, 0x00, 0x00]); // invalid CRC but we don't check it
-        return result;
-      },
-      makePat = function(options) {
-        var
-          result = [],
-          k;
-        // table_id
-        result.push(0x00);
-        // section_syntax_indicator '0' reserved section_length
-        result.push(0x80);
-        result.push(0x0d); // section_length for one program
-        // transport_stream_id
-        result.push(0x00);
-        result.push(0x00);
-        // reserved version_number current_next_indicator
-        result.push(0x01); // current_next_indicator is 1
-        // section_number
-        result.push(0x00);
-        // last_section_number
-        result.push(0x00);
-        for (k in options.programs) {
-          // program_number
-          result.push((k & 0xFF00) >>> 8);
-          result.push(k & 0x00FF);
-          // reserved program_map_pid
-          result.push((options.programs[k] & 0x1f00) >>> 8);
-          result.push(options.programs[k] & 0xff);
-        }
-        return result;
-      },
-      makePsi = function(options) {
-        var result = [];
-
-        // pointer_field
-        if (options.payloadUnitStartIndicator) {
-          result.push(0x00);
-        }
-        if (options.programs) {
-          return result.concat(makePat(options));
-        }
-        return result.concat(makePmt(options));
-      },
-      makePacket = function(options) {
-        var
-          result = [],
-          settings = extend({
-            payloadUnitStartIndicator: true,
-            pid: 0x00
-          }, options);
-
-        // header
-        // sync_byte
-        result.push(0x47);
-        // transport_error_indicator payload_unit_start_indicator transport_priority PID
-        result.push((settings.pid & 0x1f) << 8 | 0x40);
-        result.push(settings.pid & 0xff);
-        // transport_scrambling_control adaptation_field_control continuity_counter
-        result.push(0x10);
-        result = result.concat(makePsi(settings));
-
-        // ensure the resulting packet is the correct size
-        result.length = window.videojs.Hls.SegmentParser.MP2T_PACKET_LENGTH;
-        return result;
-      },
       h264Type = window.videojs.Hls.SegmentParser.STREAM_TYPES.h264,
       adtsType = window.videojs.Hls.SegmentParser.STREAM_TYPES.adts;
 
@@ -191,6 +210,22 @@
     strictEqual(parser.stream.programMapTable[adtsType], 0x03, 'audio is PID 3');
   });
 
+  test('recognizes metadata streams', function() {
+    parser.parseSegmentBinaryData(new Uint8Array(makePacket({
+      programs: {
+        0x01: [0x01]
+      }
+    }).concat(makePacket({
+      pid: 0x01,
+      pids: {
+        // Rec. ITU-T H.222.0 (06/2012), Table 2-34
+        0x02: 0x15 // Metadata carried in PES packets
+      }
+    }))));
+
+    equal(parser.stream.programMapTable[0x15], 0x02, 'metadata is PID 2');
+  });
+
   test('parses the first bipbop segment', function() {
     parser.parseSegmentBinaryData(window.bcSegment);