Add support AAC audio for HTTP-FLV

2023-01-19 21:44:15 +03:00
parent e040fb591f
commit 5c17d8fcb6
2 changed files with 295 additions and 20 deletions
@@ -0,0 +1,165 @@
+package httpflv
+
+import (
+	"encoding/binary"
+	"errors"
+	"math"
+)
+
+const (
+	TypeNumber byte = iota
+	TypeBoolean
+	TypeString
+	TypeObject
+	TypeEcmaArray = 8
+	TypeObjectEnd = 9
+)
+
+var Err = errors.New("amf0 read error")
+
+// AMF0 spec: http://download.macromedia.com/pub/labs/amf/amf0_spec_121207.pdf
+type AMF0 struct {
+	buf []byte
+	pos int
+}
+
+func NewReader(b []byte) *AMF0 {
+	return &AMF0{buf: b}
+}
+
+func (a *AMF0) ReadMetaData() map[string]interface{} {
+	if b, _ := a.ReadByte(); b != TypeString {
+		return nil
+	}
+	if s, _ := a.ReadString(); s != "onMetaData" {
+		return nil
+	}
+
+	b, _ := a.ReadByte()
+	switch b {
+	case TypeObject:
+		v, _ := a.ReadObject()
+		return v
+	case TypeEcmaArray:
+		v, _ := a.ReadEcmaArray()
+		return v
+	}
+
+	return nil
+}
+
+func (a *AMF0) ReadMap() (map[interface{}]interface{}, error) {
+	dict := make(map[interface{}]interface{})
+
+	for a.pos < len(a.buf) {
+		k, err := a.ReadItem()
+		if err != nil {
+			return nil, err
+		}
+		v, err := a.ReadItem()
+		if err != nil {
+			return nil, err
+		}
+		dict[k] = v
+	}
+
+	return dict, nil
+}
+
+func (a *AMF0) ReadItem() (interface{}, error) {
+	dataType, err := a.ReadByte()
+	if err != nil {
+		return nil, err
+	}
+
+	switch dataType {
+	case TypeNumber:
+		return a.ReadNumber()
+
+	case TypeBoolean:
+		v, err := a.ReadByte()
+		return v != 0, err
+
+	case TypeString:
+		return a.ReadString()
+
+	case TypeObject:
+		return a.ReadObject()
+
+	case TypeObjectEnd:
+		return nil, nil
+	}
+
+	return nil, Err
+}
+
+func (a *AMF0) ReadByte() (byte, error) {
+	if a.pos >= len(a.buf) {
+		return 0, Err
+	}
+
+	v := a.buf[a.pos]
+	a.pos++
+	return v, nil
+}
+
+func (a *AMF0) ReadNumber() (float64, error) {
+	if a.pos+8 >= len(a.buf) {
+		return 0, Err
+	}
+
+	v := binary.BigEndian.Uint64(a.buf[a.pos : a.pos+8])
+	a.pos += 8
+	return math.Float64frombits(v), nil
+}
+
+func (a *AMF0) ReadString() (string, error) {
+	if a.pos+2 >= len(a.buf) {
+		return "", Err
+	}
+
+	size := int(binary.BigEndian.Uint16(a.buf[a.pos:]))
+	a.pos += 2
+
+	if a.pos+size >= len(a.buf) {
+		return "", Err
+	}
+
+	s := string(a.buf[a.pos : a.pos+size])
+	a.pos += size
+
+	return s, nil
+}
+
+func (a *AMF0) ReadObject() (map[string]interface{}, error) {
+	obj := make(map[string]interface{})
+
+	for {
+		k, err := a.ReadString()
+		if err != nil {
+			return nil, err
+		}
+
+		v, err := a.ReadItem()
+		if err != nil {
+			return nil, err
+		}
+
+		if k == "" {
+			break
+		}
+
+		obj[k] = v
+	}
+
+	return obj, nil
+}
+
+func (a *AMF0) ReadEcmaArray() (map[string]interface{}, error) {
+	if a.pos+4 >= len(a.buf) {
+		return nil, Err
+	}
+	a.pos += 4 // skip size
+
+	return a.ReadObject()
+}
@@ -2,7 +2,9 @@ package httpflv

 import (
 	"bufio"
+	"bytes"
 	"github.com/deepch/vdk/av"
+	"github.com/deepch/vdk/codec/aacparser"
 	"github.com/deepch/vdk/codec/h264parser"
 	"github.com/deepch/vdk/format/flv/flvio"
 	"github.com/deepch/vdk/utils/bits/pio"
@@ -35,12 +37,19 @@ func Accept(res *http.Response) (*Conn, error) {
 		return nil, err
 	}

-	// ignore flags because Reolink cameras have a buggy realization
-	_, n, err := flvio.ParseFileHeader(c.buf)
+	flags, n, err := flvio.ParseFileHeader(c.buf)
 	if err != nil {
 		return nil, err
 	}

+	if flags&flvio.FILE_HAS_VIDEO != 0 {
+		c.videoIdx = -1
+	}
+
+	if flags&flvio.FILE_HAS_AUDIO != 0 {
+		c.audioIdx = -1
+	}
+
 	if _, err = c.reader.Discard(n); err != nil {
 		return nil, err
 	}
@@ -52,26 +61,80 @@ type Conn struct {
 	conn   io.ReadCloser
 	reader *bufio.Reader
 	buf    []byte
+
+	videoIdx int8
+	audioIdx int8
 }

 func (c *Conn) Streams() ([]av.CodecData, error) {
-	for {
+	var video, audio av.CodecData
+
+	// Normal software sends:
+	// 1. Video/audio flag in header
+	// 2. MetaData as first tag (with video/audio codec info)
+	// 3. Video/audio headers in 2nd and 3rd tag
+
+	// Reolink camera sends:
+	// 1. Empty video/audio flag
+	// 2. MedaData without stereo key for AAC
+	// 3. Audio header after Video keyframe tag
+
+	waitVideo := c.videoIdx != 0
+	waitAudio := c.audioIdx != 0
+
+	for i := 0; i < 20; i++ {
 		tag, _, err := flvio.ReadTag(c.reader, c.buf)
 		if err != nil {
 			return nil, err
 		}

-		if tag.Type != flvio.TAG_VIDEO || tag.AVCPacketType != flvio.AAC_SEQHDR {
-			continue
+		//log.Printf("[FLV] type=%d avc=%d aac=%d video=%t audio=%t", tag.Type, tag.AVCPacketType, tag.AACPacketType, video != nil, audio != nil)
+
+		switch tag.Type {
+		case flvio.TAG_SCRIPTDATA:
+			if meta := NewReader(tag.Data).ReadMetaData(); meta != nil {
+				waitVideo = meta["videocodecid"] != nil
+
+				// don't wait audio tag because parse all info from MetaData
+				waitAudio = false
+
+				audio = parseAudioConfig(meta)
+			} else {
+				waitVideo = bytes.Contains(tag.Data, []byte("videocodecid"))
+				waitAudio = bytes.Contains(tag.Data, []byte("audiocodecid"))
+			}
+
+		case flvio.TAG_VIDEO:
+			if tag.AVCPacketType == flvio.AVC_SEQHDR {
+				video, _ = h264parser.NewCodecDataFromAVCDecoderConfRecord(tag.Data)
+			}
+			waitVideo = false
+
+		case flvio.TAG_AUDIO:
+			if tag.SoundFormat == flvio.SOUND_AAC && tag.AACPacketType == flvio.AAC_SEQHDR {
+				audio, _ = aacparser.NewCodecDataFromMPEG4AudioConfigBytes(tag.Data)
+			}
+			waitAudio = false
 		}

-		stream, err := h264parser.NewCodecDataFromAVCDecoderConfRecord(tag.Data)
-		if err != nil {
-			return nil, err
+		if !waitVideo && !waitAudio {
+			break
 		}
-
-		return []av.CodecData{stream}, nil
 	}
+
+	if video != nil && audio != nil {
+		c.videoIdx = 0
+		c.audioIdx = 1
+		return []av.CodecData{video, audio}, nil
+	} else if video != nil {
+		c.videoIdx = 0
+		return []av.CodecData{video}, nil
+	} else if audio != nil {
+		c.audioIdx = 0
+		return []av.CodecData{audio}, nil
+	}
+
+	return nil, nil
 }

 func (c *Conn) ReadPacket() (av.Packet, error) {
@@ -81,20 +144,67 @@ func (c *Conn) ReadPacket() (av.Packet, error) {
 			return av.Packet{}, err
 		}

-		if tag.Type != flvio.TAG_VIDEO || tag.AVCPacketType != flvio.AVC_NALU {
-			continue
-		}
+		switch tag.Type {
+		case flvio.TAG_VIDEO:
+			if tag.AVCPacketType != flvio.AVC_NALU {
+				continue
+			}

-		return av.Packet{
-			Idx:             0,
-			Data:            tag.Data,
-			CompositionTime: flvio.TsToTime(tag.CompositionTime),
-			IsKeyFrame:      tag.FrameType == flvio.FRAME_KEY,
-			Time:            flvio.TsToTime(ts),
-		}, nil
+			return av.Packet{
+				Idx:             c.videoIdx,
+				Data:            tag.Data,
+				CompositionTime: flvio.TsToTime(tag.CompositionTime),
+				IsKeyFrame:      tag.FrameType == flvio.FRAME_KEY,
+				Time:            flvio.TsToTime(ts),
+			}, nil
+
+		case flvio.TAG_AUDIO:
+			if tag.SoundFormat != flvio.SOUND_AAC || tag.AACPacketType != flvio.AAC_RAW {
+				continue
+			}
+
+			return av.Packet{Idx: c.audioIdx, Data: tag.Data, Time: flvio.TsToTime(ts)}, nil
+		}
 	}
 }

 func (c *Conn) Close() (err error) {
 	return c.conn.Close()
 }
+
+func parseAudioConfig(meta map[string]interface{}) av.CodecData {
+	if meta["audiocodecid"] != float64(10) {
+		return nil
+	}
+
+	config := aacparser.MPEG4AudioConfig{
+		ObjectType: aacparser.AOT_AAC_LC,
+	}
+
+	switch v := meta["audiosamplerate"].(type) {
+	case float64:
+		config.SampleRate = int(v)
+	default:
+		return nil
+	}
+
+	switch meta["stereo"] {
+	case true:
+		config.ChannelConfig = 2
+		config.ChannelLayout = av.CH_STEREO
+	default:
+		// Reolink doesn't have this setting
+		config.ChannelConfig = 1
+		config.ChannelLayout = av.CH_MONO
+	}
+
+	buf := &bytes.Buffer{}
+	if err := aacparser.WriteMPEG4AudioConfig(buf, config); err != nil {
+		return nil
+	}
+
+	return aacparser.CodecData{
+		Config:      config,
+		ConfigBytes: buf.Bytes(),
+	}
+}