diff --git a/src/Common/Stamp.cpp b/src/Common/Stamp.cpp index fa449721..878fd183 100644 --- a/src/Common/Stamp.cpp +++ b/src/Common/Stamp.cpp @@ -42,8 +42,8 @@ void Stamp::setPlayBack(bool playback) { _playback = playback; } -void Stamp::makeRelation(Stamp &other){ - _related = &other; +void Stamp::syncTo(Stamp &other){ + _sync_master = &other; } void Stamp::revise(int64_t dts, int64_t pts, int64_t &dts_out, int64_t &pts_out,bool modifyStamp) { @@ -53,23 +53,21 @@ void Stamp::revise(int64_t dts, int64_t pts, int64_t &dts_out, int64_t &pts_out, return; } - if(_related && _related->_last_dts){ + if(_sync_master && _sync_master->_last_dts){ //音视频dts当前时间差 - int64_t dts_diff = _last_dts - _related->_last_dts; + int64_t dts_diff = _last_dts - _sync_master->_last_dts; if(ABS(dts_diff) < 5000){ //如果绝对时间戳小于5秒,那么说明他们的起始时间戳是一致的,那么强制同步 _last_relativeStamp = _relativeStamp; - _relativeStamp = _related->_relativeStamp + dts_diff; - dts_out += dts_diff; - pts_out += dts_diff; -// DebugL << "音视频同步事件差:" << dts_diff; + _relativeStamp = _sync_master->_relativeStamp + dts_diff; } //下次不用再强制同步 - _related = nullptr; + _sync_master = nullptr; } - if(dts_out < 0){ - //相对时间戳小于0,那么说明是同步时间戳导致的,在这个过渡期内,我们一直返回上次的结果(目的是为了防止时间戳回退) + if(dts_out < 0 || dts_out < _last_relativeStamp){ + //相对时间戳小于0,或者小于上次的时间戳, + //那么说明是同步时间戳导致的,在这个过渡期内,我们一直返回上次的结果(目的是为了防止时间戳回退) pts_out = _last_relativeStamp + (pts_out - dts_out); dts_out = _last_relativeStamp; } diff --git a/src/Common/Stamp.h b/src/Common/Stamp.h index 2b57f7c0..2e497177 100644 --- a/src/Common/Stamp.h +++ b/src/Common/Stamp.h @@ -69,9 +69,10 @@ public: void setPlayBack(bool playback = true); /** - * 产生关联,用于音视频同步用 + * 音视频同步用,音频应该同步于视频(只修改音频时间戳) + * 因为音频时间戳修改后不影响播放速度 */ - void makeRelation(Stamp &other); + void syncTo(Stamp &other); private: void revise_l(int64_t dts, int64_t pts, int64_t &dts_out, int64_t &pts_out,bool modifyStamp = false); @@ -81,7 +82,7 @@ private: int64_t _last_dts = 0; SmoothTicker _ticker; bool _playback = false; - Stamp *_related = nullptr; + Stamp *_sync_master = nullptr; }; //dts生成器, diff --git a/src/Extension/Frame.cpp b/src/Extension/Frame.cpp index 9aab3fad..1a5f8fd3 100644 --- a/src/Extension/Frame.cpp +++ b/src/Extension/Frame.cpp @@ -76,8 +76,8 @@ Frame::Ptr Frame::getCacheAbleFrame(const Frame::Ptr &frame){ } #define SWITCH_CASE(codec_id) case codec_id : return #codec_id -const char *CodecInfo::getCodecName() { - switch (getCodecId()) { +const char *getCodecName(CodecId codecId) { + switch (codecId) { SWITCH_CASE(CodecH264); SWITCH_CASE(CodecH265); SWITCH_CASE(CodecAAC); @@ -88,8 +88,8 @@ const char *CodecInfo::getCodecName() { } } -TrackType CodecInfo::getTrackType(){ - switch (getCodecId()){ +TrackType getTrackType(CodecId codecId){ + switch (codecId){ case CodecH264: case CodecH265: return TrackVideo; case CodecAAC: @@ -100,4 +100,11 @@ TrackType CodecInfo::getTrackType(){ } } +const char *CodecInfo::getCodecName() { + return mediakit::getCodecName(getCodecId()); +} + +TrackType CodecInfo::getTrackType() { + return mediakit::getTrackType(getCodecId()); +} }//namespace mediakit diff --git a/src/Extension/Frame.h b/src/Extension/Frame.h index cafa876a..b3b37a32 100644 --- a/src/Extension/Frame.h +++ b/src/Extension/Frame.h @@ -40,6 +40,16 @@ typedef enum { TrackMax = 3 } TrackType; +/** + * 获取编码器名称 + */ +const char *getCodecName(CodecId codecId); + +/** + * 获取音视频类型 + */ +TrackType getTrackType(CodecId codecId); + /** * 编码信息的抽象接口 */ diff --git a/src/Record/MP4Muxer.cpp b/src/Record/MP4Muxer.cpp index 8f3bb60d..a543126a 100644 --- a/src/Record/MP4Muxer.cpp +++ b/src/Record/MP4Muxer.cpp @@ -134,6 +134,26 @@ static uint8_t getObject(CodecId codecId){ } } +void MP4Muxer::stampSync(){ + if(_codec_to_trackid.size() < 2){ + return; + } + + Stamp *audio = nullptr, *video = nullptr; + for(auto &pr : _codec_to_trackid){ + switch (getTrackType((CodecId) pr.first)){ + case TrackAudio : audio = &pr.second.stamp; break; + case TrackVideo : video = &pr.second.stamp; break; + default : break; + } + } + + if(audio && video){ + //音频时间戳同步于视频,因为音频时间戳被修改后不影响播放 + audio->syncTo(*video); + } +} + void MP4Muxer::addTrack(const Track::Ptr &track) { auto mp4_object = getObject(track->getCodecId()); if (!mp4_object) { @@ -261,6 +281,9 @@ void MP4Muxer::addTrack(const Track::Ptr &track) { default: WarnL << "MP4录制不支持该编码格式:" << track->getCodecName(); break; } + + //尝试音视频同步 + stampSync(); } }//namespace mediakit diff --git a/src/Record/MP4Muxer.h b/src/Record/MP4Muxer.h index e5fd4807..eef8def9 100644 --- a/src/Record/MP4Muxer.h +++ b/src/Record/MP4Muxer.h @@ -45,13 +45,14 @@ public: private: void openMP4(); void closeMP4(); + void stampSync(); private: - struct track_info{ + struct track_info { int track_id = -1; Stamp stamp; }; - unordered_map _codec_to_trackid; + unordered_map _codec_to_trackid; List _frameCached; bool _started = false; bool _have_video = false; diff --git a/src/Record/TsMuxer.cpp b/src/Record/TsMuxer.cpp index 79419c74..560e2ef6 100644 --- a/src/Record/TsMuxer.cpp +++ b/src/Record/TsMuxer.cpp @@ -23,6 +23,26 @@ TsMuxer::~TsMuxer() { uninit(); } +void TsMuxer::stampSync(){ + if(_codec_to_trackid.size() < 2){ + return; + } + + Stamp *audio = nullptr, *video = nullptr; + for(auto &pr : _codec_to_trackid){ + switch (getTrackType((CodecId) pr.first)){ + case TrackAudio : audio = &pr.second.stamp; break; + case TrackVideo : video = &pr.second.stamp; break; + default : break; + } + } + + if(audio && video){ + //音频时间戳同步于视频,因为音频时间戳被修改后不影响播放 + audio->syncTo(*video); + } +} + void TsMuxer::addTrack(const Track::Ptr &track) { switch (track->getCodecId()) { case CodecH264: { @@ -52,9 +72,11 @@ void TsMuxer::addTrack(const Track::Ptr &track) { break; } - default: - break; + default: WarnL << "mpeg-ts 不支持该编码格式,已忽略:" << track->getCodecName(); break; } + + //尝试音视频同步 + stampSync(); } void TsMuxer::inputFrame(const Frame::Ptr &frame) { diff --git a/src/Record/TsMuxer.h b/src/Record/TsMuxer.h index 7f20e968..5a16f1cf 100644 --- a/src/Record/TsMuxer.h +++ b/src/Record/TsMuxer.h @@ -17,37 +17,59 @@ #include "Util/File.h" #include "Common/MediaSink.h" #include "Common/Stamp.h" - using namespace toolkit; - namespace mediakit { +//该类用于产生MPEG-TS class TsMuxer : public MediaSinkInterface { public: TsMuxer(); virtual ~TsMuxer(); + + /** + * 添加音视频轨道 + */ void addTrack(const Track::Ptr &track) override; + + /** + * 重置音视频轨道 + */ void resetTracks() override; + + /** + * 输入帧数据 + */ void inputFrame(const Frame::Ptr &frame) override; + protected: + /** + * 输出mpegts数据回调 + * @param packet mpegts数据 + * @param bytes mpegts数据长度 + * @param timestamp 时间戳,单位毫秒 + * @param is_idr_fast_packet 是否为关键帧的第一个TS包,用于确保ts切片第一帧为关键帧 + */ virtual void onTs(const void *packet, int bytes,uint32_t timestamp,bool is_idr_fast_packet) = 0; + private: void init(); void uninit(); -private: - void *_context = nullptr; - char *_tsbuf[188]; - uint32_t _timestamp = 0; + //音视频时间戳同步用 + void stampSync(); - struct track_info{ +private: + void *_context = nullptr; + char _tsbuf[188]; + uint32_t _timestamp = 0; + struct track_info { int track_id = -1; Stamp stamp; }; - unordered_map _codec_to_trackid; + unordered_map _codec_to_trackid; List _frameCached; bool _is_idr_fast_packet = false; bool _have_video = false; }; }//namespace mediakit -#endif //TSMUXER_H +#endif //TSMUXER_H \ No newline at end of file diff --git a/src/Rtmp/FlvMuxer.cpp b/src/Rtmp/FlvMuxer.cpp index 456f679b..bc29cf31 100644 --- a/src/Rtmp/FlvMuxer.cpp +++ b/src/Rtmp/FlvMuxer.cpp @@ -52,7 +52,7 @@ void FlvMuxer::start(const EventPoller::Ptr &poller,const RtmpMediaSource::Ptr & }); //音频同步于视频 - _stamp[0].makeRelation( _stamp[1]); + _stamp[0].syncTo(_stamp[1]); _ring_reader->setReadCB([weakSelf](const RtmpMediaSource::RingDataType &pkt){ auto strongSelf = weakSelf.lock(); if(!strongSelf){ diff --git a/src/Rtmp/RtmpSession.cpp b/src/Rtmp/RtmpSession.cpp index 873b1726..c2e55863 100644 --- a/src/Rtmp/RtmpSession.cpp +++ b/src/Rtmp/RtmpSession.cpp @@ -267,7 +267,7 @@ void RtmpSession::sendPlayResponse(const string &err,const RtmpMediaSource::Ptr }); //音频同步于视频 - _stamp[0].makeRelation( _stamp[1]); + _stamp[0].syncTo(_stamp[1]); _pRingReader = src->getRing()->attach(getPoller()); weak_ptr weakSelf = dynamic_pointer_cast(shared_from_this()); _pRingReader->setReadCB([weakSelf](const RtmpMediaSource::RingDataType &pkt) {