音视频传输中时间戳平滑处理

在音视频中一般时间戳从设备中系统时间得来，通常是以毫秒作为单位的linux时间。因为网络传输或者时间有时候突变的因为，造成了时间戳混乱。有必要对时间戳做一下处理。包括突变时候平滑处理，包括音视频不同步的时候的处理，下面算法解决了时间戳计算问题，在移动设备上很有好处：

static int RTMP_CanculateTimestamp(CarEye_RTMP_Handle handle, MediaFrameInfo& FrameInfo, unsigned int MediaType)
{int Diff;if(handle == NULL){return -1;}CERTMPInfo* thiz = (CERTMPInfo*)handle;if(MediaType == CAREYE_VFRAME_FLAG){if(thiz->fVideoIndex < 0){thiz->fVideoIndex++;//skip first frame}else{if(FrameInfo.TimestampInternal == 0 && thiz->fVideoPTSIncrement > 0 && thiz->fVideoIndex >= TIMESTAMP_CANCULATE_VIDEO_THRESHOLD_SIZE){unsigned int preIndex = (thiz->fVideoIndex + TIMESTAMP_CANCULATE_BUFFER_SIZE - 1)%TIMESTAMP_CANCULATE_BUFFER_SIZE;long pts = (long)(FrameInfo.Timestamp - thiz->fVideoTimestamp[preIndex]);if(pts > (thiz->fVideoPTSIncrement << 1) || pts < -(thiz->fVideoPTSIncrement << 1)){/*printf("Have an abnormal video frame! drop the Timestamp!!  fVideoIndex=%d, preIndex=%d, Timestamp=%ul, oldTimestamp=%ul, fVideoPTSIncrement=%d, pts=%ld\n", thiz->fVideoIndex,preIndex,FrameInfo.Timestamp,thiz->fVideoTimestamp[preIndex],thiz->fVideoPTSIncrement,pts);*/if((++thiz->fVideoFrameAbnornalCounts) > 1){thiz->fVideoIndex = 0;thiz->fVideoFrameAbnornalCounts = 0;thiz->fVideoPTSIncrement = 0;//重新赋值}else{return -2;}}}thiz->fVideoTimestamp[thiz->fVideoIndex++ % TIMESTAMP_CANCULATE_BUFFER_SIZE] = FrameInfo.Timestamp;if(thiz->fVideoIndex == (TIMESTAMP_CANCULATE_BUFFER_SIZE << 8)){thiz->fVideoIndex = TIMESTAMP_CANCULATE_BUFFER_SIZE;}if(TIMESTAMP_CANCULATE_VIDEO_THRESHOLD_SIZE <= thiz->fVideoIndex){int frameCount = (thiz->fVideoIndex >= TIMESTAMP_CANCULATE_BUFFER_SIZE)?TIMESTAMP_CANCULATE_BUFFER_SIZE:thiz->fVideoIndex;if(frameCount >= TIMESTAMP_CANCULATE_BUFFER_SIZE){thiz->fVideoPTSIncrement = (FrameInfo.Timestamp - thiz->fVideoTimestamp[thiz->fVideoIndex%TIMESTAMP_CANCULATE_BUFFER_SIZE] + (TIMESTAMP_CANCULATE_BUFFER_SIZE>>1))/(TIMESTAMP_CANCULATE_BUFFER_SIZE-1);}else{thiz->fVideoPTSIncrement = (FrameInfo.Timestamp - thiz->fVideoTimestamp[0] + (frameCount>>1)) / (frameCount-1);}}}}else if(MediaType == CAREYE_AFRAME_FLAG){if(thiz->fAudioIndex < 0){thiz->fAudioIndex++;//skip first frame}else{if(FrameInfo.TimestampInternal == 0 && thiz->fAudioPTSIncrement > 0 && thiz->fAudioIndex >= TIMESTAMP_CANCULATE_AUDIO_THRESHOLD_SIZE){unsigned int preIndex = (thiz->fAudioIndex + TIMESTAMP_CANCULATE_BUFFER_SIZE - 1)%TIMESTAMP_CANCULATE_BUFFER_SIZE;long pts = (long)(FrameInfo.Timestamp - thiz->fAudioTimestamp[preIndex]);if(pts > (thiz->fAudioPTSIncrement << 1) || pts < -(thiz->fAudioPTSIncrement << 1)){/*printf("Have an abnormal audio frame! drop the Timestamp!!  fAudioIndex=%d, preIndex=%d, Timestamp=%ul, oldTimestamp=%ul, fAudioPTSIncrement=%d, pts=%ld\n", thiz->fAudioIndex,preIndex,FrameInfo.Timestamp,thiz->fAudioTimestamp[preIndex],thiz->fAudioPTSIncrement,pts);*/if((++thiz->fAudioFrameAbnornalCounts) > 1){thiz->fAudioIndex = 0;thiz->fAudioFrameAbnornalCounts = 0;thiz->fAudioPTSIncrement = 0;}else{return -2;}}}thiz->fAudioTimestamp[thiz->fAudioIndex++ % TIMESTAMP_CANCULATE_BUFFER_SIZE] = FrameInfo.Timestamp;if(thiz->fAudioIndex == (TIMESTAMP_CANCULATE_BUFFER_SIZE << 8)){thiz->fAudioIndex = TIMESTAMP_CANCULATE_BUFFER_SIZE;}if(TIMESTAMP_CANCULATE_AUDIO_THRESHOLD_SIZE <= thiz->fAudioIndex){int frameCount = (thiz->fAudioIndex >= TIMESTAMP_CANCULATE_BUFFER_SIZE)?TIMESTAMP_CANCULATE_BUFFER_SIZE:thiz->fAudioIndex;if(frameCount >= TIMESTAMP_CANCULATE_BUFFER_SIZE){thiz->fAudioPTSIncrement = (FrameInfo.Timestamp - thiz->fAudioTimestamp[thiz->fAudioIndex % TIMESTAMP_CANCULATE_BUFFER_SIZE] + (TIMESTAMP_CANCULATE_BUFFER_SIZE>>1))/(TIMESTAMP_CANCULATE_BUFFER_SIZE-1);}else{thiz->fAudioPTSIncrement = (FrameInfo.Timestamp - thiz->fAudioTimestamp[0] + (frameCount>>1)) / (frameCount-1);}}}}return 0;
}

#define MAX_AUDIO_VEDIO_DIFF	1000int CRTMPPublisher::SendH264Packet(unsigned char *data, UINT32 size,bool bIsKeyFrame, UINT32 nTimeStamp, UINT32 ptzIncrement)
{UINT32 Diff;if(data == NULL && size<11){return -1;}if (data[0]==0x00 && data[1]==0x00 && data[2]==0x00 && data[3]==0x01){data +=4;size -= 4;}if (data[0]==0x00 && data[1]==0x00 && data[2]==0x01){data +=3;size -= 3;}if (m_bSentMetadata == false){if (!RTMP_IsConnected(m_pRtmp)/* && (Reconnect() == false)*/){return -2;}if(SendMetadata()==true){m_bSentMetadata = true;}else{m_bSentMetadata = false;return -3;}}int nalu_type = data[0] & 0x1F;int nalu_offset = 0;int parser_offset = 0;while (nalu_type == 0x06 || nalu_type == 0x07 || nalu_type == 0x08){nalu_offset = FindNaluHead(data + parser_offset, size - parser_offset);if (nalu_offset == -1){return -4;}parser_offset += nalu_offset;nalu_type = data[parser_offset] & 0x1F;}data += parser_offset;size -= parser_offset;if (size > m_nH264BuffSize - 10 - 30){_TRACE_LOG("SendH264Packet error! packet too large! size =%ld.\n", size);return -5;}unsigned char *body = (unsigned char*)(m_h264Buff+30);int i = 0;if(bIsKeyFrame){body[i++] = 0x17;// 1:Iframe  7:AVCif (m_bWaitingKeyFrame){m_bWaitingKeyFrame = false;}}else{body[i++] = 0x27;// 2:Pframe  7:AVC}body[i++] = 0x01;// AVC NALUbody[i++] = 0x00;body[i++] = 0x00;body[i++] = 0x00;// NALU sizebody[i++] = size>>24;body[i++] = size>>16;body[i++] = size>>8;body[i++] = size&0xff;// NALU datamemcpy(&body[i],data,size);if (!RTMP_IsConnected(m_pRtmp)/* && (Reconnect() == false)*/){return -6;}if (m_bWaitingKeyFrame){return -7;}if(ptzIncrement <= 0){//int nTick = 0;//if (m_uLastVideoPTS == 0)//{//	m_uLastVideoPTS = nTimeStamp;//	m_uVideoPTS = 0;//}//else//{//	nTick = nTimeStamp - m_uLastVideoPTS;//	if (nTick < 0 || nTick>500)//	{//		nTick = 40;//		//printf("H264 TimeStamp Exception : Tick = 0x%x\n", nTimeStamp-m_uLastVideoPTS);//	}//}//m_uVideoPTS += nTick;//m_uLastVideoPTS = nTimeStamp;//m_uVideoPTS += 100;m_uAudioPTS = m_uVideoPTS = nTimeStamp;//printf("%s m_uAudioPTS=%u, m_uVideoPTS=%u, ptzIncrement=%d\n", __FUNCTION__, m_uAudioPTS, m_uVideoPTS, ptzIncrement);}else{m_uVideoPTS += ptzIncrement;}	//增加横向音视频同步Diff = m_uAudioPTS > m_uVideoPTS ? m_uAudioPTS - m_uVideoPTS : m_uVideoPTS - m_uAudioPTS;if (Diff > MAX_AUDIO_VEDIO_DIFF){m_uAudioPTS = m_uVideoPTS = nTimeStamp;}//printf("%s m_uAudioPTS=%u, m_uVideoPTS=%u, ptzIncrement=%d\n", __FUNCTION__, m_uAudioPTS, m_uVideoPTS, ptzIncrement);int iRet = SendPacket(RTMP_PACKET_TYPE_VIDEO, body, i+size, m_uVideoPTS);return iRet;
}

第一步时间时间戳的平滑处理和突变的判断，第二步对音视频时间戳不同步进行了处理。相关代码可以从开源网站下载：https://github.com/Car-eye-team

本文来自互联网用户投稿，文章观点仅代表作者本人，不代表本站立场，不承担相关法律责任。如若转载，请注明出处。 如若内容造成侵权/违法违规/事实不符，请点击【内容举报】进行投诉反馈！