// ------------------------------------------------ // Copyright Joe Marshall 2024- All Rights Reserved // ------------------------------------------------ // // Unreal audio output implementation. // ------------------------------------------------ #include "UnrealAudioOut.h" #include "AudioDevice.h" #include "Components/ActorComponent.h" #include "Components/AudioComponent.h" #include "Runtime/Engine/Classes/Sound/AudioSettings.h" #include #include "Engine/engine.h" #include "UnrealLogging.h" static int64_t _getNSTime() { #if PLATFORM_ANDROID struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return static_cast(ts.tv_nsec) + (1000000000LL * static_cast(ts.tv_sec)); #else return -1; #endif } static int64_t _scaleTime(int64_t val, float scalar) { double dt = double(val) * double(scalar); return int64_t(dt); } void USoundWaveProceduralWithTiming::ResetAudio() { USoundWaveProcedural::ResetAudio(); QueueSilence(4096); } void USoundWaveProceduralWithTiming::QueueSilence(int64_t bytes) { UE_LOG(LogDirectVideo, VeryVerbose, TEXT("Queue Silence %d"), bytes); uint8_t silence[4096]; memset(silence, 0, 4096); int64_t count = 4096; while (bytes > 0) { if (bytes < 4096) count = bytes; bytes -= count; QueueAudio(silence, count); } } int32 USoundWaveProceduralWithTiming::GeneratePCMData(uint8 *PCMData, const int32 SamplesNeeded) { lastBufSendTime = _getNSTime(); if (!paused) { return USoundWaveProcedural::GeneratePCMData(PCMData, SamplesNeeded); } else { memset(PCMData, 0, SampleByteSize * SamplesNeeded); return SamplesNeeded; } } UnrealAudioOut::UnrealAudioOut(UActorComponent *pOwner) : audioSender(NULL), audioComponent(NULL), hasTimeOffset(false), presentationTimeOffset(0), pausedPresentationTime(-1), numChannels(0), sampleRate(0), playbackRate(1.0f), afterSeek(false), isPlaying(false), owner(pOwner) { } void UnrealAudioOut::close() { afterSeek = false; if (audioComponent.IsValid()) { audioComponent->Stop(); audioComponent->SetSound(NULL); } audioSender = NULL; } void UnrealAudioOut::init(int rate, int channels) { close(); // this object gets garbage collected at the right time because it is // assigned to a component below playbackRate = 1.0f; audioSender = NewObject(); audioSender->SetSampleRate(rate); audioSender->NumChannels = channels; audioSender->Duration = INDEFINITELY_LOOPING_DURATION; audioSender->bLooping = false; audioSender->bCanProcessAsync = true; audioSender->paused = false; if (!audioComponent.IsValid()) { if (owner) { AActor *actor = owner->GetOwner(); if (actor) { audioComponent = actor->FindComponentByClass(); } } } if (audioComponent.IsValid()) { audioComponent->SetSound(audioSender.Get()); } else { FAudioDeviceHandle audioDevice = GEngine->GetMainAudioDevice(); audioComponent = audioDevice->CreateComponent(audioSender.Get()); if (audioComponent.IsValid()) { audioComponent->bIsUISound = true; audioComponent->bAllowSpatialization = false; audioComponent->SetVolumeMultiplier(1.0f); audioComponent->AddToRoot(); audioComponent->Play(); // hold onto strong pointer to audiocomponent so it doesn't get destroyed until // we do audioComponentWeCreated = *audioComponent; } else { UE_LOGFMT(LogDirectVideo, Error, "Unable to create audio component!"); } } if (audioComponent.IsValid()) { audioComponent->Play(); } numChannels = channels; sampleRate = rate; hasTimeOffset = false; presentationTimeOffset = 0; isPlaying = true; UE_LOG(LogDirectVideo, Verbose, TEXT("Audio component: channels %d sampleRate %d"), channels, sampleRate); } void UnrealAudioOut::initSilent() { close(); UE_LOGFMT(LogDirectVideo, Verbose, "Audio init silent"); playbackRate = 1.0f; hasTimeOffset = false; } UnrealAudioOut::~UnrealAudioOut() { if (audioComponentWeCreated) { audioComponent->RemoveFromRoot(); } UE_LOGFMT(LogDirectVideo, Verbose, "Audio out destroyed"); } void UnrealAudioOut::sendBuffer(uint8_t *buf, int bufSize, int64_t presentationTimeNS, bool reset) { if (buf == NULL || bufSize == 0) { return; } auto pinnedSender = audioSender.Get(); if (!pinnedSender) { UE_LOGFMT(LogDirectVideo, Error, "No audio sender"); return; } if (reset) { // reset audio time clock // and clear audio buffers pinnedSender->ResetAudio(); } int64_t lastSendTime = pinnedSender->lastBufSendTime; if (lastSendTime != 0) { int64_t samplesInSender = pinnedSender->GetAvailableAudioByteCount() / (numChannels * 2); if (samplesInSender <= 0) { pinnedSender->QueueSilence(4096); samplesInSender = 4096 / (numChannels * 2L); } int64_t timeForBuffer = (1000000000LL * samplesInSender) / static_cast(sampleRate); int64_t thisOutputTimeNS = lastSendTime + timeForBuffer; if (afterSeek) { // we've seeked and we've seen a video frame already // so add silence if required so that we can sync on the // already displayed video frame(s) afterSeek = false; int64_t newOffset = presentationTimeNS - thisOutputTimeNS; if (newOffset > (presentationTimeOffset + 10000000L)) { UE_LOG(LogDirectVideo, VeryVerbose, TEXT("After seek silence")); // queue silence to make things line up int64_t difference = newOffset - presentationTimeOffset; int64_t silenceSamples = (difference * static_cast(sampleRate)) / 1000000000LL; pinnedSender->QueueSilence(silenceSamples * 2L * numChannels); } } else { presentationTimeOffset = presentationTimeNS - thisOutputTimeNS; } hasTimeOffset = true; UE_LOG(LogDirectVideo, VeryVerbose, TEXT("Adjusting offset (%d) - now %ld timeForBuffer %ld lastSendTime %ld " "thisSendTime %ld presentationTimeNS %ld offset %ld"), bufSize, _getNSTime(), timeForBuffer, lastSendTime, thisOutputTimeNS, presentationTimeNS, presentationTimeOffset); } pinnedSender->QueueAudio(buf, bufSize); } int64_t UnrealAudioOut::getPresentationTimeNS() { if (!hasTimeOffset) { if (!audioSender) { // silent - initialize time to zero on first query // or else we miss a frame on startup presentationTimeOffset = -_scaleTime(_getNSTime(), playbackRate); hasTimeOffset = true; } else { // with audio - initialize return -1; } } int64_t retVal = _scaleTime(_getNSTime(), playbackRate) + presentationTimeOffset; return retVal; } IAudioOut::NsTime UnrealAudioOut::getWaitTimeForPresentationTime(int64_t presentationTimeNS, int64_t maxDurationNS) { int64_t thisTime = getPresentationTimeNS(); int64_t duration = presentationTimeNS - thisTime; // give a little slack for processing time duration = (duration * 100L) / 98L; if (maxDurationNS > 0 && duration > maxDurationNS) { duration = maxDurationNS; } NsTime curTime = std::chrono::steady_clock::now(); if (duration < 0) { return curTime; } NsTime waitUntilTime = curTime + NsDuration(_scaleTime(duration, playbackRate)); return waitUntilTime; } bool UnrealAudioOut::setVolume(float volume) { if (!audioComponent.IsValid()) { return false; } if (volume > 1.0) volume = 1.0; if (volume < 0.0) volume = 0.0; audioComponent->SetVolumeMultiplier(volume); return true; } void UnrealAudioOut::setPlaying(bool playing) { if (playing == isPlaying) { return; } isPlaying = playing; if (playing) { // starting after a pause // reset presentation time offset if (pausedPresentationTime != -1 && hasTimeOffset) { presentationTimeOffset = pausedPresentationTime - _scaleTime(_getNSTime(), playbackRate); } } else { // pausing, save current prestime for restore pausedPresentationTime = getPresentationTimeNS(); } if (audioSender) { audioSender->paused = !playing; } } void UnrealAudioOut::onSeek(int64_t newPresentationTimeNs, bool resetAudio) { if (!audioSender) { presentationTimeOffset = newPresentationTimeNs - _scaleTime(_getNSTime(), playbackRate); UE_LOG(LogDirectVideo, VeryVerbose, TEXT("Seeking: pt %ld update offset %ld (rate=%f)"), newPresentationTimeNs, presentationTimeOffset, playbackRate); } else { // clear audio buffers if (resetAudio) { // after a seek, we may receive either audio or video frames // first. Here we clear the audio buffers, and reset the time // offset so that first of either audio or video frame received // will reset the clock audioSender->ResetAudio(); hasTimeOffset = false; } else { // we are looping - don't clear the audio buffer // and make a rough guess at the clock so that video // frames will still send out correctly and then resync clock when first audio frames // received int64_t lastSendTime = audioSender->lastBufSendTime; if (lastSendTime == 0) { // not actually sent anything yet - assume it will send pretty much now lastSendTime = _getNSTime(); } int64_t samplesInSender = audioSender->GetAvailableAudioByteCount() / (numChannels * 2); int64_t timeForBuffer = (1000000000LL * samplesInSender) / static_cast(sampleRate); // when the next buffer (i.e. the one at newPresentationTimeNs) // will get sent at (in clock time ) int64_t thisOutputTimeNS = lastSendTime + timeForBuffer; presentationTimeOffset = newPresentationTimeNs - _scaleTime(thisOutputTimeNS, playbackRate); } } } bool UnrealAudioOut::setRate(float newRate) { if (audioSender) { if (newRate != 1.0 && newRate != 0.0) { UE_LOGFMT(LogDirectVideo, Error, "Trying to set playback rate on video with audio, not supported yet"); return false; } else { return true; } } else { if (newRate > 0) { if (hasTimeOffset) { int64_t presTime = getPresentationTimeNS(); playbackRate = newRate; onSeek(presTime, false); } else { playbackRate = newRate; } } } return false; } int64_t UnrealAudioOut::getQueuedTimeNS() { if (audioSender) { int64_t samplesInSender = audioSender->GetAvailableAudioByteCount() / (numChannels * 2); int64_t timeTotal = (1000000000L * samplesInSender) / sampleRate; return timeTotal; } else { return -1; } } void UnrealAudioOut::onHasVideoTime(int64_t newPresentationTimeNS) { if (audioSender && !hasTimeOffset) { // if we haven't sent any audio yet, then set current time to this frame time, // and let first updating of offset add silence presentationTimeOffset = newPresentationTimeNS - _scaleTime(_getNSTime(), playbackRate); hasTimeOffset = true; afterSeek = true; } }