Sch/Cut5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

显示音频时域

Browse Source
This commit is contained in:
Sch 2023-08-19 21:14:32 +08:00
parent 50c85f073c
commit 8c0b908ce9
7 changed files with 241 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

187
Source/Cut5/Utils/FFMPEGUtils.cpp
View File

@ -1,7 +1,12 @@
#include "FFMPEGUtils.h"
#include "CanvasTypes.h"
#include "ImageUtils.h"
#include "Utils.h"
#include "Engine/Canvas.h"
#include "Engine/TextureRenderTarget2D.h"
#include "Kismet/KismetRenderingLibrary.h"
#include "Serialization/BufferArchive.h"
FString FFFMPEGUtils::LoadMedia(const FString& Path, FTimelinePropertyData* PropertyData)
@ -230,3 +235,185 @@ bool FFFMPEGUtils::ExportImage(UTexture2D* Texture2D, const FString& Path)
FImageUtils::CompressImageArray(width, height, nColors, ImgData);
return FFileHelper::SaveArrayToFile(ImgData, *Path);
}
TArray<FSlateBrush> FFFMPEGUtils::GetMovieBrush(FClipData* ClipData)
{
for (int32 i = 0; i < ClipData->MovieBrushes.Num(); i++)
{
ClipData->MovieBrushes[i].SetResourceObject(nullptr);
}
ClipData->MovieBrushes.Empty();
TArray<FSlateBrush> Result;
if (ClipData->ResourcePropertyDataPtr)
{
const float ClipLength = (ClipData->VideoEndFrame - ClipData->VideoStartFrame) * FGlobalData::DefaultTimeTickSpace;
const float FrameNum = ClipLength / 128.0;
const float TotalFrame = ClipData->ResourcePropertyDataPtr->VideoCodecContext->framerate.num * ( ClipData->ResourcePropertyDataPtr->Context->duration / AV_TIME_BASE);
for (int32 i = 0; i < FrameNum; i++)
{
// 得到视频总帧数
int64 Timestamp = av_rescale_q(i * (TotalFrame / 128) / 30.0 * AV_TIME_BASE, AVRational{1, AV_TIME_BASE}, ClipData->ResourcePropertyDataPtr->Context->streams[ClipData->ResourcePropertyDataPtr->VideoStream]->time_base);
av_seek_frame(ClipData->ResourcePropertyDataPtr->Context, ClipData->ResourcePropertyDataPtr->VideoStream, Timestamp, AVSEEK_FLAG_BACKWARD);
AVPacket Packet;
av_init_packet(&Packet);
AVFormatContext* FormatContext = ClipData->ResourcePropertyDataPtr->Context;
AVCodecContext* VideoCodecContext = ClipData->ResourcePropertyDataPtr->VideoCodecContext;
AVCodec* VideoCodec = ClipData->ResourcePropertyDataPtr->VideoCodec;
AVFrame* Frame = av_frame_alloc();
while (av_read_frame(FormatContext, &Packet) >= 0)
{
if (avcodec_send_packet(VideoCodecContext, &Packet) < 0)
{
}
if (avcodec_receive_frame(VideoCodecContext, Frame) >= 0)
{
break;
}
};
if (Frame)
{
struct SwsContext* swsCtx = sws_getContext(
Frame->width, Frame->height, VideoCodecContext->pix_fmt,
Frame->width, Frame->height, AV_PIX_FMT_RGBA,
SWS_BILINEAR, NULL, NULL, NULL
);
if (!swsCtx)
{
UE_LOG(LogTemp, Error, TEXT("Error creating swsContext"));
}
uint8* RawData = new uint8[Frame->width * Frame->height * 4];
uint8* dest[4] = {RawData, 0, 0, 0};
int32 dest_linesize[4] = {Frame->width * 4, 0, 0, 0};
sws_scale(swsCtx, Frame->data, Frame->linesize, 0, Frame->height, dest, dest_linesize);
sws_freeContext(swsCtx);
UTexture2D* Texture = UTexture2D::CreateTransient(Frame->width, Frame->height, PF_B8G8R8A8);
if (Texture)
{
void* MipData = Texture->GetPlatformData()->Mips[0].BulkData.Lock(LOCK_READ_WRITE);
FMemory::Memcpy(MipData, RawData, Frame->width * Frame->height * 4);
Texture->GetPlatformData()->Mips[0].BulkData.Unlock();
Texture->UpdateResource();
// FGuid Guid = FGuid::NewGuid();
// ExportImage(Texture, *FPaths::Combine(FUtils::GetTempPath() / Guid.ToString() + ".png"));
FSlateBrush NewBrush;
NewBrush.SetResourceObject(Texture);
Result.Add(NewBrush);
delete RawData;
}
}
}
}
return Result;
}
TArray<FSlateBrush> FFFMPEGUtils::GetAudioBrush(FClipData* ClipData)
{
const float TimeLength = (ClipData->ClipEndFrame - ClipData->ClipStartFrame) * FGlobalData::DefaultTimeTickSpace;
float MaxValue = 0;
float MinValue = 0;
for (int32 i = 0; i < ClipData->ResourcePropertyDataPtr->AudioData.Num() / 4; i++)
{
float NewFloat = *reinterpret_cast<float*>(ClipData->ResourcePropertyDataPtr->AudioData.GetData() + (i * 4));
if (NewFloat >= MaxValue)
{
MaxValue = NewFloat;
}
if (NewFloat <= MinValue)
{
MinValue = NewFloat;
}
}
auto Normalize = [MaxValue, MinValue](float CurrentValue)
{
return FMath::GetMappedRangeValueClamped(FVector2D(MinValue, MaxValue), FVector2D(FGlobalData::DefaultTrackHeight, 0), CurrentValue);
};
ClipData->AudioBrushLength.Empty();
ClipData->AudioBrushes.Empty();
int32 Index = 0;
int32 TotalLength = TimeLength;
while (TotalLength > 0)
{
int32 CurrentLength = 0;
if (TotalLength > 4096)
{
TotalLength -= 4096;
CurrentLength = 4096;
}
else
{
CurrentLength = TotalLength;
TotalLength = 0;
}
UTextureRenderTarget2D* TextureRenderTarget2D = NewObject<UTextureRenderTarget2D>();
TextureRenderTarget2D->InitCustomFormat(int32(CurrentLength), int32(FGlobalData::DefaultTrackHeight), PF_B8G8R8A8, false);
TextureRenderTarget2D->UpdateResourceImmediate();
UKismetRenderingLibrary::ClearRenderTarget2D(GWorld->GetWorld(), TextureRenderTarget2D, FLinearColor(0, 0, 0, 0));
UCanvas* Canvas;
FVector2D Size;
FDrawToRenderTargetContext RenderTargetContext;
UKismetRenderingLibrary::BeginDrawCanvasToRenderTarget(GWorld->GetWorld(), TextureRenderTarget2D, Canvas, Size, RenderTargetContext);
float LastPoint = 0.0;
int32 StartIndex = (Index * 4096 + CurrentLength) * 4;
for (int32 i = 0; i < CurrentLength; i++)
{
float CurrentData = *reinterpret_cast<float*>(ClipData->ResourcePropertyDataPtr->AudioData.GetData() + StartIndex);
float NormalizedData = Normalize(CurrentData);
Canvas->K2_DrawLine(FVector2D(i - 1, LastPoint), FVector2D(i, NormalizedData), 1, FLinearColor::White);
LastPoint = NormalizedData;
StartIndex += 4;
}
UKismetRenderingLibrary::EndDrawCanvasToRenderTarget(GWorld->GetWorld(), RenderTargetContext);
FBufferArchive Buffer;
FImageUtils::ExportRenderTarget2DAsPNG(TextureRenderTarget2D, Buffer);
FFileHelper::SaveArrayToFile(Buffer, *FPaths::Combine(FUtils::GetTempPath(), ClipData->ClipGuid.ToString(), FString::FromInt(Index) + ".png"));
FSlateDynamicImageBrush SlateBrush = FSlateDynamicImageBrush(*FPaths::Combine(FUtils::GetTempPath(), ClipData->ClipGuid.ToString(), FString::FromInt(Index) + ".png"), FVector2D(CurrentLength, FGlobalData::DefaultTrackHeight));
ClipData->AudioBrushLength.Add(CurrentLength);
ClipData->AudioBrushes.Add(SlateBrush);
Index++;
}
//
// FBufferArchive Buffer;
// FImageUtils::ExportRenderTarget2DAsPNG(TextureRenderTarget2D, Buffer);
// FFileHelper::SaveArrayToFile(Buffer, *FPaths::Combine(FUtils::GetTempPath(), "Audio.png"));
return {};
}

2
Source/Cut5/Utils/FFMPEGUtils.h
View File

@ -17,4 +17,6 @@ struct FFFMPEGUtils
static FString ConvertMediaGoPto1(const FString& Path);
static bool ExportImage(UTexture2D* Texture2D, const FString& Path);
static TArray<FSlateBrush> GetMovieBrush(struct FClipData* ClipData);
static TArray<FSlateBrush> GetAudioBrush(struct FClipData* ClipData);
};

13
Source/Cut5/Utils/Utils.cpp
View File

@ -80,6 +80,11 @@ FString FUtils::GetResourcesPath(FString ResourcesName, bool bFullPath)
return FPaths::Combine(FPaths::ProjectDir() + "/Resources/", ResourcesName);
}
FString FUtils::GetTempPath()
{
return FPaths::ConvertRelativePathToFull(FPaths::Combine(FGlobalData::BasePath / FGlobalData::CurrentProjectName / "/Temp/"));
}
FSlateDynamicImageBrush* FUtils::GetBrushFromImage(const FString& ImageName, const FVector2D Size)
{
FSlateDynamicImageBrush* Brush = new FSlateDynamicImageBrush(*ImageName, Size);
@ -106,6 +111,14 @@ bool FUtils::DetectDragTypeCanDrop(const FClipData& DraggingType, const ETrackTy
}
}
if (DropTrackType == ETrackType::AudioTrack || DropTrackType == ETrackType::AudioTrackR)
{
if (DraggingType.ClipType == ETrackType::AudioTrack)
{
return true;
}
}
return false;

1
Source/Cut5/Utils/Utils.h
View File

@ -12,6 +12,7 @@ public:
static uint8* ConvertTwoChannelSound2PortAudioSound(uint8* Channel1, uint8* Channel2, int32 Size);
static FString GetResourcesPath(FString ResourcesName, bool bFullPath = false);
static FString GetTempPath();
static FSlateDynamicImageBrush* GetBrushFromImage(const FString& ImageName, const FVector2D Size);
static TArray<FSlateDynamicImageBrush*> BrushPtr;

5
Source/Cut5/Widgets/DefineGlobal.h
View File

@ -320,6 +320,11 @@ struct CUT5_API FClipData
int32 GetClipRelativeEndFrame() const { return ClipEndFrame - ClipStartFrame; }
FPresetsCustomData PresetsCustomData;
TArray<FSlateBrush> MovieBrushes;
TArray<FSlateBrush> AudioBrushes;
TArray<float> AudioBrushLength;
enum class ECropMethod
{

6
Source/Cut5/Widgets/DragDropOperator/DragDropOperator.cpp
View File

@ -1,5 +1,6 @@
#include "DragDropOperator.h"
#include "Cut5/Utils/FFMPEGUtils.h"
#include "Cut5/Utils/OpencvUtils.h"
#include "Cut5/Utils/Utils.h"
#include "Cut5/Widgets/DefineGlobal.h"
@ -661,6 +662,10 @@ void DragDropOperator::OnDrop(const FGeometry& MyGeometry, const FDragDropEvent&
NewClipData.ClipEndFrame = NewClipData.ClipStartFrame + ClipDragOperation.TimelinePropertyData->MovieFrameLength;
NewClipData.VideoEndFrame = ClipDragOperation.TimelinePropertyData->MovieFrameLength;
NewClipData.VideoCapture = ClipDragOperation.VideoCapture;
NewClipData.MovieBrushes = FFFMPEGUtils::GetMovieBrush(&NewClipData);
if (TrackHead->TrackData.TrackType == ETrackType::LightArrayTrack)
{
NewClipData.ClipType = ETrackType::LightArrayTrack;
@ -699,6 +704,7 @@ void DragDropOperator::OnDrop(const FGeometry& MyGeometry, const FDragDropEvent&
NewClipData.ClipEndFrame = NewClipData.ClipStartFrame + ClipDragOperation.TimelinePropertyData->MovieFrameLength;
NewClipData.VideoEndFrame = ClipDragOperation.TimelinePropertyData->MovieFrameLength;
NewClipData.ResourcePropertyDataPtr = ClipDragOperation.TimelinePropertyData;
FFFMPEGUtils::GetAudioBrush(&NewClipData);
}
}

26
Source/Cut5/Widgets/STimelineClip.cpp
View File

@ -689,6 +689,32 @@ int32 STimelineClip::OnPaint(const FPaintArgs& Args, const FGeometry& AllottedGe
FSlateWindowElementList& OutDrawElements, int32 LayerId, const FWidgetStyle& InWidgetStyle,
bool bParentEnabled) const
{
const float XLength = AllottedGeometry.GetLocalSize().X;
{
int32 i = 0;
float RenderPos = 0;
for (FSlateBrush& SlateBrush : ClipData->AudioBrushes)
{
FSlateDrawElement::MakeBox(OutDrawElements, LayerId + 2, AllottedGeometry.ToPaintGeometry(FVector2f(ClipData->AudioBrushLength[i], AllottedGeometry.GetLocalSize().Y), FSlateLayoutTransform(FVector2f(RenderPos, 0))), &SlateBrush);
RenderPos += ClipData->AudioBrushLength[i];
i++;
}
}
{
int32 i = 0;
for (FSlateBrush& SlateBrush : ClipData->MovieBrushes)
{
FSlateDrawElement::MakeBox(OutDrawElements, LayerId + 3, AllottedGeometry.ToPaintGeometry(FVector2f(XLength / ClipData->MovieBrushes.Num(), AllottedGeometry.GetLocalSize().Y), FSlateLayoutTransform(FVector2f(i * (XLength / ClipData->MovieBrushes.Num()), 0))), &SlateBrush);
i++;
}
}
if (ClipData->PresetType == EPresetType::EnableProjector)
{
FSlateDrawElement::MakeText(OutDrawElements, LayerId + 2, AllottedGeometry.ToPaintGeometry(), FText::FromString(TEXT("开启投影仪")), FAppStyle::Get().GetWidgetStyle<FTextBlockStyle>("NormalText").Font, ESlateDrawEffect::None, FLinearColor::White);