From 8c0b908ce935092386fa14772f4eee40c1a764f4 Mon Sep 17 00:00:00 2001 From: Sch <3516520171@qq.com> Date: Sat, 19 Aug 2023 21:14:32 +0800 Subject: [PATCH] =?UTF-8?q?=E6=98=BE=E7=A4=BA=E9=9F=B3=E9=A2=91=E6=97=B6?= =?UTF-8?q?=E5=9F=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Source/Cut5/Utils/FFMPEGUtils.cpp | 187 ++++++++++++++++++ Source/Cut5/Utils/FFMPEGUtils.h | 4 +- Source/Cut5/Utils/Utils.cpp | 13 ++ Source/Cut5/Utils/Utils.h | 1 + Source/Cut5/Widgets/DefineGlobal.h | 5 + .../DragDropOperator/DragDropOperator.cpp | 6 + Source/Cut5/Widgets/STimelineClip.cpp | 26 +++ 7 files changed, 241 insertions(+), 1 deletion(-) diff --git a/Source/Cut5/Utils/FFMPEGUtils.cpp b/Source/Cut5/Utils/FFMPEGUtils.cpp index bc22246..c70f4ff 100644 --- a/Source/Cut5/Utils/FFMPEGUtils.cpp +++ b/Source/Cut5/Utils/FFMPEGUtils.cpp @@ -1,7 +1,12 @@ #include "FFMPEGUtils.h" +#include "CanvasTypes.h" #include "ImageUtils.h" #include "Utils.h" +#include "Engine/Canvas.h" +#include "Engine/TextureRenderTarget2D.h" +#include "Kismet/KismetRenderingLibrary.h" +#include "Serialization/BufferArchive.h" FString FFFMPEGUtils::LoadMedia(const FString& Path, FTimelinePropertyData* PropertyData) @@ -230,3 +235,185 @@ bool FFFMPEGUtils::ExportImage(UTexture2D* Texture2D, const FString& Path) FImageUtils::CompressImageArray(width, height, nColors, ImgData); return FFileHelper::SaveArrayToFile(ImgData, *Path); } + +TArray FFFMPEGUtils::GetMovieBrush(FClipData* ClipData) +{ + for (int32 i = 0; i < ClipData->MovieBrushes.Num(); i++) + { + ClipData->MovieBrushes[i].SetResourceObject(nullptr); + } + ClipData->MovieBrushes.Empty(); + + + + + TArray Result; + if (ClipData->ResourcePropertyDataPtr) + { + const float ClipLength = (ClipData->VideoEndFrame - ClipData->VideoStartFrame) * FGlobalData::DefaultTimeTickSpace; + const float FrameNum = ClipLength / 128.0; + const float TotalFrame = ClipData->ResourcePropertyDataPtr->VideoCodecContext->framerate.num * ( ClipData->ResourcePropertyDataPtr->Context->duration / AV_TIME_BASE); + for (int32 i = 0; i < FrameNum; i++) + { + // 得到视频总帧数 + + int64 Timestamp = av_rescale_q(i * (TotalFrame / 128) / 30.0 * AV_TIME_BASE, AVRational{1, AV_TIME_BASE}, ClipData->ResourcePropertyDataPtr->Context->streams[ClipData->ResourcePropertyDataPtr->VideoStream]->time_base); + av_seek_frame(ClipData->ResourcePropertyDataPtr->Context, ClipData->ResourcePropertyDataPtr->VideoStream, Timestamp, AVSEEK_FLAG_BACKWARD); + AVPacket Packet; + av_init_packet(&Packet); + AVFormatContext* FormatContext = ClipData->ResourcePropertyDataPtr->Context; + AVCodecContext* VideoCodecContext = ClipData->ResourcePropertyDataPtr->VideoCodecContext; + AVCodec* VideoCodec = ClipData->ResourcePropertyDataPtr->VideoCodec; + AVFrame* Frame = av_frame_alloc(); + + while (av_read_frame(FormatContext, &Packet) >= 0) + { + if (avcodec_send_packet(VideoCodecContext, &Packet) < 0) + { + + } + if (avcodec_receive_frame(VideoCodecContext, Frame) >= 0) + { + break; + } + }; + + + + + if (Frame) + { + struct SwsContext* swsCtx = sws_getContext( + Frame->width, Frame->height, VideoCodecContext->pix_fmt, + Frame->width, Frame->height, AV_PIX_FMT_RGBA, + SWS_BILINEAR, NULL, NULL, NULL + ); + if (!swsCtx) + { + UE_LOG(LogTemp, Error, TEXT("Error creating swsContext")); + } + uint8* RawData = new uint8[Frame->width * Frame->height * 4]; + uint8* dest[4] = {RawData, 0, 0, 0}; + int32 dest_linesize[4] = {Frame->width * 4, 0, 0, 0}; + sws_scale(swsCtx, Frame->data, Frame->linesize, 0, Frame->height, dest, dest_linesize); + sws_freeContext(swsCtx); + + + UTexture2D* Texture = UTexture2D::CreateTransient(Frame->width, Frame->height, PF_B8G8R8A8); + if (Texture) + { + void* MipData = Texture->GetPlatformData()->Mips[0].BulkData.Lock(LOCK_READ_WRITE); + FMemory::Memcpy(MipData, RawData, Frame->width * Frame->height * 4); + Texture->GetPlatformData()->Mips[0].BulkData.Unlock(); + Texture->UpdateResource(); + + // FGuid Guid = FGuid::NewGuid(); + // ExportImage(Texture, *FPaths::Combine(FUtils::GetTempPath() / Guid.ToString() + ".png")); + + FSlateBrush NewBrush; + NewBrush.SetResourceObject(Texture); + Result.Add(NewBrush); + delete RawData; + } + } + } + } + return Result; + +} + +TArray FFFMPEGUtils::GetAudioBrush(FClipData* ClipData) +{ + const float TimeLength = (ClipData->ClipEndFrame - ClipData->ClipStartFrame) * FGlobalData::DefaultTimeTickSpace; + + float MaxValue = 0; + float MinValue = 0; + for (int32 i = 0; i < ClipData->ResourcePropertyDataPtr->AudioData.Num() / 4; i++) + { + float NewFloat = *reinterpret_cast(ClipData->ResourcePropertyDataPtr->AudioData.GetData() + (i * 4)); + + if (NewFloat >= MaxValue) + { + MaxValue = NewFloat; + } + if (NewFloat <= MinValue) + { + MinValue = NewFloat; + } + } + + + auto Normalize = [MaxValue, MinValue](float CurrentValue) + { + return FMath::GetMappedRangeValueClamped(FVector2D(MinValue, MaxValue), FVector2D(FGlobalData::DefaultTrackHeight, 0), CurrentValue); + }; + + ClipData->AudioBrushLength.Empty(); + ClipData->AudioBrushes.Empty(); + int32 Index = 0; + int32 TotalLength = TimeLength; + while (TotalLength > 0) + { + int32 CurrentLength = 0; + if (TotalLength > 4096) + { + TotalLength -= 4096; + CurrentLength = 4096; + } + else + { + CurrentLength = TotalLength; + TotalLength = 0; + } + UTextureRenderTarget2D* TextureRenderTarget2D = NewObject(); + TextureRenderTarget2D->InitCustomFormat(int32(CurrentLength), int32(FGlobalData::DefaultTrackHeight), PF_B8G8R8A8, false); + TextureRenderTarget2D->UpdateResourceImmediate(); + UKismetRenderingLibrary::ClearRenderTarget2D(GWorld->GetWorld(), TextureRenderTarget2D, FLinearColor(0, 0, 0, 0)); + UCanvas* Canvas; + FVector2D Size; + FDrawToRenderTargetContext RenderTargetContext; + UKismetRenderingLibrary::BeginDrawCanvasToRenderTarget(GWorld->GetWorld(), TextureRenderTarget2D, Canvas, Size, RenderTargetContext); + + + float LastPoint = 0.0; + int32 StartIndex = (Index * 4096 + CurrentLength) * 4; + + for (int32 i = 0; i < CurrentLength; i++) + { + float CurrentData = *reinterpret_cast(ClipData->ResourcePropertyDataPtr->AudioData.GetData() + StartIndex); + float NormalizedData = Normalize(CurrentData); + Canvas->K2_DrawLine(FVector2D(i - 1, LastPoint), FVector2D(i, NormalizedData), 1, FLinearColor::White); + LastPoint = NormalizedData; + + + StartIndex += 4; + } + + + + + + UKismetRenderingLibrary::EndDrawCanvasToRenderTarget(GWorld->GetWorld(), RenderTargetContext); + + + FBufferArchive Buffer; + FImageUtils::ExportRenderTarget2DAsPNG(TextureRenderTarget2D, Buffer); + FFileHelper::SaveArrayToFile(Buffer, *FPaths::Combine(FUtils::GetTempPath(), ClipData->ClipGuid.ToString(), FString::FromInt(Index) + ".png")); + + FSlateDynamicImageBrush SlateBrush = FSlateDynamicImageBrush(*FPaths::Combine(FUtils::GetTempPath(), ClipData->ClipGuid.ToString(), FString::FromInt(Index) + ".png"), FVector2D(CurrentLength, FGlobalData::DefaultTrackHeight)); + ClipData->AudioBrushLength.Add(CurrentLength); + ClipData->AudioBrushes.Add(SlateBrush); + Index++; + } + + + + + + // + // FBufferArchive Buffer; + // FImageUtils::ExportRenderTarget2DAsPNG(TextureRenderTarget2D, Buffer); + // FFileHelper::SaveArrayToFile(Buffer, *FPaths::Combine(FUtils::GetTempPath(), "Audio.png")); + + return {}; +} diff --git a/Source/Cut5/Utils/FFMPEGUtils.h b/Source/Cut5/Utils/FFMPEGUtils.h index 96398a7..0dc1c0d 100644 --- a/Source/Cut5/Utils/FFMPEGUtils.h +++ b/Source/Cut5/Utils/FFMPEGUtils.h @@ -16,5 +16,7 @@ struct FFFMPEGUtils static FString LoadMedia(const FString& Path, FTimelinePropertyData* PropertyData); static FString ConvertMediaGoPto1(const FString& Path); static bool ExportImage(UTexture2D* Texture2D, const FString& Path); - + + static TArray GetMovieBrush(struct FClipData* ClipData); + static TArray GetAudioBrush(struct FClipData* ClipData); }; diff --git a/Source/Cut5/Utils/Utils.cpp b/Source/Cut5/Utils/Utils.cpp index 0fcdd6c..f07176b 100644 --- a/Source/Cut5/Utils/Utils.cpp +++ b/Source/Cut5/Utils/Utils.cpp @@ -80,6 +80,11 @@ FString FUtils::GetResourcesPath(FString ResourcesName, bool bFullPath) return FPaths::Combine(FPaths::ProjectDir() + "/Resources/", ResourcesName); } +FString FUtils::GetTempPath() +{ + return FPaths::ConvertRelativePathToFull(FPaths::Combine(FGlobalData::BasePath / FGlobalData::CurrentProjectName / "/Temp/")); +} + FSlateDynamicImageBrush* FUtils::GetBrushFromImage(const FString& ImageName, const FVector2D Size) { FSlateDynamicImageBrush* Brush = new FSlateDynamicImageBrush(*ImageName, Size); @@ -105,6 +110,14 @@ bool FUtils::DetectDragTypeCanDrop(const FClipData& DraggingType, const ETrackTy return true; } } + + if (DropTrackType == ETrackType::AudioTrack || DropTrackType == ETrackType::AudioTrackR) + { + if (DraggingType.ClipType == ETrackType::AudioTrack) + { + return true; + } + } return false; diff --git a/Source/Cut5/Utils/Utils.h b/Source/Cut5/Utils/Utils.h index 5c6f6da..aa5bbda 100644 --- a/Source/Cut5/Utils/Utils.h +++ b/Source/Cut5/Utils/Utils.h @@ -12,6 +12,7 @@ public: static uint8* ConvertTwoChannelSound2PortAudioSound(uint8* Channel1, uint8* Channel2, int32 Size); static FString GetResourcesPath(FString ResourcesName, bool bFullPath = false); + static FString GetTempPath(); static FSlateDynamicImageBrush* GetBrushFromImage(const FString& ImageName, const FVector2D Size); static TArray BrushPtr; diff --git a/Source/Cut5/Widgets/DefineGlobal.h b/Source/Cut5/Widgets/DefineGlobal.h index 96fa066..7de3376 100644 --- a/Source/Cut5/Widgets/DefineGlobal.h +++ b/Source/Cut5/Widgets/DefineGlobal.h @@ -320,6 +320,11 @@ struct CUT5_API FClipData int32 GetClipRelativeEndFrame() const { return ClipEndFrame - ClipStartFrame; } FPresetsCustomData PresetsCustomData; + TArray MovieBrushes; + + TArray AudioBrushes; + TArray AudioBrushLength; + enum class ECropMethod { diff --git a/Source/Cut5/Widgets/DragDropOperator/DragDropOperator.cpp b/Source/Cut5/Widgets/DragDropOperator/DragDropOperator.cpp index f34489f..7a0f329 100644 --- a/Source/Cut5/Widgets/DragDropOperator/DragDropOperator.cpp +++ b/Source/Cut5/Widgets/DragDropOperator/DragDropOperator.cpp @@ -1,5 +1,6 @@ #include "DragDropOperator.h" +#include "Cut5/Utils/FFMPEGUtils.h" #include "Cut5/Utils/OpencvUtils.h" #include "Cut5/Utils/Utils.h" #include "Cut5/Widgets/DefineGlobal.h" @@ -661,6 +662,10 @@ void DragDropOperator::OnDrop(const FGeometry& MyGeometry, const FDragDropEvent& NewClipData.ClipEndFrame = NewClipData.ClipStartFrame + ClipDragOperation.TimelinePropertyData->MovieFrameLength; NewClipData.VideoEndFrame = ClipDragOperation.TimelinePropertyData->MovieFrameLength; NewClipData.VideoCapture = ClipDragOperation.VideoCapture; + + + NewClipData.MovieBrushes = FFFMPEGUtils::GetMovieBrush(&NewClipData); + if (TrackHead->TrackData.TrackType == ETrackType::LightArrayTrack) { NewClipData.ClipType = ETrackType::LightArrayTrack; @@ -699,6 +704,7 @@ void DragDropOperator::OnDrop(const FGeometry& MyGeometry, const FDragDropEvent& NewClipData.ClipEndFrame = NewClipData.ClipStartFrame + ClipDragOperation.TimelinePropertyData->MovieFrameLength; NewClipData.VideoEndFrame = ClipDragOperation.TimelinePropertyData->MovieFrameLength; NewClipData.ResourcePropertyDataPtr = ClipDragOperation.TimelinePropertyData; + FFFMPEGUtils::GetAudioBrush(&NewClipData); } } diff --git a/Source/Cut5/Widgets/STimelineClip.cpp b/Source/Cut5/Widgets/STimelineClip.cpp index ac85881..457800c 100644 --- a/Source/Cut5/Widgets/STimelineClip.cpp +++ b/Source/Cut5/Widgets/STimelineClip.cpp @@ -689,6 +689,32 @@ int32 STimelineClip::OnPaint(const FPaintArgs& Args, const FGeometry& AllottedGe FSlateWindowElementList& OutDrawElements, int32 LayerId, const FWidgetStyle& InWidgetStyle, bool bParentEnabled) const { + + const float XLength = AllottedGeometry.GetLocalSize().X; + { + int32 i = 0; + float RenderPos = 0; + for (FSlateBrush& SlateBrush : ClipData->AudioBrushes) + { + FSlateDrawElement::MakeBox(OutDrawElements, LayerId + 2, AllottedGeometry.ToPaintGeometry(FVector2f(ClipData->AudioBrushLength[i], AllottedGeometry.GetLocalSize().Y), FSlateLayoutTransform(FVector2f(RenderPos, 0))), &SlateBrush); + + RenderPos += ClipData->AudioBrushLength[i]; + i++; + } + } + + + { + int32 i = 0; + for (FSlateBrush& SlateBrush : ClipData->MovieBrushes) + { + FSlateDrawElement::MakeBox(OutDrawElements, LayerId + 3, AllottedGeometry.ToPaintGeometry(FVector2f(XLength / ClipData->MovieBrushes.Num(), AllottedGeometry.GetLocalSize().Y), FSlateLayoutTransform(FVector2f(i * (XLength / ClipData->MovieBrushes.Num()), 0))), &SlateBrush); + i++; + } + } + + + if (ClipData->PresetType == EPresetType::EnableProjector) { FSlateDrawElement::MakeText(OutDrawElements, LayerId + 2, AllottedGeometry.ToPaintGeometry(), FText::FromString(TEXT("开启投影仪")), FAppStyle::Get().GetWidgetStyle("NormalText").Font, ESlateDrawEffect::None, FLinearColor::White);