显示音频时域

Source/Cut5/Utils/FFMPEGUtils.cpp

@@ -1,7 +1,12 @@
 #include "FFMPEGUtils.h"
 
+#include "CanvasTypes.h"
 #include "ImageUtils.h"
 #include "Utils.h"
+#include "Engine/Canvas.h"
+#include "Engine/TextureRenderTarget2D.h"
+#include "Kismet/KismetRenderingLibrary.h"
+#include "Serialization/BufferArchive.h"
 
 
 FString FFFMPEGUtils::LoadMedia(const FString& Path, FTimelinePropertyData* PropertyData)
@@ -230,3 +235,185 @@ bool FFFMPEGUtils::ExportImage(UTexture2D* Texture2D, const FString& Path)
 	FImageUtils::CompressImageArray(width, height, nColors, ImgData);
 	return FFileHelper::SaveArrayToFile(ImgData, *Path);
 }
+
+TArray<FSlateBrush> FFFMPEGUtils::GetMovieBrush(FClipData* ClipData)
+{
+	for (int32 i = 0; i < ClipData->MovieBrushes.Num(); i++)
+	{
+		ClipData->MovieBrushes[i].SetResourceObject(nullptr);
+	}
+	ClipData->MovieBrushes.Empty();
+
+
+
+	
+	TArray<FSlateBrush> Result;
+	if (ClipData->ResourcePropertyDataPtr)
+	{
+		const float ClipLength = (ClipData->VideoEndFrame - ClipData->VideoStartFrame) * FGlobalData::DefaultTimeTickSpace;
+		const float FrameNum = ClipLength / 128.0;
+		const float TotalFrame = ClipData->ResourcePropertyDataPtr->VideoCodecContext->framerate.num * ( ClipData->ResourcePropertyDataPtr->Context->duration / AV_TIME_BASE);
+		for (int32 i = 0; i < FrameNum; i++)
+		{
+			// 得到视频总帧数
+			
+			int64 Timestamp = av_rescale_q(i * (TotalFrame / 128) / 30.0 * AV_TIME_BASE, AVRational{1, AV_TIME_BASE}, ClipData->ResourcePropertyDataPtr->Context->streams[ClipData->ResourcePropertyDataPtr->VideoStream]->time_base);
+			av_seek_frame(ClipData->ResourcePropertyDataPtr->Context, ClipData->ResourcePropertyDataPtr->VideoStream, Timestamp, AVSEEK_FLAG_BACKWARD);
+			AVPacket Packet;
+			av_init_packet(&Packet);
+			AVFormatContext* FormatContext = ClipData->ResourcePropertyDataPtr->Context;
+			AVCodecContext* VideoCodecContext = ClipData->ResourcePropertyDataPtr->VideoCodecContext;
+			AVCodec* VideoCodec = ClipData->ResourcePropertyDataPtr->VideoCodec;
+			AVFrame* Frame = av_frame_alloc();
+
+			while (av_read_frame(FormatContext, &Packet) >= 0)
+			{
+				if (avcodec_send_packet(VideoCodecContext, &Packet) < 0)
+				{
+				
+				}
+				if (avcodec_receive_frame(VideoCodecContext, Frame) >= 0)
+				{
+					break;
+				}
+			};
+
+
+
+			
+			if (Frame)
+			{
+				struct SwsContext* swsCtx = sws_getContext(
+				Frame->width, Frame->height, VideoCodecContext->pix_fmt,
+				Frame->width, Frame->height, AV_PIX_FMT_RGBA,
+				SWS_BILINEAR, NULL, NULL, NULL
+				);
+				if (!swsCtx)
+				{
+					UE_LOG(LogTemp, Error, TEXT("Error creating swsContext"));
+				}
+				uint8* RawData = new uint8[Frame->width * Frame->height * 4];
+				uint8* dest[4] = {RawData, 0, 0, 0};
+				int32 dest_linesize[4] = {Frame->width * 4, 0, 0, 0};
+				sws_scale(swsCtx, Frame->data, Frame->linesize, 0, Frame->height, dest, dest_linesize);
+				sws_freeContext(swsCtx);
+
+			
+				UTexture2D* Texture = UTexture2D::CreateTransient(Frame->width, Frame->height, PF_B8G8R8A8);
+				if (Texture)
+				{
+					void* MipData = Texture->GetPlatformData()->Mips[0].BulkData.Lock(LOCK_READ_WRITE);
+					FMemory::Memcpy(MipData, RawData, Frame->width * Frame->height * 4);
+					Texture->GetPlatformData()->Mips[0].BulkData.Unlock();
+					Texture->UpdateResource();
+
+					// FGuid Guid = FGuid::NewGuid();
+					// ExportImage(Texture, *FPaths::Combine(FUtils::GetTempPath() / Guid.ToString() + ".png"));
+
+					FSlateBrush NewBrush;
+					NewBrush.SetResourceObject(Texture);
+					Result.Add(NewBrush);
+					delete RawData;
+				}
+			}
+		}
+	}
+	return Result;
+
+}
+
+TArray<FSlateBrush> FFFMPEGUtils::GetAudioBrush(FClipData* ClipData)
+{
+	const float TimeLength = (ClipData->ClipEndFrame - ClipData->ClipStartFrame) * FGlobalData::DefaultTimeTickSpace;
+
+	float MaxValue = 0;
+	float MinValue = 0;
+	for (int32 i = 0; i < ClipData->ResourcePropertyDataPtr->AudioData.Num() / 4; i++)
+	{
+		float NewFloat = *reinterpret_cast<float*>(ClipData->ResourcePropertyDataPtr->AudioData.GetData() + (i * 4));
+
+		if (NewFloat >= MaxValue)
+		{
+			MaxValue = NewFloat;
+		}
+		if (NewFloat <= MinValue)
+		{
+			MinValue = NewFloat;
+		}
+	}
+	
+
+	auto Normalize = [MaxValue, MinValue](float CurrentValue)
+	{
+		return FMath::GetMappedRangeValueClamped(FVector2D(MinValue, MaxValue), FVector2D(FGlobalData::DefaultTrackHeight, 0), CurrentValue);
+	};
+	
+	ClipData->AudioBrushLength.Empty();
+	ClipData->AudioBrushes.Empty();
+	int32 Index = 0;
+	int32 TotalLength = TimeLength;
+	while (TotalLength > 0)
+	{
+		int32 CurrentLength = 0;
+		if (TotalLength > 4096)
+		{
+			TotalLength -= 4096;
+			CurrentLength = 4096;
+		}
+		else
+		{
+			CurrentLength = TotalLength;
+			TotalLength = 0;
+		}
+		UTextureRenderTarget2D* TextureRenderTarget2D = NewObject<UTextureRenderTarget2D>();
+		TextureRenderTarget2D->InitCustomFormat(int32(CurrentLength), int32(FGlobalData::DefaultTrackHeight), PF_B8G8R8A8, false);
+		TextureRenderTarget2D->UpdateResourceImmediate();
+		UKismetRenderingLibrary::ClearRenderTarget2D(GWorld->GetWorld(), TextureRenderTarget2D, FLinearColor(0, 0, 0, 0));
+		UCanvas* Canvas;
+		FVector2D Size;
+		FDrawToRenderTargetContext RenderTargetContext;
+		UKismetRenderingLibrary::BeginDrawCanvasToRenderTarget(GWorld->GetWorld(), TextureRenderTarget2D, Canvas, Size, RenderTargetContext);
+		
+		
+		float LastPoint = 0.0;
+		int32 StartIndex = (Index * 4096 + CurrentLength) * 4;
+		
+		for (int32 i = 0; i < CurrentLength; i++)
+		{
+			float CurrentData = *reinterpret_cast<float*>(ClipData->ResourcePropertyDataPtr->AudioData.GetData() + StartIndex);
+			float NormalizedData = Normalize(CurrentData);
+			Canvas->K2_DrawLine(FVector2D(i - 1, LastPoint), FVector2D(i, NormalizedData), 1, FLinearColor::White);
+			LastPoint = NormalizedData;
+
+
+			StartIndex += 4;
+		}
+		
+		
+		
+		
+		
+		UKismetRenderingLibrary::EndDrawCanvasToRenderTarget(GWorld->GetWorld(), RenderTargetContext);
+
+		
+		FBufferArchive Buffer;
+		FImageUtils::ExportRenderTarget2DAsPNG(TextureRenderTarget2D, Buffer);
+		FFileHelper::SaveArrayToFile(Buffer, *FPaths::Combine(FUtils::GetTempPath(), ClipData->ClipGuid.ToString(), FString::FromInt(Index) + ".png"));
+
+		FSlateDynamicImageBrush SlateBrush = FSlateDynamicImageBrush(*FPaths::Combine(FUtils::GetTempPath(), ClipData->ClipGuid.ToString(), FString::FromInt(Index) + ".png"), FVector2D(CurrentLength, FGlobalData::DefaultTrackHeight));
+		ClipData->AudioBrushLength.Add(CurrentLength);
+		ClipData->AudioBrushes.Add(SlateBrush);
+		Index++;
+	}
+	
+	
+	
+	
+
+	//
+	// FBufferArchive Buffer;
+	// FImageUtils::ExportRenderTarget2DAsPNG(TextureRenderTarget2D, Buffer);
+	// FFileHelper::SaveArrayToFile(Buffer, *FPaths::Combine(FUtils::GetTempPath(), "Audio.png"));
+
+	return {};
+}

Source/Cut5/Utils/FFMPEGUtils.h

@@ -17,4 +17,6 @@ struct FFFMPEGUtils
 	static FString ConvertMediaGoPto1(const FString& Path);
 	static bool ExportImage(UTexture2D* Texture2D, const FString& Path);
 	
+	static TArray<FSlateBrush> GetMovieBrush(struct FClipData* ClipData);
+	static TArray<FSlateBrush> GetAudioBrush(struct FClipData* ClipData);
 };

Source/Cut5/Utils/Utils.cpp

@@ -80,6 +80,11 @@ FString FUtils::GetResourcesPath(FString ResourcesName, bool bFullPath)
 	return FPaths::Combine(FPaths::ProjectDir() + "/Resources/", ResourcesName);
 }
 
+FString FUtils::GetTempPath()
+{
+	return FPaths::ConvertRelativePathToFull(FPaths::Combine(FGlobalData::BasePath / FGlobalData::CurrentProjectName / "/Temp/"));
+}
+
 FSlateDynamicImageBrush* FUtils::GetBrushFromImage(const FString& ImageName, const FVector2D Size)
 {
 	FSlateDynamicImageBrush* Brush = new FSlateDynamicImageBrush(*ImageName, Size);
@@ -106,6 +111,14 @@ bool FUtils::DetectDragTypeCanDrop(const FClipData& DraggingType, const ETrackTy
 		}
 	}
 
+	if (DropTrackType == ETrackType::AudioTrack || DropTrackType == ETrackType::AudioTrackR)
+	{
+		if (DraggingType.ClipType == ETrackType::AudioTrack)
+		{
+			return true;
+		}
+	}
+	
 
 	return false;
 

Source/Cut5/Utils/Utils.h

@@ -12,6 +12,7 @@ public:
 	static uint8* ConvertTwoChannelSound2PortAudioSound(uint8* Channel1, uint8* Channel2, int32 Size);
 
 	static FString GetResourcesPath(FString ResourcesName, bool bFullPath = false);
+	static FString GetTempPath();
 	static FSlateDynamicImageBrush* GetBrushFromImage(const FString& ImageName, const FVector2D Size);
 	static TArray<FSlateDynamicImageBrush*> BrushPtr;
 

Source/Cut5/Widgets/DefineGlobal.h

@@ -320,6 +320,11 @@ struct CUT5_API FClipData
 	
 	int32 GetClipRelativeEndFrame() const { return ClipEndFrame - ClipStartFrame; }
 	FPresetsCustomData PresetsCustomData;
+	TArray<FSlateBrush> MovieBrushes;
+	
+	TArray<FSlateBrush> AudioBrushes;
+	TArray<float> AudioBrushLength;
+
 	
 	enum class ECropMethod
 	{

Source/Cut5/Widgets/DragDropOperator/DragDropOperator.cpp

@@ -1,5 +1,6 @@
 #include "DragDropOperator.h"
 
+#include "Cut5/Utils/FFMPEGUtils.h"
 #include "Cut5/Utils/OpencvUtils.h"
 #include "Cut5/Utils/Utils.h"
 #include "Cut5/Widgets/DefineGlobal.h"
@@ -661,6 +662,10 @@ void DragDropOperator::OnDrop(const FGeometry& MyGeometry, const FDragDropEvent&
 				NewClipData.ClipEndFrame = NewClipData.ClipStartFrame + ClipDragOperation.TimelinePropertyData->MovieFrameLength;
 				NewClipData.VideoEndFrame = ClipDragOperation.TimelinePropertyData->MovieFrameLength;
 				NewClipData.VideoCapture = ClipDragOperation.VideoCapture;
+
+				
+				NewClipData.MovieBrushes = FFFMPEGUtils::GetMovieBrush(&NewClipData);
+				
 				if (TrackHead->TrackData.TrackType == ETrackType::LightArrayTrack)
 				{
 					NewClipData.ClipType = ETrackType::LightArrayTrack;
@@ -699,6 +704,7 @@ void DragDropOperator::OnDrop(const FGeometry& MyGeometry, const FDragDropEvent&
 				NewClipData.ClipEndFrame = NewClipData.ClipStartFrame + ClipDragOperation.TimelinePropertyData->MovieFrameLength;
 				NewClipData.VideoEndFrame = ClipDragOperation.TimelinePropertyData->MovieFrameLength;
 				NewClipData.ResourcePropertyDataPtr = ClipDragOperation.TimelinePropertyData;
+				FFFMPEGUtils::GetAudioBrush(&NewClipData);
 			}
 		}
 		

Source/Cut5/Widgets/STimelineClip.cpp

@@ -689,6 +689,32 @@ int32 STimelineClip::OnPaint(const FPaintArgs& Args, const FGeometry& AllottedGe
 	FSlateWindowElementList& OutDrawElements, int32 LayerId, const FWidgetStyle& InWidgetStyle,
 	bool bParentEnabled) const
 {
+
+	const float XLength = AllottedGeometry.GetLocalSize().X;
+	{
+		int32 i = 0;
+		float RenderPos = 0;
+		for (FSlateBrush& SlateBrush : ClipData->AudioBrushes)
+		{
+			FSlateDrawElement::MakeBox(OutDrawElements, LayerId + 2, AllottedGeometry.ToPaintGeometry(FVector2f(ClipData->AudioBrushLength[i], AllottedGeometry.GetLocalSize().Y), FSlateLayoutTransform(FVector2f(RenderPos, 0))), &SlateBrush);
+			
+			RenderPos += ClipData->AudioBrushLength[i];
+			i++;
+		}
+	}
+
+
+	{
+		int32 i = 0;
+		for (FSlateBrush& SlateBrush : ClipData->MovieBrushes)
+		{
+			FSlateDrawElement::MakeBox(OutDrawElements, LayerId + 3, AllottedGeometry.ToPaintGeometry(FVector2f(XLength / ClipData->MovieBrushes.Num(), AllottedGeometry.GetLocalSize().Y), FSlateLayoutTransform(FVector2f(i * (XLength / ClipData->MovieBrushes.Num()), 0))), &SlateBrush);
+			i++;
+		}
+	}
+
+
+	
 	if (ClipData->PresetType == EPresetType::EnableProjector)
 	{
 		FSlateDrawElement::MakeText(OutDrawElements, LayerId + 2, AllottedGeometry.ToPaintGeometry(), FText::FromString(TEXT("开启投影仪")), FAppStyle::Get().GetWidgetStyle<FTextBlockStyle>("NormalText").Font, ESlateDrawEffect::None, FLinearColor::White);