说明

  本文发布较早,查看最新动态,请关注 TypeScript 版本。(2020 年 1 月 注)

在线演示: 音频可视化(TypeScript)

准备

  IDE:Visual Studio

  Nuget包:SharpDx.XAudio2

  Nuget包:Win2D.UWP

  了解学习:Win2D 官方博客

  了解学习:Win2D 官方示例

第一节 波形

  获取实时时域数据。

Imports SharpDX.Multimedia
Imports SharpDX.XAudio2
Public Class AudioPlayer
Public Event WavePlaying(e As WavePlayingEventArgs)
Public Property Device As XAudio2
Public Property Voice As SourceVoice
Private CurrentFormat As WaveFormat
Private CurrentBuffer As AudioBuffer
Private PacketsInfo As UInteger()
Public Sub New()
Device = New XAudio2()
Device.StartEngine()
Dim mv As New MasteringVoice(Device)
End Sub
Public Async Function LoadFile(fileName As String) As Task(Of Boolean)
Try
Voice = Await CreateVoiceFromFile(Device, fileName)
LoadBuffer()
Return True
Catch
Return False
End Try
End Function
Public Sub Play(Optional volume As Single = .0F)
Voice?.SetVolume(volume)
Voice?.Start()
ReadBuffer()
End Sub
Public Sub [Stop]()
Voice?.Stop()
End Sub
Protected Async Function CreateVoiceFromFile(device As XAudio2, fileName As String) As Task(Of SourceVoice)
Dim file = Await Package.Current.InstalledLocation.GetFileAsync(fileName)
Dim streamWithContentType = Await file.OpenReadAsync()
Dim st = streamWithContentType.AsStreamForRead()
Using stream = New SoundStream(st)
CurrentFormat = stream.Format
CurrentBuffer = New AudioBuffer() With {
.Stream = stream.ToDataStream(),
.AudioBytes = CInt(stream.Length),
.Flags = BufferFlags.EndOfStream
}
PacketsInfo = stream.DecodedPacketsInfo
End Using
Dim sourceVoice = New SourceVoice(device, CurrentFormat, True)
Return sourceVoice
End Function
Protected Sub LoadBuffer()
Voice?.FlushSourceBuffers()
Voice?.SubmitSourceBuffer(CurrentBuffer, PacketsInfo)
End Sub
''' <summary>
''' 从流中读取当前播放的数据
''' </summary>
Private Async Sub ReadBuffer()
Try
Dim count As Integer = CurrentFormat?.AverageBytesPerSecond /
While Voice.State.BuffersQueued >
If Voice.State.SamplesPlayed * CurrentFormat.BlockAlign > CurrentBuffer.Stream.Position + count Then
Dim byteArr(count - ) As Byte
Await CurrentBuffer.Stream.ReadAsync(byteArr, , count)
RaiseEvent WavePlaying(New WavePlayingEventArgs(byteArr, CurrentFormat))
Else
Await Task.Delay()
End If
End While
Catch
Return
End Try
End Sub
Public Function Position() As Integer
Return Voice.State.SamplesPlayed / CurrentFormat.SampleRate
End Function
Protected Overrides Sub Finalize()
Try
Dispose(False)
Finally
MyBase.Finalize()
End Try
End Sub
Public Sub Dispose()
Dispose(True)
GC.SuppressFinalize(Me)
End Sub
Private Sub Dispose(isDisposing As Boolean)
If Not isDisposing Then
Return
End If
Voice.DestroyVoice()
Voice.Dispose()
CurrentBuffer.Stream.Dispose()
End Sub
End Class

VB.NET

//由于在线转换工具对异步代码段支持不好,该处代码请参考VB.

C#

图1-1 实时波谱

第二节 频谱

  通过快速傅里叶变换将时域信号转换为频域信号。

    ''' <summary>
''' 快速傅里叶变换
''' </summary>
''' <param name="data">指定的数据</param>
''' <param name="count"></param>
''' <returns></returns>
Public Shared Function FFT(ByVal data() As Single, ByVal count As Integer) As Single() '快速傅里叶变换
Dim fftCount As Integer = count
Dim j As Integer
Dim k As Integer
Dim NM1 As Integer
Dim ND2 As Integer
Dim M, L, LE, LE2, IP, JM1 As Integer
Dim TR, TI, SR, SI, UR, UI As Double
Dim IMX(fftCount - ) As Double
Dim REX(fftCount - ) As Double
Dim n As Integer = fftCount
Array.Copy(data, REX, fftCount)
NM1 = n -
ND2 = n /
M = CInt(Math.Log(n) / Math.Log())
j = ND2 For i = To n -
If i < j Then
TR = REX(j)
TI = IMX(j)
REX(j) = REX(i)
IMX(j) = IMX(i)
REX(i) = TR
IMX(i) = TI
End If
k = ND2
While (k <= j)
j = j - k
k = k /
End While
j = j + k
Next i For L = To M
LE = CInt( ^ L)
LE2 = LE /
UR =
UI =
SR = Math.Cos(Math.PI / LE2)
SI = -Math.Sin(Math.PI / LE2)
For j = To LE2
JM1 = j -
For i = JM1 To NM1 Step LE
IP = i + LE2
TR = REX(IP) * UR - IMX(IP) * UI
TI = REX(IP) * UI + IMX(IP) * UR
REX(IP) = REX(i) - TR
IMX(IP) = IMX(i) - TI
REX(i) = REX(i) + TR
IMX(i) = IMX(i) + TI
Next i
TR = UR
UR = TR * SR - UI * SI
UI = TR * SI + UI * SR
Next j
Next L
'
Dim w() As Single
ReDim w(fftCount / ) '取有效值
''公式:F=Kf/N F=频率;k=位置;f=取样频率;N=样本数;有效数据(N/2+1)个.
For i = To fftCount /
w(i) = Math.Sqrt(REX(i) * REX(i) + IMX(i) * IMX(i))
Next
Return w
End Function

VB.NET

/// <summary>
/// 快速傅里叶变换
/// </summary>
/// <param name="data">指定的数据</param>
/// <param name="count"></param>
/// <returns></returns>
public static float[] FFT(float[] data, int count)
{
//快速傅里叶变换
int fftCount = count;
int j = ;
int k = ;
int NM1 = ;
int ND2 = ;
int M = ;
int L = ;
int LE = ;
int LE2 = ;
int IP = ;
int JM1 = ;
double TR = ;
double TI = ;
double SR = ;
double SI = ;
double UR = ;
double UI = ;
double[] IMX = new double[fftCount];
double[] REX = new double[fftCount];
int n = fftCount;
Array.Copy(data, REX, fftCount);
NM1 = n - ;
ND2 = n / ;
M = Convert.ToInt32(Math.Log(n) / Math.Log());
j = ND2; for (i = ; i <= n - ; i++) {
if (i < j) {
TR = REX[j];
TI = IMX[j];
REX[j] = REX[i];
IMX[j] = IMX[i];
REX[i] = TR;
IMX[i] = TI;
}
k = ND2;
while ((k <= j)) {
j = j - k;
k = k / ;
}
j = j + k;
} for (L = ; L <= M; L++) {
LE = Convert.ToInt32(Math.Pow(, L));
LE2 = LE / ;
UR = ;
UI = ;
SR = Math.Cos(Math.PI / LE2);
SI = -Math.Sin(Math.PI / LE2);
for (j = ; j <= LE2; j++) {
JM1 = j - ;
for (i = JM1; i <= NM1; i += LE) {
IP = i + LE2;
TR = REX[IP] * UR - IMX[IP] * UI;
TI = REX[IP] * UI + IMX[IP] * UR;
REX[IP] = REX[i] - TR;
IMX[IP] = IMX[i] - TI;
REX[i] = REX[i] + TR;
IMX[i] = IMX[i] + TI;
}
TR = UR;
UR = TR * SR - UI * SI;
UI = TR * SI + UI * SR;
}
}
//
float[] w = null;
w = new float[fftCount / + ];
//取有效值
//'公式:F=Kf/N F=频率;k=位置;f=取样频率;N=样本数;有效数据(N/2+1)个.
for (i = ; i <= fftCount / ; i++) {
w[i] = Math.Sqrt(REX[i] * REX[i] + IMX[i] * IMX[i]);
}
return w;
}

C#

图2-1 实时频谱

第三节 简化

  简化频域数据,将相邻的若干组值相加后取平均值。

  频谱也可以绘制成圆环状。

    Public DataWave As ConcurrentQueue(Of Single)
Public DataFFT() As Single
Public DataFFTExtra() As Single
Private Sub CalcFFT()
Dim count As Integer = '最小时域数据的长度
If DataWave.Count < count Then Return
Dim tempD(count - ) As Single
For i = To count -
tempD(i) = DataWave(DataWave.Count - count + i)
Next
DataFFT = SignalMath.FFT(tempD, count) '原始频谱 Dim extraCount As Integer = '简化的长度
Dim TempList As New List(Of Single)
Dim sCount As Integer = DataFFT.Count / extraCount
TempList.Add()
For i = To extraCount -
Dim TempSingle As Single =
For j = To sCount -
TempSingle += DataFFT(i * sCount + j)
Next
TempSingle = TempSingle / sCount /
TempList.Add(TempSingle)
Next
DataFFTExtra = TempList.ToArray '简化后的频谱
WaveSignal.Energy = DataFFTExtra.ToList.IndexOf(DataFFTExtra.Max) '不精确的基音位置
End Sub

VB.NET

public ConcurrentQueue<float> DataWave;
public float[] DataFFT;
public float[] DataFFTExtra;
private void CalcFFT()
{
int count = ;
//最小时域数据的长度
if (DataWave.Count < count)
return;
float[] tempD = new float[count];
for (i = ; i <= count - ; i++) {
tempD[i] = DataWave(DataWave.Count - count + i);
}
DataFFT = SignalMath.FFT(tempD, count);
//原始频谱 int extraCount = ;
//简化的长度
List<float> TempList = new List<float>();
int sCount = DataFFT.Count / extraCount;
TempList.Add();
for (i = ; i <= extraCount - ; i++) {
float TempSingle = ;
for (j = ; j <= sCount - ; j++) {
TempSingle += DataFFT[i * sCount + j];
}
TempSingle = TempSingle / sCount / ;
TempList.Add(TempSingle);
}
DataFFTExtra = TempList.ToArray();//简化后的频谱
}

C#

图3-1 频谱变形

第四节 场景

  粒子

  加入飞舞的粒子。粒子运动速度与实时音频的基音大小相关。

  若用贴图取代圆点,可生成效果逼真的烟雾或者飞雪等场景。

Imports System.Numerics
Imports Windows.UI
''' <summary>
''' 粒子类,表示一个拥有加速度、加速度和位置矢量的抽象粒子
''' </summary>
Public Class Partical
Public Property Location As Vector2 '位置矢量
Public Property Velocity As Vector2 '速度
Public Property Acceleration As Vector2 '加速度
Public Property Mass As Single = 10.0 '质量大小
Public Property Age As Single = '生命周期
Public Property Alpha As Single =
Public Property Size As Single =
Public Property Radius As Single =
Public Property RadiusX As Single =
Public Property RadiusY As Single =
Public Property ImageSize As Single = '粒子图像的大小
Public Property Color As Color '粒子颜色
Public Shared Rnd As New Random
''' <summary>
''' 初始化一个粒子
''' </summary>
Public Sub New(loc As Vector2)
Location = loc
Velocity = New Vector2(, )
Acceleration = New Vector2(, )
End Sub
''' <summary>
''' 指定的力作用于当前对象
''' </summary>
''' <param name="forceVec">指定的力</param>
Public Sub ApplyForce(forceVec As Vector2)
Acceleration = Acceleration + forceVec / Mass
End Sub
''' <summary>
''' 更新粒子位置,重绘每帧图像前调用该方法
''' </summary>
Public Sub Move()
Velocity += Acceleration * WaveSignal.Energy
'Velocity.LimitMag(20)
Location += Velocity '更新位置
Acceleration = Vector2.Zero
End Sub
Public Sub StartNew(Loc As Vector2)
Location = Loc
Velocity.SetMag()
End Sub
End Class

VB.NET

using Microsoft.VisualBasic;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Data;
using System.Diagnostics;
using System.Numerics;
using Windows.UI;
/// <summary>
/// 粒子类,表示一个拥有加速度、加速度和位置矢量的抽象粒子
/// </summary>
public class Partical
{
public Vector2 Location { get; set; }
//位置矢量
public Vector2 Velocity { get; set; }
//速度
public Vector2 Acceleration { get; set; }
//加速度
public float Mass { get; set; }
//质量大小
public float Age { get; set; }
//生命周期
public float Alpha { get; set; }
public float Size { get; set; }
public float Radius { get; set; }
public float RadiusX { get; set; }
public float RadiusY { get; set; }
public float ImageSize { get; set; }
//粒子图像的大小
public Color Color { get; set; }
//粒子颜色
public static Random Rnd = new Random();
/// <summary>
/// 初始化一个粒子
/// </summary>
public Partical(Vector2 loc)
{
Location = loc;
Velocity = new Vector2(, );
Acceleration = new Vector2(, );
}
/// <summary>
/// 指定的力作用于当前对象
/// </summary>
/// <param name="forceVec">指定的力</param>
public void ApplyForce(Vector2 forceVec)
{
Acceleration = Acceleration + forceVec / Mass;
}
/// <summary>
/// 更新粒子位置,重绘每帧图像前调用该方法
/// </summary>
public void Move()
{
Velocity += Acceleration * WaveSignal.Energy;//与基音位置相关
//Velocity.LimitMag(20)
Location += Velocity;
//更新位置
Acceleration = Vector2.Zero;
}
public void StartNew(Vector2 Loc)
{
Location = Loc;
Velocity.SetMag();
}
}

C#

  背景

  加入背景图片。背景高斯模糊与实时音频的基音大小相关。

  也可以让背景随机抖动或者旋转。

    Private Sub DrawBackGround(DrawingSession As CanvasDrawingSession)
Using cmdList = New CanvasCommandList(DrawingSession)
Using dl = cmdList.CreateDrawingSession
dl.DrawImage(ImageManager.GetResource(ImageResourceID.back1))
End Using
Using blur1 = New Effects.GaussianBlurEffect With {.Source = cmdList, .BlurAmount = + WaveSignal.Energy / }
DrawingSession.DrawImage(blur1)
End Using
End Using
End Sub

VB.NET

private void DrawBackGround(CanvasDrawingSession DrawingSession)
{
using (cmdList == new CanvasCommandList(DrawingSession)) {
using (dl == cmdList.CreateDrawingSession) {
dl.DrawImage(ImageManager.GetResource(ImageResourceID.back1));
}
using (blur1 == new Effects.GaussianBlurEffect {Source = cmdList,BlurAmount = + WaveSignal.Energy / }) {
DrawingSession.DrawImage(blur1);
}
}
}

C#

  文字

  加入歌曲名称信息。

  具体的文字效果由你设置,请参考 Win2D 的 Microsoft.Graphics.Canvas.Effects 文档。

Imports Microsoft.Graphics.Canvas
Imports Windows.UI
Public Class StaticStringView
Inherits TypedGameView(Of StaticString)
Public Sub New(Target As StaticString)
MyBase.New(Target)
End Sub
Public Overrides Sub Draw(DrawingSession As CanvasDrawingSession)
Using cmdList = New CanvasCommandList(DrawingSession)
Using dl = cmdList.CreateDrawingSession
DrawText(dl)
End Using
Using blur1 = New Effects.GaussianBlurEffect With {.Source = cmdList, .BlurAmount = }
DrawingSession.DrawImage(blur1)
DrawingSession.DrawImage(cmdList)
End Using
End Using
End Sub
Dim TextFormat = New Text.CanvasTextFormat() With {.FontFamily = "微软雅黑",
.FontSize = ,
.HorizontalAlignment = Microsoft.Graphics.Canvas.Text.CanvasHorizontalAlignment.Center,
.VerticalAlignment = Microsoft.Graphics.Canvas.Text.CanvasVerticalAlignment.Center}
Public Sub DrawText(Dl As CanvasDrawingSession)
Dl.DrawText(Target.Str, Target.Location, Colors.White, TextFormat)
End Sub
End Class

VB.NET

using Microsoft.VisualBasic;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Data;
using System.Diagnostics;
using Microsoft.Graphics.Canvas;
using Windows.UI;
public class StaticStringView : TypedGameView<StaticString>
{
public StaticStringView(StaticString Target) : base(Target)
{
}
public override void Draw(CanvasDrawingSession DrawingSession)
{
using (cmdList == new CanvasCommandList(DrawingSession)) {
using (dl == cmdList.CreateDrawingSession) {
DrawText(dl);
}
using (blur1 == new Effects.GaussianBlurEffect {Source = cmdList,BlurAmount = }) {
DrawingSession.DrawImage(blur1);
DrawingSession.DrawImage(cmdList);
}
}
}
TextFormat = new Text.CanvasTextFormat {
FontFamily = "微软雅黑",
FontSize = ,
HorizontalAlignment = Microsoft.Graphics.Canvas.Text.CanvasHorizontalAlignment.Center,
VerticalAlignment = Microsoft.Graphics.Canvas.Text.CanvasVerticalAlignment.Center
};
public void DrawText(CanvasDrawingSession Dl)
{
Dl.DrawText(Target.Str, Target.Location, Colors.White, TextFormat);
}
}

C#

图4-1 最终效果

附录

  开源:MusicVideoGenerator (已失效)

  相同类型的MV:

  Over The Horizon

  Tuesdays

  Supernova(Original Mix)

  Lovers

  Life

UWP简单示例(一):快速合成音乐MV的更多相关文章

  1. UWP简单示例(一):快速合成音乐MV

    准备 IDE:Visual Studio 2015 为你的项目安装Nuget包 SharpDx.XAudio2 为你的项目安装Nuget包 Win2D.UWP 了解并学习:Win2D官方博客 了解并学 ...

  2. UWP简单示例(三):快速开发2D游戏引擎

    准备 IDE:VisualStudio 2015 Language:VB.NET/C# 图形API:Win2D MSDN教程:UWP游戏开发 游戏开发涉及哪些技术? 游戏开发是一门复杂的艺术,编码方面 ...

  3. UWP简单示例(二):快速开始你的3D编程

    准备 IDE:Visual Studio 2015 了解并学习:SharpDx官方GitHub 推荐Demo:SharpDX_D3D12HelloWorld 第一节 世界 世界坐标系是一个特殊的坐标系 ...

  4. UWP简单示例(三):快速开发2D游戏引擎

    准备 IDE:Visual Studio 图形 API:Win2D MSDN 教程:UWP游戏开发 游戏开发涉及哪些技术? 游戏开发是一门复杂的艺术,编码方面你需要考虑图形.输入和网络 以及相对独立的 ...

  5. UWP简单示例(二):快速开始你的3D编程

    准备 IDE:Visual Studio 开源库:GitHub.SharpDx 入门示例:SharpDX_D3D12HelloWorld 为什么选择 SharpDx? SharpDx 库与 UWP 兼 ...

  6. Redis 安装与简单示例

    Redis 安装与简单示例 一.Redis的安装 Redis下载地址如下:https://github.com/dmajkic/redis/downloads 解压后根据自己机器的实际情况选择32位或 ...

  7. springMVC源码分析--异常处理机制HandlerExceptionResolver简单示例(一)

    springMVC对Controller执行过程中出现的异常提供了统一的处理机制,其实这种处理机制也简单,只要抛出的异常在DispatcherServlet中都会进行捕获,这样就可以统一的对异常进行处 ...

  8. Optaplanner规划引擎的工作原理及简单示例(2)

    开篇 在前面一篇关于规划引擎Optapalnner的文章里(Optaplanner规划引擎的工作原理及简单示例(1)),老农介绍了应用Optaplanner过程中需要掌握的一些基本概念,这些概念有且于 ...

  9. AMQP消息队列之RabbitMQ简单示例

    前面一篇文章讲了如何快速搭建一个ActiveMQ的示例程序,ActiveMQ是JMS的实现,那这篇文章就再看下另外一种消息队列AMQP的代表实现RabbitMQ的简单示例吧.在具体讲解之前,先通过一个 ...

随机推荐

  1. Oracle EBS PO rcv_shipment_headers 数据缺失

    Datafix : How to Recreate Missing Receipt or Shipment Header Records (RCV_SHIPMENT_HEADERS table) (D ...

  2. ip 命令的使用

    网上相似的资源很多,可以参考如下资料: man ip ip help 博客链接: https://linoxide.com/linux-command/use-ip-command-linux/ ht ...

  3. python基础学习14----正则表达式

    正则表达式是对字符串操作的一种逻辑公式,就是用事先定义好的一些特定字符.及这些特定字符的组合,组成一个“规则字符串”,这个“规则字符串”用来表达对字符串的一种过滤逻辑. 在python中正则表达式被封 ...

  4. [Python_6] Python 配置 MySQL 访问

    0. 说明 Python 访问 MySQL 数据库,需要安装 MySQL 的 Python 插件. 1. 安装 MySQL 插件 pip install PyMySQL 2. 编写代码 # -*-co ...

  5. 【转】Mysql学习---SQL的优化

    [原文]https://www.toutiao.com/i6594314336913588743/ mysql如何处理亿级数据,第一个阶段--优化SQL语句 1.应尽量避免在 where 子句中使用! ...

  6. xml的xsi:type序列化和反序列化

    最近在做HL7V3的对接,关于XML的序列化和反序列化遇到xsi:type的问题解决方法 实体类定义: public class HL7V3_ProviderOrganization { public ...

  7. windows中使用git和开源中国

    现学现卖,学了忘忘了学. 非常感谢OSC提供了这么好的一个国内的免费的git托管平台.这里简单说下TortoiseGit操作的流程.很傻瓜了首先你要准备两个软件,分别是msysgit和tortoise ...

  8. PyQt5--MessageBox

    # -*- coding:utf-8 -*- ''' Created on Sep 13, 2018 @author: SaShuangYiBing ''' import sys from PyQt5 ...

  9. 基于Map的简易记忆化缓存

    背景 在应用程序中,时常会碰到需要维护一个map,从中读取一些数据避免重复计算,如果还没有值则计算一下塞到map里的的小需求(没错,其实就是简易的缓存或者说实现记忆化).在公司项目里看到过有些代码中写 ...

  10. 网络编程_TCP协议_客户端与服务端

    客户端发数据到服务端 Tcp传输,客户端建立的过程. 1,创建tcp客户端socket服务.使用的是Socket对象.建议该对象一创建就明确目的地.要连接的主机. 2,如果连接建立成功,说明数据传输通 ...