rnentjes/vst-chip
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor MIDI handling and improve audio processing

Browse Source 
Replaced `uInt8ArrayOf` with simplified integer arrays for MIDI messages. Introduced `TimedMidiMessage` and buffer handling for better synchronization in audio processing. Updated Gradle dependencies and added timing-aware MIDI utilities.
This commit is contained in:
rnentjes
2024-12-17 20:51:32 +01:00
parent 4c00356dff
commit fbba6d1422
8 changed files with 153 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
audio-worklet/build.gradle.kts
View File

@@ -41,6 +41,7 @@ kotlin {
val commonMain by getting {
dependencies {
implementation("nl.astraeus:vst-worklet-base:1.0.1")
implementation("nl.astraeus:midi-arrays:0.3.2")
}
}
val jsMain by getting

138
audio-worklet/src/jsMain/kotlin/nl/astraeus/vst/chip/ChipProcessor.kt
View File

@@ -2,14 +2,15 @@
package nl.astraeus.vst.chip
import nl.astraeus.midi.message.SortedTimedMidiMessageList
import nl.astraeus.midi.message.TimedMidiMessage
import nl.astraeus.tba.SlicedByteArray
import nl.astraeus.vst.ADSR
import nl.astraeus.vst.AudioWorkletProcessor
import nl.astraeus.vst.currentTime
import nl.astraeus.vst.registerProcessor
import nl.astraeus.vst.sampleRate
import org.khronos.webgl.Float32Array
import org.khronos.webgl.Int32Array
import org.khronos.webgl.Uint8Array
import org.khronos.webgl.get
import org.khronos.webgl.set
import org.w3c.dom.MessageEvent
@@ -65,6 +66,7 @@ enum class RecordingState {
@JsExport
class VstChipProcessor : AudioWorkletProcessor() {
var midiChannel = 0
val midiMessageBuffer = SortedTimedMidiMessageList()
val notes = Array<PlayingNote?>(POLYPHONICS) { null }
var waveform = Waveform.SINE.ordinal
@@ -101,15 +103,15 @@ class VstChipProcessor : AudioWorkletProcessor() {
}
private fun handleMessage(message: MessageEvent) {
//console.log("VstChipProcessor: Received message:", message.data)
//console.log("VstChipProcessor: Received message:", currentTime)
val data = message.data
try {
when (data) {
is String -> {
when (data) {
"start_recording" -> {
when {
data == "start_recording" -> {
port.postMessage(recordingBuffer)
if (recordingState == RecordingState.STOPPED) {
recordingState = RecordingState.WAITING_TO_START
@@ -117,34 +119,43 @@ class VstChipProcessor : AudioWorkletProcessor() {
}
}
else ->
if (data.startsWith("set_channel")) {
val parts = data.split('\n')
if (parts.size == 2) {
midiChannel = parts[1].toInt()
println("Setting channel: $midiChannel")
}
} else if (data.startsWith("waveform")) {
val parts = data.split('\n')
if (parts.size == 2) {
waveform = parts[1].toInt()
println("Setting waveform: $waveform")
}
data.startsWith("set_channel") -> {
val parts = data.split('\n')
if (parts.size == 2) {
midiChannel = parts[1].toInt()
println("Setting channel: $midiChannel")
}
}
data.startsWith("waveform") -> {
val parts = data.split('\n')
if (parts.size == 2) {
waveform = parts[1].toInt()
println("Setting waveform: $waveform")
}
}
}
}
is Uint8Array -> {
val data32 = Int32Array(data.length)
for (i in 0 until data.length) {
data32[i] = (data[i].toInt() and 0xff)
}
playMidi(data32)
is ByteArray -> {
val message1 = TimedMidiMessage(data)
console.log("Message as bytearray: ", message1.timeToPlay, data)
midiMessageBuffer.add(message1)
playBuffer()
}
/*
is Uint8Array -> {
val data32 = Int32Array(data.length)
for (i in 0 until data.length) {
data32[i] = (data[i].toInt() and 0xff)
}
playMidi(data32)
}
is Int32Array -> {
playMidi(data)
}
is Int32Array -> {
playMidi(data)
}
*/
else ->
console.error("Don't kow how to handle message", message)
@@ -154,16 +165,52 @@ class VstChipProcessor : AudioWorkletProcessor() {
}
}
private fun playMidi(bytes: Int32Array) {
//console.log("playMidi", bytes)
private fun playBuffer() {
while (midiMessageBuffer.isNotEmpty() && (midiMessageBuffer.nextTimestamp()
?: 0.0) < currentTime
) {
val midi = midiMessageBuffer.read()
console.log("Message", currentTime, midi)
playMidi(midi.midi)
}
}
private fun playMidi(bytes: SlicedByteArray) {
var index = 0
console.log(
"--playMidi",
bytes.size,
index,
bytes[index + 0],
bytes[index + 1],
bytes[index + 2]
)
while (index < bytes.size && bytes[index].toUByte() > 0u) {
console.log("playMidi", bytes, index, bytes[index + 0], bytes[index + 1], bytes[index + 2])
index += playMidiFromBuffer(bytes, index)
}
}
private fun playMidiFromBuffer(bytes: SlicedByteArray, index: Int): Int {
if (bytes[index] == 0.toByte()) {
return 0
}
if (bytes.length > 0) {
var cmdByte = bytes[0]
var cmdByte = bytes[0].toInt() and 0xff
val channelCmd = ((cmdByte shr 4) and 0xf) != 0xf0
val channel = cmdByte and 0xf
//println("Channel cmd: $channelCmd")
val byteLength = when (cmdByte) {
0x90, 0xb0, 0xe0 -> 3
0x80, 0xc9 -> 2
else -> throw IllegalArgumentException("Unknown command: $cmdByte")
}
if (channelCmd && channel != midiChannel) {
console.log("Wrong channel", midiChannel, bytes)
return
return byteLength
}
cmdByte = cmdByte and 0xf0
@@ -171,13 +218,15 @@ class VstChipProcessor : AudioWorkletProcessor() {
//console.log("Received", bytes)
when (cmdByte) {
0x90 -> {
if (bytes.length == 3) {
val note = bytes[1]
val velocity = bytes[2]
if (bytes.length >= 3) {
val note = bytes[1].toInt() and 0xff
val velocity = bytes[2].toInt() and 0xff
if (velocity > 0) {
console.log("Note on", note, velocity)
noteOn(note, velocity)
} else {
console.log("Note off", note)
noteOff(note)
}
}
@@ -185,15 +234,16 @@ class VstChipProcessor : AudioWorkletProcessor() {
0x80 -> {
if (bytes.length >= 2) {
val note = bytes[1]
val note = bytes[1].toInt() and 0xff
console.log("Note off", note)
noteOff(note)
}
}
0xc9 -> {
if (bytes.length >= 1) {
val waveform = bytes[1]
val waveform = bytes[1].toInt() and 0xff
if (waveform < 4) {
this.waveform = waveform
@@ -202,9 +252,9 @@ class VstChipProcessor : AudioWorkletProcessor() {
}
0xb0 -> {
if (bytes.length == 3) {
val knob = bytes[1]
val value = bytes[2]
if (bytes.length >= 3) {
val knob = bytes[1].toInt() and 0xff
val value = bytes[2].toInt() and 0xff
when (knob) {
7 -> {
@@ -272,7 +322,7 @@ class VstChipProcessor : AudioWorkletProcessor() {
}
0xe0 -> {
if (bytes.length == 3) {
if (bytes.length >= 3) {
val lsb = bytes[1]
val msb = bytes[2]
@@ -280,7 +330,11 @@ class VstChipProcessor : AudioWorkletProcessor() {
}
}
}
return byteLength
}
throw IllegalArgumentException("Unable to handle empty byte array")
}
private fun noteOn(note: Int, velocity: Int) {
@@ -332,6 +386,8 @@ class VstChipProcessor : AudioWorkletProcessor() {
recordingStart = 0
}
playBuffer()
for ((index, note) in notes.withIndex()) {
if (note != null) {
val midiNote = Note.fromMidi(note.note)
@@ -468,5 +524,5 @@ class VstChipProcessor : AudioWorkletProcessor() {
fun main() {
registerProcessor("vst-chip-processor", VstChipProcessor::class.js)
println("'vst-chip-processor' registered!")
console.log("'vst-chip-processor' registered!", currentTime)
}

1
build.gradle.kts
View File

@@ -59,6 +59,7 @@ kotlin {
implementation("nl.astraeus:kotlin-css-generator:1.0.10")
implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.6.0")
implementation("nl.astraeus:vst-ui-base:1.1.2")
implementation("nl.astraeus:midi-arrays:0.3.2")
}
}
val jsMain by getting {

3
common.gradle.kts
View File

@@ -5,6 +5,9 @@ allprojects {
repositories {
mavenLocal()
mavenCentral()
maven {
url = uri("https://gitea.astraeus.nl/api/packages/rnentjes/maven")
}
maven {
url = uri("https://gitea.astraeus.nl:8443/api/packages/rnentjes/maven")
}

9
src/jsMain/kotlin/nl/astraeus/vst/chip/audio/AudioContextHandler.kt
View File

@@ -1,10 +1,5 @@
package nl.astraeus.vst.chip.audio
import nl.astraeus.vst.chip.AudioContext
object AudioContextHandler {
val audioContext: dynamic = AudioContext()
}
val audioContext: dynamic = js("new AudioContext()")
}

18
src/jsMain/kotlin/nl/astraeus/vst/chip/audio/AudioModule.kt
View File

@@ -1,5 +1,6 @@
package nl.astraeus.vst.chip.audio
import nl.astraeus.midi.message.TimedMidiMessage
import nl.astraeus.vst.chip.AudioWorkletNode
import nl.astraeus.vst.chip.AudioWorkletNodeParameters
import nl.astraeus.vst.chip.audio.AudioContextHandler.audioContext
@@ -53,11 +54,27 @@ abstract class AudioNode(
abstract fun onMessage(message: MessageEvent)
open fun postMessage(vararg data: Int) {
if (port == null) {
console.log("postMessage port is NULL!")
}
val array = ByteArray(data.size) { data[it].toByte() }
port?.postMessage(
TimedMidiMessage(
audioContext.currentTime,
*array
).data.buffer.toByteArray()
)
}
open fun postMessage(msg: Any) {
if (port == null) {
console.log("postMessage port is NULL!")
}
port?.postMessage(msg)
//console.log("Posted message", audioContext.currentTime)
}
// call from user gesture
@@ -83,6 +100,7 @@ abstract class AudioNode(
port = node.port as? MessagePort
created = true
console.log("Created node: ${audioContext.currentTime}")
done(node)
}

52
src/jsMain/kotlin/nl/astraeus/vst/chip/audio/VstChipWorklet.kt
View File

@@ -5,12 +5,8 @@ package nl.astraeus.vst.chip.audio
import nl.astraeus.vst.chip.PatchDTO
import nl.astraeus.vst.chip.view.MainView
import nl.astraeus.vst.chip.view.WaveformView
import nl.astraeus.vst.ui.util.uInt8ArrayOf
import org.khronos.webgl.Float32Array
import org.khronos.webgl.Uint8Array
import org.khronos.webgl.get
import org.w3c.dom.MessageEvent
import kotlin.experimental.and
object VstChipWorklet : AudioNode(
"/vst-chip-worklet.js",
@@ -33,63 +29,63 @@ object VstChipWorklet : AudioNode(
set(value) {
field = value
super.postMessage(
uInt8ArrayOf(0xb0 + midiChannel, 7, (value * 127).toInt())
0xb0 + midiChannel, 7, (value * 127).toInt()
)
}
var dutyCycle = 0.5
set(value) {
field = value
super.postMessage(
uInt8ArrayOf(0xb0 + midiChannel, 0x47, (value * 127).toInt())
0xb0 + midiChannel, 0x47, (value * 127).toInt()
)
}
var fmModFreq = 0.0
set(value) {
field = value
super.postMessage(
uInt8ArrayOf(0xb0 + midiChannel, 0x40, (value * 127).toInt())
0xb0 + midiChannel, 0x40, (value * 127).toInt()
)
}
var fmModAmp = 0.0
set(value) {
field = value
super.postMessage(
uInt8ArrayOf(0xb0 + midiChannel, 0x41, (value * 127).toInt())
0xb0 + midiChannel, 0x41, (value * 127).toInt()
)
}
var amModFreq = 0.0
set(value) {
field = value
super.postMessage(
uInt8ArrayOf(0xb0 + midiChannel, 0x42, (value * 127).toInt())
0xb0 + midiChannel, 0x42, (value * 127).toInt()
)
}
var amModAmp = 0.0
set(value) {
field = value
super.postMessage(
uInt8ArrayOf(0xb0 + midiChannel, 0x43, (value * 127).toInt())
0xb0 + midiChannel, 0x43, (value * 127).toInt()
)
}
var feedback = 0.0
set(value) {
field = value
super.postMessage(
uInt8ArrayOf(0xb0 + midiChannel, 0x50, (value * 127).toInt())
0xb0 + midiChannel, 0x50, (value * 127).toInt()
)
}
var delay = 0.0
set(value) {
field = value
super.postMessage(
uInt8ArrayOf(0xb0 + midiChannel, 0x4e, (value * 127).toInt())
0xb0 + midiChannel, 0x4e, (value * 127).toInt()
)
}
var delayDepth = 0.0
set(value) {
field = value
super.postMessage(
uInt8ArrayOf(0xb0 + midiChannel, 0x4f, (value * 127).toInt())
0xb0 + midiChannel, 0x4f, (value * 127).toInt()
)
}
@@ -97,28 +93,28 @@ object VstChipWorklet : AudioNode(
set(value) {
field = value
super.postMessage(
uInt8ArrayOf(0xb0 + midiChannel, 0x49, (value * 127).toInt())
0xb0 + midiChannel, 0x49, (value * 127).toInt()
)
}
var decay = 0.2
set(value) {
field = value
super.postMessage(
uInt8ArrayOf(0xb0 + midiChannel, 0x4b, (value * 127).toInt())
0xb0 + midiChannel, 0x4b, (value * 127).toInt()
)
}
var sustain = 0.5
set(value) {
field = value
super.postMessage(
uInt8ArrayOf(0xb0 + midiChannel, 0x46, (value * 127).toInt())
0xb0 + midiChannel, 0x46, (value * 127).toInt()
)
}
var release = 0.2
set(value) {
field = value
super.postMessage(
uInt8ArrayOf(0xb0 + midiChannel, 0x48, (value * 127).toInt())
0xb0 + midiChannel, 0x48, (value * 127).toInt()
)
}
@@ -138,20 +134,16 @@ object VstChipWorklet : AudioNode(
super.postMessage(msg)
}
override fun postMessage(msg: Any) {
if (msg is Uint8Array) {
if (
msg.length == 3
&& (msg[0] and 0xf == midiChannel.toByte())
&& (msg[0] and 0xf0.toByte() == 0xb0.toByte())
) {
val knob = msg[1]
val value = msg[2]
override fun postMessage(vararg msg: Int) {
if (
msg.size == 3
&& (msg[0] and 0xf == midiChannel)
&& (msg[0] and 0xf0 == 0xb0)
) {
val knob = msg[1]
val value = msg[2]
handleIncomingMidi(knob, value)
} else {
super.postMessage(msg)
}
handleIncomingMidi(knob.toByte(), value.toByte())
} else {
super.postMessage(msg)
}

11
src/jsMain/kotlin/nl/astraeus/vst/chip/midi/Midi.kt
View File

@@ -1,6 +1,8 @@
package nl.astraeus.vst.chip.midi
import kotlinx.browser.window
import nl.astraeus.midi.message.TimedMidiMessage
import nl.astraeus.vst.chip.audio.AudioContextHandler
import nl.astraeus.vst.chip.audio.VstChipWorklet
import nl.astraeus.vst.chip.view.MainView
import org.khronos.webgl.Uint8Array
@@ -124,9 +126,14 @@ object Midi {
hex.append(data[index].toString(16))
hex.append(" ")
}
//console.log("Midi message:", hex)
console.log("Midi message:", hex, message)
val midiData = ByteArray(message.data.length) { data[it].toByte() }
val timeMessage = TimedMidiMessage(
AudioContextHandler.audioContext.currentTime,
*midiData
)
VstChipWorklet.postMessage(
message.data
timeMessage.data.buffer.toByteArray()
)
}