Refactor MIDI handling and improve audio processing

Replaced `uInt8ArrayOf` with simplified integer arrays for MIDI messages. Introduced `TimedMidiMessage` and buffer handling for better synchronization in audio processing. Updated Gradle dependencies and added timing-aware MIDI utilities.

audio-worklet/build.gradle.kts

@@ -41,6 +41,7 @@ kotlin {
         val commonMain by getting {
             dependencies {
                 implementation("nl.astraeus:vst-worklet-base:1.0.1")
+                implementation("nl.astraeus:midi-arrays:0.3.2")
             }
         }
         val jsMain by getting

audio-worklet/src/jsMain/kotlin/nl/astraeus/vst/chip/ChipProcessor.kt

@@ -2,14 +2,15 @@
 
 package nl.astraeus.vst.chip
 
+import nl.astraeus.midi.message.SortedTimedMidiMessageList
+import nl.astraeus.midi.message.TimedMidiMessage
+import nl.astraeus.tba.SlicedByteArray
 import nl.astraeus.vst.ADSR
 import nl.astraeus.vst.AudioWorkletProcessor
 import nl.astraeus.vst.currentTime
 import nl.astraeus.vst.registerProcessor
 import nl.astraeus.vst.sampleRate
 import org.khronos.webgl.Float32Array
-import org.khronos.webgl.Int32Array
-import org.khronos.webgl.Uint8Array
 import org.khronos.webgl.get
 import org.khronos.webgl.set
 import org.w3c.dom.MessageEvent
@@ -65,6 +66,7 @@ enum class RecordingState {
 @JsExport
 class VstChipProcessor : AudioWorkletProcessor() {
   var midiChannel = 0
+  val midiMessageBuffer = SortedTimedMidiMessageList()
   val notes = Array<PlayingNote?>(POLYPHONICS) { null }
 
   var waveform = Waveform.SINE.ordinal
@@ -101,15 +103,15 @@ class VstChipProcessor : AudioWorkletProcessor() {
   }
 
   private fun handleMessage(message: MessageEvent) {
-    //console.log("VstChipProcessor: Received message:", message.data)
+    //console.log("VstChipProcessor: Received message:", currentTime)
 
     val data = message.data
 
     try {
       when (data) {
         is String -> {
-          when (data) {
+          when {
-            "start_recording" -> {
+            data == "start_recording" -> {
               port.postMessage(recordingBuffer)
               if (recordingState == RecordingState.STOPPED) {
                 recordingState = RecordingState.WAITING_TO_START
@@ -117,34 +119,43 @@ class VstChipProcessor : AudioWorkletProcessor() {
               }
             }
 
-            else ->
+            data.startsWith("set_channel") -> {
-              if (data.startsWith("set_channel")) {
+              val parts = data.split('\n')
-                val parts = data.split('\n')
+              if (parts.size == 2) {
-                if (parts.size == 2) {
+                midiChannel = parts[1].toInt()
-                  midiChannel = parts[1].toInt()
+                println("Setting channel: $midiChannel")
-                  println("Setting channel: $midiChannel")
-                }
-              } else if (data.startsWith("waveform")) {
-                val parts = data.split('\n')
-                if (parts.size == 2) {
-                  waveform = parts[1].toInt()
-                  println("Setting waveform: $waveform")
-                }
               }
+            }
+
+            data.startsWith("waveform") -> {
+              val parts = data.split('\n')
+              if (parts.size == 2) {
+                waveform = parts[1].toInt()
+                println("Setting waveform: $waveform")
+              }
+            }
           }
         }
 
-        is Uint8Array -> {
+        is ByteArray -> {
-          val data32 = Int32Array(data.length)
+          val message1 = TimedMidiMessage(data)
-          for (i in 0 until data.length) {
+          console.log("Message as bytearray: ", message1.timeToPlay, data)
-            data32[i] = (data[i].toInt() and 0xff)
+          midiMessageBuffer.add(message1)
-          }
+          playBuffer()
-          playMidi(data32)
         }
+        /*
+                is Uint8Array -> {
+                  val data32 = Int32Array(data.length)
+                  for (i in 0 until data.length) {
+                    data32[i] = (data[i].toInt() and 0xff)
+                  }
+                  playMidi(data32)
+                }
 
-        is Int32Array -> {
+                is Int32Array -> {
-          playMidi(data)
+                  playMidi(data)
-        }
+                }
+        */
 
         else ->
           console.error("Don't kow how to handle message", message)
@@ -154,16 +165,52 @@ class VstChipProcessor : AudioWorkletProcessor() {
     }
   }
 
-  private fun playMidi(bytes: Int32Array) {
+  private fun playBuffer() {
-    //console.log("playMidi", bytes)
+    while (midiMessageBuffer.isNotEmpty() && (midiMessageBuffer.nextTimestamp()
+        ?: 0.0) < currentTime
+    ) {
+      val midi = midiMessageBuffer.read()
+      console.log("Message", currentTime, midi)
+      playMidi(midi.midi)
+    }
+  }
+
+  private fun playMidi(bytes: SlicedByteArray) {
+    var index = 0
+
+    console.log(
+      "--playMidi",
+      bytes.size,
+      index,
+      bytes[index + 0],
+      bytes[index + 1],
+      bytes[index + 2]
+    )
+    while (index < bytes.size && bytes[index].toUByte() > 0u) {
+      console.log("playMidi", bytes, index, bytes[index + 0], bytes[index + 1], bytes[index + 2])
+      index += playMidiFromBuffer(bytes, index)
+    }
+  }
+
+  private fun playMidiFromBuffer(bytes: SlicedByteArray, index: Int): Int {
+    if (bytes[index] == 0.toByte()) {
+      return 0
+    }
+
     if (bytes.length > 0) {
-      var cmdByte = bytes[0]
+      var cmdByte = bytes[0].toInt() and 0xff
       val channelCmd = ((cmdByte shr 4) and 0xf) != 0xf0
       val channel = cmdByte and 0xf
       //println("Channel cmd: $channelCmd")
+      val byteLength = when (cmdByte) {
+        0x90, 0xb0, 0xe0 -> 3
+        0x80, 0xc9 -> 2
+        else -> throw IllegalArgumentException("Unknown command: $cmdByte")
+      }
+
       if (channelCmd && channel != midiChannel) {
         console.log("Wrong channel", midiChannel, bytes)
-        return
+        return byteLength
       }
 
       cmdByte = cmdByte and 0xf0
@@ -171,13 +218,15 @@ class VstChipProcessor : AudioWorkletProcessor() {
       //console.log("Received", bytes)
       when (cmdByte) {
         0x90 -> {
-          if (bytes.length == 3) {
+          if (bytes.length >= 3) {
-            val note = bytes[1]
+            val note = bytes[1].toInt() and 0xff
-            val velocity = bytes[2]
+            val velocity = bytes[2].toInt() and 0xff
 
             if (velocity > 0) {
+              console.log("Note on", note, velocity)
               noteOn(note, velocity)
             } else {
+              console.log("Note off", note)
               noteOff(note)
             }
           }
@@ -185,15 +234,16 @@ class VstChipProcessor : AudioWorkletProcessor() {
 
         0x80 -> {
           if (bytes.length >= 2) {
-            val note = bytes[1]
+            val note = bytes[1].toInt() and 0xff
 
+            console.log("Note off", note)
             noteOff(note)
           }
         }
 
         0xc9 -> {
           if (bytes.length >= 1) {
-            val waveform = bytes[1]
+            val waveform = bytes[1].toInt() and 0xff
 
             if (waveform < 4) {
               this.waveform = waveform
@@ -202,9 +252,9 @@ class VstChipProcessor : AudioWorkletProcessor() {
         }
 
         0xb0 -> {
-          if (bytes.length == 3) {
+          if (bytes.length >= 3) {
-            val knob = bytes[1]
+            val knob = bytes[1].toInt() and 0xff
-            val value = bytes[2]
+            val value = bytes[2].toInt() and 0xff
 
             when (knob) {
               7 -> {
@@ -272,7 +322,7 @@ class VstChipProcessor : AudioWorkletProcessor() {
         }
 
         0xe0 -> {
-          if (bytes.length == 3) {
+          if (bytes.length >= 3) {
             val lsb = bytes[1]
             val msb = bytes[2]
 
@@ -280,7 +330,11 @@ class VstChipProcessor : AudioWorkletProcessor() {
           }
         }
       }
+
+      return byteLength
     }
+
+    throw IllegalArgumentException("Unable to handle empty byte array")
   }
 
   private fun noteOn(note: Int, velocity: Int) {
@@ -332,6 +386,8 @@ class VstChipProcessor : AudioWorkletProcessor() {
       recordingStart = 0
     }
 
+    playBuffer()
+
     for ((index, note) in notes.withIndex()) {
       if (note != null) {
         val midiNote = Note.fromMidi(note.note)
@@ -468,5 +524,5 @@ class VstChipProcessor : AudioWorkletProcessor() {
 fun main() {
   registerProcessor("vst-chip-processor", VstChipProcessor::class.js)
 
-  println("'vst-chip-processor' registered!")
+  console.log("'vst-chip-processor' registered!", currentTime)
 }

build.gradle.kts

@@ -59,6 +59,7 @@ kotlin {
         implementation("nl.astraeus:kotlin-css-generator:1.0.10")
         implementation("org.jetbrains.kotlinx:kotlinx-datetime:0.6.0")
         implementation("nl.astraeus:vst-ui-base:1.1.2")
+        implementation("nl.astraeus:midi-arrays:0.3.2")
       }
     }
     val jsMain by getting {

common.gradle.kts

@@ -5,6 +5,9 @@ allprojects {
   repositories {
     mavenLocal()
     mavenCentral()
+    maven {
+      url = uri("https://gitea.astraeus.nl/api/packages/rnentjes/maven")
+    }
     maven {
       url = uri("https://gitea.astraeus.nl:8443/api/packages/rnentjes/maven")
     }

src/jsMain/kotlin/nl/astraeus/vst/chip/audio/AudioContextHandler.kt

@@ -1,10 +1,5 @@
 package nl.astraeus.vst.chip.audio
 
-import nl.astraeus.vst.chip.AudioContext
-
 object AudioContextHandler {
-  val audioContext: dynamic = AudioContext()
+  val audioContext: dynamic = js("new AudioContext()")
-
+}
-
-
-}

src/jsMain/kotlin/nl/astraeus/vst/chip/audio/AudioModule.kt

@@ -1,5 +1,6 @@
 package nl.astraeus.vst.chip.audio
 
+import nl.astraeus.midi.message.TimedMidiMessage
 import nl.astraeus.vst.chip.AudioWorkletNode
 import nl.astraeus.vst.chip.AudioWorkletNodeParameters
 import nl.astraeus.vst.chip.audio.AudioContextHandler.audioContext
@@ -53,11 +54,27 @@ abstract class AudioNode(
 
   abstract fun onMessage(message: MessageEvent)
 
+  open fun postMessage(vararg data: Int) {
+    if (port == null) {
+      console.log("postMessage port is NULL!")
+    }
+
+    val array = ByteArray(data.size) { data[it].toByte() }
+
+    port?.postMessage(
+      TimedMidiMessage(
+        audioContext.currentTime,
+        *array
+      ).data.buffer.toByteArray()
+    )
+  }
+
   open fun postMessage(msg: Any) {
     if (port == null) {
       console.log("postMessage port is NULL!")
     }
     port?.postMessage(msg)
+    //console.log("Posted message", audioContext.currentTime)
   }
 
   // call from user gesture
@@ -83,6 +100,7 @@ abstract class AudioNode(
       port = node.port as? MessagePort
 
       created = true
+      console.log("Created node: ${audioContext.currentTime}")
 
       done(node)
     }

src/jsMain/kotlin/nl/astraeus/vst/chip/audio/VstChipWorklet.kt

@@ -5,12 +5,8 @@ package nl.astraeus.vst.chip.audio
 import nl.astraeus.vst.chip.PatchDTO
 import nl.astraeus.vst.chip.view.MainView
 import nl.astraeus.vst.chip.view.WaveformView
-import nl.astraeus.vst.ui.util.uInt8ArrayOf
 import org.khronos.webgl.Float32Array
-import org.khronos.webgl.Uint8Array
-import org.khronos.webgl.get
 import org.w3c.dom.MessageEvent
-import kotlin.experimental.and
 
 object VstChipWorklet : AudioNode(
   "/vst-chip-worklet.js",
@@ -33,63 +29,63 @@ object VstChipWorklet : AudioNode(
     set(value) {
       field = value
       super.postMessage(
-        uInt8ArrayOf(0xb0 + midiChannel, 7, (value * 127).toInt())
+        0xb0 + midiChannel, 7, (value * 127).toInt()
       )
     }
   var dutyCycle = 0.5
     set(value) {
       field = value
       super.postMessage(
-        uInt8ArrayOf(0xb0 + midiChannel, 0x47, (value * 127).toInt())
+        0xb0 + midiChannel, 0x47, (value * 127).toInt()
       )
     }
   var fmModFreq = 0.0
     set(value) {
       field = value
       super.postMessage(
-        uInt8ArrayOf(0xb0 + midiChannel, 0x40, (value * 127).toInt())
+        0xb0 + midiChannel, 0x40, (value * 127).toInt()
       )
     }
   var fmModAmp = 0.0
     set(value) {
       field = value
       super.postMessage(
-        uInt8ArrayOf(0xb0 + midiChannel, 0x41, (value * 127).toInt())
+        0xb0 + midiChannel, 0x41, (value * 127).toInt()
       )
     }
   var amModFreq = 0.0
     set(value) {
       field = value
       super.postMessage(
-        uInt8ArrayOf(0xb0 + midiChannel, 0x42, (value * 127).toInt())
+        0xb0 + midiChannel, 0x42, (value * 127).toInt()
       )
     }
   var amModAmp = 0.0
     set(value) {
       field = value
       super.postMessage(
-        uInt8ArrayOf(0xb0 + midiChannel, 0x43, (value * 127).toInt())
+        0xb0 + midiChannel, 0x43, (value * 127).toInt()
       )
     }
   var feedback = 0.0
     set(value) {
       field = value
       super.postMessage(
-        uInt8ArrayOf(0xb0 + midiChannel, 0x50, (value * 127).toInt())
+        0xb0 + midiChannel, 0x50, (value * 127).toInt()
       )
     }
   var delay = 0.0
     set(value) {
       field = value
       super.postMessage(
-        uInt8ArrayOf(0xb0 + midiChannel, 0x4e, (value * 127).toInt())
+        0xb0 + midiChannel, 0x4e, (value * 127).toInt()
       )
     }
   var delayDepth = 0.0
     set(value) {
       field = value
       super.postMessage(
-        uInt8ArrayOf(0xb0 + midiChannel, 0x4f, (value * 127).toInt())
+        0xb0 + midiChannel, 0x4f, (value * 127).toInt()
       )
     }
 
@@ -97,28 +93,28 @@ object VstChipWorklet : AudioNode(
     set(value) {
       field = value
       super.postMessage(
-        uInt8ArrayOf(0xb0 + midiChannel, 0x49, (value * 127).toInt())
+        0xb0 + midiChannel, 0x49, (value * 127).toInt()
       )
     }
   var decay = 0.2
     set(value) {
       field = value
       super.postMessage(
-        uInt8ArrayOf(0xb0 + midiChannel, 0x4b, (value * 127).toInt())
+        0xb0 + midiChannel, 0x4b, (value * 127).toInt()
       )
     }
   var sustain = 0.5
     set(value) {
       field = value
       super.postMessage(
-        uInt8ArrayOf(0xb0 + midiChannel, 0x46, (value * 127).toInt())
+        0xb0 + midiChannel, 0x46, (value * 127).toInt()
       )
     }
   var release = 0.2
     set(value) {
       field = value
       super.postMessage(
-        uInt8ArrayOf(0xb0 + midiChannel, 0x48, (value * 127).toInt())
+        0xb0 + midiChannel, 0x48, (value * 127).toInt()
       )
     }
 
@@ -138,20 +134,16 @@ object VstChipWorklet : AudioNode(
     super.postMessage(msg)
   }
 
-  override fun postMessage(msg: Any) {
+  override fun postMessage(vararg msg: Int) {
-    if (msg is Uint8Array) {
+    if (
-      if (
+      msg.size == 3
-        msg.length == 3
+      && (msg[0] and 0xf == midiChannel)
-        && (msg[0] and 0xf == midiChannel.toByte())
+      && (msg[0] and 0xf0 == 0xb0)
-        && (msg[0] and 0xf0.toByte() == 0xb0.toByte())
+    ) {
-      ) {
+      val knob = msg[1]
-        val knob = msg[1]
+      val value = msg[2]
-        val value = msg[2]
 
-        handleIncomingMidi(knob, value)
+      handleIncomingMidi(knob.toByte(), value.toByte())
-      } else {
-        super.postMessage(msg)
-      }
     } else {
       super.postMessage(msg)
     }

src/jsMain/kotlin/nl/astraeus/vst/chip/midi/Midi.kt

@@ -1,6 +1,8 @@
 package nl.astraeus.vst.chip.midi
 
 import kotlinx.browser.window
+import nl.astraeus.midi.message.TimedMidiMessage
+import nl.astraeus.vst.chip.audio.AudioContextHandler
 import nl.astraeus.vst.chip.audio.VstChipWorklet
 import nl.astraeus.vst.chip.view.MainView
 import org.khronos.webgl.Uint8Array
@@ -124,9 +126,14 @@ object Midi {
         hex.append(data[index].toString(16))
         hex.append(" ")
       }
-      //console.log("Midi message:", hex)
+      console.log("Midi message:", hex, message)
+      val midiData = ByteArray(message.data.length) { data[it].toByte() }
+      val timeMessage = TimedMidiMessage(
+        AudioContextHandler.audioContext.currentTime,
+        *midiData
+      )
       VstChipWorklet.postMessage(
-        message.data
+        timeMessage.data.buffer.toByteArray()
       )
     }