Connect multiple audio worklets

build.gradle.kts

@@ -1,10 +1,9 @@
-import org.jetbrains.kotlin.gradle.targets.js.webpack.KotlinWebpackOutput.Target.VAR
+import org.jetbrains.kotlin.gradle.dsl.KotlinJsCompile
-import org.jetbrains.kotlin.gradle.plugin.KotlinJsCompilerType.LEGACY
 import org.jetbrains.kotlin.gradle.plugin.KotlinJsCompilerType.IR
-import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+import org.jetbrains.kotlin.gradle.targets.js.webpack.KotlinWebpackOutput.Target.VAR
 
 plugins {
-    kotlin("multiplatform") version "1.8.10"
+    kotlin("multiplatform") version "1.9.0"
     application
 }
 
@@ -18,6 +17,13 @@ repositories {
 
 val jsMode = IR
 
+tasks.withType<KotlinJsCompile>().configureEach {
+    kotlinOptions {
+        moduleKind = "es"
+        useEsClasses = true
+    }
+}
+
 kotlin {
     jvm {
         compilations.all {

src/commonMain/kotlin/Note.kt

@@ -8,7 +8,6 @@ import kotlin.math.pow
  * Time: 11:50
  */
 
-var sampleRate: Int = 44100
 
 enum class Note(
   val description: String
@@ -146,6 +145,10 @@ enum class Note(
     this
   }
 
+  companion object {
+    var sampleRate: Int = 44100
+  }
+
   /*
    * Amount of one cycle to advance per sample
    */

src/jsAudioWorkletMain/kotlin/AudioProcessor.kt

@@ -0,0 +1,111 @@
+import org.khronos.webgl.Float32Array
+import org.khronos.webgl.set
+import org.w3c.dom.MessageEvent
+import kotlin.math.PI
+import kotlin.math.sin
+
+const val PI2 = PI * 2
+
+@ExperimentalJsExport
+@JsExport
+class AudioProcessor : AudioWorkletProcessor() {
+  private var started = true
+  private var counter: Int = 0
+  private var note = Note.C2
+  private var offset = 0.0
+
+  private var note_length = 2500
+  private var harmonics = 3
+  private var transpose = 0
+
+  init {
+    this.port.onmessage = ::handleMessage
+  }
+
+  private fun handleMessage(message: MessageEvent) {
+    console.log("WorkletProcessor: Received message", message)
+
+    val data = message.data
+    if (data is String) {
+      val parts = data.split("\n")
+      when (parts[0]) {
+        "start" -> {
+          println("Start worklet!")
+
+          Note.sampleRate = sampleRate
+          started = true
+        }
+        "stop" -> {
+          println("Stop worklet!")
+
+          started = false
+        }
+        "set_note_length" -> {
+          note_length = parts[1].toInt()
+        }
+        "harmonics" -> {
+          harmonics = parts[1].toInt()
+        }
+        "transpose" -> {
+          transpose = parts[1].toInt()
+        }
+        else ->
+          console.error("Don't kow how to handle message", message)
+      }
+    }
+  }
+
+  override fun process(
+    inputs: Array<Array<Float32Array>>,
+    outputs: Array<Array<Float32Array>>,
+    parameters: dynamic
+  ) : Boolean {
+    if (started) {
+      //console.log("process called", inputs, outputs, parameters, port)
+      //console.log("sample rate", sampleRate)//console.log("WorkletProcessor: process", samples, left, right)
+
+      check(outputs.size == 1) {
+        "Expected 1 output got ${outputs.size}"
+      }
+      check(outputs[0].size == 2) {
+        "Expected 2 output channels, got ${outputs.size}"
+      }
+
+      var delta = note.sampleDelta
+      val samples = outputs[0][0].length
+      val left = outputs[0][0]
+      val right = outputs[0][1]
+
+      //console.log("left/right", left, right)
+
+      for (sample in 0..<samples) {
+        var value = sin(offset * PI2)
+
+        for(index in 0..<harmonics) {
+          value += sin(offset * (index + 2) * PI2) * (3.0 / (index+2))
+        }
+        offset += delta
+
+        // new note every NOTE_LENGTH samples
+        val noteProgress = counter % note_length
+        if (noteProgress == 0) {
+          note = note.transpose(1)
+          if (note.ordinal >= Note.C7.transpose(transpose).ordinal) {
+            note = Note.C2.transpose(transpose)
+          }
+          delta = note.sampleDelta
+        }
+        // simple envelop from max to 0 every note
+        value *= (1.0 - noteProgress / note_length.toDouble())
+
+        left[sample] = value.toFloat()
+        right[sample] = value.toFloat()
+
+        counter++
+      }
+    }
+
+    return true
+  }
+
+}

src/jsAudioWorkletMain/kotlin/Externals.kt

@@ -0,0 +1,21 @@
+import org.khronos.webgl.Float32Array
+import org.w3c.dom.MessagePort
+
+abstract external class AudioWorkletProcessor {
+  /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/AudioWorkletNode/parameters) */
+  //val parameters: AudioParamMap;
+  /** [MDN Reference](https://developer.mozilla.org/docs/Web/API/AudioWorkletNode/port) */
+  @JsName("port")
+  val port: MessagePort
+
+  @JsName("process")
+  open fun process (
+    inputs: Array<Array<Float32Array>>,
+    outputs: Array<Array<Float32Array>>,
+    parameters: dynamic
+  ) : Boolean
+
+}
+
+external fun registerProcessor(name: String, processorCtor: JsClass<*>)
+external val sampleRate: Int

src/jsAudioWorkletMain/kotlin/Main.kt

@@ -0,0 +1,7 @@
+
+
+@OptIn(ExperimentalJsExport::class)
+fun main() {
+  registerProcessor("audio-processor", AudioProcessor::class.js)
+  registerProcessor("mixer-processor", MixerProcessor::class.js)
+}

src/jsAudioWorkletMain/kotlin/MixerProcessor.kt

@@ -0,0 +1,65 @@
+import org.khronos.webgl.Float32Array
+import org.khronos.webgl.get
+import org.khronos.webgl.set
+import org.w3c.dom.MessageEvent
+
+
+@ExperimentalJsExport
+@JsExport
+class MixerProcessor : AudioWorkletProcessor() {
+  var started = false
+
+  init {
+    this.port.onmessage = ::handleMessage
+  }
+
+  private fun handleMessage(message: MessageEvent) {
+    console.log("WorkletProcessor: Received message", message)
+
+    val data = message.data
+    if (data is String) {
+      val parts = data.split("\n")
+      when (parts[0]) {
+        "start" -> {
+          println("Start worklet!")
+
+          started = true
+        }
+        "stop" -> {
+          println("Stop worklet!")
+
+          started = false
+        }
+        else ->
+          console.error("Don't kow how to handle message", message)
+      }
+    }
+  }
+
+  override fun process(
+    inputs: Array<Array<Float32Array>>,
+    outputs: Array<Array<Float32Array>>,
+    parameters: dynamic
+  ) : Boolean {
+    if (started) {
+      check(inputs.size == 2) {
+        "Expected 2 inputs, got ${inputs.size}"
+      }
+      check(outputs.size == 1) {
+        "Expected 1 output, got ${outputs.size}"
+      }
+      val left = outputs[0][0]
+      val right = outputs[0][1]
+
+      for (input in inputs.indices) {
+        for (index in 0 ..< left.length) {
+          left[index] = left[index] + inputs[input][0][index]
+          right[index] = right[index] + inputs[input][1][index]
+        }
+      }
+    }
+
+    return true
+  }
+
+}

src/jsAudioWorkletMain/kotlin/WorkletProcessor.kt

@@ -1,96 +0,0 @@
-@file:OptIn(ExperimentalJsExport::class)
-
-import org.khronos.webgl.Float64Array
-import org.khronos.webgl.set
-import org.w3c.dom.MessageEvent
-import org.w3c.dom.MessagePort
-import kotlin.math.PI
-import kotlin.math.sin
-
-const val PI2 = PI * 2
-
-@ExperimentalJsExport
-@JsExport
-object WorkletProcessor {
-  private var port: MessagePort? = null
-
-  private var counter: Int = 0
-  private var note = Note.C2
-  private var offset = 0.0
-
-  private var started = false
-  private var note_length = 2500
-  private var harmonics = 3
-
-  @JsName("setPort")
-  fun setPort(port: MessagePort) {
-    WorkletProcessor.port = port
-    WorkletProcessor.port?.onmessage = WorkletProcessor::onMessage
-  }
-
-  @JsName("onMessage")
-  fun onMessage(message: MessageEvent) {
-    console.log("WorkletProcessor: Received message", message)
-
-    val data = message.data
-    if (data is String) {
-      val parts = data.split("\n")
-      when (parts[0]) {
-        "start" -> {
-          println("Start worklet!")
-
-          started = true
-        }
-        "stop" -> {
-          println("Stop worklet!")
-
-          started = false
-        }
-        "set_note_length" -> {
-          note_length = parts[1].toInt()
-        }
-        "harmonics" -> {
-          harmonics = parts[1].toInt()
-        }
-        else ->
-          console.error("Don't kow how to handle message", message)
-      }
-    }
-  }
-
-  @JsName("process")
-  fun process(samples: Int, left: Float64Array, right: Float64Array) {
-    if (started) {
-      var tmpCounter = counter
-      var delta = note.sampleDelta
-
-      for (sample in 0 until samples) {
-        var value = sin(offset * PI2);
-
-        for(index in 0 until harmonics) {
-          value += sin(offset * (index + 2) * PI2) * (1.0 / (index+2))
-        }
-        offset += delta
-
-        // new note every NOTE_LENGTH samples
-        val noteProgress = tmpCounter % note_length
-        if (noteProgress == 0) {
-          note = note.transpose(1)
-          if (note == Note.C7) {
-            note = Note.C2
-          }
-          delta = note.sampleDelta
-        }
-        // simple envelop from max to 0 every note
-        value *= (1.0 - noteProgress / note_length.toDouble())
-
-        left[sample] = value
-        right[sample] = value
-        tmpCounter++
-      }
-
-      counter += samples
-    }
-  }
-
-}

src/jsAudioWorkletMain/resources/worklet-processor.js

@@ -1,54 +0,0 @@
-; // make sure kotlin js code last statement is ended
-
-class WorkletProcessor extends AudioWorkletProcessor {
-
-    // Static getter to define AudioParam objects in this custom processor.
-    static get parameterDescriptors() {
-        return [{
-            name: 'volume',
-            defaultValue: 0.75
-        }];
-    }
-
-    constructor() {
-        super();
-
-        console.log("worklet-processor.constructor", this, audioWorklet);
-
-        audioWorklet.WorkletProcessor.setPort(this.port);
-
-        console.log("STARTED worklet-processor.js");
-    }
-
-    process(inputs, outputs, parameters) {
-        let result = true;
-        let samplesToProcess = 0;
-
-        if (outputs.length !== 1) {
-            result = false;
-            console.log("Unexpected number of outputs!", outputs)
-        } else {
-            let channels = outputs[0].length;
-
-            if (channels !== 2) {
-                result = false;
-                console.log("Unexpected number of channels!", outputs[0]);
-            } else if (outputs[0][0].length !== outputs[0][1].length) {
-                result = false;
-                console.log("Channels have different lengths!!", outputs[0]);
-            } else {
-                samplesToProcess = outputs[0][0].length;
-
-                audioWorklet.WorkletProcessor.process(
-                    samplesToProcess,
-                    outputs[0][0],
-                    outputs[0][1]
-                );
-            }
-        }
-
-        return result;
-    }
-}
-
-registerProcessor('worklet-processor', WorkletProcessor);

src/jsMain/kotlin/nl/astraeus/AudioProcessorNode.kt

@@ -0,0 +1,55 @@
+package nl.astraeus
+
+import nl.astraeus.handler.AudioNode
+import org.w3c.dom.MessageEvent
+
+class AudioProcessorNode(
+  audioModule: dynamic,
+  destination: dynamic,
+  outputIndex: Int = 0,
+  inputIndex: Int = 0
+) : AudioNode(
+  audioModule,
+  "audio-processor",
+  0,
+  arrayOf(2),
+  destination,
+  outputIndex,
+  inputIndex
+) {
+  override fun onMessage(message: MessageEvent) {
+    console.log("Got message from audio worklet", message)
+  }
+
+  fun start() {
+    if (!created) {
+      create {
+        start()
+      }
+    } else {
+      node?.port.postMessage("start")
+    }
+  }
+
+  fun stop() {
+    if (!created) {
+      create {
+        stop()
+      }
+    } else {
+      node?.port.postMessage("stop")
+    }
+  }
+
+  fun harmonic(i: Int) {
+    node?.port.postMessage("harmonics\n$i")
+  }
+
+  fun transpose(i: Int) {
+    node?.port.postMessage("transpose\n$i")
+  }
+
+  fun length(i: Int) {
+    node?.port.postMessage("set_note_length\n$i")
+  }
+}

src/jsMain/kotlin/nl/astraeus/Main.kt

@@ -1,31 +1,43 @@
 package nl.astraeus
 
 import kotlinx.browser.document
-import nl.astraeus.handler.AudioWorkletHandler
+import nl.astraeus.handler.AudioModule
 import org.w3c.dom.HTMLInputElement
 
 fun main() {
-  AudioWorkletHandler.loadCode()
+  val audioModule = AudioModule("static/audio-worklet.js")
+  val mixer = MixerProcessorNode(audioModule)
+  var node1: AudioProcessorNode? = null
+  var node2: AudioProcessorNode? = null
 
-  println("Ok")
+
+  document.getElementById("createButton")?.also {
+    it.addEventListener("click", {
+      mixer.create {
+        node1 = AudioProcessorNode(audioModule, mixer.node)
+        node2 = AudioProcessorNode(audioModule, mixer.node, 0,1)
+
+        node1?.create {
+          println("node 1 created")
+        }
+        node2?.create {
+          println("node 2 created")
+
+          node2?.transpose(7)
+        }
+      }
+    }, "")
+  }
 
   document.getElementById("startButton")?.also {
     it.addEventListener("click", {
-      if (AudioWorkletHandler.audioContext == null) {
+      mixer.start()
-        AudioWorkletHandler.createContext {
-          println("Created context")
-
-          AudioWorkletHandler.start()
-        }
-      } else {
-        AudioWorkletHandler.start()
-      }
     }, "")
   }
 
   document.getElementById("stopButton")?.also {
     it.addEventListener("click", {
-        AudioWorkletHandler.stop()
+      mixer.stop()
     }, "")
   }
 
@@ -33,7 +45,8 @@ fun main() {
     it.addEventListener("change", {
       val target = it.target
       if (target is HTMLInputElement) {
-        AudioWorkletHandler.setNoteLength(target.value.toInt())
+        node1?.length(target.value.toInt())
+        node2?.length(target.value.toInt())
       }
     }, "")
   }
@@ -41,7 +54,8 @@ fun main() {
     it.addEventListener("change", {
       val target = it.target
       if (target is HTMLInputElement) {
-        AudioWorkletHandler.setHarmonics(target.value.toInt())
+        node1?.harmonic(target.value.toInt())
+        node2?.harmonic(target.value.toInt())
       }
     }, "")
   }

src/jsMain/kotlin/nl/astraeus/MixerProcessorNode.kt

@@ -0,0 +1,37 @@
+package nl.astraeus
+
+import nl.astraeus.handler.AudioNode
+import org.w3c.dom.MessageEvent
+
+class MixerProcessorNode(
+  audioModule: dynamic
+) : AudioNode(
+  audioModule,
+  "mixer-processor",
+  2,
+  arrayOf(2)
+) {
+  override fun onMessage(message: MessageEvent) {
+    console.log("Got message from audio worklet", message)
+  }
+
+  fun start() {
+    if (!created) {
+      create {
+        start()
+      }
+    } else {
+      node?.port.postMessage("start")
+    }
+  }
+
+  fun stop() {
+    if (!created) {
+      create {
+        stop()
+      }
+    } else {
+      node?.port.postMessage("stop")
+    }
+  }
+}

src/jsMain/kotlin/nl/astraeus/handler/AudioModule.kt

@@ -0,0 +1,92 @@
+package nl.astraeus.handler
+
+import org.w3c.dom.MessageEvent
+import org.w3c.dom.MessagePort
+
+private val audioModules = mutableMapOf<String, AudioModule>()
+
+fun loadAudioModule(jsFile: String) {
+  val module = audioModules.getOrPut(jsFile) {
+    AudioModule(jsFile)
+  }
+}
+
+enum class ModuleStatus {
+  INIT,
+  LOADING,
+  READY
+}
+
+class AudioModule(
+  val jsFile: String
+) {
+  var status = ModuleStatus.INIT
+  var audioContext: dynamic = null
+  var module: dynamic = null
+
+  // call from user gesture
+  fun doAction(action: () -> Unit) {
+    if (module == null && status == ModuleStatus.INIT) {
+      status = ModuleStatus.LOADING
+      if (audioContext == null) {
+        audioContext = AudioContext()
+      }
+      module = audioContext.audioWorklet.addModule(
+        jsFile
+      )
+      module.then {
+        status = ModuleStatus.READY
+        action()
+      }
+    } else if (status == ModuleStatus.READY) {
+      action()
+    } else {
+      console.log("Module not yet loaded")
+    }
+  }
+}
+
+abstract class AudioNode(
+  val module: AudioModule,
+  val processorName: String,
+  val numberOfInputs: Int = 0,
+  val outputChannelCount: Array<Int> = arrayOf(2),
+  val destination: dynamic = null,
+  val outputIndex: Int = 0,
+  val inputIndex: Int = 0
+) {
+  var created = false
+  var node: dynamic = null
+  var port: MessagePort? = null
+
+  abstract fun onMessage(message: MessageEvent)
+
+  // call from user gesture
+  fun create(done: (node: dynamic) -> Unit) {
+    module.doAction {
+      node = AudioWorkletNode(
+        module.audioContext,
+        processorName,
+        AudioWorkletNodeParameters(
+          numberOfInputs,
+          outputChannelCount
+        )
+      )
+
+      if (destination == null) {
+        node.connect(module.audioContext.destination)
+      } else {
+        node.connect(destination, outputIndex, inputIndex)
+       }
+
+      node.port.onmessage = ::onMessage
+
+      port = node.port as? MessagePort
+
+      created = true
+
+      done(node)
+    }
+  }
+
+}

src/jsMain/kotlin/nl/astraeus/handler/AudioWorkletHandler.kt

@@ -1,173 +0,0 @@
-package nl.astraeus.handler
-
-import kotlinx.browser.window
-import org.w3c.dom.MessageEvent
-import org.w3c.dom.MessagePort
-import org.w3c.dom.url.URL
-import org.w3c.files.Blob
-import org.w3c.files.FilePropertyBag
-
-enum class WorkletState {
-  INIT,
-  LOADING,
-  LOADED,
-  READY
-}
-
-abstract class AudioWorklet(
-  val jsCodeFile: String,
-  val kotlinCodeFile: String
-) {
-  var audioContext: dynamic = null
-  var state = WorkletState.INIT
-  var processingCode: Blob? = null
-  var sampleRate: Int = 44100
-  var audioWorkletMessagePort: MessagePort? = null
-
-  abstract fun createNode(audioContext: dynamic): dynamic
-
-  abstract fun onAudioWorkletMessage(message: MessageEvent)
-
-  abstract fun onCodeLoaded()
-
-  fun loadCode() {
-    // hack
-    // concat kotlin js and note-processor.js because
-    // audio worklet es6 is not supported in kotlin yet
-    state = WorkletState.LOADING
-
-    window.fetch(jsCodeFile).then { daResponse ->
-      if (daResponse.ok) {
-        daResponse.text().then { daText ->
-          window.fetch(kotlinCodeFile).then { npResponse ->
-            if (npResponse.ok) {
-              npResponse.text().then { npText ->
-                processingCode = Blob(
-                  arrayOf(daText, npText),
-                  FilePropertyBag(type = "application/javascript")
-                )
-
-                state = WorkletState.LOADED
-
-                println("Loaded $this code")
-
-                onCodeLoaded()
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-
-  fun createContext(callback: () -> Unit) {
-    js("window.AudioContext = window.AudioContext || window.webkitAudioContext")
-
-    audioContext = js("new window.AudioContext()")
-    sampleRate = audioContext.sampleRate as Int
-
-    check(state == WorkletState.LOADED) {
-      "Can not createContext when code is not yet loaded, call loadCode first"
-    }
-
-    val module = audioContext.audioWorklet.addModule(
-      URL.createObjectURL(processingCode!!)
-    )
-
-    module.then {
-      val node: dynamic = createNode(audioContext)
-
-      node.connect(audioContext.destination)
-
-      node.port.onmessage = ::onAudioWorkletMessage
-
-      audioWorkletMessagePort = node.port as? MessagePort
-
-      state = WorkletState.READY
-
-      //postBatchedRequests()
-
-      callback()
-
-      "dynamic"
-    }
-  }
-
-  fun isResumed(): Boolean = audioContext?.state == "running"
-
-  fun resume() {
-    check(state == WorkletState.READY) {
-      "Unable to resume, state is not READY [$state]"
-    }
-
-    audioContext?.resume()
-  }
-
-/*  fun postRequest(request: WorkerRequest) {
-    batchedRequests.add(request)
-
-    postBatchedRequests()
-  }
-
-  private fun postBatchedRequests() {
-    val port = audioWorkletMessagePort
-
-    if (port != null) {
-      for (request in batchedRequests) {
-        val message = when (serializer) {
-          is StringFormat -> {
-            serializer.encodeToString(WorkerRequest.serializer(), request)
-          }
-
-          is BinaryFormat -> {
-            serializer.encodeToByteArray(WorkerRequest.serializer(), request)
-          }
-
-          else -> {
-            error("Unknown serializer format ${serializer::class.simpleName}")
-          }
-        }
-
-        port.postMessage(message)
-      }
-
-      batchedRequests.clear()
-    }
-  }*/
-}
-
-object AudioWorkletHandler : AudioWorklet(
-  "static/worklet-processor.js",
-  "static/audio-worklet.js"
-) {
-
-  override fun createNode(audioContext: dynamic): dynamic = js(
-    // worklet-processor as defined in de javascript:
-    // registerProcessor('worklet-processor', WorkletProcessor);
-  "new AudioWorkletNode(audioContext, 'worklet-processor', { numberOfInputs: 0, outputChannelCount: [2] })"
-  )
-
-  override fun onAudioWorkletMessage(message: MessageEvent) {
-    console.log("Received message from audio worklet: ", message)
-  }
-
-  override fun onCodeLoaded() {
-    println("Audio worklet code is loaded.")
-  }
-
-  fun start() {
-    audioWorkletMessagePort?.postMessage("start")
-  }
-
-  fun stop() {
-    audioWorkletMessagePort?.postMessage("stop")
-  }
-
-  fun setNoteLength(length: Int) {
-    audioWorkletMessagePort?.postMessage("set_note_length\n$length")
-  }
-
-  fun setHarmonics(length: Int) {
-    audioWorkletMessagePort?.postMessage("harmonics\n$length")
-  }
-}

src/jsMain/kotlin/nl/astraeus/handler/Externals.kt

@@ -0,0 +1,18 @@
+package nl.astraeus.handler
+
+external class AudioContext {
+  var sampleRate: Int
+}
+
+external class AudioWorkletNode(
+  audioContext: dynamic,
+  name: String,
+  options: dynamic
+)
+
+class AudioWorkletNodeParameters(
+  @JsName("numberOfInputs")
+  val numberOfInputs: Int,
+  @JsName("outputChannelCount")
+  val outputChannelCount: Array<Int>
+)

src/jvmMain/kotlin/Server.kt

@@ -17,6 +17,12 @@ fun HTML.index() {
       +"We need a button to start because we can only start audio from a user event:"
     }
     div("button_div") {
+      span("button") {
+        id = "createButton"
+
+        +"Create"
+      }
+
       span("button") {
         id = "startButton"