Communication iframe -> audio worklet etc.

build.gradle.kts

@@ -3,7 +3,7 @@ import org.jetbrains.kotlin.gradle.plugin.KotlinJsCompilerType.IR
 import org.jetbrains.kotlin.gradle.targets.js.webpack.KotlinWebpackOutput.Target.VAR
 
 plugins {
-    kotlin("multiplatform") version "1.9.0"
+    kotlin("multiplatform") version "1.9.10"
     application
 }
 
@@ -38,6 +38,15 @@ kotlin {
         binaries.executable()
         browser()
     }
+    js("jsWorklet", jsMode) {
+        binaries.executable()
+
+        browser {
+            commonWebpackConfig {
+                outputFileName = "html-worklet.js"
+            }
+        }
+    }
     js("jsAudioWorklet", jsMode) {
         binaries.executable()
 
@@ -75,6 +84,7 @@ tasks.named<Copy>("jvmProcessResources") {
     val jsBrowserDistribution = tasks.named("jsBrowserDevelopmentWebpack")
     from(jsBrowserDistribution)
     from(tasks.named("jsAudioWorkletBrowserDistribution"))
+    from(tasks.named("jsWorkletBrowserDistribution"))
 }
 
 tasks.named<JavaExec>("run") {

src/jsAudioWorkletMain/kotlin/AudioProcessor.kt

@@ -1,31 +1,39 @@
 import org.khronos.webgl.Float32Array
 import org.khronos.webgl.set
 import org.w3c.dom.MessageEvent
+import org.w3c.dom.MessagePort
 import kotlin.math.PI
 import kotlin.math.sin
 
 const val PI2 = PI * 2
 
+class NoteState(
+  note: Note,
+  counter: Int
+)
+
 @ExperimentalJsExport
 @JsExport
 class AudioProcessor : AudioWorkletProcessor() {
   private var started = true
   private var counter: Int = 0
-  private var note = Note.C2
+  private var note: Note? = null
   private var offset = 0.0
 
   private var note_length = 2500
   private var harmonics = 3
   private var transpose = 0
 
+  private var workletPort: MessagePort? = null
+
   init {
     this.port.onmessage = ::handleMessage
   }
 
   private fun handleMessage(message: MessageEvent) {
-    console.log("WorkletProcessor: Received message", message)
+    console.log("AudioProcessor: Received message", message)
 
-    val data = message.data
+    val data: Any? = message.data
     if (data is String) {
       val parts = data.split("\n")
       when (parts[0]) {
@@ -49,9 +57,24 @@ class AudioProcessor : AudioWorkletProcessor() {
         "transpose" -> {
           transpose = parts[1].toInt()
         }
+        "play" -> {
+          note = Note.valueOf(parts[1])
+          counter=1
+          offset = 0.0
+        }
         else ->
           console.error("Don't kow how to handle message", message)
       }
+    } else {
+      val dynData: dynamic = message.data
+      if (dynData.command == "audio-iframe-message-port") {
+        workletPort = dynData.port
+        workletPort?.onmessage = {
+          console.log("AudioProcessor: Received message from iframe", it)
+
+          handleMessage(it)
+        }
+      }
     }
   }
 
@@ -61,9 +84,6 @@ class AudioProcessor : AudioWorkletProcessor() {
     parameters: dynamic
   ) : Boolean {
     if (started) {
-      //console.log("process called", inputs, outputs, parameters, port)
-      //console.log("sample rate", sampleRate)//console.log("WorkletProcessor: process", samples, left, right)
-
       check(outputs.size == 1) {
         "Expected 1 output got ${outputs.size}"
       }
@@ -71,37 +91,33 @@ class AudioProcessor : AudioWorkletProcessor() {
         "Expected 2 output channels, got ${outputs.size}"
       }
 
-      var delta = note.sampleDelta
+      note?.also { activeNote ->
-      val samples = outputs[0][0].length
+        var delta = activeNote.sampleDelta
-      val left = outputs[0][0]
+        val samples = outputs[0][0].length
-      val right = outputs[0][1]
+        val left = outputs[0][0]
+        val right = outputs[0][1]
 
-      //console.log("left/right", left, right)
+        for (sample in 0..<samples) {
+          var value = sin(offset * PI2)
 
-      for (sample in 0..<samples) {
+          for (index in 0..<harmonics) {
-        var value = sin(offset * PI2)
+            value += sin(offset * (index + 2) * PI2) * (3.0 / (index + 2))
-
-        for(index in 0..<harmonics) {
-          value += sin(offset * (index + 2) * PI2) * (3.0 / (index+2))
-        }
-        offset += delta
-
-        // new note every NOTE_LENGTH samples
-        val noteProgress = counter % note_length
-        if (noteProgress == 0) {
-          note = note.transpose(1)
-          if (note.ordinal >= Note.C7.transpose(transpose).ordinal) {
-            note = Note.C2.transpose(transpose)
           }
-          delta = note.sampleDelta
+          offset += delta
+
+          // new note every NOTE_LENGTH samples
+          val noteProgress = counter % note_length
+          // simple envelop from max to 0 every note
+          value *= (1.0 - noteProgress / note_length.toDouble())
+
+          left[sample] = value.toFloat()
+          right[sample] = value.toFloat()
+
+          if (counter >= note_length) {
+            note = null
+          }
+          counter++
         }
-        // simple envelop from max to 0 every note
-        value *= (1.0 - noteProgress / note_length.toDouble())
-
-        left[sample] = value.toFloat()
-        right[sample] = value.toFloat()
-
-        counter++
       }
     }
 

src/jsAudioWorkletMain/kotlin/MixerProcessor.kt

@@ -3,18 +3,19 @@ import org.khronos.webgl.get
 import org.khronos.webgl.set
 import org.w3c.dom.MessageEvent
 
-
 @ExperimentalJsExport
 @JsExport
 class MixerProcessor : AudioWorkletProcessor() {
   var started = false
+  val leftBuffer = Float32Array(sampleRate * 2)
+  val rightBuffer = Float32Array(sampleRate * 2)
 
   init {
     this.port.onmessage = ::handleMessage
   }
 
   private fun handleMessage(message: MessageEvent) {
-    console.log("WorkletProcessor: Received message", message)
+    console.log("MixerProcessor: Received message", message)
 
     val data = message.data
     if (data is String) {

src/jsMain/kotlin/nl/astraeus/AudioProcessorNode.kt

@@ -1,7 +1,9 @@
 package nl.astraeus
 
+import Note
 import nl.astraeus.handler.AudioNode
 import org.w3c.dom.MessageEvent
+import org.w3c.dom.MessagePort
 
 class AudioProcessorNode(
   audioModule: dynamic,
@@ -52,4 +54,18 @@ class AudioProcessorNode(
   fun length(i: Int) {
     node?.port.postMessage("set_note_length\n$i")
   }
+
+  fun play(note: Note) {
+    node?.port.postMessage("play\n$note")
+  }
+
+  fun setWorkletPort(port2: MessagePort) {
+    node?.port.postMessage(
+      createWorkletSetPortMessage(
+        "audio-iframe-message-port",
+        port2
+      ),
+      arrayOf(port2)
+    )
+  }
 }

src/jsMain/kotlin/nl/astraeus/Main.kt

@@ -1,15 +1,54 @@
 package nl.astraeus
 
 import kotlinx.browser.document
+import kotlinx.browser.window
 import nl.astraeus.handler.AudioModule
+import org.w3c.dom.HTMLIFrameElement
 import org.w3c.dom.HTMLInputElement
+import org.w3c.dom.MessageChannel
+import org.w3c.dom.MessagePort
+
+
+
+
+fun createWorkletSetPortMessage(
+  command: String,
+  port: MessagePort
+): dynamic {
+  val result = js("{}")
+
+  result.command = command
+  result.port = port
+
+  return result
+}
 
 fun main() {
   val audioModule = AudioModule("static/audio-worklet.js")
   val mixer = MixerProcessorNode(audioModule)
   var node1: AudioProcessorNode? = null
   var node2: AudioProcessorNode? = null
+  val iframeWorkletChannel = MessageChannel()
 
+  window.addEventListener("message", { event ->
+    console.log("Main Received message: ", event)
+  }, "")
+  //window.postMessage("Hello from Main 1",  window.location.origin + "/worklet.html")
+
+  document.getElementById("iframe")?.also {
+    val iframe = it as? HTMLIFrameElement
+    iframe?.also {
+      it.contentWindow?.postMessage("Hello from Main 2",  window.location.origin + "/worklet.html")
+      it.contentWindow?.postMessage(
+        createWorkletSetPortMessage(
+          "audio-processor-message-port",
+          iframeWorkletChannel.port1
+        ),
+        window.location.origin + "/worklet.html",
+        arrayOf(iframeWorkletChannel.port1)
+      )
+    }
+  }
 
   document.getElementById("createButton")?.also {
     it.addEventListener("click", {
@@ -19,6 +58,7 @@ fun main() {
 
         node1?.create {
           println("node 1 created")
+          node1?.setWorkletPort(iframeWorkletChannel.port2)
         }
         node2?.create {
           println("node 2 created")
@@ -50,13 +90,22 @@ fun main() {
       }
     }, "")
   }
-  document.getElementById("harmonics")?.also {
+  document.getElementById("note_c3")?.also {
-    it.addEventListener("change", {
+    it.addEventListener("click", {
-      val target = it.target
+      node1?.play(Note.C3)
-      if (target is HTMLInputElement) {
+      node2?.play(Note.C3)
-        node1?.harmonic(target.value.toInt())
+    }, "")
-        node2?.harmonic(target.value.toInt())
+    }
-      }
+  document.getElementById("note_e3")?.also {
+    it.addEventListener("click", {
+      node1?.play(Note.C3)
+      node2?.play(Note.G3)
+    }, "")
+  }
+  document.getElementById("note_g3")?.also {
+    it.addEventListener("click", {
+      node1?.play(Note.G3)
+      node2?.play(Note.G3)
     }, "")
   }
 

src/jsMain/kotlin/nl/astraeus/handler/AudioModule.kt

@@ -3,14 +3,6 @@ package nl.astraeus.handler
 import org.w3c.dom.MessageEvent
 import org.w3c.dom.MessagePort
 
-private val audioModules = mutableMapOf<String, AudioModule>()
-
-fun loadAudioModule(jsFile: String) {
-  val module = audioModules.getOrPut(jsFile) {
-    AudioModule(jsFile)
-  }
-}
-
 enum class ModuleStatus {
   INIT,
   LOADING,

src/jsWorkletMain/kotlin/nl/astraeus/worklet/Main.kt

@@ -0,0 +1,43 @@
+package nl.astraeus.worklet
+
+import kotlinx.browser.document
+import kotlinx.browser.window
+import org.w3c.dom.HTMLInputElement
+import org.w3c.dom.MessageEvent
+import org.w3c.dom.MessagePort
+
+fun main() {
+  var port: MessagePort? = null
+  console.log("Worklet", document.location)
+
+  window.addEventListener("message", { event ->
+    console.log("Worklet xxx Received message: ", event)
+
+    if (event is MessageEvent) {
+      val data: dynamic = event.data
+
+      console.log("DATA: ", data, data?.command == "audio-processor-message-port")
+      if (data?.command == "audio-processor-message-port") {
+        port = data.port
+        port?.onmessage = {
+          console.log("Worklet Received message from audio worklet: ", it)
+        }
+      }
+    }
+  }, "")
+
+  window.parent.postMessage(
+    "Hello from Worklet",
+    window.location.origin + "/index.html"
+  )
+
+  document.getElementById("harmonics")?.also {
+    it.addEventListener("change", {
+      val target = it.target
+      if (target is HTMLInputElement) {
+        console.log("Sending: ", target.value, port)
+        port?.postMessage("harmonics\n${target.value}")
+      }
+    }, "")
+  }
+}

src/jvmMain/kotlin/IndexHtml.kt

@@ -0,0 +1,101 @@
+import kotlinx.html.HTML
+import kotlinx.html.InputType
+import kotlinx.html.body
+import kotlinx.html.div
+import kotlinx.html.head
+import kotlinx.html.id
+import kotlinx.html.iframe
+import kotlinx.html.input
+import kotlinx.html.label
+import kotlinx.html.link
+import kotlinx.html.script
+import kotlinx.html.span
+import kotlinx.html.style
+import kotlinx.html.title
+
+fun HTML.index() {
+  head {
+    title("Hello from Ktor!")
+    link("static/worklet.css", "stylesheet" ,"text/css")
+  }
+  body {
+    div {
+      +"We need a button to start because we can only start audio from a user event:"
+    }
+    div("button_div") {
+      span("button") {
+        id = "createButton"
+
+        +"Create"
+      }
+
+      span("button") {
+        id = "startButton"
+
+        +"Start"
+      }
+
+      span("button") {
+        id = "stopButton"
+
+        +"Stop"
+      }
+    }
+    div {
+      + "An example of how to interact with the audioworklet:"
+    }
+    div {
+      label {
+        htmlFor = "noteLength"
+        +"Note length (in samples):"
+      }
+      input {
+        id = "noteLength"
+        type = InputType.number
+        value = "2500"
+        min = "1"
+        max = "100000"
+        step = "100"
+      }
+    }
+    div {
+      label {
+        htmlFor = "harmonics"
+        +"Number of harmonics:"
+      }
+      input {
+        id = "harmonics"
+        type = InputType.number
+        value = "3"
+        min = "1"
+        max = "10"
+      }
+    }
+    div {
+      input {
+        id = "note_c3"
+        type = InputType.button
+        value = "C3"
+      }
+      input {
+        id = "note_e3"
+        type = InputType.button
+        value = "E3"
+      }
+      input {
+        id = "note_g3"
+        type = InputType.button
+        value = "G3"
+      }
+    }
+
+    iframe {
+      id = "iframe"
+      src = "/worklet.html"
+      style = "width: 100%; height: 500px; background-color: #ddf;"
+    }
+
+    script(src = "/static/kotlin-audioworklet.js") {}
+  }
+}
+

src/jvmMain/kotlin/Server.kt

@@ -4,77 +4,22 @@ import io.ktor.server.engine.embeddedServer
 import io.ktor.server.html.*
 import io.ktor.server.http.content.*
 import io.ktor.server.netty.Netty
+import io.ktor.server.response.*
 import io.ktor.server.routing.*
 import kotlinx.html.*
 
-fun HTML.index() {
-  head {
-    title("Hello from Ktor!")
-    link("static/worklet.css", "stylesheet" ,"text/css")
-  }
-  body {
-    div {
-      +"We need a button to start because we can only start audio from a user event:"
-    }
-    div("button_div") {
-      span("button") {
-        id = "createButton"
-
-        +"Create"
-      }
-
-      span("button") {
-        id = "startButton"
-
-        +"Start"
-      }
-
-      span("button") {
-        id = "stopButton"
-
-        +"Stop"
-      }
-    }
-    div {
-      + "An example of how to interact with the audioworklet:"
-    }
-    div {
-      label {
-        htmlFor = "noteLength"
-        +"Note length (in samples):"
-      }
-      input {
-        id = "noteLength"
-        type = InputType.number
-        value = "2500"
-        min = "1"
-        max = "100000"
-        step = "100"
-      }
-    }
-    div {
-      label {
-        htmlFor = "harmonics"
-        +"Number of harmonics:"
-      }
-      input {
-        id = "harmonics"
-        type = InputType.number
-        value = "3"
-        min = "1"
-        max = "10"
-      }
-    }
-    script(src = "/static/kotlin-audioworklet.js") {}
-  }
-}
-
 fun main() {
   embeddedServer(Netty, port = 8080, host = "127.0.0.1") {
     routing {
       get("/") {
+        call.respondRedirect("/index.html")
+      }
+      get("/index.html") {
         call.respondHtml(HttpStatusCode.OK, HTML::index)
       }
+      get("/worklet.html") {
+        call.respondHtml(HttpStatusCode.OK, HTML::worklet)
+      }
       static("/static") {
         resources()
       }

src/jvmMain/kotlin/WorkletHtml.kt

@@ -0,0 +1,40 @@
+import kotlinx.html.HTML
+import kotlinx.html.InputType
+import kotlinx.html.body
+import kotlinx.html.div
+import kotlinx.html.head
+import kotlinx.html.id
+import kotlinx.html.iframe
+import kotlinx.html.input
+import kotlinx.html.label
+import kotlinx.html.link
+import kotlinx.html.script
+import kotlinx.html.span
+import kotlinx.html.title
+
+fun HTML.worklet() {
+  head {
+    title("Hello from Ktor!")
+    link("static/worklet.css", "stylesheet" ,"text/css")
+  }
+  body {
+    div {
+      +"IFrame js set worklet parameters:"
+    }
+    div {
+      label {
+        htmlFor = "harmonics"
+        +"Number of harmonics:"
+      }
+      input {
+        id = "harmonics"
+        type = InputType.number
+        value = "3"
+        min = "1"
+        max = "10"
+      }
+    }
+
+    script(src = "/static/html-worklet.js") {}
+  }
+}