Add detailed documentation in README.md for setup and usage

- Add check on references

.gitignore

@@ -3,6 +3,7 @@ build/
 !gradle/wrapper/gradle-wrapper.jar
 !**/src/main/**/build/
 !**/src/test/**/build/
+gradle.properties
 
 ### IntelliJ IDEA ###
 .idea/modules.xml

.idea/kotlinc.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="KotlinJpsPluginSettings">
-    <option name="version" value="2.0.0-RC2" />
+    <option name="version" value="2.1.21" />
   </component>
 </project>

.idea/misc.xml

@@ -4,7 +4,7 @@
   <component name="FrameworkDetectionExcludesConfiguration">
     <file type="web" url="file://$PROJECT_DIR$" />
   </component>
-  <component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" default="true" project-jdk-name="1.8" project-jdk-type="JavaSDK">
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_15" default="true" project-jdk-name="11" project-jdk-type="JavaSDK">
     <output url="file://$PROJECT_DIR$/out" />
   </component>
   <component name="accountSettings">

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Rien Nentjes
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

build.gradle.kts

@@ -1,9 +1,14 @@
+import com.vanniktech.maven.publish.SonatypeHost
+
 plugins {
-    kotlin("jvm") version "2.0.0-RC2"
+    kotlin("jvm") version "2.1.21"
+    signing
+    id("org.jetbrains.dokka") version "2.0.0"
+    id("com.vanniktech.maven.publish") version "0.31.0"
 }
 
 group = "nl.astraeus"
-version = "1.0-SNAPSHOT"
+version = "1.1.5-SNAPSHOT"
 
 repositories {
     mavenCentral()
@@ -15,7 +20,85 @@ dependencies {
 
 tasks.test {
     useJUnitPlatform()
+
+    minHeapSize = "128m"
+    maxHeapSize = "1g"
+}
+
+val javadocJar by tasks.registering(Jar::class) {
+    archiveClassifier.set("javadoc")
+}
+
+val sourcesJar by tasks.registering(Jar::class) {
+    archiveClassifier.set("sources")  // This sets the classifier to 'sources'
+    from(sourceSets.main.get().allSource)  // Assumes you are using the 'main' source set
+}
+
+publishing {
+    repositories {
+        mavenLocal()
+        maven {
+            name = "gitea"
+            setUrl("https://gitea.astraeus.nl/api/packages/rnentjes/maven")
+
+            credentials {
+                val giteaUsername: String? by project
+                val giteaPassword: String? by project
+
+                username = giteaUsername
+                password = giteaPassword
+            }
+        }
+        maven {
+            name = "gitea8443"
+            setUrl("https://gitea.astraeus.nl:8443/api/packages/rnentjes/maven")
+
+            credentials {
+                val giteaUsername: String? by project
+                val giteaPassword: String? by project
+
+                username = giteaUsername
+                password = giteaPassword
+            }
+        }
+    }
+}
+
+tasks.withType<AbstractPublishToMaven> {
+    dependsOn(tasks.withType<Sign>())
+}
+
+signing {
+    sign(publishing.publications)
+}
+
+mavenPublishing {
+    publishToMavenCentral(SonatypeHost.CENTRAL_PORTAL)
+
+    signAllPublications()
+
+    coordinates(group.toString(), name, version.toString())
+
+    pom {
+        name = "simple-persistence-kotlin"
+        description = "Simple persistence"
+        inceptionYear = "2024"
+        url = "https://gitea.astraeus.nl/rnentjes/simple-persistence-kotlin"
+        licenses {
+            license {
+                name = "MIT"
+                url = "https://opensource.org/licenses/MIT"
+            }
+        }
+        developers {
+            developer {
+                id = "rnentjes"
+                name = "Rien Nentjes"
+                email = "info@nentjes.com"
+            }
+        }
+        scm {
+            url = "https://gitea.astraeus.nl/rnentjes/simple-persistence-kotlin"
+        }
+    }
 }
-kotlin {
-    jvmToolchain(17)
-}

gradle.properties

@@ -1 +0,0 @@
-kotlin.code.style=official

readme.md

@@ -0,0 +1,182 @@
+# Simple Persistence for Kotlin
+
+A lightweight, file-based persistence library for Kotlin applications that provides a simple way to store, retrieve, and query objects.
+
+## Available on Maven Central
+
+```kotlin
+// Gradle Kotlin DSL
+implementation("nl.astraeus:simple-persistence-kotlin:1.1.5")
+```
+
+```groovy
+// Gradle Groovy DSL
+implementation 'nl.astraeus:simple-persistence-kotlin:1.1.5'
+```
+
+```xml
+<!-- Maven -->
+<dependency>
+    <groupId>nl.astraeus</groupId>
+    <artifactId>simple-persistence-kotlin</artifactId>
+    <version>1.1.5</version>
+</dependency>
+```
+
+## Features
+
+- Simple and intuitive API for persisting Kotlin objects
+- Transaction support with ACID properties
+- Optimistic locking to handle concurrent modifications
+- Flexible indexing for efficient querying
+- Object references and collections handling
+- Encryption support for sensitive data
+- Snapshot and transaction log capabilities
+- File-based storage with automatic cleanup
+
+## Quick Start
+
+### Define your model
+
+```kotlin
+class Person(
+  override var id: Long = 0,
+  override var version: Long = 0,
+  val name: String,
+  val age: Int,
+  company: Company? = null
+) : Persistable, Cloneable {
+  var company: Company? by nullableReference(company)
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+}
+
+class Company(
+  override var id: Long = 0,
+  override var version: Long = 0,
+  val name: String
+) : Persistable, Cloneable {
+  val persons: MutableCollection<Person> = referenceCollection()
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+}
+```
+
+### Initialize the persistence manager
+
+```kotlin
+val persistent = Persistent(
+  directory = File("data", "my-app"),
+  indexes = arrayOf(
+    index<Person>("name") { p -> (p as? Person)?.name ?: "" },
+    index<Person>("age") { p -> (p as? Person)?.age ?: -1 },
+    index<Company>("name") { p -> (p as? Company)?.name ?: "" }
+  )
+)
+```
+
+### Perform CRUD operations
+
+```kotlin
+// Create or update
+persistent.transaction {
+  val person = Person(
+    id = 1L,
+    name = "John Doe",
+    age = 30
+  )
+
+  val company = Company(
+    id = 1L,
+    name = "ACME Inc."
+  )
+
+  person.company = company
+  company.persons.add(person)
+
+  store(person)
+  store(company)
+}
+
+// Read
+persistent.query {
+  val person = find<Person>(1L)
+  println("Found person: $person")
+
+  // Query by index
+  val persons: List<Person> = findByIndex("age", 30)
+
+  // Search with predicate
+  val results = search(Person::class) { p -> p.name.startsWith("John") }
+}
+
+// Delete
+persistent.transaction {
+  val person = find<Person>(1L)
+  if (person != null) {
+    delete(person)
+  }
+}
+```
+
+## How it works
+
+- In-memory model: Your objects live in memory during normal operation for fast reads and writes.
+- Durable writes: Every create/update/delete inside a transaction is appended to on-disk transaction log files (*.trn). This is a write-ahead log approach.
+- Startup recovery: On application startup, the library replays the transaction logs (and any snapshot, if present) to rebuild the in-memory model.
+- Snapshots: You can create a compact on-disk snapshot (*.snp) of the current in-memory state. On the next startup, the snapshot is loaded first and only the newer transaction log files are replayed. This keeps startup time low without losing durability.
+
+Tip: Call snapshot() periodically (or on controlled shutdown) and use removeOldFiles() to prune obsolete logs after a successful snapshot.
+
+## Advanced Features
+
+### Indexing
+
+```kotlin
+// Define complex indexes
+val persistent = Persistent(
+  directory = File("data", "my-app"),
+  indexes = arrayOf(
+    index<Person>("ageOver30") { p -> ((p as? Person)?.age ?: 0) > 30 }
+  )
+)
+
+// Query using indexes
+persistent.query {
+  findByIndex<Person>("ageOver30", true).forEach { person ->
+    println("Person over 30: ${person.name}")
+  }
+}
+```
+
+### Transactions
+
+```kotlin
+// Read-only query
+persistent.query {
+  // Only read operations allowed
+}
+
+// Read-write transaction
+persistent.transaction {
+  // Both read and write operations allowed
+}
+```
+
+### Snapshots and Maintenance
+
+```kotlin
+// Create a snapshot of the current state
+persistent.snapshot()
+
+// Remove old transaction log files
+persistent.removeOldFiles()
+```
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.

src/main/kotlin/Main.kt

@@ -1,5 +0,0 @@
-package nl.astraeus
-
-fun main() {
-  println("Hello World!")
-}

src/main/kotlin/nl/astraeus/persistence/Datastore.kt

@@ -1,4 +1,4 @@
-package nl.astraeus.nl.astraeus.persistence
+package nl.astraeus.persistence
 
 import java.io.File
 import java.io.ObjectInputStream
@@ -7,6 +7,7 @@ import java.io.Serializable
 import java.text.DecimalFormat
 import java.util.*
 import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.atomic.AtomicLong
 import kotlin.reflect.KClass
 
 enum class ActionType {
@@ -15,30 +16,41 @@ enum class ActionType {
 }
 
 class TypeData(
-  var nextId: Long = 1L,
-  val data: MutableMap<Any, Persistable> = ConcurrentHashMap(),
-) : Serializable
+  var nextId: AtomicLong = AtomicLong(1L),
+  val data: MutableMap<Serializable, Persistable> = ConcurrentHashMap(),
+) : Serializable {
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+}
 
 class Action(
   val type: ActionType,
   val obj: Persistable
-) : Serializable
+) : Serializable {
+  override fun toString(): String {
+    return "Action(type=$type, obj=$obj)"
+  }
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+}
 
 class Datastore(
   private val directory: File,
+  val enableOptimisticLocking: Boolean = false,
+  val decryptionKey: String? = null,
+  val encryptionKey: String? = null,
   indexes: Array<PersistableIndex> = arrayOf(),
 ) {
   private val fileManager = FileManager(directory)
   private val transactionFormatter = DecimalFormat("#")
   private var nextTransactionNumber = 1L
-  private val data: MutableMap<Class<*>, TypeData> = ConcurrentHashMap()
+  private var data: MutableMap<Class<*>, TypeData> = ConcurrentHashMap()
   private val indexes: MutableMap<Class<*>, MutableMap<String, PersistableIndex>> = ConcurrentHashMap()
 
   init {
-    if (!directory.exists()) {
-      directory.mkdirs()
-    }
-
     for (index in indexes) {
       this.indexes.getOrPut(index.cls) {
         ConcurrentHashMap()
@@ -48,57 +60,90 @@ class Datastore(
     loadTransactions()
   }
 
-  private fun loadTransactions() {
-    synchronized(this) {
-      val snapshots: Array<File>? = directory.listFiles { _, name -> name.startsWith("transaction-") && name.endsWith(".snp") }
-      val files: Array<File>? = directory.listFiles { _, name -> name.startsWith("transaction-") && name.endsWith(".trn") }
-
-      var lastSnapshot: Long? = null
-      var lastSnapshotFile: File? = null
-
-      snapshots?.let {
-        it.forEach {
-          val trnx = getTrnx(it)
-          if (lastSnapshot == null || trnx > (lastSnapshot ?: 0L)) {
-            lastSnapshot = trnx
-            lastSnapshotFile = it
-          }
+  fun getNextId(javaClass: Class<Persistable>): Long {
+    if (data[javaClass] == null) {
+      synchronized(this) {
+        if (data[javaClass] == null) {
+          data[javaClass] = TypeData()
         }
       }
+    }
 
-      val lastSnapshotFile2 = fileManager.findLastSnapshot()
+    return data[javaClass]!!.nextId.get()
+  }
 
-      if (lastSnapshotFile != null) {
-        ObjectInputStream(lastSnapshotFile?.inputStream()).use { ois ->
-          readSnapshot(ois)
+  fun getNextIdAndIncrement(javaClass: Class<Persistable>): Long {
+    if (data[javaClass] == null) {
+      synchronized(this) {
+        if (data[javaClass] == null) {
+          data[javaClass] = TypeData()
         }
       }
+    }
 
-      val trns = fileManager.findTransactionsAfter(lastSnapshot ?: 0L)
+    return data[javaClass]!!.nextId.getAndIncrement()
+  }
 
-      files?.also { snaphotFiles ->
-        Arrays.sort(snaphotFiles) { o1, o2 -> if (getTrnx(o1) > getTrnx(o2)) 1 else -1}
+  fun setMaxId(javaClass: Class<Persistable>, id: Long) {
+    val nextId = data.getOrPut(javaClass) { TypeData() }.nextId
+    val current = nextId.get()
+    if (current <= id) nextId.addAndGet(id - current)
+  }
 
-        snaphotFiles.forEach { file ->
-          if (getTrnx(file) > (lastSnapshot ?: 0L)) {
-            ObjectInputStream(file.inputStream()).use { ois ->
-              val transactionNumber = ois.readLong()
-              nextTransactionNumber = transactionNumber + 1
-              val actions = ois.readObject() as MutableList<Action>
-              execute(actions)
-            }
-          }
-        }
+  override fun toString(): String {
+    return "Datastore(directory=${fileManager.directory}, classes=${data.keys.size}, indexes=${indexes.keys.size})"
+  }
+
+  // print status, show number of entries for each class and index
+  fun printStatus() {
+    println(this)
+    for ((cls, typeData) in data) {
+      println("  ${cls.simpleName}: ${typeData.data.size}")
+      for ((name, index) in indexes.getOrDefault(cls, mutableMapOf())) {
+        println("    $name: ${index.index.size}")
       }
     }
   }
 
-  private fun getTrnx(file: File): Long {
-    return file.name.substringAfterLast('/').substringAfter("transaction-").substringBefore(".").toLong()
+  private fun loadTransactions() {
+    val start = System.nanoTime()
+
+    synchronized(this) {
+      val (lastSnapshot, lastSnapshotFile) = fileManager.findLastSnapshot()
+
+      if (lastSnapshotFile != null) {
+        ObjectInputStream(DecryptingInputStream(lastSnapshotFile.inputStream(), decryptionKey)).use { ois ->
+          readSnapshot(ois)
+        }
+      }
+
+      val transactions = fileManager.findTransactionsAfter(lastSnapshot ?: 0L)
+
+      transactions?.forEach { file ->
+        ObjectInputStream(DecryptingInputStream(file.inputStream(), decryptionKey)).use { ois ->
+          readTransaction(ois)
+        }
+      }
+    }
+
+    Logger.debug("Loaded transactions in %6.3fms", ((System.nanoTime() - start) / 1_000_000f))
   }
 
-  fun execute(actions: MutableList<Action>) {
+  private fun execute(actions: List<Action>) {
     synchronized(this) {
+      if (enableOptimisticLocking) {
+        for (action in actions) {
+          val typeData = data.getOrPut(action.obj::class.java) {
+            TypeData()
+          }
+          if (action.type == ActionType.STORE) {
+            if ((typeData.data[action.obj.id]?.version ?: -1L) >= action.obj.version) {
+              throw OptimisticLockingException(action.obj)
+            }
+          }
+        }
+      }
+
       for (action in actions) {
         val typeData = data.getOrPut(action.obj::class.java) {
           TypeData()
@@ -106,13 +151,15 @@ class Datastore(
 
         when (action.type) {
           ActionType.STORE -> {
-            if (action.obj.id == 0L) {
-              action.obj.id = typeData.nextId++
-            }
             typeData.data[action.obj.id] = action.obj
 
+            if (action.obj.id >= typeData.nextId.get()) {
+              typeData.nextId.set(action.obj.id + 1)
+            }
+
             for (index in indexes[action.obj::class.java]?.values ?: listOf()) {
-              index.add(action.obj as Persistable)
+              index.remove(action.obj.id)
+              index.add(action.obj)
             }
           }
 
@@ -120,7 +167,7 @@ class Datastore(
             typeData.data.remove(action.obj.id)
 
             for (index in indexes[action.obj::class.java]?.values ?: listOf()) {
-              index.remove(action.obj)
+              index.remove(action.obj.id)
             }
           }
         }
@@ -128,6 +175,14 @@ class Datastore(
     }
   }
 
+  fun <T : Persistable> count(clazz: KClass<T>): Int {
+    val typeData = data.getOrPut(clazz.java) {
+      TypeData()
+    }
+
+    return typeData.data.size
+  }
+
   fun <T : Persistable> find(clazz: KClass<T>, id: Long): T? {
     val typeData = data.getOrPut(clazz.java) {
       TypeData()
@@ -149,6 +204,7 @@ class Datastore(
     return typeData.data.values
       .filter { search(it as T) }
       .map { o -> o.copy() as T }
+      .sortedBy { it.id }
   }
 
   fun findIndex(
@@ -158,24 +214,52 @@ class Datastore(
     return indexes[kClass.java]?.get(indexName)
   }
 
-  fun storeActions(actions: MutableList<Action>) {
+  fun storeAndExecute(actions: List<Action>) {
     if (actions.isNotEmpty()) {
       synchronized(this) {
-        val number = transactionFormatter.format(nextTransactionNumber)
-        val file = File(directory, "transaction-$number.trn")
-        ObjectOutputStream(file.outputStream()).use { oos ->
-          oos.writeLong(nextTransactionNumber++)
-          oos.writeObject(actions)
-        }
+        execute(actions)
+        writeTransaction(actions)
       }
     }
   }
 
+  private fun readTransaction(ois: ObjectInputStream) {
+    val versionNumber = ois.readInt()
+    check(versionNumber == 1) { "Unsupported version number: $versionNumber" }
+    val transactionNumber = ois.readLong()
+    nextTransactionNumber = transactionNumber + 1
+    val actions = ois.readObject()
+    when(actions) {
+      is Set<*> -> {
+        val list = LinkedList(actions as Set<Action>)
+        execute(list)
+      }
+      is List<*> -> {
+        execute(actions as List<Action>)
+      }
+    }
+  }
+
+  private fun writeTransaction(actions: List<Action>) {
+    val number = transactionFormatter.format(nextTransactionNumber)
+    val file = File(directory, "transaction-$number.trn-tmp")
+    ObjectOutputStream(EncryptingOutputStream(file.outputStream(), encryptionKey)).use { oos ->
+      // version number
+      oos.writeInt(1)
+      oos.writeLong(nextTransactionNumber++)
+      oos.writeObject(actions)
+    }
+    file.renameTo(File(directory, "transaction-$number.trn"))
+  }
+
   fun snapshot() {
+    val start = System.nanoTime()
     synchronized(this) {
       val number = transactionFormatter.format(nextTransactionNumber)
-      val file = File(directory, "transaction-$number.snp")
-      ObjectOutputStream(file.outputStream()).use { oos ->
+      val file = File(directory, "transaction-$number.snp-tmp")
+      ObjectOutputStream(EncryptingOutputStream(file.outputStream(), encryptionKey)).use { oos ->
+        // version number
+        oos.writeInt(1)
         oos.writeLong(nextTransactionNumber++)
         oos.writeObject(data)
         oos.writeInt(indexes.size)
@@ -188,13 +272,17 @@ class Datastore(
           }
         }
       }
+      file.renameTo(File(directory, "transaction-$number.snp"))
     }
+    Logger.debug("Snapshot in %6.3fms", ((System.nanoTime() - start) / 1_000_000f))
   }
 
   private fun readSnapshot(ois: ObjectInputStream) {
+    val versionNumber = ois.readInt()
+    check(versionNumber == 1) { "Unsupported version number: $versionNumber" }
     nextTransactionNumber = ois.readLong() + 1
-    data.clear()
-    data.putAll(ois.readObject() as MutableMap<Class<*>, TypeData>)
+    val dataObj = ois.readObject()
+    data = dataObj as ConcurrentHashMap<Class<*>, TypeData>
 
     val foundIndexes = mutableMapOf<Class<*>, MutableList<String>>()
     val numberOfClassesWithIndex = ois.readInt()
@@ -226,4 +314,8 @@ class Datastore(
     }
   }
 
+  fun removeOldFiles() {
+    fileManager.removeOldFiles()
+  }
+
 }

src/main/kotlin/nl/astraeus/persistence/Encryption.kt

@@ -0,0 +1,150 @@
+@file:OptIn(ExperimentalEncodingApi::class, ExperimentalEncodingApi::class)
+
+package nl.astraeus.persistence
+
+import java.io.ByteArrayOutputStream
+import java.io.InputStream
+import java.io.OutputStream
+import java.security.SecureRandom
+import javax.crypto.Cipher
+import javax.crypto.KeyGenerator
+import javax.crypto.SecretKey
+import javax.crypto.spec.IvParameterSpec
+import javax.crypto.spec.SecretKeySpec
+import kotlin.io.encoding.Base64
+import kotlin.io.encoding.ExperimentalEncodingApi
+
+class Encryptor(
+  base64EncryptionKey: String?,
+  base64DecryptionKey: String?,
+) {
+  private var decryptionKey: SecretKey? = null
+  private var encryptionKey: SecretKey? = null
+
+  init {
+    if (base64EncryptionKey?.isNotEmpty() == true) {
+      encryptionKey = SecretKeySpec(Base64.UrlSafe.decode(base64EncryptionKey), "AES")
+    }
+    if (base64DecryptionKey?.isNotEmpty() == true) {
+      decryptionKey = SecretKeySpec(Base64.UrlSafe.decode(base64DecryptionKey), "AES")
+    }
+  }
+
+  fun encrypt(data: ByteArray): ByteArray {
+    if (encryptionKey == null) {
+      return data
+    }
+
+    val prePaddedData =  ByteArray(16) + data
+    val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding")
+
+    // Generate a new IV (Initialization Vector)
+    val secureRandom = SecureRandom()
+    val iv = ByteArray(cipher.blockSize)
+    secureRandom.nextBytes(iv)
+    val ivParams = IvParameterSpec(iv)
+
+    cipher.init(Cipher.ENCRYPT_MODE, encryptionKey, ivParams)
+    return cipher.doFinal(prePaddedData)
+  }
+
+  fun decrypt(data: ByteArray): ByteArray {
+    if (decryptionKey == null) {
+      return data
+    }
+
+    val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding")
+    val secureRandom = SecureRandom()
+    val iv = ByteArray(cipher.blockSize)
+    secureRandom.nextBytes(iv)
+    cipher.init(Cipher.DECRYPT_MODE, decryptionKey, IvParameterSpec(iv))
+    val completeData = cipher.doFinal(data)
+
+    return completeData.sliceArray(16 until completeData.size)
+  }
+}
+
+fun generateBase64Key(): String {
+  val keyGen: KeyGenerator = KeyGenerator.getInstance("AES")
+  keyGen.init(256) // for AES-256
+  val secretKey: SecretKey = keyGen.generateKey()
+
+  return Base64.UrlSafe.encode(secretKey.encoded)
+}
+
+/*
+object Encryption {
+  var encryptor = Encryptor(
+    System.getenv().getOrDefault("SPK_ENCRYPTION_KEY", ""),
+    System.getenv().getOrDefault("SPK_DECRYPTION_KEY", "")
+  )
+}
+*/
+
+class DecryptingInputStream(
+  val input: InputStream,
+  val base64DecryptionKey: String?
+) : InputStream() {
+  val bytes: ByteArray
+  var index = 0
+
+  init {
+    val encryptedBytes = input.readAllBytes()
+    if (base64DecryptionKey?.isBlank() == true) {
+      bytes = encryptedBytes
+    } else {
+      val encryptor = Encryptor(
+        base64EncryptionKey = null,
+        base64DecryptionKey = base64DecryptionKey
+      )
+      bytes = encryptor.decrypt(encryptedBytes)
+    }
+  }
+
+/*  override fun readAllBytes(): ByteArray {
+    index = bytes.size
+    return bytes
+  }*/
+
+  override fun read(): Int {
+    return if (index < bytes.size) {
+      bytes[index++].toUByte().toInt()
+    } else {
+      -1
+    }
+  }
+
+  override fun close() {
+    input.close()
+  }
+}
+
+class EncryptingOutputStream(
+  val output: OutputStream,
+  val base64EncryptionKey: String?
+) : OutputStream() {
+  val baos = ByteArrayOutputStream()
+
+  override fun write(b: Int) {
+    baos.write(b)
+  }
+
+  override fun close() {
+    if (base64EncryptionKey?.isBlank() == true) {
+      output.write(baos.toByteArray())
+    } else {
+      val encryptor = Encryptor(
+        base64EncryptionKey = base64EncryptionKey,
+        base64DecryptionKey = null
+      )
+      val encryptedBytes = encryptor.encrypt(baos.toByteArray())
+      output.write(encryptedBytes)
+    }
+    output.flush()
+    output.close()
+  }
+
+  override fun flush() {
+    // no flush
+  }
+}

src/main/kotlin/nl/astraeus/persistence/FileManager.kt

@@ -0,0 +1,62 @@
+package nl.astraeus.persistence
+
+import java.io.File
+
+class FileManager(
+  val directory: File
+) {
+
+  init {
+    if (!directory.exists()) {
+      directory.mkdirs()
+    }
+  }
+
+  fun findLastSnapshot(): Pair<Long?, File?> {
+    directory.listFiles { _, name ->
+      name.startsWith("transaction-") && name.endsWith(".snp")
+    }?.maxByOrNull {
+      getTrnx(it)
+    }?.also { file ->
+      return getTrnx(file) to file
+    }
+
+    return null to null
+  }
+
+  fun findTransactionsAfter(trnx: Long): List<File>? {
+    return directory.listFiles { _, name ->
+      name.startsWith("transaction-") && name.endsWith(".trn")
+    }?.filter {
+      getTrnx(it) > trnx
+    }?.sortedBy {
+      getTrnx(it)
+    }
+  }
+
+  private fun getTrnx(file: File): Long {
+    // todo: add checks, improve performance
+    return file.name
+      .substringAfterLast('/')
+      .substringAfter("transaction-")
+      .substringBefore(".")
+      .toLong()
+  }
+
+  fun removeOldFiles() {
+    val (lastSnapshot, _) = findLastSnapshot()
+
+    if (lastSnapshot != null) {
+      val files = directory.listFiles { _, name ->
+        name.startsWith("transaction-")
+      }?.filter {
+        getTrnx(it) < lastSnapshot
+      }
+
+      files?.forEach {
+        it.delete()
+      }
+    }
+  }
+
+}

src/main/kotlin/nl/astraeus/persistence/Indexing.kt

@@ -1,18 +1,32 @@
-package nl.astraeus.nl.astraeus.persistence
+package nl.astraeus.persistence
 
 import java.io.Serializable
 import kotlin.reflect.KClass
 
 typealias PersistableIndex = Index<out Persistable>
 
+inline fun <reified T : Persistable> index(
+  name: String,
+  noinline value: (Persistable) -> Serializable?
+): Index<T> = Index(
+  T::class,
+  name,
+  value
+)
+
 class Index<T : Persistable>(
-  kcls: KClass<T>,
+  val cls: Class<T>,
   val name: String,
   val value: (Persistable) -> Serializable?,
 ) : Serializable {
-  val cls: Class<T> = kcls.java
   val index = mutableMapOf<Serializable, MutableSet<Long>>()
 
+  constructor(
+    cls: KClass<T>,
+    name: String,
+    value: (Persistable) -> Serializable?
+  ) : this(cls.java, name, value)
+
   fun add(obj: Persistable) {
     val key = value(obj)
 
@@ -27,11 +41,17 @@ class Index<T : Persistable>(
     index[key]?.remove(obj.id)
   }
 
+  fun remove(id: Long) {
+    for ((_, ids) in index) {
+      ids.remove(id)
+    }
+  }
+
   fun find(key: Any): List<T> {
     return index[key]?.mapNotNull { currentTransaction()?.find(cls.kotlin, it) } ?: emptyList()
   }
 
-  fun matches(obj: Persistable, value: Any): Boolean {
+  fun matches(obj: Persistable, value: Serializable): Boolean {
     return value(obj) == value
   }
 }

src/main/kotlin/nl/astraeus/persistence/Logger.kt

@@ -0,0 +1,62 @@
+package nl.astraeus.persistence
+
+enum class LogLevel {
+  TRACE,
+  DEBUG,
+  INFO,
+  WARN,
+  ERROR
+}
+
+object Logger {
+  var level: LogLevel = LogLevel.DEBUG
+
+  var tracePrinter: (String) -> Unit = { println(it) }
+  var debugPrinter: (String) -> Unit = { println(it) }
+  var infoPrinter: (String) -> Unit = { println(it) }
+  var warnPrinter: (String) -> Unit = { println(it) }
+  var errorPrinter: (String) -> Unit = { System.err.println(it) }
+
+  fun trace(message: String, vararg parameters: Any?) {
+    if (level <= LogLevel.TRACE) {
+      writeLogMessage(LogLevel.TRACE, message, *parameters)
+    }
+  }
+
+  fun debug(message: String, vararg parameters: Any?) {
+    if (level <= LogLevel.DEBUG) {
+      writeLogMessage(LogLevel.DEBUG, message, *parameters)
+    }
+  }
+
+  fun info(message: String, vararg parameters: Any?) {
+    if (level <= LogLevel.INFO) {
+      writeLogMessage(LogLevel.INFO, message, *parameters)
+    }
+  }
+
+  fun warn(message: String, vararg parameters: Any?) {
+    if (level <= LogLevel.DEBUG) {
+      writeLogMessage(LogLevel.DEBUG, message, *parameters)
+    }
+  }
+
+  fun error(message: String, vararg parameters: Any?) {
+    if (level <= LogLevel.ERROR) {
+      writeLogMessage(LogLevel.ERROR, message, *parameters)
+    }
+  }
+
+  private fun writeLogMessage(level: LogLevel, message: String, vararg parameters: Any?) {
+    val formattedMessage = "[${level}] - ${message.format(*parameters)}"
+
+    when (level) {
+      LogLevel.TRACE -> tracePrinter(formattedMessage)
+      LogLevel.DEBUG -> debugPrinter(formattedMessage)
+      LogLevel.INFO -> infoPrinter(formattedMessage)
+      LogLevel.WARN -> warnPrinter(formattedMessage)
+      LogLevel.ERROR -> errorPrinter(formattedMessage)
+    }
+
+  }
+}

src/main/kotlin/nl/astraeus/persistence/OptimisticLockingException.kt

@@ -0,0 +1,18 @@
+package nl.astraeus.persistence
+
+class OptimisticLockingException : Exception {
+  constructor(
+    obj: Persistable
+  ) : this("Optimistic locking failed for ${obj.javaClass.simpleName} with id ${obj.id}, version ${obj.version}")
+
+  constructor() : super()
+  constructor(message: String?) : super(message)
+  constructor(message: String?, cause: Throwable?) : super(message, cause)
+  constructor(cause: Throwable?) : super(cause)
+  constructor(message: String?, cause: Throwable?, enableSuppression: Boolean, writableStackTrace: Boolean) : super(
+    message,
+    cause,
+    enableSuppression,
+    writableStackTrace
+  )
+}

src/main/kotlin/nl/astraeus/persistence/Persistable.kt

@@ -1,4 +1,4 @@
-package nl.astraeus.nl.astraeus.persistence
+package nl.astraeus.persistence
 
 import java.io.ByteArrayInputStream
 import java.io.ByteArrayOutputStream
@@ -6,7 +6,7 @@ import java.io.ObjectInputStream
 import java.io.ObjectOutputStream
 import java.io.Serializable
 
-interface Persistable : Serializable, Cloneable {
+interface Persistable : Serializable {
   var id: Long
   var version: Long
 
@@ -17,9 +17,21 @@ interface Persistable : Serializable, Cloneable {
       }
       ByteArrayInputStream(baos.toByteArray()).use { bais ->
         ObjectInputStream(bais).use { ois ->
-          return ois.readObject() as Persistable
+          val result = ois.readObject() as Persistable
+          result.version++
+          return result
         }
       }
     }
   }
 }
+
+abstract class AbstractPersistable : Persistable {
+  override fun copy(): Persistable {
+    return super.copy()
+  }
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+}

src/main/kotlin/nl/astraeus/persistence/Persistent.kt

@@ -1,4 +1,4 @@
-package nl.astraeus.nl.astraeus.persistence
+package nl.astraeus.persistence
 
 import java.io.File
 
@@ -10,9 +10,34 @@ fun currentTransaction(): Transaction? {
 
 class Persistent(
   directory: File,
+  enableOptimisticLocking: Boolean = false,
+  decryptionKey: String? = null,
+  encryptionKey: String? = null,
   indexes: Array<PersistableIndex> = arrayOf(),
 ) {
-  val datastore: Datastore = Datastore(directory, indexes)
+  val datastore: Datastore = Datastore(
+    directory,
+    enableOptimisticLocking,
+    decryptionKey,
+    encryptionKey,
+    indexes
+  )
+
+  fun <T> query(block: Query.() -> T): T {
+    var cleanup = false
+    if (transactions.get() == null) {
+      transactions.set(Transaction(this))
+      cleanup = true
+    }
+
+    try {
+      return block(Query(this))
+    } finally {
+      if (cleanup) {
+        transactions.remove()
+      }
+    }
+  }
 
   fun transaction(block: Transaction.() -> Unit) {
     var cleanup = false
@@ -24,7 +49,9 @@ class Persistent(
     try {
       block(transactions.get())
 
-      transactions.get().commit()
+      if (cleanup) {
+        transactions.get().commit()
+      }
     } finally {
       if (cleanup) {
         transactions.remove()
@@ -35,4 +62,8 @@ class Persistent(
   fun snapshot() {
     datastore.snapshot()
   }
+
+  fun removeOldFiles() {
+    datastore.removeOldFiles()
+  }
 }

src/main/kotlin/nl/astraeus/persistence/Reference.kt

@@ -1,133 +0,0 @@
-package nl.astraeus.nl.astraeus.persistence
-
-import java.io.Serializable
-import kotlin.reflect.KProperty
-
-class Reference<S : Persistable, H : Persistable>(
-  val cls: Class<S>,
-) : Serializable {
-  companion object {
-    private const val serialVersionUID: Long = 1L
-  }
-
-  var id: Long = 0
-
-  operator fun getValue(thisRef: H, property: KProperty<*>): S {
-    return currentTransaction()?.find(cls.kotlin, id) ?: throw IllegalStateException("Reference not found")
-  }
-
-  operator fun setValue(thisRef: H, property: KProperty<*>, value: S) {
-    id = value.id
-    currentTransaction()?.store(value)
-  }
-
-}
-
-class ListReference<S : Persistable, H : Persistable>(
-  val cls: Class<S>,
-) : Serializable {
-  companion object {
-    private const val serialVersionUID: Long = 1L
-  }
-
-  var ids: ReferenceList<S> = ReferenceList(cls)
-
-  operator fun getValue(thisRef: H, property: KProperty<*>): ReferenceList<S> {
-    return ids
-  }
-
-  operator fun setValue(thisRef: H, property: KProperty<*>, value: List<S>) {
-    this.ids.clear()
-    this.ids.addAll(value)
-  }
-}
-
-class ReferenceList<T : Persistable>(
-  val cls: Class<T>,
-) : MutableList<T> {
-  val ids = ArrayList<Long>()
-
-  private fun checkElementIsPersisted(element: T) {
-    if (currentTransaction()?.find(cls.kotlin, element.id) == null) {
-      currentTransaction()?.store(element)
-    }
-  }
-
-  override val size: Int = ids.size
-  override fun clear() = ids.clear()
-
-  override fun addAll(elements: Collection<T>): Boolean {
-    TODO("Not yet implemented")
-  }
-
-  override fun addAll(index: Int, elements: Collection<T>): Boolean {
-    TODO("Not yet implemented")
-  }
-
-  override fun add(index: Int, element: T) {
-    ids.add(index, element.id)
-  }
-
-  override fun add(element: T): Boolean {
-    return ids.add(element.id)
-  }
-
-  override fun get(index: Int): T = currentTransaction()?.find(cls.kotlin, ids[index]) ?: throw IllegalStateException("Reference not found")
-
-  override fun isEmpty(): Boolean = ids.isEmpty()
-
-  override fun iterator(): MutableIterator<T> {
-    TODO("Not yet implemented")
-  }
-
-  override fun listIterator(): MutableListIterator<T> {
-    TODO("Not yet implemented")
-  }
-
-  override fun listIterator(index: Int): MutableListIterator<T> {
-    TODO("Not yet implemented")
-  }
-
-  override fun removeAt(index: Int): T {
-    val id = ids.removeAt(index)
-
-    return currentTransaction()?.find(cls.kotlin, id) ?: throw IllegalStateException("Reference not found")
-  }
-
-  override fun set(index: Int, element: T): T {
-    TODO("Not yet implemented")
-  }
-
-  override fun retainAll(elements: Collection<T>): Boolean {
-    TODO("Not yet implemented")
-  }
-
-  override fun removeAll(elements: Collection<T>): Boolean {
-    TODO("Not yet implemented")
-  }
-
-  override fun remove(element: T): Boolean {
-    TODO("Not yet implemented")
-  }
-
-  override fun subList(fromIndex: Int, toIndex: Int): MutableList<T> {
-    TODO("Not yet implemented")
-  }
-
-  override fun lastIndexOf(element: T): Int {
-    TODO("Not yet implemented")
-  }
-
-  override fun indexOf(element: T): Int {
-    TODO("Not yet implemented")
-  }
-
-  override fun containsAll(elements: Collection<T>): Boolean {
-    TODO("Not yet implemented")
-  }
-
-  override fun contains(element: T): Boolean {
-    TODO("Not yet implemented")
-  }
-
-}

src/main/kotlin/nl/astraeus/persistence/Transaction.kt

@@ -1,14 +1,113 @@
-package nl.astraeus.nl.astraeus.persistence
+package nl.astraeus.persistence
 
 import java.io.Serializable
 import kotlin.reflect.KClass
 
-class Transaction(
+inline fun <reified T : Persistable> Query.count(): Int = this.count(T::class)
+inline fun <reified T : Persistable> Query.find(id: Long): T? = this.find(T::class, id)
+inline fun <reified T : Persistable> Query.all(): List<T> = this.all(T::class)
+inline fun <reified T : Persistable> Query.search(noinline search: (T) -> Boolean): List<T> =
+  this.search(T::class, search)
+inline fun <reified T : Persistable> Query.findByIndex(
+  indexName: String,
+  search: Serializable
+): List<T> = this.findByIndex(T::class, indexName, search)
+inline fun <reified T : Persistable> Query.searchIndex(
+  indexName: String,
+  noinline search: (Serializable) -> Boolean,
+): List<T> = this.searchIndex(T::class, indexName, search)
+
+inline fun <reified T : Persistable> Transaction.count(): Int = this.count(T::class)
+inline fun <reified T : Persistable> Transaction.find(id: Long): T? = this.find(T::class, id)
+inline fun <reified T : Persistable> Transaction.search(noinline search: (T) -> Boolean): List<T> =
+  this.search(T::class, search)
+inline fun <reified T : Persistable> Transaction.findByIndex(
+  indexName: String,
+  search: Serializable
+): List<T> = this.findByIndex(T::class, indexName, search)
+inline fun <reified T : Persistable> Transaction.searchIndex(
+  indexName: String,
+  noinline search: (Serializable) -> Boolean,
+): List<T> = this.searchIndex(T::class, indexName, search)
+
+open class Query(
   val persistent: Persistent,
 ) : Serializable {
-  private val actions = mutableListOf<Action>()
+
+  fun <T : Persistable> count(clazz: Class<T>): Int = count(clazz.kotlin)
+  fun <T : Persistable> count(clazz: KClass<T>): Int = persistent.datastore.count(clazz)
+
+  fun <T : Persistable> find(clazz: Class<T>, id: Long): T? {
+    return find(clazz.kotlin, id)
+  }
+
+  open fun <T : Persistable> find(clazz: KClass<T>, id: Long): T? = persistent.datastore.find(clazz, id)
+
+  open fun <T : Persistable> all(clazz: Class<T>): List<T> = all(clazz.kotlin)
+  open fun <T : Persistable> all(clazz: KClass<T>): List<T> = search(clazz) { _ -> true }
+
+  open fun <T : Persistable> search(clazz: KClass<T>, search: (T) -> Boolean): List<T> = persistent.datastore.search(clazz, search)
+
+  fun <T : Persistable> findByIndex(
+    kcls: KClass<T>,
+    indexName: String,
+    search: Serializable
+  ): List<T> = findByIndex(kcls.java, indexName, search)
+
+  open fun <T : Persistable> findByIndex(
+    cls: Class<T>,
+    indexName: String,
+    search: Serializable
+  ): List<T> {
+    val result = mutableListOf<T>()
+    val index = persistent.datastore.findIndex(cls.kotlin, indexName)
+      ?: throw IllegalArgumentException("Index with name $indexName not found for class ${cls.simpleName}")
+
+    index.find(search).forEach { id ->
+      result.add(id as T)
+    }
+
+    return result
+  }
+
+  fun <T : Persistable> searchIndex(
+    kcls: KClass<T>,
+    indexName: String,
+    search: (Serializable) -> Boolean,
+  ): List<T> = searchIndex(kcls.java, indexName, search)
+
+  open fun <T : Persistable> searchIndex(
+    cls: Class<T>,
+    indexName: String,
+    search: (Serializable) -> Boolean,
+  ): List<T> {
+    val result = mutableListOf<T>()
+    val index = persistent.datastore.findIndex(cls.kotlin, indexName) ?: throw IllegalArgumentException("Index not found")
+
+    index.index.keys.forEach { key ->
+      if (search(key)) {
+        index.find(key).forEach { id ->
+          result.add(id as T)
+        }
+      }
+    }
+
+    return result
+  }
+}
+
+class Transaction(
+  persistent: Persistent,
+) : Query(persistent), Serializable {
+  private val actions = ArrayList<Action>()
 
   fun store(obj: Persistable) {
+    if (obj.id == 0L) {
+      obj.id = persistent.datastore.getNextIdAndIncrement(obj.javaClass)
+    } else if (obj.id > persistent.datastore.getNextId(obj.javaClass)) {
+      persistent.datastore.setMaxId(obj.javaClass, obj.id + 1)
+    }
+
     actions.add(Action(ActionType.STORE, obj))
   }
 
@@ -16,23 +115,23 @@ class Transaction(
     actions.add(Action(ActionType.DELETE, obj))
   }
 
-  fun <T : Persistable> find(clazz: KClass<T>, id: Long): T? {
-    var result: T? = persistent.datastore.find(clazz, id)
+  fun commit() {
+    persistent.datastore.storeAndExecute(actions)
+    actions.clear()
+  }
+
+  override fun <T : Persistable> find(clazz: KClass<T>, id: Long): T? {
+    var result = super.find(clazz, id)
 
     for (action in actions) {
       if (action.obj::class == clazz && action.obj.id == id) {
-        result = when {
-          action.type == ActionType.DELETE -> {
+        result = when(action.type) {
+          ActionType.DELETE -> {
             null
           }
-
-          action.type == ActionType.STORE -> {
+          ActionType.STORE -> {
             action.obj as? T
           }
-
-          else -> {
-            result
-          }
         }
       }
     }
@@ -40,10 +139,9 @@ class Transaction(
     return result
   }
 
-  fun <T : Persistable> search(clazz: KClass<T>, search: (T) -> Boolean): List<T> {
-    val fromDatastore: List<T> = persistent.datastore.search(clazz, search)
+  override fun <T : Persistable> search(clazz: KClass<T>, search: (T) -> Boolean): List<T> {
     val result = mutableListOf<T>()
-    result.addAll(fromDatastore)
+    result.addAll(super.search(clazz, search))
 
     for (obj in result) {
       for (action in actions) {
@@ -61,26 +159,13 @@ class Transaction(
     return result
   }
 
-  fun commit() {
-    persistent.datastore.storeActions(actions)
-    persistent.datastore.execute(actions)
-    actions.clear()
-  }
-
-  fun <T : Persistable> findByIndex(
-    kClass: KClass<T>,
-    indexName: String,
-    search: Any
-  ): List<T> {
+  override fun <T : Persistable> findByIndex(cls: Class<T>, indexName: String, search: Serializable): List<T> {
     val result = mutableListOf<T>()
-    val index = persistent.datastore.findIndex(kClass, indexName) ?: throw IllegalArgumentException("Index not found")
-
-    index.find(search).forEach { id ->
-      result.add(id as T)
-    }
+    val index = persistent.datastore.findIndex(cls.kotlin, indexName) ?: throw IllegalArgumentException("Index not found")
+    result.addAll(super.findByIndex(cls, indexName, search))
 
     for (action in actions) {
-      if (action.obj::class == kClass) {
+      if (action.obj::class == cls.kotlin) {
         if (action.type == ActionType.DELETE) {
           if (index.matches(action.obj, search)) {
             result.remove(action.obj as T)
@@ -97,4 +182,25 @@ class Transaction(
     return result
   }
 
+  override fun <T : Persistable> searchIndex(cls: Class<T>, indexName: String, search: (Serializable) -> Boolean): List<T> {
+    val result = mutableListOf<T>()
+    val index = persistent.datastore.findIndex(cls.kotlin, indexName) ?: throw IllegalArgumentException("Index not found")
+    result.addAll(super.searchIndex(cls, indexName, search))
+
+    for (action in actions) {
+      if (action.obj::class == cls.kotlin) {
+        val indexedValue = index.value(action.obj)
+        if (indexedValue != null && index.matches(action.obj, indexedValue)) {
+          if (action.type == ActionType.DELETE) {
+            result.remove(action.obj as T)
+          } else if (action.type == ActionType.STORE) {
+            result.remove(action.obj)
+            result.add(action.obj as T)
+          }
+        }
+      }
+    }
+
+    return result
+  }
 }

src/main/kotlin/nl/astraeus/persistence/TransactionLog.kt

@@ -0,0 +1,68 @@
+package nl.astraeus.persistence
+
+import java.io.File
+import java.io.ObjectInputStream
+import java.util.*
+
+class TransactionLog(
+  directory: File,
+  val decryptionKey: String? = null,
+) {
+  val fileManager = FileManager(directory)
+
+  fun showTransactions(printer: (String) -> Unit = ::println) {
+    fileManager.findLastSnapshot().let { (after, snapshot) ->
+      printer("Last snapshot: $snapshot")
+
+      printer("Snapshot:")
+      snapshot?.inputStream()?.use { input ->
+        ObjectInputStream(DecryptingInputStream(input, decryptionKey)).use { ois ->
+          val versionNumber = ois.readInt()
+          check(versionNumber == 1) {
+            "Unsupported version number: $versionNumber"
+          }
+          val transactionNumber = ois.readLong()
+          printer("[$versionNumber]  $transactionNumber")
+          val dataObj = ois.readObject()
+          val data = dataObj as MutableMap<Class<*>, TypeData>
+          printer("Data:")
+          printer("\tClasses:")
+          for ((cls, entries) in  data.entries) {
+            printer("\t\t- $cls: ${entries.data.keys.size}")
+          }
+        }
+      }
+
+      val transactions = fileManager.findTransactionsAfter(after ?: 0L)
+
+      printer("Transactions:")
+      transactions?.forEach { transaction ->
+        transaction.inputStream().use { input ->
+          ObjectInputStream(DecryptingInputStream(input, decryptionKey)).use { ois ->
+            val versionNumber = ois.readInt()
+            check(versionNumber == 1) {
+              "Unsupported version number: $versionNumber"
+            }
+            val transactionNumber = ois.readLong()
+            val actions = ois.readObject()
+            val actionList = when(actions) {
+              is Set<*> -> {
+                LinkedList(actions as Set<Action>)
+              }
+              is List<*> -> {
+                actions as List<Action>
+              }
+              else -> {
+                emptyList()
+              }
+            }
+            printer("\t[$transactionNumber]")
+            for (action in actionList) {
+              printer("\t\t- $action")
+            }
+          }
+        }
+      }
+    }
+  }
+}

src/main/kotlin/nl/astraeus/persistence/reference/Reference.kt

@@ -0,0 +1,69 @@
+package nl.astraeus.persistence.reference
+
+import nl.astraeus.persistence.Persistable
+import nl.astraeus.persistence.currentTransaction
+import java.io.Serializable
+import kotlin.reflect.KProperty
+
+inline fun <reified T : Persistable> reference(
+  initialValue:T
+) = Reference(T::class.java, initialValue)
+
+inline fun <reified T : Persistable> nullableReference(
+  initialValue:T? = null
+) = NullableReference(T::class.java, initialValue)
+
+class Reference<S : Persistable>(
+  val cls: Class<S>,
+  initialValue: S
+) : Serializable {
+  var id: Long = initialValue.id
+
+  operator fun getValue(thisRef: Persistable, property: KProperty<*>): S {
+    check(currentTransaction() != null) {
+      "No transaction available"
+    }
+    return currentTransaction()?.find(cls.kotlin, id) ?: throw IllegalStateException("Reference not found")
+  }
+
+  operator fun setValue(thisRef: Persistable, property: KProperty<*>, value: S) {
+    check(currentTransaction() != null) {
+      "No transaction available"
+    }
+    currentTransaction()?.store(value)
+    id = value.id
+  }
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+}
+
+class NullableReference<S : Persistable>(
+  val cls: Class<S>,
+  initialValue: S? = null
+) : Serializable {
+  var id: Long? = initialValue?.id
+
+  operator fun getValue(thisRef: Persistable, property: KProperty<*>): S? {
+    check(currentTransaction() != null) {
+      "No transaction available"
+    }
+    return currentTransaction()?.find(cls.kotlin, id ?: 0L)
+  }
+
+  operator fun setValue(thisRef: Persistable, property: KProperty<*>, value: S?) {
+    check(currentTransaction() != null) {
+      "No transaction available"
+    }
+    if (value != null) {
+      // todo: only store if not already stored?
+      currentTransaction()?.store(value)
+    }
+    id = value?.id
+  }
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+}

src/main/kotlin/nl/astraeus/persistence/reference/ReferenceCollection.kt

@@ -0,0 +1,82 @@
+package nl.astraeus.persistence.reference
+
+import nl.astraeus.persistence.Persistable
+import nl.astraeus.persistence.currentTransaction
+import java.io.Serializable
+
+inline fun <reified T : Persistable> referenceCollection(
+  ids: MutableCollection<Long> = mutableListOf()
+) = ReferenceCollection(T::class.java, ids)
+
+open class ReferenceCollection<T : Persistable>(
+  val cls: Class<T>,
+  val ids: MutableCollection<Long> = mutableListOf()
+) : MutableCollection<T>, Serializable {
+
+  protected fun checkElementIsPersisted(element: T) {
+    if (currentTransaction()?.find(cls.kotlin, element.id) == null) {
+      currentTransaction()?.store(element)
+    }
+  }
+
+  override val size: Int
+    get() {
+      return ids.size
+    }
+
+  override fun clear() {
+    ids.clear()
+  }
+
+  override fun isEmpty(): Boolean = ids.isEmpty()
+
+  override fun iterator(): MutableIterator<T> {
+    return object : MutableIterator<T> {
+      private var idsIterator = ids.iterator()
+
+      override fun hasNext(): Boolean = idsIterator.hasNext()
+
+      override fun next(): T = idsIterator.next().let {
+        currentTransaction()?.find(cls.kotlin, it)
+      } ?: throw IllegalStateException("Reference not found")
+
+      override fun remove() = idsIterator.remove()
+    }
+  }
+
+  override fun retainAll(elements: Collection<T>): Boolean {
+    return ids.retainAll(elements.map { it.id }.toSet())
+  }
+
+  override fun removeAll(elements: Collection<T>): Boolean {
+    return ids.removeAll(elements.map { it.id }.toSet())
+  }
+
+  override fun remove(element: T): Boolean {
+    return ids.remove(element.id)
+  }
+
+  override fun containsAll(elements: Collection<T>): Boolean {
+    return ids.containsAll(elements.map { it.id })
+  }
+
+  override fun contains(element: T): Boolean {
+    return ids.contains(element.id)
+  }
+
+  override fun addAll(elements: Collection<T>): Boolean {
+    for (element in elements) {
+      checkElementIsPersisted(element)
+    }
+    return ids.addAll(elements.map { it.id })
+  }
+
+  override fun add(element: T): Boolean {
+    checkElementIsPersisted(element)
+    return ids.add(element.id)
+  }
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+}

src/main/kotlin/nl/astraeus/persistence/reference/ReferenceList.kt

@@ -0,0 +1,66 @@
+package nl.astraeus.persistence.reference
+
+import nl.astraeus.persistence.Persistable
+import nl.astraeus.persistence.currentTransaction
+import java.io.Serializable
+
+inline fun <reified T : Persistable> referenceList(
+  ids: MutableList<Long> = mutableListOf()
+) = ReferenceList(T::class.java, ids)
+
+class ReferenceList<T : Persistable>(
+  cls: Class<T>,
+  val idsList: MutableList<Long> = mutableListOf()
+) : ReferenceCollection<T>(cls, idsList), MutableList<T>, Serializable {
+
+  override fun add(index: Int, element: T) {
+    checkElementIsPersisted(element)
+    idsList.add(index, element.id)
+  }
+
+  override fun addAll(index: Int, elements: Collection<T>): Boolean {
+    for(element in elements) {
+      checkElementIsPersisted(element)
+    }
+    return idsList.addAll(index, elements.map { it.id })
+  }
+
+  override fun get(index: Int): T = currentTransaction()?.find(cls.kotlin, idsList[index]) ?: throw IllegalStateException("Reference not found")
+  override fun indexOf(element: T): Int {
+    return idsList.indexOf(element.id)
+  }
+
+  override fun isEmpty(): Boolean = ids.isEmpty()
+  override fun lastIndexOf(element: T): Int {
+    return idsList.lastIndexOf(element.id)
+  }
+
+  override fun listIterator(): MutableListIterator<T> {
+    return ReferenceListIterator(cls, idsList)
+  }
+
+  override fun listIterator(index: Int): MutableListIterator<T> {
+    return ReferenceListIterator(cls, idsList, index)
+  }
+
+  override fun removeAt(index: Int): T {
+    val id = idsList.removeAt(index)
+
+    return currentTransaction()?.find(cls.kotlin, id) ?: throw IllegalStateException("Reference not found")
+  }
+
+  override fun subList(fromIndex: Int, toIndex: Int): MutableList<T> {
+    return ReferenceList(cls, idsList.subList(fromIndex, toIndex))
+  }
+
+  override fun set(index: Int, element: T): T {
+    checkElementIsPersisted(element)
+    idsList[index] = element.id
+    return element
+  }
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+
+}

src/main/kotlin/nl/astraeus/persistence/reference/ReferenceListIterator.kt

@@ -0,0 +1,47 @@
+package nl.astraeus.persistence.reference
+
+import nl.astraeus.persistence.Persistable
+import nl.astraeus.persistence.currentTransaction
+
+class ReferenceListIterator<T : Persistable>(
+  private val cls: Class<T>,
+  idsList: MutableList<Long>,
+  index: Int = 0
+) : MutableListIterator<T> {
+  private val idsIterator = idsList.listIterator(index)
+
+  private fun checkElementIsPersisted(element: T) {
+    if (currentTransaction()?.find(cls.kotlin, element.id) == null) {
+      currentTransaction()?.store(element)
+    }
+  }
+
+  override fun add(element: T) {
+    checkElementIsPersisted(element)
+    idsIterator.add(element.id)
+  }
+
+  override fun hasNext(): Boolean = idsIterator.hasNext()
+  override fun hasPrevious(): Boolean = idsIterator.hasPrevious()
+
+  override fun next(): T = idsIterator.next().let {
+    currentTransaction()?.find(cls.kotlin, it)
+  } ?: throw IllegalStateException("Reference not found")
+
+  override fun nextIndex(): Int = idsIterator.nextIndex()
+
+  override fun previous(): T = idsIterator.previous().let {
+    currentTransaction()?.find(cls.kotlin, it)
+  } ?: throw IllegalStateException("Reference not found")
+
+  override fun previousIndex(): Int = idsIterator.previousIndex()
+
+  override fun remove() {
+    idsIterator.remove()
+  }
+
+  override fun set(element: T) {
+    checkElementIsPersisted(element)
+    idsIterator.set(element.id)
+  }
+}

src/test/java/nl/astraeus/persistence/TestPersistenceJava.java

@@ -0,0 +1,125 @@
+package nl.astraeus.persistence;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.io.Serial;
+import java.util.List;
+
+public class TestPersistenceJava {
+
+    static class Person extends AbstractPersistable {
+        @Serial
+        private static final long serialVersionUID = 1L;
+
+        private long id = 0;
+        private long version = 0;
+        private String name;
+        private int age;
+
+        public Person(String name, int age) {
+            this.name = name;
+            this.age = age;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public void setName(String name) {
+            this.name = name;
+        }
+
+        public int getAge() {
+            return age;
+        }
+
+        public void setAge(int age) {
+            this.age = age;
+        }
+
+        @Override
+        public long getId() {
+            return id;
+        }
+
+        @Override
+        public void setId(long id) {
+            this.id = id;
+        }
+
+        @Override
+        public long getVersion() {
+            return version;
+        }
+
+        @Override
+        public void setVersion(long version) {
+            this.version = version;
+        }
+    }
+
+    @Test
+    void testPersistence() {
+        System.out.println("TestPersistenceJava.testPersistence");
+
+        Persistent persistent = new Persistent(
+                new File("data", "java-test"),
+                false,
+                null,
+                null,
+                new Index[] {
+                        new Index<>(
+                                Person.class,
+                                "name",
+                                (p) -> ((Person)p).getName()
+                        )
+                }
+        );
+
+        persistent.transaction((t) -> {
+            Person person = t.find(Person.class, 1L);
+
+            if (person != null) {
+                System.out.println(
+                        "Person: "
+                        + person.getName() + " is "
+                        + person.getAge() + " years old."
+                );
+            }
+
+            return null;
+        });
+
+        persistent.transaction((t) -> {
+            Person person = new Person("John Doe", 42);
+
+            t.store(person);
+
+            return null;
+        });
+
+        persistent.query((t) -> {
+            List<Person> persons = t.findByIndex(
+                    Person.class,
+                    "name",
+                    "John Doe"
+            );
+
+            for (Person person : persons) {
+                System.out.println(
+                        "Person: "
+                        + person.getName() + " is "
+                        + person.getAge() + " years old."
+                );
+            }
+
+            return null;
+        });
+
+        persistent.snapshot();
+        persistent.getDatastore().printStatus();
+        persistent.removeOldFiles();
+    }
+
+}

src/test/kotlin/nl/astraeus/persistence/EncryptionTest.kt

@@ -0,0 +1,54 @@
+package nl.astraeus.persistence
+
+import org.junit.jupiter.api.Test
+
+import org.junit.jupiter.api.Assertions.*
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
+import java.security.SecureRandom
+
+class EncryptionTest {
+
+  @Test
+  fun testKeyGen() {
+    println(generateBase64Key())
+  }
+
+  @Test
+  fun testEncryptDecrypt() {
+    val random = SecureRandom()
+    val randomBytes = ByteArray(random.nextInt(10000))
+    random.nextBytes(randomBytes)
+    val base64Key = generateBase64Key()
+    val encryptor = Encryptor(
+      base64Key,
+      base64Key,
+    )
+
+    val encrypted = encryptor.encrypt(randomBytes)
+    val decrypted = encryptor.decrypt(encrypted)
+
+    assertArrayEquals(randomBytes, decrypted)
+  }
+
+  @Test
+  fun `test encryption-decryption streams`() {
+    val random = SecureRandom()
+    val key = generateBase64Key()
+    val baos = ByteArrayOutputStream()
+    val encryptionStream = EncryptingOutputStream(baos, key)
+    val bytes = ByteArray(random.nextInt(10000))
+    random.nextBytes(bytes)
+
+    encryptionStream.use {
+      it.write(bytes)
+    }
+
+    val bais = ByteArrayInputStream(baos.toByteArray())
+    val decryptingStream = DecryptingInputStream(bais, key)
+
+    val decryptedBytes = decryptingStream.readAllBytes()
+
+    assertArrayEquals(bytes, decryptedBytes)
+  }
+}

src/test/kotlin/nl/astraeus/persistence/TestOptimisticLocking.kt

@@ -0,0 +1,95 @@
+package nl.astraeus.persistence
+
+import nl.astraeus.persistence.domain.Person
+import org.junit.jupiter.api.Assertions.assertNotNull
+import org.junit.jupiter.api.assertThrows
+import java.io.File
+import kotlin.test.Test
+
+class TestOptimisticLocking {
+
+  @Test
+  fun showTransactions() {
+    val log = TransactionLog(File("data", "test-locking"))
+
+    log.showTransactions()
+  }
+
+  @Test
+  fun testOptimisticLocking() {
+    println("Test locking")
+
+    val pst = Persistent(
+      directory = File("data", "test-locking"),
+      true,
+      null,
+      null,
+      arrayOf(
+        index<Person>("name") { p -> (p as? Person)?.name ?: "" },
+      ),
+    )
+
+    pst.transaction {
+      val person = find<Person>(1L) ?: Person(
+        id = 1L,
+        name = "John Doe",
+        age = 25
+      )
+
+      store(person)
+
+      findByIndex<Person>("name", "John Doe").forEach { p ->
+        println("Found person by name: ${p.name} - ${p.age}")
+      }
+    }
+
+    pst.query {
+      val person = find<Person>(1L)
+
+      assertNotNull(person)
+    }
+
+    val threads = Array(2) { index ->
+      Thread {
+        println("Start thread $index")
+        var person: Person? = null
+
+        pst.transaction {
+          Thread.sleep(10L)
+          person = find<Person>(1L)
+          println("Thread $index -> ${person?.version}")
+        }
+
+        if (person != null) {
+          Thread.sleep((index + 1) * 10L)
+
+          if (index == 1) {
+            assertThrows<OptimisticLockingException> {
+              println("Store thread $index -> ${person!!.version}")
+              pst.transaction {
+                store(person!!)
+              }
+            }
+          } else {
+            println("Store thread $index -> ${person!!.version}")
+            pst.transaction {
+              store(person!!)
+            }
+          }
+        }
+      }
+    }
+
+    for (thread in threads) {
+      thread.start()
+    }
+
+    for (thread in threads) {
+      thread.join()
+    }
+
+    pst.datastore.printStatus()
+    //pst.snapshot()
+    pst.removeOldFiles()
+  }
+}

src/test/kotlin/nl/astraeus/persistence/TestPersistence.kt

@@ -1,37 +1,19 @@
 package nl.astraeus.persistence
 
-import nl.astraeus.nl.astraeus.persistence.Index
-import nl.astraeus.nl.astraeus.persistence.Persistable
-import nl.astraeus.nl.astraeus.persistence.Persistent
-import nl.astraeus.nl.astraeus.persistence.Reference
+import nl.astraeus.persistence.domain.Company
+import nl.astraeus.persistence.domain.Person
+import org.junit.jupiter.api.Assertions.assertNotNull
+import org.junit.jupiter.api.Assertions.assertNull
 import java.io.File
 import kotlin.test.Test
 
 class TestPersistence {
 
-  class Company(
-    override var id: Long = 0,
-    override var version: Long = 0,
-    val name: String
-  ) : Persistable, Cloneable {
-    //var persons: MutableList<Person> by ListReference<Person, Company>(Person::class.java)
+  @Test
+  fun showTransactions() {
+    val log = TransactionLog(File("data", "test-persistence"))
 
-    companion object {
-      private const val serialVersionUID: Long = 1L
-    }
-  }
-
-  class Person(
-    override var id: Long = 0,
-    override var version: Long = 0,
-    val name: String,
-    val age: Int,
-  ) : Persistable, Cloneable {
-    var company: Company by Reference<Company, Person>(Company::class.java)
-
-    companion object {
-      private const val serialVersionUID: Long = 1L
-    }
+    log.showTransactions()
   }
 
   @Test
@@ -39,31 +21,31 @@ class TestPersistence {
     println("Test persistence")
 
     val pst = Persistent(
-      directory = File("data"),
-      arrayOf(
-        Index(Person::class, "name") { p -> (p as? Person)?.name ?: "" },
-        Index(Person::class, "age") { p -> (p as? Person)?.age ?: -1 },
-        Index(Person::class, "ageGt20") { p -> ((p as? Person)?.age ?: 0) > 20 },
-        Index(Person::class, "ageGt23") { p -> ((p as? Person)?.age ?: 0) > 23 },
-        Index(Person::class, "ageOnlyGt20") { p ->
+      directory = File("data", "test-persistence"),
+      indexes = arrayOf(
+        index<Person>("name") { p -> (p as? Person)?.name ?: "" },
+        index<Person>("age") { p -> (p as? Person)?.age ?: -1 },
+        index<Person>("ageGt20") { p -> ((p as? Person)?.age ?: 0) > 20 },
+        index<Person>("ageGt23") { p -> ((p as? Person)?.age ?: 0) > 23 },
+        index<Person>("ageOnlyGt20") { p ->
           if (((p as? Person)?.age ?: 0) > 20) {
             true
           } else {
             null
           }
         },
-        Index(Company::class, "name") { p -> (p as? Company)?.name ?: "" },
+        index<Company>("name") { p -> (p as? Company)?.name ?: "" },
       )
     )
 
     pst.transaction {
-      val person = find(Person::class, 1L) ?: Person(
+      val person = find<Person>(1L) ?: Person(
         id = 1L,
         name = "John Doe",
         age = 25
       )
 
-      val company = find(Company::class, 1L) ?: Company(
+      val company = find<Company>(1L) ?: Company(
         id = 1L,
         name = "ACME"
       )
@@ -83,14 +65,19 @@ class TestPersistence {
         age = 18
       ))
 
-      findByIndex(Person::class, "name", "John Doe").forEach { p ->
+      findByIndex<Person>("name", "John Doe").forEach { p ->
         println("Found person by name: ${p.name} - ${p.age}")
       }
 
-      findByIndex(Person::class, "age", 23).forEach { p ->
+      val persons: List<Person> = findByIndex("age", 23)
+
+      persons.forEach { p ->
         println("Found person by age: ${p.name} - ${p.age}")
       }
 
+      val companies: List<Company> = findByIndex("name", "ACME")
+      assert(companies.isNotEmpty())
+
       findByIndex(Person::class, "ageGt20", true).forEach { p ->
         println("Found person by age > 20: ${p.name} - ${p.age}")
       }
@@ -112,6 +99,20 @@ class TestPersistence {
       assert(c2 != null)
     }
 
+    pst.query {
+      val person = find<Person>(1L)
+
+      assertNotNull(person)
+    }
+
+    pst.transaction {
+      val p1 = find<Person>(1L)
+      val p2 = find<Person>(1L)
+
+      store(p2!!)
+      store(p1!!)
+    }
+
     pst.transaction {
       val person = find(Person::class, 1L)
 
@@ -136,17 +137,48 @@ class TestPersistence {
       store(
         Person(
           id = 10L,
-          name = "Pipo",
+          name = "John Pipo",
           age = 23
         )
       )
+      val person = Person(
+        id = 11L,
+        name = "John Clown",
+        age = 18
+      )
+
+      store(person)
+      assertNotNull(find(Person::class, person.id))
+      delete(person)
+      assertNull(find(Person::class, person.id))
+      store(person)
+      assertNotNull(find(Person::class, person.id))
+
+      searchIndex(Person::class, "name") {
+        name -> (name as? String)?.startsWith("John") == true
+      }.forEach { p ->
+        println("Found person where name starts with 'John': ${p.name} - ${p.age}")
+      }
+    }
+
+    pst.transaction {
+      store(
+        Person(
+          id = 15L,
+          name = "Mama",
+          age = 26
+        )
+      )
       store(
         Person(
-          id = 11L,
-          name = "Clown",
-          age = 18
+          id = 16L,
+          name = "Loe",
+          age = 16
         )
       )
     }
+
+    pst.datastore.printStatus()
+    pst.removeOldFiles()
   }
 }

src/test/kotlin/nl/astraeus/persistence/TestPersistenceJavaInKotlin.kt

@@ -0,0 +1,66 @@
+package nl.astraeus.persistence
+
+import org.junit.jupiter.api.Test
+import java.io.File
+
+class TestPersistenceJavaInKotlin {
+
+  internal class Person(
+    var name: String,
+    var age: Int
+  ) : Persistable {
+    override var id: Long = 0
+    override var version: Long = 0
+
+    companion object {
+      private const val serialVersionUID = 1L
+    }
+  }
+
+  @Test
+  fun testPersistence() {
+    println("TestPersistenceJavaInKotlin.testPersistence")
+
+    val persistent = Persistent(
+      File("data", "java-kotlin-test"),
+      enableOptimisticLocking = false,
+      indexes = arrayOf(
+        Index(
+          Person::class,
+          "name"
+        ) { p -> (p as Person).name }
+      ),
+    )
+
+    persistent.transaction {
+      val person = find(Person::class.java, 1L)
+
+      if (person != null) {
+        println(
+          "Person: ${person.name} is ${person.age} years old."
+        )
+      }
+    }
+
+    persistent.transaction {
+      val person = Person("John Doe", 42)
+
+      store(person)
+    }
+
+    persistent.query {
+      val persons = findByIndex(
+        Person::class.java,
+        "name",
+        "John Doe"
+      )
+      for (person in persons) {
+        println("Person: ${person.name}  is ${person.age} years old.")
+      }
+    }
+
+    persistent.snapshot()
+    persistent.datastore.printStatus()
+    persistent.removeOldFiles()
+  }
+}

src/test/kotlin/nl/astraeus/persistence/TestReferences.kt

@@ -0,0 +1,71 @@
+package nl.astraeus.persistence
+
+import nl.astraeus.persistence.domain.Company
+import nl.astraeus.persistence.reference.reference
+import org.junit.jupiter.api.assertThrows
+import java.io.File
+import kotlin.test.Test
+
+class TestReferences {
+
+  class Person(
+    override var id: Long = 0,
+    override var version: Long = 0,
+    val name: String,
+    company: Company
+  ) : Persistable, Cloneable {
+    var company: Company by reference(company)
+
+    companion object {
+      private const val serialVersionUID: Long = 1L
+    }
+
+    override fun toString(): String {
+      return "Person(id=$id, version=$version, name='$name')"
+    }
+  }
+
+  @Test
+  fun showTransactions() {
+    val log = TransactionLog(File("data", "test-references"))
+
+    log.showTransactions()
+  }
+
+  @Test
+  fun testSerializeDeSerializeReferenceList() {
+    val pst = Persistent(
+      directory = File("data", "test-references"),
+    )
+
+    pst.transaction {
+      val company = Company(
+        id = 1L,
+        name = "ACME"
+      )
+
+      val person = Person(
+        id = 0L,
+        name = "John Doe",
+        company = company
+      )
+      store(person)
+
+      store(company)
+
+      for (p in company.persons) {
+        println("Person: $p")
+      }
+
+      delete(company)
+
+      // company is gone, can't get it through person anymore
+      assertThrows<IllegalStateException> {
+        println("Company in person: ${person.company}")
+      }
+
+      delete(person)
+    }
+  }
+
+}

src/test/kotlin/nl/astraeus/persistence/TestThreaded.kt

@@ -0,0 +1,119 @@
+package nl.astraeus.persistence
+
+import nl.astraeus.persistence.domain.Company
+import nl.astraeus.persistence.domain.Person
+import java.io.File
+import kotlin.random.Random
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+class TestThreaded {
+
+  @Test
+  fun showTransactions() {
+    val log = TransactionLog(File("data", "test-threaded"))
+
+    log.showTransactions()
+  }
+
+  @Test
+  fun testThreaded() {
+    println("Test threaded")
+
+    val pst = Persistent(
+      directory = File("data", "test-threaded"),
+      indexes = arrayOf(
+        index<Person>("name") { p -> (p as? Person)?.name ?: "" },
+        index<Person>("age") { p -> (p as? Person)?.age ?: -1 },
+        index<Person>("ageGt20") { p -> ((p as? Person)?.age ?: 0) > 20 },
+        index<Person>("ageGt23") { p -> ((p as? Person)?.age ?: 0) > 23 },
+        index<Person>("ageOnlyGt20") { p ->
+          if (((p as? Person)?.age ?: 0) > 20) {
+            true
+          } else {
+            null
+          }
+        },
+        index<Person>("nameAndAge") { p ->
+          val person = p as? Person
+
+          if (person == null) {
+            null
+          } else {
+            person.name to person.age
+          }
+        },
+        index<Person>("personCompanyId") { p -> (p as? Person)?.company?.id ?: 0L },
+        index<Company>("name") { p -> (p as? Company)?.name ?: "" },
+      )
+    )
+
+    val companyNames = arrayOf("Company A", "Company B", "Company C", "Company D", "Company E")
+    val names = arrayOf("John Doe", "Jane Doe", "John Smith", "Jane Smith", "John Johnson", "Jane Johnson")
+    val random = Random(System.currentTimeMillis())
+
+    val empty = pst.query {
+      count<Person>() == 0
+    }
+
+    if (empty) {
+      val runnable = {
+        repeat(10) {
+          pst.transaction {
+            val company = Company(
+              id = 0L,
+              name = companyNames[random.nextInt(companyNames.size)]
+            )
+            repeat(10) {
+              val person = Person(
+                id = 0L,
+                name = names[random.nextInt(names.size)],
+                age = random.nextInt(0, 100),
+              )
+              person.company = company
+
+              store(person)
+            }
+          }
+        }
+      }
+
+      val threads = Array(25) {
+        Thread(runnable)
+      }
+
+      val start = System.nanoTime()
+      for (thread in threads) {
+        thread.start()
+      }
+
+      for (thread in threads) {
+        thread.join()
+      }
+      println("Store elapsed time: ${(System.nanoTime() - start) / 1_000_000}ms")
+    }
+
+    var start = 0L
+    repeat(10) {
+      start = System.nanoTime()
+      val withoutIndex = pst.query {
+        search<Person> { person ->
+          person.age == 20
+        }
+      }
+      println("withoutIndex elapsed time: ${(System.nanoTime() - start) / 1_000_000f}ms")
+
+      start = System.nanoTime()
+      val withIndex = pst.query {
+        searchIndex<Person>("age") { age -> (age as? Int ?: -1) == 20 }
+      }
+      println("withIndex elapsed time: ${(System.nanoTime() - start) / 1_000_000f}ms")
+
+      assertEquals(withIndex.size, withoutIndex.size)
+    }
+
+    pst.snapshot()
+    pst.datastore.printStatus()
+    pst.removeOldFiles()
+  }
+}

src/test/kotlin/nl/astraeus/persistence/domain/Company.kt

@@ -0,0 +1,20 @@
+package nl.astraeus.persistence.domain
+
+import nl.astraeus.persistence.Persistable
+import nl.astraeus.persistence.reference.referenceCollection
+
+class Company(
+  override var id: Long = 0,
+  override var version: Long = 0,
+  val name: String
+) : Persistable, Cloneable {
+  val persons: MutableCollection<Person> = referenceCollection()
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+
+  override fun toString(): String {
+    return "Company(id=$id, version=$version, name='$name', persons=${persons.size})"
+  }
+}

src/test/kotlin/nl/astraeus/persistence/domain/Person.kt

@@ -0,0 +1,22 @@
+package nl.astraeus.persistence.domain
+
+import nl.astraeus.persistence.Persistable
+import nl.astraeus.persistence.reference.nullableReference
+
+class Person(
+  override var id: Long = 0,
+  override var version: Long = 0,
+  val name: String,
+  val age: Int,
+  company: Company? = null
+) : Persistable, Cloneable {
+  var company: Company? by nullableReference(company)
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+
+  override fun toString(): String {
+    return "Person(id=$id, version=$version, name='$name', age=$age)"
+  }
+}