Add detailed documentation in README.md for setup and usage

- Add check on references

.gitignore

@@ -3,6 +3,7 @@ build/
 !gradle/wrapper/gradle-wrapper.jar
 !**/src/main/**/build/
 !**/src/test/**/build/
+gradle.properties
 
 ### IntelliJ IDEA ###
 .idea/modules.xml

.idea/kotlinc.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="KotlinJpsPluginSettings">
-    <option name="version" value="1.9.23" />
+    <option name="version" value="2.1.21" />
   </component>
 </project>

.idea/misc.xml

@@ -4,7 +4,7 @@
   <component name="FrameworkDetectionExcludesConfiguration">
     <file type="web" url="file://$PROJECT_DIR$" />
   </component>
-  <component name="ProjectRootManager" version="2" languageLevel="JDK_17" project-jdk-name="1.8" project-jdk-type="JavaSDK">
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_15" default="true" project-jdk-name="11" project-jdk-type="JavaSDK">
     <output url="file://$PROJECT_DIR$/out" />
   </component>
   <component name="accountSettings">

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Rien Nentjes
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

build.gradle.kts

@@ -1,9 +1,14 @@
+import com.vanniktech.maven.publish.SonatypeHost
+
 plugins {
-    kotlin("jvm") version "1.9.23"
+    kotlin("jvm") version "2.1.21"
+    signing
+    id("org.jetbrains.dokka") version "2.0.0"
+    id("com.vanniktech.maven.publish") version "0.31.0"
 }
 
 group = "nl.astraeus"
-version = "1.0.0-SNAPSHOT"
+version = "1.1.5-SNAPSHOT"
 
 repositories {
     mavenCentral()
@@ -15,8 +20,85 @@ dependencies {
 
 tasks.test {
     useJUnitPlatform()
+
+    minHeapSize = "128m"
+    maxHeapSize = "1g"
 }
 
-kotlin {
-    jvmToolchain(17)
+val javadocJar by tasks.registering(Jar::class) {
+    archiveClassifier.set("javadoc")
+}
+
+val sourcesJar by tasks.registering(Jar::class) {
+    archiveClassifier.set("sources")  // This sets the classifier to 'sources'
+    from(sourceSets.main.get().allSource)  // Assumes you are using the 'main' source set
+}
+
+publishing {
+    repositories {
+        mavenLocal()
+        maven {
+            name = "gitea"
+            setUrl("https://gitea.astraeus.nl/api/packages/rnentjes/maven")
+
+            credentials {
+                val giteaUsername: String? by project
+                val giteaPassword: String? by project
+
+                username = giteaUsername
+                password = giteaPassword
+            }
+        }
+        maven {
+            name = "gitea8443"
+            setUrl("https://gitea.astraeus.nl:8443/api/packages/rnentjes/maven")
+
+            credentials {
+                val giteaUsername: String? by project
+                val giteaPassword: String? by project
+
+                username = giteaUsername
+                password = giteaPassword
+            }
+        }
+    }
+}
+
+tasks.withType<AbstractPublishToMaven> {
+    dependsOn(tasks.withType<Sign>())
+}
+
+signing {
+    sign(publishing.publications)
+}
+
+mavenPublishing {
+    publishToMavenCentral(SonatypeHost.CENTRAL_PORTAL)
+
+    signAllPublications()
+
+    coordinates(group.toString(), name, version.toString())
+
+    pom {
+        name = "simple-persistence-kotlin"
+        description = "Simple persistence"
+        inceptionYear = "2024"
+        url = "https://gitea.astraeus.nl/rnentjes/simple-persistence-kotlin"
+        licenses {
+            license {
+                name = "MIT"
+                url = "https://opensource.org/licenses/MIT"
+            }
+        }
+        developers {
+            developer {
+                id = "rnentjes"
+                name = "Rien Nentjes"
+                email = "info@nentjes.com"
+            }
+        }
+        scm {
+            url = "https://gitea.astraeus.nl/rnentjes/simple-persistence-kotlin"
+        }
+    }
 }

gradle.properties

@@ -1 +0,0 @@
-kotlin.code.style=official

readme.md

@@ -0,0 +1,182 @@
+# Simple Persistence for Kotlin
+
+A lightweight, file-based persistence library for Kotlin applications that provides a simple way to store, retrieve, and query objects.
+
+## Available on Maven Central
+
+```kotlin
+// Gradle Kotlin DSL
+implementation("nl.astraeus:simple-persistence-kotlin:1.1.5")
+```
+
+```groovy
+// Gradle Groovy DSL
+implementation 'nl.astraeus:simple-persistence-kotlin:1.1.5'
+```
+
+```xml
+<!-- Maven -->
+<dependency>
+    <groupId>nl.astraeus</groupId>
+    <artifactId>simple-persistence-kotlin</artifactId>
+    <version>1.1.5</version>
+</dependency>
+```
+
+## Features
+
+- Simple and intuitive API for persisting Kotlin objects
+- Transaction support with ACID properties
+- Optimistic locking to handle concurrent modifications
+- Flexible indexing for efficient querying
+- Object references and collections handling
+- Encryption support for sensitive data
+- Snapshot and transaction log capabilities
+- File-based storage with automatic cleanup
+
+## Quick Start
+
+### Define your model
+
+```kotlin
+class Person(
+  override var id: Long = 0,
+  override var version: Long = 0,
+  val name: String,
+  val age: Int,
+  company: Company? = null
+) : Persistable, Cloneable {
+  var company: Company? by nullableReference(company)
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+}
+
+class Company(
+  override var id: Long = 0,
+  override var version: Long = 0,
+  val name: String
+) : Persistable, Cloneable {
+  val persons: MutableCollection<Person> = referenceCollection()
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+}
+```
+
+### Initialize the persistence manager
+
+```kotlin
+val persistent = Persistent(
+  directory = File("data", "my-app"),
+  indexes = arrayOf(
+    index<Person>("name") { p -> (p as? Person)?.name ?: "" },
+    index<Person>("age") { p -> (p as? Person)?.age ?: -1 },
+    index<Company>("name") { p -> (p as? Company)?.name ?: "" }
+  )
+)
+```
+
+### Perform CRUD operations
+
+```kotlin
+// Create or update
+persistent.transaction {
+  val person = Person(
+    id = 1L,
+    name = "John Doe",
+    age = 30
+  )
+
+  val company = Company(
+    id = 1L,
+    name = "ACME Inc."
+  )
+
+  person.company = company
+  company.persons.add(person)
+
+  store(person)
+  store(company)
+}
+
+// Read
+persistent.query {
+  val person = find<Person>(1L)
+  println("Found person: $person")
+
+  // Query by index
+  val persons: List<Person> = findByIndex("age", 30)
+
+  // Search with predicate
+  val results = search(Person::class) { p -> p.name.startsWith("John") }
+}
+
+// Delete
+persistent.transaction {
+  val person = find<Person>(1L)
+  if (person != null) {
+    delete(person)
+  }
+}
+```
+
+## How it works
+
+- In-memory model: Your objects live in memory during normal operation for fast reads and writes.
+- Durable writes: Every create/update/delete inside a transaction is appended to on-disk transaction log files (*.trn). This is a write-ahead log approach.
+- Startup recovery: On application startup, the library replays the transaction logs (and any snapshot, if present) to rebuild the in-memory model.
+- Snapshots: You can create a compact on-disk snapshot (*.snp) of the current in-memory state. On the next startup, the snapshot is loaded first and only the newer transaction log files are replayed. This keeps startup time low without losing durability.
+
+Tip: Call snapshot() periodically (or on controlled shutdown) and use removeOldFiles() to prune obsolete logs after a successful snapshot.
+
+## Advanced Features
+
+### Indexing
+
+```kotlin
+// Define complex indexes
+val persistent = Persistent(
+  directory = File("data", "my-app"),
+  indexes = arrayOf(
+    index<Person>("ageOver30") { p -> ((p as? Person)?.age ?: 0) > 30 }
+  )
+)
+
+// Query using indexes
+persistent.query {
+  findByIndex<Person>("ageOver30", true).forEach { person ->
+    println("Person over 30: ${person.name}")
+  }
+}
+```
+
+### Transactions
+
+```kotlin
+// Read-only query
+persistent.query {
+  // Only read operations allowed
+}
+
+// Read-write transaction
+persistent.transaction {
+  // Both read and write operations allowed
+}
+```
+
+### Snapshots and Maintenance
+
+```kotlin
+// Create a snapshot of the current state
+persistent.snapshot()
+
+// Remove old transaction log files
+persistent.removeOldFiles()
+```
+
+## License
+
+This project is licensed under the MIT License - see the LICENSE file for details.

src/main/kotlin/Main.kt

@@ -1,5 +0,0 @@
-package nl.astraeus
-
-fun main() {
-  println("Hello World!")
-}

src/main/kotlin/nl/astraeus/persistence/Datastore.kt

@@ -1,10 +1,11 @@
-package nl.astraeus.nl.astraeus.persistence
+package nl.astraeus.persistence
 
 import java.io.File
 import java.io.ObjectInputStream
 import java.io.ObjectOutputStream
 import java.io.Serializable
 import java.text.DecimalFormat
+import java.util.*
 import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.atomic.AtomicLong
 import kotlin.reflect.KClass
@@ -17,21 +18,36 @@ enum class ActionType {
 class TypeData(
   var nextId: AtomicLong = AtomicLong(1L),
   val data: MutableMap<Serializable, Persistable> = ConcurrentHashMap(),
-) : Serializable
+) : Serializable {
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+}
 
 class Action(
   val type: ActionType,
   val obj: Persistable
-) : Serializable
+) : Serializable {
+  override fun toString(): String {
+    return "Action(type=$type, obj=$obj)"
+  }
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+}
 
 class Datastore(
   private val directory: File,
+  val enableOptimisticLocking: Boolean = false,
+  val decryptionKey: String? = null,
+  val encryptionKey: String? = null,
   indexes: Array<PersistableIndex> = arrayOf(),
 ) {
   private val fileManager = FileManager(directory)
   private val transactionFormatter = DecimalFormat("#")
   private var nextTransactionNumber = 1L
-  private val data: MutableMap<Class<*>, TypeData> = ConcurrentHashMap()
+  private var data: MutableMap<Class<*>, TypeData> = ConcurrentHashMap()
   private val indexes: MutableMap<Class<*>, MutableMap<String, PersistableIndex>> = ConcurrentHashMap()
 
   init {
@@ -53,12 +69,25 @@ class Datastore(
       }
     }
 
+    return data[javaClass]!!.nextId.get()
+  }
+
+  fun getNextIdAndIncrement(javaClass: Class<Persistable>): Long {
+    if (data[javaClass] == null) {
+      synchronized(this) {
+        if (data[javaClass] == null) {
+          data[javaClass] = TypeData()
+        }
+      }
+    }
+
     return data[javaClass]!!.nextId.getAndIncrement()
   }
 
   fun setMaxId(javaClass: Class<Persistable>, id: Long) {
     val nextId = data.getOrPut(javaClass) { TypeData() }.nextId
-    if (nextId.get() <= id) nextId.set(id + 1)
+    val current = nextId.get()
+    if (current <= id) nextId.addAndGet(id - current)
   }
 
   override fun toString(): String {
@@ -83,7 +112,7 @@ class Datastore(
       val (lastSnapshot, lastSnapshotFile) = fileManager.findLastSnapshot()
 
       if (lastSnapshotFile != null) {
-        ObjectInputStream(lastSnapshotFile.inputStream()).use { ois ->
+        ObjectInputStream(DecryptingInputStream(lastSnapshotFile.inputStream(), decryptionKey)).use { ois ->
           readSnapshot(ois)
         }
       }
@@ -91,21 +120,30 @@ class Datastore(
       val transactions = fileManager.findTransactionsAfter(lastSnapshot ?: 0L)
 
       transactions?.forEach { file ->
-        ObjectInputStream(file.inputStream()).use { ois ->
+        ObjectInputStream(DecryptingInputStream(file.inputStream(), decryptionKey)).use { ois ->
           readTransaction(ois)
         }
       }
     }
 
-    Logger.debug("Loaded transactions in ${(System.nanoTime() - start) / 1_000_000}ms")
+    Logger.debug("Loaded transactions in %6.3fms", ((System.nanoTime() - start) / 1_000_000f))
   }
 
-  private fun getTrnx(file: File): Long {
-    return file.name.substringAfterLast('/').substringAfter("transaction-").substringBefore(".").toLong()
-  }
-
-  fun execute(actions: MutableList<Action>) {
+  private fun execute(actions: List<Action>) {
     synchronized(this) {
+      if (enableOptimisticLocking) {
+        for (action in actions) {
+          val typeData = data.getOrPut(action.obj::class.java) {
+            TypeData()
+          }
+          if (action.type == ActionType.STORE) {
+            if ((typeData.data[action.obj.id]?.version ?: -1L) >= action.obj.version) {
+              throw OptimisticLockingException(action.obj)
+            }
+          }
+        }
+      }
+
       for (action in actions) {
         val typeData = data.getOrPut(action.obj::class.java) {
           TypeData()
@@ -137,7 +175,6 @@ class Datastore(
     }
   }
 
-
   fun <T : Persistable> count(clazz: KClass<T>): Int {
     val typeData = data.getOrPut(clazz.java) {
       TypeData()
@@ -167,6 +204,7 @@ class Datastore(
     return typeData.data.values
       .filter { search(it as T) }
       .map { o -> o.copy() as T }
+      .sortedBy { it.id }
   }
 
   fun findIndex(
@@ -176,41 +214,50 @@ class Datastore(
     return indexes[kClass.java]?.get(indexName)
   }
 
-  fun storeAndExecute(actions: MutableList<Action>) {
+  fun storeAndExecute(actions: List<Action>) {
     if (actions.isNotEmpty()) {
       synchronized(this) {
-        writeTransaction(actions)
         execute(actions)
+        writeTransaction(actions)
       }
     }
   }
 
   private fun readTransaction(ois: ObjectInputStream) {
     val versionNumber = ois.readInt()
-    check (versionNumber == 1) { "Unsupported version number: $versionNumber" }
+    check(versionNumber == 1) { "Unsupported version number: $versionNumber" }
     val transactionNumber = ois.readLong()
     nextTransactionNumber = transactionNumber + 1
-    val actions = ois.readObject() as MutableList<Action>
-    execute(actions)
+    val actions = ois.readObject()
+    when(actions) {
+      is Set<*> -> {
+        val list = LinkedList(actions as Set<Action>)
+        execute(list)
+      }
+      is List<*> -> {
+        execute(actions as List<Action>)
+      }
+    }
   }
 
-  private fun writeTransaction(actions: MutableList<Action>) {
+  private fun writeTransaction(actions: List<Action>) {
     val number = transactionFormatter.format(nextTransactionNumber)
-    val file = File(directory, "transaction-$number.trn")
-    ObjectOutputStream(file.outputStream()).use { oos ->
+    val file = File(directory, "transaction-$number.trn-tmp")
+    ObjectOutputStream(EncryptingOutputStream(file.outputStream(), encryptionKey)).use { oos ->
       // version number
       oos.writeInt(1)
       oos.writeLong(nextTransactionNumber++)
       oos.writeObject(actions)
     }
+    file.renameTo(File(directory, "transaction-$number.trn"))
   }
 
   fun snapshot() {
     val start = System.nanoTime()
     synchronized(this) {
       val number = transactionFormatter.format(nextTransactionNumber)
-      val file = File(directory, "transaction-$number.snp")
-      ObjectOutputStream(file.outputStream()).use { oos ->
+      val file = File(directory, "transaction-$number.snp-tmp")
+      ObjectOutputStream(EncryptingOutputStream(file.outputStream(), encryptionKey)).use { oos ->
         // version number
         oos.writeInt(1)
         oos.writeLong(nextTransactionNumber++)
@@ -225,16 +272,17 @@ class Datastore(
           }
         }
       }
+      file.renameTo(File(directory, "transaction-$number.snp"))
     }
-    Logger.debug("Snapshot in ${(System.nanoTime() - start) / 1_000_000}ms")
+    Logger.debug("Snapshot in %6.3fms", ((System.nanoTime() - start) / 1_000_000f))
   }
 
   private fun readSnapshot(ois: ObjectInputStream) {
     val versionNumber = ois.readInt()
-    check (versionNumber == 1) { "Unsupported version number: $versionNumber" }
+    check(versionNumber == 1) { "Unsupported version number: $versionNumber" }
     nextTransactionNumber = ois.readLong() + 1
-    data.clear()
-    data.putAll(ois.readObject() as MutableMap<Class<*>, TypeData>)
+    val dataObj = ois.readObject()
+    data = dataObj as ConcurrentHashMap<Class<*>, TypeData>
 
     val foundIndexes = mutableMapOf<Class<*>, MutableList<String>>()
     val numberOfClassesWithIndex = ois.readInt()

src/main/kotlin/nl/astraeus/persistence/Encryption.kt

@@ -0,0 +1,150 @@
+@file:OptIn(ExperimentalEncodingApi::class, ExperimentalEncodingApi::class)
+
+package nl.astraeus.persistence
+
+import java.io.ByteArrayOutputStream
+import java.io.InputStream
+import java.io.OutputStream
+import java.security.SecureRandom
+import javax.crypto.Cipher
+import javax.crypto.KeyGenerator
+import javax.crypto.SecretKey
+import javax.crypto.spec.IvParameterSpec
+import javax.crypto.spec.SecretKeySpec
+import kotlin.io.encoding.Base64
+import kotlin.io.encoding.ExperimentalEncodingApi
+
+class Encryptor(
+  base64EncryptionKey: String?,
+  base64DecryptionKey: String?,
+) {
+  private var decryptionKey: SecretKey? = null
+  private var encryptionKey: SecretKey? = null
+
+  init {
+    if (base64EncryptionKey?.isNotEmpty() == true) {
+      encryptionKey = SecretKeySpec(Base64.UrlSafe.decode(base64EncryptionKey), "AES")
+    }
+    if (base64DecryptionKey?.isNotEmpty() == true) {
+      decryptionKey = SecretKeySpec(Base64.UrlSafe.decode(base64DecryptionKey), "AES")
+    }
+  }
+
+  fun encrypt(data: ByteArray): ByteArray {
+    if (encryptionKey == null) {
+      return data
+    }
+
+    val prePaddedData =  ByteArray(16) + data
+    val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding")
+
+    // Generate a new IV (Initialization Vector)
+    val secureRandom = SecureRandom()
+    val iv = ByteArray(cipher.blockSize)
+    secureRandom.nextBytes(iv)
+    val ivParams = IvParameterSpec(iv)
+
+    cipher.init(Cipher.ENCRYPT_MODE, encryptionKey, ivParams)
+    return cipher.doFinal(prePaddedData)
+  }
+
+  fun decrypt(data: ByteArray): ByteArray {
+    if (decryptionKey == null) {
+      return data
+    }
+
+    val cipher = Cipher.getInstance("AES/CBC/PKCS5Padding")
+    val secureRandom = SecureRandom()
+    val iv = ByteArray(cipher.blockSize)
+    secureRandom.nextBytes(iv)
+    cipher.init(Cipher.DECRYPT_MODE, decryptionKey, IvParameterSpec(iv))
+    val completeData = cipher.doFinal(data)
+
+    return completeData.sliceArray(16 until completeData.size)
+  }
+}
+
+fun generateBase64Key(): String {
+  val keyGen: KeyGenerator = KeyGenerator.getInstance("AES")
+  keyGen.init(256) // for AES-256
+  val secretKey: SecretKey = keyGen.generateKey()
+
+  return Base64.UrlSafe.encode(secretKey.encoded)
+}
+
+/*
+object Encryption {
+  var encryptor = Encryptor(
+    System.getenv().getOrDefault("SPK_ENCRYPTION_KEY", ""),
+    System.getenv().getOrDefault("SPK_DECRYPTION_KEY", "")
+  )
+}
+*/
+
+class DecryptingInputStream(
+  val input: InputStream,
+  val base64DecryptionKey: String?
+) : InputStream() {
+  val bytes: ByteArray
+  var index = 0
+
+  init {
+    val encryptedBytes = input.readAllBytes()
+    if (base64DecryptionKey?.isBlank() == true) {
+      bytes = encryptedBytes
+    } else {
+      val encryptor = Encryptor(
+        base64EncryptionKey = null,
+        base64DecryptionKey = base64DecryptionKey
+      )
+      bytes = encryptor.decrypt(encryptedBytes)
+    }
+  }
+
+/*  override fun readAllBytes(): ByteArray {
+    index = bytes.size
+    return bytes
+  }*/
+
+  override fun read(): Int {
+    return if (index < bytes.size) {
+      bytes[index++].toUByte().toInt()
+    } else {
+      -1
+    }
+  }
+
+  override fun close() {
+    input.close()
+  }
+}
+
+class EncryptingOutputStream(
+  val output: OutputStream,
+  val base64EncryptionKey: String?
+) : OutputStream() {
+  val baos = ByteArrayOutputStream()
+
+  override fun write(b: Int) {
+    baos.write(b)
+  }
+
+  override fun close() {
+    if (base64EncryptionKey?.isBlank() == true) {
+      output.write(baos.toByteArray())
+    } else {
+      val encryptor = Encryptor(
+        base64EncryptionKey = base64EncryptionKey,
+        base64DecryptionKey = null
+      )
+      val encryptedBytes = encryptor.encrypt(baos.toByteArray())
+      output.write(encryptedBytes)
+    }
+    output.flush()
+    output.close()
+  }
+
+  override fun flush() {
+    // no flush
+  }
+}

src/main/kotlin/nl/astraeus/persistence/FileManager.kt

@@ -1,4 +1,4 @@
-package nl.astraeus.nl.astraeus.persistence
+package nl.astraeus.persistence
 
 import java.io.File
 

src/main/kotlin/nl/astraeus/persistence/Indexing.kt

@@ -1,4 +1,4 @@
-package nl.astraeus.nl.astraeus.persistence
+package nl.astraeus.persistence
 
 import java.io.Serializable
 import kotlin.reflect.KClass

src/main/kotlin/nl/astraeus/persistence/Logger.kt

@@ -1,8 +1,62 @@
-package nl.astraeus.nl.astraeus.persistence
+package nl.astraeus.persistence
+
+enum class LogLevel {
+  TRACE,
+  DEBUG,
+  INFO,
+  WARN,
+  ERROR
+}
 
 object Logger {
-  var debug: (String) -> Unit = { println("DEBUG: $it") }
-  var info: (String) -> Unit = { println("INFO: $it") }
-  var warn: (String) -> Unit = { println("WARN: $it") }
-  var error: (String) -> Unit = { println("ERROR: $it") }
-}
+  var level: LogLevel = LogLevel.DEBUG
+
+  var tracePrinter: (String) -> Unit = { println(it) }
+  var debugPrinter: (String) -> Unit = { println(it) }
+  var infoPrinter: (String) -> Unit = { println(it) }
+  var warnPrinter: (String) -> Unit = { println(it) }
+  var errorPrinter: (String) -> Unit = { System.err.println(it) }
+
+  fun trace(message: String, vararg parameters: Any?) {
+    if (level <= LogLevel.TRACE) {
+      writeLogMessage(LogLevel.TRACE, message, *parameters)
+    }
+  }
+
+  fun debug(message: String, vararg parameters: Any?) {
+    if (level <= LogLevel.DEBUG) {
+      writeLogMessage(LogLevel.DEBUG, message, *parameters)
+    }
+  }
+
+  fun info(message: String, vararg parameters: Any?) {
+    if (level <= LogLevel.INFO) {
+      writeLogMessage(LogLevel.INFO, message, *parameters)
+    }
+  }
+
+  fun warn(message: String, vararg parameters: Any?) {
+    if (level <= LogLevel.DEBUG) {
+      writeLogMessage(LogLevel.DEBUG, message, *parameters)
+    }
+  }
+
+  fun error(message: String, vararg parameters: Any?) {
+    if (level <= LogLevel.ERROR) {
+      writeLogMessage(LogLevel.ERROR, message, *parameters)
+    }
+  }
+
+  private fun writeLogMessage(level: LogLevel, message: String, vararg parameters: Any?) {
+    val formattedMessage = "[${level}] - ${message.format(*parameters)}"
+
+    when (level) {
+      LogLevel.TRACE -> tracePrinter(formattedMessage)
+      LogLevel.DEBUG -> debugPrinter(formattedMessage)
+      LogLevel.INFO -> infoPrinter(formattedMessage)
+      LogLevel.WARN -> warnPrinter(formattedMessage)
+      LogLevel.ERROR -> errorPrinter(formattedMessage)
+    }
+
+  }
+}

src/main/kotlin/nl/astraeus/persistence/OptimisticLockingException.kt

@@ -0,0 +1,18 @@
+package nl.astraeus.persistence
+
+class OptimisticLockingException : Exception {
+  constructor(
+    obj: Persistable
+  ) : this("Optimistic locking failed for ${obj.javaClass.simpleName} with id ${obj.id}, version ${obj.version}")
+
+  constructor() : super()
+  constructor(message: String?) : super(message)
+  constructor(message: String?, cause: Throwable?) : super(message, cause)
+  constructor(cause: Throwable?) : super(cause)
+  constructor(message: String?, cause: Throwable?, enableSuppression: Boolean, writableStackTrace: Boolean) : super(
+    message,
+    cause,
+    enableSuppression,
+    writableStackTrace
+  )
+}

src/main/kotlin/nl/astraeus/persistence/Persistable.kt

@@ -1,4 +1,4 @@
-package nl.astraeus.nl.astraeus.persistence
+package nl.astraeus.persistence
 
 import java.io.ByteArrayInputStream
 import java.io.ByteArrayOutputStream
@@ -17,7 +17,9 @@ interface Persistable : Serializable {
       }
       ByteArrayInputStream(baos.toByteArray()).use { bais ->
         ObjectInputStream(bais).use { ois ->
-          return ois.readObject() as Persistable
+          val result = ois.readObject() as Persistable
+          result.version++
+          return result
         }
       }
     }

src/main/kotlin/nl/astraeus/persistence/Persistent.kt

@@ -1,4 +1,4 @@
-package nl.astraeus.nl.astraeus.persistence
+package nl.astraeus.persistence
 
 import java.io.File
 
@@ -10,12 +10,33 @@ fun currentTransaction(): Transaction? {
 
 class Persistent(
   directory: File,
+  enableOptimisticLocking: Boolean = false,
+  decryptionKey: String? = null,
+  encryptionKey: String? = null,
   indexes: Array<PersistableIndex> = arrayOf(),
 ) {
-  val datastore: Datastore = Datastore(directory, indexes)
+  val datastore: Datastore = Datastore(
+    directory,
+    enableOptimisticLocking,
+    decryptionKey,
+    encryptionKey,
+    indexes
+  )
 
   fun <T> query(block: Query.() -> T): T {
-    return block(Query(this))
+    var cleanup = false
+    if (transactions.get() == null) {
+      transactions.set(Transaction(this))
+      cleanup = true
+    }
+
+    try {
+      return block(Query(this))
+    } finally {
+      if (cleanup) {
+        transactions.remove()
+      }
+    }
   }
 
   fun transaction(block: Transaction.() -> Unit) {
@@ -28,7 +49,9 @@ class Persistent(
     try {
       block(transactions.get())
 
-      transactions.get().commit()
+      if (cleanup) {
+        transactions.get().commit()
+      }
     } finally {
       if (cleanup) {
         transactions.remove()

src/main/kotlin/nl/astraeus/persistence/Reference.kt

@@ -1,133 +0,0 @@
-package nl.astraeus.nl.astraeus.persistence
-
-import java.io.Serializable
-import kotlin.reflect.KProperty
-
-class Reference<S : Persistable, H : Persistable>(
-  val cls: Class<S>,
-) : Serializable {
-  companion object {
-    private const val serialVersionUID: Long = 1L
-  }
-
-  var id: Long = 0
-
-  operator fun getValue(thisRef: H, property: KProperty<*>): S {
-    return currentTransaction()?.find(cls.kotlin, id) ?: throw IllegalStateException("Reference not found")
-  }
-
-  operator fun setValue(thisRef: H, property: KProperty<*>, value: S) {
-    id = value.id
-    currentTransaction()?.store(value)
-  }
-
-}
-
-class ListReference<S : Persistable, H : Persistable>(
-  val cls: Class<S>,
-) : Serializable {
-  companion object {
-    private const val serialVersionUID: Long = 1L
-  }
-
-  var ids: ReferenceList<S> = ReferenceList(cls)
-
-  operator fun getValue(thisRef: H, property: KProperty<*>): ReferenceList<S> {
-    return ids
-  }
-
-  operator fun setValue(thisRef: H, property: KProperty<*>, value: List<S>) {
-    this.ids.clear()
-    this.ids.addAll(value)
-  }
-}
-
-class ReferenceList<T : Persistable>(
-  val cls: Class<T>,
-) : MutableList<T> {
-  val ids = ArrayList<Long>()
-
-  private fun checkElementIsPersisted(element: T) {
-    if (currentTransaction()?.find(cls.kotlin, element.id) == null) {
-      currentTransaction()?.store(element)
-    }
-  }
-
-  override val size: Int = ids.size
-  override fun clear() = ids.clear()
-
-  override fun addAll(elements: Collection<T>): Boolean {
-    TODO("Not yet implemented")
-  }
-
-  override fun addAll(index: Int, elements: Collection<T>): Boolean {
-    TODO("Not yet implemented")
-  }
-
-  override fun add(index: Int, element: T) {
-    ids.add(index, element.id)
-  }
-
-  override fun add(element: T): Boolean {
-    return ids.add(element.id)
-  }
-
-  override fun get(index: Int): T = currentTransaction()?.find(cls.kotlin, ids[index]) ?: throw IllegalStateException("Reference not found")
-
-  override fun isEmpty(): Boolean = ids.isEmpty()
-
-  override fun iterator(): MutableIterator<T> {
-    TODO("Not yet implemented")
-  }
-
-  override fun listIterator(): MutableListIterator<T> {
-    TODO("Not yet implemented")
-  }
-
-  override fun listIterator(index: Int): MutableListIterator<T> {
-    TODO("Not yet implemented")
-  }
-
-  override fun removeAt(index: Int): T {
-    val id = ids.removeAt(index)
-
-    return currentTransaction()?.find(cls.kotlin, id) ?: throw IllegalStateException("Reference not found")
-  }
-
-  override fun set(index: Int, element: T): T {
-    TODO("Not yet implemented")
-  }
-
-  override fun retainAll(elements: Collection<T>): Boolean {
-    TODO("Not yet implemented")
-  }
-
-  override fun removeAll(elements: Collection<T>): Boolean {
-    TODO("Not yet implemented")
-  }
-
-  override fun remove(element: T): Boolean {
-    TODO("Not yet implemented")
-  }
-
-  override fun subList(fromIndex: Int, toIndex: Int): MutableList<T> {
-    TODO("Not yet implemented")
-  }
-
-  override fun lastIndexOf(element: T): Int {
-    TODO("Not yet implemented")
-  }
-
-  override fun indexOf(element: T): Int {
-    TODO("Not yet implemented")
-  }
-
-  override fun containsAll(elements: Collection<T>): Boolean {
-    TODO("Not yet implemented")
-  }
-
-  override fun contains(element: T): Boolean {
-    TODO("Not yet implemented")
-  }
-
-}

src/main/kotlin/nl/astraeus/persistence/Transaction.kt

@@ -1,10 +1,11 @@
-package nl.astraeus.nl.astraeus.persistence
+package nl.astraeus.persistence
 
 import java.io.Serializable
 import kotlin.reflect.KClass
 
 inline fun <reified T : Persistable> Query.count(): Int = this.count(T::class)
 inline fun <reified T : Persistable> Query.find(id: Long): T? = this.find(T::class, id)
+inline fun <reified T : Persistable> Query.all(): List<T> = this.all(T::class)
 inline fun <reified T : Persistable> Query.search(noinline search: (T) -> Boolean): List<T> =
   this.search(T::class, search)
 inline fun <reified T : Persistable> Query.findByIndex(
@@ -42,6 +43,9 @@ open class Query(
 
   open fun <T : Persistable> find(clazz: KClass<T>, id: Long): T? = persistent.datastore.find(clazz, id)
 
+  open fun <T : Persistable> all(clazz: Class<T>): List<T> = all(clazz.kotlin)
+  open fun <T : Persistable> all(clazz: KClass<T>): List<T> = search(clazz) { _ -> true }
+
   open fun <T : Persistable> search(clazz: KClass<T>, search: (T) -> Boolean): List<T> = persistent.datastore.search(clazz, search)
 
   fun <T : Persistable> findByIndex(
@@ -95,11 +99,13 @@ open class Query(
 class Transaction(
   persistent: Persistent,
 ) : Query(persistent), Serializable {
-  private val actions = mutableListOf<Action>()
+  private val actions = ArrayList<Action>()
 
   fun store(obj: Persistable) {
     if (obj.id == 0L) {
-      obj.id = persistent.datastore.getNextId(obj.javaClass)
+      obj.id = persistent.datastore.getNextIdAndIncrement(obj.javaClass)
+    } else if (obj.id > persistent.datastore.getNextId(obj.javaClass)) {
+      persistent.datastore.setMaxId(obj.javaClass, obj.id + 1)
     }
 
     actions.add(Action(ActionType.STORE, obj))

src/main/kotlin/nl/astraeus/persistence/TransactionLog.kt

@@ -0,0 +1,68 @@
+package nl.astraeus.persistence
+
+import java.io.File
+import java.io.ObjectInputStream
+import java.util.*
+
+class TransactionLog(
+  directory: File,
+  val decryptionKey: String? = null,
+) {
+  val fileManager = FileManager(directory)
+
+  fun showTransactions(printer: (String) -> Unit = ::println) {
+    fileManager.findLastSnapshot().let { (after, snapshot) ->
+      printer("Last snapshot: $snapshot")
+
+      printer("Snapshot:")
+      snapshot?.inputStream()?.use { input ->
+        ObjectInputStream(DecryptingInputStream(input, decryptionKey)).use { ois ->
+          val versionNumber = ois.readInt()
+          check(versionNumber == 1) {
+            "Unsupported version number: $versionNumber"
+          }
+          val transactionNumber = ois.readLong()
+          printer("[$versionNumber]  $transactionNumber")
+          val dataObj = ois.readObject()
+          val data = dataObj as MutableMap<Class<*>, TypeData>
+          printer("Data:")
+          printer("\tClasses:")
+          for ((cls, entries) in  data.entries) {
+            printer("\t\t- $cls: ${entries.data.keys.size}")
+          }
+        }
+      }
+
+      val transactions = fileManager.findTransactionsAfter(after ?: 0L)
+
+      printer("Transactions:")
+      transactions?.forEach { transaction ->
+        transaction.inputStream().use { input ->
+          ObjectInputStream(DecryptingInputStream(input, decryptionKey)).use { ois ->
+            val versionNumber = ois.readInt()
+            check(versionNumber == 1) {
+              "Unsupported version number: $versionNumber"
+            }
+            val transactionNumber = ois.readLong()
+            val actions = ois.readObject()
+            val actionList = when(actions) {
+              is Set<*> -> {
+                LinkedList(actions as Set<Action>)
+              }
+              is List<*> -> {
+                actions as List<Action>
+              }
+              else -> {
+                emptyList()
+              }
+            }
+            printer("\t[$transactionNumber]")
+            for (action in actionList) {
+              printer("\t\t- $action")
+            }
+          }
+        }
+      }
+    }
+  }
+}

src/main/kotlin/nl/astraeus/persistence/reference/Reference.kt

@@ -0,0 +1,69 @@
+package nl.astraeus.persistence.reference
+
+import nl.astraeus.persistence.Persistable
+import nl.astraeus.persistence.currentTransaction
+import java.io.Serializable
+import kotlin.reflect.KProperty
+
+inline fun <reified T : Persistable> reference(
+  initialValue:T
+) = Reference(T::class.java, initialValue)
+
+inline fun <reified T : Persistable> nullableReference(
+  initialValue:T? = null
+) = NullableReference(T::class.java, initialValue)
+
+class Reference<S : Persistable>(
+  val cls: Class<S>,
+  initialValue: S
+) : Serializable {
+  var id: Long = initialValue.id
+
+  operator fun getValue(thisRef: Persistable, property: KProperty<*>): S {
+    check(currentTransaction() != null) {
+      "No transaction available"
+    }
+    return currentTransaction()?.find(cls.kotlin, id) ?: throw IllegalStateException("Reference not found")
+  }
+
+  operator fun setValue(thisRef: Persistable, property: KProperty<*>, value: S) {
+    check(currentTransaction() != null) {
+      "No transaction available"
+    }
+    currentTransaction()?.store(value)
+    id = value.id
+  }
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+}
+
+class NullableReference<S : Persistable>(
+  val cls: Class<S>,
+  initialValue: S? = null
+) : Serializable {
+  var id: Long? = initialValue?.id
+
+  operator fun getValue(thisRef: Persistable, property: KProperty<*>): S? {
+    check(currentTransaction() != null) {
+      "No transaction available"
+    }
+    return currentTransaction()?.find(cls.kotlin, id ?: 0L)
+  }
+
+  operator fun setValue(thisRef: Persistable, property: KProperty<*>, value: S?) {
+    check(currentTransaction() != null) {
+      "No transaction available"
+    }
+    if (value != null) {
+      // todo: only store if not already stored?
+      currentTransaction()?.store(value)
+    }
+    id = value?.id
+  }
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+}

src/main/kotlin/nl/astraeus/persistence/reference/ReferenceCollection.kt

@@ -0,0 +1,82 @@
+package nl.astraeus.persistence.reference
+
+import nl.astraeus.persistence.Persistable
+import nl.astraeus.persistence.currentTransaction
+import java.io.Serializable
+
+inline fun <reified T : Persistable> referenceCollection(
+  ids: MutableCollection<Long> = mutableListOf()
+) = ReferenceCollection(T::class.java, ids)
+
+open class ReferenceCollection<T : Persistable>(
+  val cls: Class<T>,
+  val ids: MutableCollection<Long> = mutableListOf()
+) : MutableCollection<T>, Serializable {
+
+  protected fun checkElementIsPersisted(element: T) {
+    if (currentTransaction()?.find(cls.kotlin, element.id) == null) {
+      currentTransaction()?.store(element)
+    }
+  }
+
+  override val size: Int
+    get() {
+      return ids.size
+    }
+
+  override fun clear() {
+    ids.clear()
+  }
+
+  override fun isEmpty(): Boolean = ids.isEmpty()
+
+  override fun iterator(): MutableIterator<T> {
+    return object : MutableIterator<T> {
+      private var idsIterator = ids.iterator()
+
+      override fun hasNext(): Boolean = idsIterator.hasNext()
+
+      override fun next(): T = idsIterator.next().let {
+        currentTransaction()?.find(cls.kotlin, it)
+      } ?: throw IllegalStateException("Reference not found")
+
+      override fun remove() = idsIterator.remove()
+    }
+  }
+
+  override fun retainAll(elements: Collection<T>): Boolean {
+    return ids.retainAll(elements.map { it.id }.toSet())
+  }
+
+  override fun removeAll(elements: Collection<T>): Boolean {
+    return ids.removeAll(elements.map { it.id }.toSet())
+  }
+
+  override fun remove(element: T): Boolean {
+    return ids.remove(element.id)
+  }
+
+  override fun containsAll(elements: Collection<T>): Boolean {
+    return ids.containsAll(elements.map { it.id })
+  }
+
+  override fun contains(element: T): Boolean {
+    return ids.contains(element.id)
+  }
+
+  override fun addAll(elements: Collection<T>): Boolean {
+    for (element in elements) {
+      checkElementIsPersisted(element)
+    }
+    return ids.addAll(elements.map { it.id })
+  }
+
+  override fun add(element: T): Boolean {
+    checkElementIsPersisted(element)
+    return ids.add(element.id)
+  }
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+}

src/main/kotlin/nl/astraeus/persistence/reference/ReferenceList.kt

@@ -0,0 +1,66 @@
+package nl.astraeus.persistence.reference
+
+import nl.astraeus.persistence.Persistable
+import nl.astraeus.persistence.currentTransaction
+import java.io.Serializable
+
+inline fun <reified T : Persistable> referenceList(
+  ids: MutableList<Long> = mutableListOf()
+) = ReferenceList(T::class.java, ids)
+
+class ReferenceList<T : Persistable>(
+  cls: Class<T>,
+  val idsList: MutableList<Long> = mutableListOf()
+) : ReferenceCollection<T>(cls, idsList), MutableList<T>, Serializable {
+
+  override fun add(index: Int, element: T) {
+    checkElementIsPersisted(element)
+    idsList.add(index, element.id)
+  }
+
+  override fun addAll(index: Int, elements: Collection<T>): Boolean {
+    for(element in elements) {
+      checkElementIsPersisted(element)
+    }
+    return idsList.addAll(index, elements.map { it.id })
+  }
+
+  override fun get(index: Int): T = currentTransaction()?.find(cls.kotlin, idsList[index]) ?: throw IllegalStateException("Reference not found")
+  override fun indexOf(element: T): Int {
+    return idsList.indexOf(element.id)
+  }
+
+  override fun isEmpty(): Boolean = ids.isEmpty()
+  override fun lastIndexOf(element: T): Int {
+    return idsList.lastIndexOf(element.id)
+  }
+
+  override fun listIterator(): MutableListIterator<T> {
+    return ReferenceListIterator(cls, idsList)
+  }
+
+  override fun listIterator(index: Int): MutableListIterator<T> {
+    return ReferenceListIterator(cls, idsList, index)
+  }
+
+  override fun removeAt(index: Int): T {
+    val id = idsList.removeAt(index)
+
+    return currentTransaction()?.find(cls.kotlin, id) ?: throw IllegalStateException("Reference not found")
+  }
+
+  override fun subList(fromIndex: Int, toIndex: Int): MutableList<T> {
+    return ReferenceList(cls, idsList.subList(fromIndex, toIndex))
+  }
+
+  override fun set(index: Int, element: T): T {
+    checkElementIsPersisted(element)
+    idsList[index] = element.id
+    return element
+  }
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+
+}

src/main/kotlin/nl/astraeus/persistence/reference/ReferenceListIterator.kt

@@ -0,0 +1,47 @@
+package nl.astraeus.persistence.reference
+
+import nl.astraeus.persistence.Persistable
+import nl.astraeus.persistence.currentTransaction
+
+class ReferenceListIterator<T : Persistable>(
+  private val cls: Class<T>,
+  idsList: MutableList<Long>,
+  index: Int = 0
+) : MutableListIterator<T> {
+  private val idsIterator = idsList.listIterator(index)
+
+  private fun checkElementIsPersisted(element: T) {
+    if (currentTransaction()?.find(cls.kotlin, element.id) == null) {
+      currentTransaction()?.store(element)
+    }
+  }
+
+  override fun add(element: T) {
+    checkElementIsPersisted(element)
+    idsIterator.add(element.id)
+  }
+
+  override fun hasNext(): Boolean = idsIterator.hasNext()
+  override fun hasPrevious(): Boolean = idsIterator.hasPrevious()
+
+  override fun next(): T = idsIterator.next().let {
+    currentTransaction()?.find(cls.kotlin, it)
+  } ?: throw IllegalStateException("Reference not found")
+
+  override fun nextIndex(): Int = idsIterator.nextIndex()
+
+  override fun previous(): T = idsIterator.previous().let {
+    currentTransaction()?.find(cls.kotlin, it)
+  } ?: throw IllegalStateException("Reference not found")
+
+  override fun previousIndex(): Int = idsIterator.previousIndex()
+
+  override fun remove() {
+    idsIterator.remove()
+  }
+
+  override fun set(element: T) {
+    checkElementIsPersisted(element)
+    idsIterator.set(element.id)
+  }
+}

src/test/java/nl/astraeus/persistence/TestPersistenceJava.java

@@ -1,19 +1,16 @@
 package nl.astraeus.persistence;
 
-import nl.astraeus.nl.astraeus.persistence.AbstractPersistable;
-import nl.astraeus.nl.astraeus.persistence.Index;
-import nl.astraeus.nl.astraeus.persistence.Persistable;
-import nl.astraeus.nl.astraeus.persistence.Persistent;
-import org.jetbrains.annotations.NotNull;
 import org.junit.jupiter.api.Test;
 
 import java.io.File;
+import java.io.Serial;
 import java.util.List;
 
 public class TestPersistenceJava {
 
     static class Person extends AbstractPersistable {
-        private static Long serialVersionUID = 1L;
+        @Serial
+        private static final long serialVersionUID = 1L;
 
         private long id = 0;
         private long version = 0;
@@ -68,6 +65,9 @@ public class TestPersistenceJava {
 
         Persistent persistent = new Persistent(
                 new File("data", "java-test"),
+                false,
+                null,
+                null,
                 new Index[] {
                         new Index<>(
                                 Person.class,
@@ -99,8 +99,7 @@ public class TestPersistenceJava {
             return null;
         });
 
-
-        persistent.transaction((t) -> {
+        persistent.query((t) -> {
             List<Person> persons = t.findByIndex(
                     Person.class,
                     "name",

src/test/kotlin/nl/astraeus/persistence/EncryptionTest.kt

@@ -0,0 +1,54 @@
+package nl.astraeus.persistence
+
+import org.junit.jupiter.api.Test
+
+import org.junit.jupiter.api.Assertions.*
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
+import java.security.SecureRandom
+
+class EncryptionTest {
+
+  @Test
+  fun testKeyGen() {
+    println(generateBase64Key())
+  }
+
+  @Test
+  fun testEncryptDecrypt() {
+    val random = SecureRandom()
+    val randomBytes = ByteArray(random.nextInt(10000))
+    random.nextBytes(randomBytes)
+    val base64Key = generateBase64Key()
+    val encryptor = Encryptor(
+      base64Key,
+      base64Key,
+    )
+
+    val encrypted = encryptor.encrypt(randomBytes)
+    val decrypted = encryptor.decrypt(encrypted)
+
+    assertArrayEquals(randomBytes, decrypted)
+  }
+
+  @Test
+  fun `test encryption-decryption streams`() {
+    val random = SecureRandom()
+    val key = generateBase64Key()
+    val baos = ByteArrayOutputStream()
+    val encryptionStream = EncryptingOutputStream(baos, key)
+    val bytes = ByteArray(random.nextInt(10000))
+    random.nextBytes(bytes)
+
+    encryptionStream.use {
+      it.write(bytes)
+    }
+
+    val bais = ByteArrayInputStream(baos.toByteArray())
+    val decryptingStream = DecryptingInputStream(bais, key)
+
+    val decryptedBytes = decryptingStream.readAllBytes()
+
+    assertArrayEquals(bytes, decryptedBytes)
+  }
+}

src/test/kotlin/nl/astraeus/persistence/TestOptimisticLocking.kt

@@ -0,0 +1,95 @@
+package nl.astraeus.persistence
+
+import nl.astraeus.persistence.domain.Person
+import org.junit.jupiter.api.Assertions.assertNotNull
+import org.junit.jupiter.api.assertThrows
+import java.io.File
+import kotlin.test.Test
+
+class TestOptimisticLocking {
+
+  @Test
+  fun showTransactions() {
+    val log = TransactionLog(File("data", "test-locking"))
+
+    log.showTransactions()
+  }
+
+  @Test
+  fun testOptimisticLocking() {
+    println("Test locking")
+
+    val pst = Persistent(
+      directory = File("data", "test-locking"),
+      true,
+      null,
+      null,
+      arrayOf(
+        index<Person>("name") { p -> (p as? Person)?.name ?: "" },
+      ),
+    )
+
+    pst.transaction {
+      val person = find<Person>(1L) ?: Person(
+        id = 1L,
+        name = "John Doe",
+        age = 25
+      )
+
+      store(person)
+
+      findByIndex<Person>("name", "John Doe").forEach { p ->
+        println("Found person by name: ${p.name} - ${p.age}")
+      }
+    }
+
+    pst.query {
+      val person = find<Person>(1L)
+
+      assertNotNull(person)
+    }
+
+    val threads = Array(2) { index ->
+      Thread {
+        println("Start thread $index")
+        var person: Person? = null
+
+        pst.transaction {
+          Thread.sleep(10L)
+          person = find<Person>(1L)
+          println("Thread $index -> ${person?.version}")
+        }
+
+        if (person != null) {
+          Thread.sleep((index + 1) * 10L)
+
+          if (index == 1) {
+            assertThrows<OptimisticLockingException> {
+              println("Store thread $index -> ${person!!.version}")
+              pst.transaction {
+                store(person!!)
+              }
+            }
+          } else {
+            println("Store thread $index -> ${person!!.version}")
+            pst.transaction {
+              store(person!!)
+            }
+          }
+        }
+      }
+    }
+
+    for (thread in threads) {
+      thread.start()
+    }
+
+    for (thread in threads) {
+      thread.join()
+    }
+
+    pst.datastore.printStatus()
+    //pst.snapshot()
+    pst.removeOldFiles()
+  }
+}

src/test/kotlin/nl/astraeus/persistence/TestPersistence.kt

@@ -1,11 +1,7 @@
 package nl.astraeus.persistence
 
-import nl.astraeus.nl.astraeus.persistence.Persistable
-import nl.astraeus.nl.astraeus.persistence.Persistent
-import nl.astraeus.nl.astraeus.persistence.Reference
-import nl.astraeus.nl.astraeus.persistence.find
-import nl.astraeus.nl.astraeus.persistence.findByIndex
-import nl.astraeus.nl.astraeus.persistence.index
+import nl.astraeus.persistence.domain.Company
+import nl.astraeus.persistence.domain.Person
 import org.junit.jupiter.api.Assertions.assertNotNull
 import org.junit.jupiter.api.Assertions.assertNull
 import java.io.File
@@ -13,29 +9,11 @@ import kotlin.test.Test
 
 class TestPersistence {
 
-  class Company(
-    override var id: Long = 0,
-    override var version: Long = 0,
-    val name: String
-  ) : Persistable, Cloneable {
-    //var persons: MutableList<Person> by ListReference<Person, Company>(Person::class.java)
+  @Test
+  fun showTransactions() {
+    val log = TransactionLog(File("data", "test-persistence"))
 
-    companion object {
-      private const val serialVersionUID: Long = 1L
-    }
-  }
-
-  class Person(
-    override var id: Long = 0,
-    override var version: Long = 0,
-    val name: String,
-    val age: Int,
-  ) : Persistable, Cloneable {
-    var company: Company by Reference<Company, Person>(Company::class.java)
-
-    companion object {
-      private const val serialVersionUID: Long = 1L
-    }
+    log.showTransactions()
   }
 
   @Test
@@ -44,7 +22,7 @@ class TestPersistence {
 
     val pst = Persistent(
       directory = File("data", "test-persistence"),
-      arrayOf(
+      indexes = arrayOf(
         index<Person>("name") { p -> (p as? Person)?.name ?: "" },
         index<Person>("age") { p -> (p as? Person)?.age ?: -1 },
         index<Person>("ageGt20") { p -> ((p as? Person)?.age ?: 0) > 20 },
@@ -127,6 +105,14 @@ class TestPersistence {
       assertNotNull(person)
     }
 
+    pst.transaction {
+      val p1 = find<Person>(1L)
+      val p2 = find<Person>(1L)
+
+      store(p2!!)
+      store(p1!!)
+    }
+
     pst.transaction {
       val person = find(Person::class, 1L)
 
@@ -193,6 +179,6 @@ class TestPersistence {
     }
 
     pst.datastore.printStatus()
-    //pst.removeOldFiles()
+    pst.removeOldFiles()
   }
 }

src/test/kotlin/nl/astraeus/persistence/TestPersistenceJavaInKotlin.kt

@@ -0,0 +1,66 @@
+package nl.astraeus.persistence
+
+import org.junit.jupiter.api.Test
+import java.io.File
+
+class TestPersistenceJavaInKotlin {
+
+  internal class Person(
+    var name: String,
+    var age: Int
+  ) : Persistable {
+    override var id: Long = 0
+    override var version: Long = 0
+
+    companion object {
+      private const val serialVersionUID = 1L
+    }
+  }
+
+  @Test
+  fun testPersistence() {
+    println("TestPersistenceJavaInKotlin.testPersistence")
+
+    val persistent = Persistent(
+      File("data", "java-kotlin-test"),
+      enableOptimisticLocking = false,
+      indexes = arrayOf(
+        Index(
+          Person::class,
+          "name"
+        ) { p -> (p as Person).name }
+      ),
+    )
+
+    persistent.transaction {
+      val person = find(Person::class.java, 1L)
+
+      if (person != null) {
+        println(
+          "Person: ${person.name} is ${person.age} years old."
+        )
+      }
+    }
+
+    persistent.transaction {
+      val person = Person("John Doe", 42)
+
+      store(person)
+    }
+
+    persistent.query {
+      val persons = findByIndex(
+        Person::class.java,
+        "name",
+        "John Doe"
+      )
+      for (person in persons) {
+        println("Person: ${person.name}  is ${person.age} years old.")
+      }
+    }
+
+    persistent.snapshot()
+    persistent.datastore.printStatus()
+    persistent.removeOldFiles()
+  }
+}

src/test/kotlin/nl/astraeus/persistence/TestReferences.kt

@@ -0,0 +1,71 @@
+package nl.astraeus.persistence
+
+import nl.astraeus.persistence.domain.Company
+import nl.astraeus.persistence.reference.reference
+import org.junit.jupiter.api.assertThrows
+import java.io.File
+import kotlin.test.Test
+
+class TestReferences {
+
+  class Person(
+    override var id: Long = 0,
+    override var version: Long = 0,
+    val name: String,
+    company: Company
+  ) : Persistable, Cloneable {
+    var company: Company by reference(company)
+
+    companion object {
+      private const val serialVersionUID: Long = 1L
+    }
+
+    override fun toString(): String {
+      return "Person(id=$id, version=$version, name='$name')"
+    }
+  }
+
+  @Test
+  fun showTransactions() {
+    val log = TransactionLog(File("data", "test-references"))
+
+    log.showTransactions()
+  }
+
+  @Test
+  fun testSerializeDeSerializeReferenceList() {
+    val pst = Persistent(
+      directory = File("data", "test-references"),
+    )
+
+    pst.transaction {
+      val company = Company(
+        id = 1L,
+        name = "ACME"
+      )
+
+      val person = Person(
+        id = 0L,
+        name = "John Doe",
+        company = company
+      )
+      store(person)
+
+      store(company)
+
+      for (p in company.persons) {
+        println("Person: $p")
+      }
+
+      delete(company)
+
+      // company is gone, can't get it through person anymore
+      assertThrows<IllegalStateException> {
+        println("Company in person: ${person.company}")
+      }
+
+      delete(person)
+    }
+  }
+
+}

src/test/kotlin/nl/astraeus/persistence/TestThreaded.kt

@@ -1,41 +1,19 @@
 package nl.astraeus.persistence
 
-import nl.astraeus.nl.astraeus.persistence.Persistable
-import nl.astraeus.nl.astraeus.persistence.Persistent
-import nl.astraeus.nl.astraeus.persistence.Reference
-import nl.astraeus.nl.astraeus.persistence.count
-import nl.astraeus.nl.astraeus.persistence.index
-import nl.astraeus.nl.astraeus.persistence.searchIndex
+import nl.astraeus.persistence.domain.Company
+import nl.astraeus.persistence.domain.Person
 import java.io.File
 import kotlin.random.Random
 import kotlin.test.Test
+import kotlin.test.assertEquals
 
 class TestThreaded {
 
-  class Company(
-    override var id: Long = 0,
-    override var version: Long = 0,
-    val name: String,
-    val adres: String = "Blaat"
-  ) : Persistable, Cloneable {
-    //var persons: MutableList<Person> by ListReference<Person, Company>(Person::class.java)
+  @Test
+  fun showTransactions() {
+    val log = TransactionLog(File("data", "test-threaded"))
 
-    companion object {
-      private const val serialVersionUID: Long = 1L
-    }
-  }
-
-  class Person(
-    override var id: Long = 0,
-    override var version: Long = 0,
-    val name: String,
-    val age: Int,
-  ) : Persistable, Cloneable {
-    var company: Company by Reference<Company, Person>(Company::class.java)
-
-    companion object {
-      private const val serialVersionUID: Long = 1L
-    }
+    log.showTransactions()
   }
 
   @Test
@@ -44,7 +22,7 @@ class TestThreaded {
 
     val pst = Persistent(
       directory = File("data", "test-threaded"),
-      arrayOf(
+      indexes = arrayOf(
         index<Person>("name") { p -> (p as? Person)?.name ?: "" },
         index<Person>("age") { p -> (p as? Person)?.age ?: -1 },
         index<Person>("ageGt20") { p -> ((p as? Person)?.age ?: 0) > 20 },
@@ -65,6 +43,7 @@ class TestThreaded {
             person.name to person.age
           }
         },
+        index<Person>("personCompanyId") { p -> (p as? Person)?.company?.id ?: 0L },
         index<Company>("name") { p -> (p as? Company)?.name ?: "" },
       )
     )
@@ -103,6 +82,7 @@ class TestThreaded {
         Thread(runnable)
       }
 
+      val start = System.nanoTime()
       for (thread in threads) {
         thread.start()
       }
@@ -110,16 +90,26 @@ class TestThreaded {
       for (thread in threads) {
         thread.join()
       }
+      println("Store elapsed time: ${(System.nanoTime() - start) / 1_000_000}ms")
     }
 
-    pst.query {
-      searchIndex<Person>("nameAndAge") { nameAndAge ->
-        val (name, age) = nameAndAge as Pair<String, Int>
-
-        name.contains("mit") && age > 80
-      }.forEach { p ->
-        println("Found person by name and age: ${p.id}: ${p.name} - ${p.age}")
+    var start = 0L
+    repeat(10) {
+      start = System.nanoTime()
+      val withoutIndex = pst.query {
+        search<Person> { person ->
+          person.age == 20
+        }
       }
+      println("withoutIndex elapsed time: ${(System.nanoTime() - start) / 1_000_000f}ms")
+
+      start = System.nanoTime()
+      val withIndex = pst.query {
+        searchIndex<Person>("age") { age -> (age as? Int ?: -1) == 20 }
+      }
+      println("withIndex elapsed time: ${(System.nanoTime() - start) / 1_000_000f}ms")
+
+      assertEquals(withIndex.size, withoutIndex.size)
     }
 
     pst.snapshot()

src/test/kotlin/nl/astraeus/persistence/domain/Company.kt

@@ -0,0 +1,20 @@
+package nl.astraeus.persistence.domain
+
+import nl.astraeus.persistence.Persistable
+import nl.astraeus.persistence.reference.referenceCollection
+
+class Company(
+  override var id: Long = 0,
+  override var version: Long = 0,
+  val name: String
+) : Persistable, Cloneable {
+  val persons: MutableCollection<Person> = referenceCollection()
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+
+  override fun toString(): String {
+    return "Company(id=$id, version=$version, name='$name', persons=${persons.size})"
+  }
+}

src/test/kotlin/nl/astraeus/persistence/domain/Person.kt

@@ -0,0 +1,22 @@
+package nl.astraeus.persistence.domain
+
+import nl.astraeus.persistence.Persistable
+import nl.astraeus.persistence.reference.nullableReference
+
+class Person(
+  override var id: Long = 0,
+  override var version: Long = 0,
+  val name: String,
+  val age: Int,
+  company: Company? = null
+) : Persistable, Cloneable {
+  var company: Company? by nullableReference(company)
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+
+  override fun toString(): String {
+    return "Person(id=$id, version=$version, name='$name', age=$age)"
+  }
+}