Fix multi-threading, add Query interface, java test

.idea/kotlinc.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="KotlinJpsPluginSettings">
-    <option name="version" value="2.0.0-RC2" />
+    <option name="version" value="1.9.23" />
   </component>
 </project>

.idea/misc.xml

@@ -4,7 +4,7 @@
   <component name="FrameworkDetectionExcludesConfiguration">
     <file type="web" url="file://$PROJECT_DIR$" />
   </component>
-  <component name="ProjectRootManager" version="2" languageLevel="JDK_1_8" default="true" project-jdk-name="1.8" project-jdk-type="JavaSDK">
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_17" project-jdk-name="1.8" project-jdk-type="JavaSDK">
     <output url="file://$PROJECT_DIR$/out" />
   </component>
   <component name="accountSettings">

build.gradle.kts

@@ -1,9 +1,9 @@
 plugins {
-    kotlin("jvm") version "2.0.0-RC2"
+    kotlin("jvm") version "1.9.23"
 }
 
 group = "nl.astraeus"
-version = "1.0-SNAPSHOT"
+version = "1.0.0-SNAPSHOT"
 
 repositories {
     mavenCentral()
@@ -16,6 +16,7 @@ dependencies {
 tasks.test {
     useJUnitPlatform()
 }
+
 kotlin {
     jvmToolchain(17)
 }

src/main/kotlin/nl/astraeus/persistence/Datastore.kt

@@ -5,8 +5,8 @@ import java.io.ObjectInputStream
 import java.io.ObjectOutputStream
 import java.io.Serializable
 import java.text.DecimalFormat
-import java.util.*
 import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.atomic.AtomicLong
 import kotlin.reflect.KClass
 
 enum class ActionType {
@@ -15,8 +15,8 @@ enum class ActionType {
 }
 
 class TypeData(
-  var nextId: Long = 1L,
+  var nextId: AtomicLong = AtomicLong(1L),
-  val data: MutableMap<Any, Persistable> = ConcurrentHashMap(),
+  val data: MutableMap<Serializable, Persistable> = ConcurrentHashMap(),
 ) : Serializable
 
 class Action(
@@ -35,10 +35,6 @@ class Datastore(
   private val indexes: MutableMap<Class<*>, MutableMap<String, PersistableIndex>> = ConcurrentHashMap()
 
   init {
-    if (!directory.exists()) {
-      directory.mkdirs()
-    }
-
     for (index in indexes) {
       this.indexes.getOrPut(index.cls) {
         ConcurrentHashMap()
@@ -48,49 +44,60 @@ class Datastore(
     loadTransactions()
   }
 
-  private fun loadTransactions() {
+  fun getNextId(javaClass: Class<Persistable>): Long {
+    if (data[javaClass] == null) {
       synchronized(this) {
-      val snapshots: Array<File>? = directory.listFiles { _, name -> name.startsWith("transaction-") && name.endsWith(".snp") }
+        if (data[javaClass] == null) {
-      val files: Array<File>? = directory.listFiles { _, name -> name.startsWith("transaction-") && name.endsWith(".trn") }
+          data[javaClass] = TypeData()
-
-      var lastSnapshot: Long? = null
-      var lastSnapshotFile: File? = null
-
-      snapshots?.let {
-        it.forEach {
-          val trnx = getTrnx(it)
-          if (lastSnapshot == null || trnx > (lastSnapshot ?: 0L)) {
-            lastSnapshot = trnx
-            lastSnapshotFile = it
         }
       }
     }
 
-      val lastSnapshotFile2 = fileManager.findLastSnapshot()
+    return data[javaClass]!!.nextId.getAndIncrement()
+  }
+
+  fun setMaxId(javaClass: Class<Persistable>, id: Long) {
+    val nextId = data.getOrPut(javaClass) { TypeData() }.nextId
+    if (nextId.get() <= id) nextId.set(id + 1)
+  }
+
+  override fun toString(): String {
+    return "Datastore(directory=${fileManager.directory}, classes=${data.keys.size}, indexes=${indexes.keys.size})"
+  }
+
+  // print status, show number of entries for each class and index
+  fun printStatus() {
+    println(this)
+    for ((cls, typeData) in data) {
+      println("  ${cls.simpleName}: ${typeData.data.size}")
+      for ((name, index) in indexes.getOrDefault(cls, mutableMapOf())) {
+        println("    $name: ${index.index.size}")
+      }
+    }
+  }
+
+  private fun loadTransactions() {
+    val start = System.nanoTime()
+
+    synchronized(this) {
+      val (lastSnapshot, lastSnapshotFile) = fileManager.findLastSnapshot()
 
       if (lastSnapshotFile != null) {
-        ObjectInputStream(lastSnapshotFile?.inputStream()).use { ois ->
+        ObjectInputStream(lastSnapshotFile.inputStream()).use { ois ->
           readSnapshot(ois)
         }
       }
 
-      val trns = fileManager.findTransactionsAfter(lastSnapshot ?: 0L)
+      val transactions = fileManager.findTransactionsAfter(lastSnapshot ?: 0L)
 
-      files?.also { snaphotFiles ->
+      transactions?.forEach { file ->
-        Arrays.sort(snaphotFiles) { o1, o2 -> if (getTrnx(o1) > getTrnx(o2)) 1 else -1}
-
-        snaphotFiles.forEach { file ->
-          if (getTrnx(file) > (lastSnapshot ?: 0L)) {
         ObjectInputStream(file.inputStream()).use { ois ->
-              val transactionNumber = ois.readLong()
+          readTransaction(ois)
-              nextTransactionNumber = transactionNumber + 1
-              val actions = ois.readObject() as MutableList<Action>
-              execute(actions)
-            }
-          }
         }
       }
     }
+
+    Logger.debug("Loaded transactions in ${(System.nanoTime() - start) / 1_000_000}ms")
   }
 
   private fun getTrnx(file: File): Long {
@@ -106,13 +113,15 @@ class Datastore(
 
         when (action.type) {
           ActionType.STORE -> {
-            if (action.obj.id == 0L) {
-              action.obj.id = typeData.nextId++
-            }
             typeData.data[action.obj.id] = action.obj
 
+            if (action.obj.id >= typeData.nextId.get()) {
+              typeData.nextId.set(action.obj.id + 1)
+            }
+
             for (index in indexes[action.obj::class.java]?.values ?: listOf()) {
-              index.add(action.obj as Persistable)
+              index.remove(action.obj.id)
+              index.add(action.obj)
             }
           }
 
@@ -120,7 +129,7 @@ class Datastore(
             typeData.data.remove(action.obj.id)
 
             for (index in indexes[action.obj::class.java]?.values ?: listOf()) {
-              index.remove(action.obj)
+              index.remove(action.obj.id)
             }
           }
         }
@@ -128,6 +137,15 @@ class Datastore(
     }
   }
 
+
+  fun <T : Persistable> count(clazz: KClass<T>): Int {
+    val typeData = data.getOrPut(clazz.java) {
+      TypeData()
+    }
+
+    return typeData.data.size
+  }
+
   fun <T : Persistable> find(clazz: KClass<T>, id: Long): T? {
     val typeData = data.getOrPut(clazz.java) {
       TypeData()
@@ -158,24 +176,43 @@ class Datastore(
     return indexes[kClass.java]?.get(indexName)
   }
 
-  fun storeActions(actions: MutableList<Action>) {
+  fun storeAndExecute(actions: MutableList<Action>) {
     if (actions.isNotEmpty()) {
       synchronized(this) {
-        val number = transactionFormatter.format(nextTransactionNumber)
+        writeTransaction(actions)
-        val file = File(directory, "transaction-$number.trn")
+        execute(actions)
-        ObjectOutputStream(file.outputStream()).use { oos ->
-          oos.writeLong(nextTransactionNumber++)
-          oos.writeObject(actions)
-        }
       }
     }
   }
 
+  private fun readTransaction(ois: ObjectInputStream) {
+    val versionNumber = ois.readInt()
+    check (versionNumber == 1) { "Unsupported version number: $versionNumber" }
+    val transactionNumber = ois.readLong()
+    nextTransactionNumber = transactionNumber + 1
+    val actions = ois.readObject() as MutableList<Action>
+    execute(actions)
+  }
+
+  private fun writeTransaction(actions: MutableList<Action>) {
+    val number = transactionFormatter.format(nextTransactionNumber)
+    val file = File(directory, "transaction-$number.trn")
+    ObjectOutputStream(file.outputStream()).use { oos ->
+      // version number
+      oos.writeInt(1)
+      oos.writeLong(nextTransactionNumber++)
+      oos.writeObject(actions)
+    }
+  }
+
   fun snapshot() {
+    val start = System.nanoTime()
     synchronized(this) {
       val number = transactionFormatter.format(nextTransactionNumber)
       val file = File(directory, "transaction-$number.snp")
       ObjectOutputStream(file.outputStream()).use { oos ->
+        // version number
+        oos.writeInt(1)
         oos.writeLong(nextTransactionNumber++)
         oos.writeObject(data)
         oos.writeInt(indexes.size)
@@ -189,9 +226,12 @@ class Datastore(
         }
       }
     }
+    Logger.debug("Snapshot in ${(System.nanoTime() - start) / 1_000_000}ms")
   }
 
   private fun readSnapshot(ois: ObjectInputStream) {
+    val versionNumber = ois.readInt()
+    check (versionNumber == 1) { "Unsupported version number: $versionNumber" }
     nextTransactionNumber = ois.readLong() + 1
     data.clear()
     data.putAll(ois.readObject() as MutableMap<Class<*>, TypeData>)
@@ -226,4 +266,8 @@ class Datastore(
     }
   }
 
+  fun removeOldFiles() {
+    fileManager.removeOldFiles()
+  }
+
 }

src/main/kotlin/nl/astraeus/persistence/FileManager.kt

@@ -0,0 +1,62 @@
+package nl.astraeus.nl.astraeus.persistence
+
+import java.io.File
+
+class FileManager(
+  val directory: File
+) {
+
+  init {
+    if (!directory.exists()) {
+      directory.mkdirs()
+    }
+  }
+
+  fun findLastSnapshot(): Pair<Long?, File?> {
+    directory.listFiles { _, name ->
+      name.startsWith("transaction-") && name.endsWith(".snp")
+    }?.maxByOrNull {
+      getTrnx(it)
+    }?.also { file ->
+      return getTrnx(file) to file
+    }
+
+    return null to null
+  }
+
+  fun findTransactionsAfter(trnx: Long): List<File>? {
+    return directory.listFiles { _, name ->
+      name.startsWith("transaction-") && name.endsWith(".trn")
+    }?.filter {
+      getTrnx(it) > trnx
+    }?.sortedBy {
+      getTrnx(it)
+    }
+  }
+
+  private fun getTrnx(file: File): Long {
+    // todo: add checks, improve performance
+    return file.name
+      .substringAfterLast('/')
+      .substringAfter("transaction-")
+      .substringBefore(".")
+      .toLong()
+  }
+
+  fun removeOldFiles() {
+    val (lastSnapshot, _) = findLastSnapshot()
+
+    if (lastSnapshot != null) {
+      val files = directory.listFiles { _, name ->
+        name.startsWith("transaction-")
+      }?.filter {
+        getTrnx(it) < lastSnapshot
+      }
+
+      files?.forEach {
+        it.delete()
+      }
+    }
+  }
+
+}

src/main/kotlin/nl/astraeus/persistence/Indexing.kt

@@ -5,14 +5,28 @@ import kotlin.reflect.KClass
 
 typealias PersistableIndex = Index<out Persistable>
 
+inline fun <reified T : Persistable> index(
+  name: String,
+  noinline value: (Persistable) -> Serializable?
+): Index<T> = Index(
+  T::class,
+  name,
+  value
+)
+
 class Index<T : Persistable>(
-  kcls: KClass<T>,
+  val cls: Class<T>,
   val name: String,
   val value: (Persistable) -> Serializable?,
 ) : Serializable {
-  val cls: Class<T> = kcls.java
   val index = mutableMapOf<Serializable, MutableSet<Long>>()
 
+  constructor(
+    cls: KClass<T>,
+    name: String,
+    value: (Persistable) -> Serializable?
+  ) : this(cls.java, name, value)
+
   fun add(obj: Persistable) {
     val key = value(obj)
 
@@ -27,11 +41,17 @@ class Index<T : Persistable>(
     index[key]?.remove(obj.id)
   }
 
+  fun remove(id: Long) {
+    for ((_, ids) in index) {
+      ids.remove(id)
+    }
+  }
+
   fun find(key: Any): List<T> {
     return index[key]?.mapNotNull { currentTransaction()?.find(cls.kotlin, it) } ?: emptyList()
   }
 
-  fun matches(obj: Persistable, value: Any): Boolean {
+  fun matches(obj: Persistable, value: Serializable): Boolean {
     return value(obj) == value
   }
 }

src/main/kotlin/nl/astraeus/persistence/Logger.kt

@@ -0,0 +1,8 @@
+package nl.astraeus.nl.astraeus.persistence
+
+object Logger {
+  var debug: (String) -> Unit = { println("DEBUG: $it") }
+  var info: (String) -> Unit = { println("INFO: $it") }
+  var warn: (String) -> Unit = { println("WARN: $it") }
+  var error: (String) -> Unit = { println("ERROR: $it") }
+}

src/main/kotlin/nl/astraeus/persistence/Persistable.kt

@@ -6,7 +6,7 @@ import java.io.ObjectInputStream
 import java.io.ObjectOutputStream
 import java.io.Serializable
 
-interface Persistable : Serializable, Cloneable {
+interface Persistable : Serializable {
   var id: Long
   var version: Long
 
@@ -23,3 +23,13 @@ interface Persistable : Serializable, Cloneable {
     }
   }
 }
+
+abstract class AbstractPersistable : Persistable {
+  override fun copy(): Persistable {
+    return super.copy()
+  }
+
+  companion object {
+    private const val serialVersionUID: Long = 1L
+  }
+}

src/main/kotlin/nl/astraeus/persistence/Persistent.kt

@@ -14,6 +14,10 @@ class Persistent(
 ) {
   val datastore: Datastore = Datastore(directory, indexes)
 
+  fun <T> query(block: Query.() -> T): T {
+    return block(Query(this))
+  }
+
   fun transaction(block: Transaction.() -> Unit) {
     var cleanup = false
     if (transactions.get() == null) {
@@ -35,4 +39,8 @@ class Persistent(
   fun snapshot() {
     datastore.snapshot()
   }
+
+  fun removeOldFiles() {
+    datastore.removeOldFiles()
+  }
 }

src/main/kotlin/nl/astraeus/persistence/Transaction.kt

@@ -3,12 +3,105 @@ package nl.astraeus.nl.astraeus.persistence
 import java.io.Serializable
 import kotlin.reflect.KClass
 
-class Transaction(
+inline fun <reified T : Persistable> Query.count(): Int = this.count(T::class)
+inline fun <reified T : Persistable> Query.find(id: Long): T? = this.find(T::class, id)
+inline fun <reified T : Persistable> Query.search(noinline search: (T) -> Boolean): List<T> =
+  this.search(T::class, search)
+inline fun <reified T : Persistable> Query.findByIndex(
+  indexName: String,
+  search: Serializable
+): List<T> = this.findByIndex(T::class, indexName, search)
+inline fun <reified T : Persistable> Query.searchIndex(
+  indexName: String,
+  noinline search: (Serializable) -> Boolean,
+): List<T> = this.searchIndex(T::class, indexName, search)
+
+inline fun <reified T : Persistable> Transaction.count(): Int = this.count(T::class)
+inline fun <reified T : Persistable> Transaction.find(id: Long): T? = this.find(T::class, id)
+inline fun <reified T : Persistable> Transaction.search(noinline search: (T) -> Boolean): List<T> =
+  this.search(T::class, search)
+inline fun <reified T : Persistable> Transaction.findByIndex(
+  indexName: String,
+  search: Serializable
+): List<T> = this.findByIndex(T::class, indexName, search)
+inline fun <reified T : Persistable> Transaction.searchIndex(
+  indexName: String,
+  noinline search: (Serializable) -> Boolean,
+): List<T> = this.searchIndex(T::class, indexName, search)
+
+open class Query(
   val persistent: Persistent,
 ) : Serializable {
+
+  fun <T : Persistable> count(clazz: Class<T>): Int = count(clazz.kotlin)
+  fun <T : Persistable> count(clazz: KClass<T>): Int = persistent.datastore.count(clazz)
+
+  fun <T : Persistable> find(clazz: Class<T>, id: Long): T? {
+    return find(clazz.kotlin, id)
+  }
+
+  open fun <T : Persistable> find(clazz: KClass<T>, id: Long): T? = persistent.datastore.find(clazz, id)
+
+  open fun <T : Persistable> search(clazz: KClass<T>, search: (T) -> Boolean): List<T> = persistent.datastore.search(clazz, search)
+
+  fun <T : Persistable> findByIndex(
+    kcls: KClass<T>,
+    indexName: String,
+    search: Serializable
+  ): List<T> = findByIndex(kcls.java, indexName, search)
+
+  open fun <T : Persistable> findByIndex(
+    cls: Class<T>,
+    indexName: String,
+    search: Serializable
+  ): List<T> {
+    val result = mutableListOf<T>()
+    val index = persistent.datastore.findIndex(cls.kotlin, indexName)
+      ?: throw IllegalArgumentException("Index with name $indexName not found for class ${cls.simpleName}")
+
+    index.find(search).forEach { id ->
+      result.add(id as T)
+    }
+
+    return result
+  }
+
+  fun <T : Persistable> searchIndex(
+    kcls: KClass<T>,
+    indexName: String,
+    search: (Serializable) -> Boolean,
+  ): List<T> = searchIndex(kcls.java, indexName, search)
+
+  open fun <T : Persistable> searchIndex(
+    cls: Class<T>,
+    indexName: String,
+    search: (Serializable) -> Boolean,
+  ): List<T> {
+    val result = mutableListOf<T>()
+    val index = persistent.datastore.findIndex(cls.kotlin, indexName) ?: throw IllegalArgumentException("Index not found")
+
+    index.index.keys.forEach { key ->
+      if (search(key)) {
+        index.find(key).forEach { id ->
+          result.add(id as T)
+        }
+      }
+    }
+
+    return result
+  }
+}
+
+class Transaction(
+  persistent: Persistent,
+) : Query(persistent), Serializable {
   private val actions = mutableListOf<Action>()
 
   fun store(obj: Persistable) {
+    if (obj.id == 0L) {
+      obj.id = persistent.datastore.getNextId(obj.javaClass)
+    }
+
     actions.add(Action(ActionType.STORE, obj))
   }
 
@@ -16,23 +109,23 @@ class Transaction(
     actions.add(Action(ActionType.DELETE, obj))
   }
 
-  fun <T : Persistable> find(clazz: KClass<T>, id: Long): T? {
+  fun commit() {
-    var result: T? = persistent.datastore.find(clazz, id)
+    persistent.datastore.storeAndExecute(actions)
+    actions.clear()
+  }
+
+  override fun <T : Persistable> find(clazz: KClass<T>, id: Long): T? {
+    var result = super.find(clazz, id)
 
     for (action in actions) {
       if (action.obj::class == clazz && action.obj.id == id) {
-        result = when {
+        result = when(action.type) {
-          action.type == ActionType.DELETE -> {
+          ActionType.DELETE -> {
             null
           }
-
+          ActionType.STORE -> {
-          action.type == ActionType.STORE -> {
             action.obj as? T
           }
-
-          else -> {
-            result
-          }
         }
       }
     }
@@ -40,10 +133,9 @@ class Transaction(
     return result
   }
 
-  fun <T : Persistable> search(clazz: KClass<T>, search: (T) -> Boolean): List<T> {
+  override fun <T : Persistable> search(clazz: KClass<T>, search: (T) -> Boolean): List<T> {
-    val fromDatastore: List<T> = persistent.datastore.search(clazz, search)
     val result = mutableListOf<T>()
-    result.addAll(fromDatastore)
+    result.addAll(super.search(clazz, search))
 
     for (obj in result) {
       for (action in actions) {
@@ -61,26 +153,13 @@ class Transaction(
     return result
   }
 
-  fun commit() {
+  override fun <T : Persistable> findByIndex(cls: Class<T>, indexName: String, search: Serializable): List<T> {
-    persistent.datastore.storeActions(actions)
-    persistent.datastore.execute(actions)
-    actions.clear()
-  }
-
-  fun <T : Persistable> findByIndex(
-    kClass: KClass<T>,
-    indexName: String,
-    search: Any
-  ): List<T> {
     val result = mutableListOf<T>()
-    val index = persistent.datastore.findIndex(kClass, indexName) ?: throw IllegalArgumentException("Index not found")
+    val index = persistent.datastore.findIndex(cls.kotlin, indexName) ?: throw IllegalArgumentException("Index not found")
-
+    result.addAll(super.findByIndex(cls, indexName, search))
-    index.find(search).forEach { id ->
-      result.add(id as T)
-    }
 
     for (action in actions) {
-      if (action.obj::class == kClass) {
+      if (action.obj::class == cls.kotlin) {
         if (action.type == ActionType.DELETE) {
           if (index.matches(action.obj, search)) {
             result.remove(action.obj as T)
@@ -97,4 +176,25 @@ class Transaction(
     return result
   }
 
+  override fun <T : Persistable> searchIndex(cls: Class<T>, indexName: String, search: (Serializable) -> Boolean): List<T> {
+    val result = mutableListOf<T>()
+    val index = persistent.datastore.findIndex(cls.kotlin, indexName) ?: throw IllegalArgumentException("Index not found")
+    result.addAll(super.searchIndex(cls, indexName, search))
+
+    for (action in actions) {
+      if (action.obj::class == cls.kotlin) {
+        val indexedValue = index.value(action.obj)
+        if (indexedValue != null && index.matches(action.obj, indexedValue)) {
+          if (action.type == ActionType.DELETE) {
+            result.remove(action.obj as T)
+          } else if (action.type == ActionType.STORE) {
+            result.remove(action.obj)
+            result.add(action.obj as T)
+          }
+        }
+      }
+    }
+
+    return result
+  }
 }

src/test/java/nl/astraeus/persistence/TestPersistenceJava.java

@@ -0,0 +1,126 @@
+package nl.astraeus.persistence;
+
+import nl.astraeus.nl.astraeus.persistence.AbstractPersistable;
+import nl.astraeus.nl.astraeus.persistence.Index;
+import nl.astraeus.nl.astraeus.persistence.Persistable;
+import nl.astraeus.nl.astraeus.persistence.Persistent;
+import org.jetbrains.annotations.NotNull;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.util.List;
+
+public class TestPersistenceJava {
+
+    static class Person extends AbstractPersistable {
+        private static Long serialVersionUID = 1L;
+
+        private long id = 0;
+        private long version = 0;
+        private String name;
+        private int age;
+
+        public Person(String name, int age) {
+            this.name = name;
+            this.age = age;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public void setName(String name) {
+            this.name = name;
+        }
+
+        public int getAge() {
+            return age;
+        }
+
+        public void setAge(int age) {
+            this.age = age;
+        }
+
+        @Override
+        public long getId() {
+            return id;
+        }
+
+        @Override
+        public void setId(long id) {
+            this.id = id;
+        }
+
+        @Override
+        public long getVersion() {
+            return version;
+        }
+
+        @Override
+        public void setVersion(long version) {
+            this.version = version;
+        }
+    }
+
+    @Test
+    void testPersistence() {
+        System.out.println("TestPersistenceJava.testPersistence");
+
+        Persistent persistent = new Persistent(
+                new File("data", "java-test"),
+                new Index[] {
+                        new Index<>(
+                                Person.class,
+                                "name",
+                                (p) -> ((Person)p).getName()
+                        )
+                }
+        );
+
+        persistent.transaction((t) -> {
+            Person person = t.find(Person.class, 1L);
+
+            if (person != null) {
+                System.out.println(
+                        "Person: "
+                        + person.getName() + " is "
+                        + person.getAge() + " years old."
+                );
+            }
+
+            return null;
+        });
+
+        persistent.transaction((t) -> {
+            Person person = new Person("John Doe", 42);
+
+            t.store(person);
+
+            return null;
+        });
+
+
+        persistent.transaction((t) -> {
+            List<Person> persons = t.findByIndex(
+                    Person.class,
+                    "name",
+                    "John Doe"
+            );
+
+            for (Person person : persons) {
+                System.out.println(
+                        "Person: "
+                        + person.getName() + " is "
+                        + person.getAge() + " years old."
+                );
+            }
+
+            return null;
+        });
+
+        persistent.snapshot();
+        persistent.getDatastore().printStatus();
+        persistent.removeOldFiles();
+    }
+
+}

src/test/kotlin/nl/astraeus/persistence/TestPersistence.kt

@@ -1,9 +1,13 @@
 package nl.astraeus.persistence
 
-import nl.astraeus.nl.astraeus.persistence.Index
 import nl.astraeus.nl.astraeus.persistence.Persistable
 import nl.astraeus.nl.astraeus.persistence.Persistent
 import nl.astraeus.nl.astraeus.persistence.Reference
+import nl.astraeus.nl.astraeus.persistence.find
+import nl.astraeus.nl.astraeus.persistence.findByIndex
+import nl.astraeus.nl.astraeus.persistence.index
+import org.junit.jupiter.api.Assertions.assertNotNull
+import org.junit.jupiter.api.Assertions.assertNull
 import java.io.File
 import kotlin.test.Test
 
@@ -39,31 +43,31 @@ class TestPersistence {
     println("Test persistence")
 
     val pst = Persistent(
-      directory = File("data"),
+      directory = File("data", "test-persistence"),
       arrayOf(
-        Index(Person::class, "name") { p -> (p as? Person)?.name ?: "" },
+        index<Person>("name") { p -> (p as? Person)?.name ?: "" },
-        Index(Person::class, "age") { p -> (p as? Person)?.age ?: -1 },
+        index<Person>("age") { p -> (p as? Person)?.age ?: -1 },
-        Index(Person::class, "ageGt20") { p -> ((p as? Person)?.age ?: 0) > 20 },
+        index<Person>("ageGt20") { p -> ((p as? Person)?.age ?: 0) > 20 },
-        Index(Person::class, "ageGt23") { p -> ((p as? Person)?.age ?: 0) > 23 },
+        index<Person>("ageGt23") { p -> ((p as? Person)?.age ?: 0) > 23 },
-        Index(Person::class, "ageOnlyGt20") { p ->
+        index<Person>("ageOnlyGt20") { p ->
           if (((p as? Person)?.age ?: 0) > 20) {
             true
           } else {
             null
           }
         },
-        Index(Company::class, "name") { p -> (p as? Company)?.name ?: "" },
+        index<Company>("name") { p -> (p as? Company)?.name ?: "" },
       )
     )
 
     pst.transaction {
-      val person = find(Person::class, 1L) ?: Person(
+      val person = find<Person>(1L) ?: Person(
         id = 1L,
         name = "John Doe",
         age = 25
       )
 
-      val company = find(Company::class, 1L) ?: Company(
+      val company = find<Company>(1L) ?: Company(
         id = 1L,
         name = "ACME"
       )
@@ -83,14 +87,19 @@ class TestPersistence {
         age = 18
       ))
 
-      findByIndex(Person::class, "name", "John Doe").forEach { p ->
+      findByIndex<Person>("name", "John Doe").forEach { p ->
         println("Found person by name: ${p.name} - ${p.age}")
       }
 
-      findByIndex(Person::class, "age", 23).forEach { p ->
+      val persons: List<Person> = findByIndex("age", 23)
+
+      persons.forEach { p ->
         println("Found person by age: ${p.name} - ${p.age}")
       }
 
+      val companies: List<Company> = findByIndex("name", "ACME")
+      assert(companies.isNotEmpty())
+
       findByIndex(Person::class, "ageGt20", true).forEach { p ->
         println("Found person by age > 20: ${p.name} - ${p.age}")
       }
@@ -112,6 +121,12 @@ class TestPersistence {
       assert(c2 != null)
     }
 
+    pst.query {
+      val person = find<Person>(1L)
+
+      assertNotNull(person)
+    }
+
     pst.transaction {
       val person = find(Person::class, 1L)
 
@@ -136,17 +151,48 @@ class TestPersistence {
       store(
         Person(
           id = 10L,
-          name = "Pipo",
+          name = "John Pipo",
           age = 23
         )
       )
+      val person = Person(
+        id = 11L,
+        name = "John Clown",
+        age = 18
+      )
+
+      store(person)
+      assertNotNull(find(Person::class, person.id))
+      delete(person)
+      assertNull(find(Person::class, person.id))
+      store(person)
+      assertNotNull(find(Person::class, person.id))
+
+      searchIndex(Person::class, "name") {
+        name -> (name as? String)?.startsWith("John") == true
+      }.forEach { p ->
+        println("Found person where name starts with 'John': ${p.name} - ${p.age}")
+      }
+    }
+
+    pst.transaction {
+      store(
+        Person(
+          id = 15L,
+          name = "Mama",
+          age = 26
+        )
+      )
       store(
         Person(
-          id = 11L,
+          id = 16L,
-          name = "Clown",
+          name = "Loe",
-          age = 18
+          age = 16
         )
       )
     }
+
+    pst.datastore.printStatus()
+    //pst.removeOldFiles()
   }
 }

src/test/kotlin/nl/astraeus/persistence/TestThreaded.kt

@@ -0,0 +1,129 @@
+package nl.astraeus.persistence
+
+import nl.astraeus.nl.astraeus.persistence.Persistable
+import nl.astraeus.nl.astraeus.persistence.Persistent
+import nl.astraeus.nl.astraeus.persistence.Reference
+import nl.astraeus.nl.astraeus.persistence.count
+import nl.astraeus.nl.astraeus.persistence.index
+import nl.astraeus.nl.astraeus.persistence.searchIndex
+import java.io.File
+import kotlin.random.Random
+import kotlin.test.Test
+
+class TestThreaded {
+
+  class Company(
+    override var id: Long = 0,
+    override var version: Long = 0,
+    val name: String,
+    val adres: String = "Blaat"
+  ) : Persistable, Cloneable {
+    //var persons: MutableList<Person> by ListReference<Person, Company>(Person::class.java)
+
+    companion object {
+      private const val serialVersionUID: Long = 1L
+    }
+  }
+
+  class Person(
+    override var id: Long = 0,
+    override var version: Long = 0,
+    val name: String,
+    val age: Int,
+  ) : Persistable, Cloneable {
+    var company: Company by Reference<Company, Person>(Company::class.java)
+
+    companion object {
+      private const val serialVersionUID: Long = 1L
+    }
+  }
+
+  @Test
+  fun testThreaded() {
+    println("Test threaded")
+
+    val pst = Persistent(
+      directory = File("data", "test-threaded"),
+      arrayOf(
+        index<Person>("name") { p -> (p as? Person)?.name ?: "" },
+        index<Person>("age") { p -> (p as? Person)?.age ?: -1 },
+        index<Person>("ageGt20") { p -> ((p as? Person)?.age ?: 0) > 20 },
+        index<Person>("ageGt23") { p -> ((p as? Person)?.age ?: 0) > 23 },
+        index<Person>("ageOnlyGt20") { p ->
+          if (((p as? Person)?.age ?: 0) > 20) {
+            true
+          } else {
+            null
+          }
+        },
+        index<Person>("nameAndAge") { p ->
+          val person = p as? Person
+
+          if (person == null) {
+            null
+          } else {
+            person.name to person.age
+          }
+        },
+        index<Company>("name") { p -> (p as? Company)?.name ?: "" },
+      )
+    )
+
+    val companyNames = arrayOf("Company A", "Company B", "Company C", "Company D", "Company E")
+    val names = arrayOf("John Doe", "Jane Doe", "John Smith", "Jane Smith", "John Johnson", "Jane Johnson")
+    val random = Random(System.currentTimeMillis())
+
+    val empty = pst.query {
+      count<Person>() == 0
+    }
+
+    if (empty) {
+      val runnable = {
+        repeat(10) {
+          pst.transaction {
+            val company = Company(
+              id = 0L,
+              name = companyNames[random.nextInt(companyNames.size)]
+            )
+            repeat(10) {
+              val person = Person(
+                id = 0L,
+                name = names[random.nextInt(names.size)],
+                age = random.nextInt(0, 100),
+              )
+              person.company = company
+
+              store(person)
+            }
+          }
+        }
+      }
+
+      val threads = Array(25) {
+        Thread(runnable)
+      }
+
+      for (thread in threads) {
+        thread.start()
+      }
+
+      for (thread in threads) {
+        thread.join()
+      }
+    }
+
+    pst.query {
+      searchIndex<Person>("nameAndAge") { nameAndAge ->
+        val (name, age) = nameAndAge as Pair<String, Int>
+
+        name.contains("mit") && age > 80
+      }.forEach { p ->
+        println("Found person by name and age: ${p.id}: ${p.name} - ${p.age}")
+      }
+    }
+
+    pst.snapshot()
+    pst.datastore.printStatus()
+    pst.removeOldFiles()
+  }
+}