Text segmentation tool
Interactive chunking tool with customizable size and overlap settings.
Essential for preparing large texts for AI processing and RAG systems.
// Text Chunking Implementation
class Chunker {
fun chunkText(text: String, maxSize: Int, overlap: Int): List<Chunk> {
val chunks = mutableListOf<Chunk>()
var startIndex = 0
while (startIndex < text.length) {
val endIndex = minOf(startIndex + maxSize, text.length)
val chunkText = text.substring(startIndex, endIndex)
chunks.add(Chunk(
content = chunkText,
startIndex = startIndex,
endIndex = endIndex
))
startIndex += maxSize - overlap
}
return chunks
}
fun chunkBySentence(text: String, targetSize: Int): List<Chunk> {
val sentences = text.split(". ")
return sentences.chunked(targetSize) { group ->
Chunk(content = group.joinToString(". "))
}
}
}