Reference · Snippet · python

Chunk text by words

ragpythonchunking

def chunk_words(text: str, size: int = 120, overlap: int = 20) -> list[str]:
    words = text.split()
    chunks = []
    i = 0
    while i < len(words):
        piece = words[i : i + size]
        chunks.append(" ".join(piece))
        i += max(1, size - overlap)
    return chunks