Text Splitters

from langchain_text_splitters import (
    RecursiveCharacterTextSplitter,
    MarkdownHeaderTextSplitter,
    HTMLHeaderTextSplitter,
    PythonCodeTextSplitter,
    Language,
    TokenTextSplitter,
    NLTKTextSplitter,
    SpacyTextSplitter,
    CharacterTextSplitter,
)
from langchain_core.documents import Document

# 1) Recursive character — sensible default
splitter = RecursiveCharacterTextSplitter(
    chunk_size      = 1000,                # characters
    chunk_overlap   = 200,                 # 10-20% overlap is typical
    length_function = len,
    separators      = ['\n\n', '\n', '. ', ' ', ''],    # try in order
    add_start_index = True,
)
chunks = splitter.split_text(long_text)
# Or with Documents:
chunks = splitter.split_documents(docs)

# 2) Token-based — what the LLM actually counts
splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(
    encoding_name = 'cl100k_base',         # GPT-4 / 3.5
    chunk_size    = 500,                   # tokens
    chunk_overlap = 100,
)
# Or pure token splitter:
ts = TokenTextSplitter(chunk_size=500, chunk_overlap=100)

# 3) Markdown — split on heading boundaries
md_splitter = MarkdownHeaderTextSplitter(
    headers_to_split_on = [
        ('#',   'h1'),
        ('##',  'h2'),
        ('###', 'h3'),
    ],
    strip_headers = False,                 # keep headers in the chunk
)
md_chunks = md_splitter.split_text(markdown_text)
# Each chunk has metadata: {'h1': '...', 'h2': '...', 'h3': '...'}

# Common combo: split by headings first, then by chars to enforce a max size
rct = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
final_chunks = rct.split_documents(md_chunks)

# 4) HTML — split on element boundaries
html_splitter = HTMLHeaderTextSplitter(headers_to_split_on=[('h1','h1'),('h2','h2'),('h3','h3')])
html_chunks = html_splitter.split_text(html_string)

# 5) Code — language-aware
py_splitter = PythonCodeTextSplitter(chunk_size=2000, chunk_overlap=200)
py_chunks = py_splitter.split_text(python_source)

# More languages
for lang in [Language.JS, Language.TS, Language.GO, Language.RUST, Language.JAVA]:
    sp = RecursiveCharacterTextSplitter.from_language(language=lang, chunk_size=2000, chunk_overlap=200)

# 6) Sentence-level — NLTK / spaCy
nltk = NLTKTextSplitter(chunk_size=1000, chunk_overlap=100)
# pip install nltk; python -m nltk.downloader punkt
sp = SpacyTextSplitter(pipeline='en_core_web_sm', chunk_size=1000, chunk_overlap=100)

# 7) Custom — for domain-specific structure (e.g. legal clauses, dialogue turns)
class DialogueSplitter:
    def split_text(self, text):
        return text.split('\nSPEAKER:')

# 8) Best practices — chunk-size decision tree
# - Token-based always more reliable than char-based for LLMs
# - Chunk size 500-1000 tokens for most prose
# - Overlap 10-20% of chunk size — prevents losing concepts at boundaries
# - For code: bigger chunks (1500-2000) to keep functions together
# - For Q&A pairs: small chunks (300-500); chunks should be self-contained

# 9) Preserve metadata across splits
for c in chunks:
    print(c.metadata)
# split_documents copies parent metadata to each chunk. Use it for source filtering at search time.

# 10) Hybrid splitting — combine for production RAG
# Step 1: split by markdown headings (semantic boundaries)
md_chunks = MarkdownHeaderTextSplitter(headers_to_split_on=[('#','h1'),('##','h2')]).split_text(text)

# Step 2: enforce a max token size within each header section
tok = RecursiveCharacterTextSplitter.from_tiktoken_encoder(chunk_size=500, chunk_overlap=80)
final = tok.split_documents(md_chunks)
# Each chunk is bounded in size AND retains semantic context.

# 11) Edge cases
# Very long code blocks inside markdown — keep them together if they exceed chunk_size
# Tables — splitters mangle them; preprocess to keep rows together
# PDFs with footers / page numbers — strip noise before splitting
# Legal contracts with numbered sections — split on numbering pattern

# 12) Evaluate splits — sanity check
import tiktoken
enc = tiktoken.get_encoding('cl100k_base')

for c in chunks[:5]:
    tokens = len(enc.encode(c.page_content))
    print(f'{tokens} tokens: {c.page_content[:80]}...')

# Watch for:
# - Chunks that end mid-sentence
# - Tiny chunks (10 tokens) — splitter granularity too small
# - Huge chunks (3000+ tokens) — splitter not actually splitting

# 13) Retrieval-aware tuning
# - Bigger chunks → fewer chunks → cheaper to embed + retrieve, less precise
# - Smaller chunks → more chunks → more precise retrieval, more tokens to LLM context
# Test BOTH on your eval set; pick the chunk size where recall@k stops improving.

# 14) When NOT to chunk
# Short documents (under 1000 tokens) — embed whole
# Tabular data — load as a tool / SQL, not as text chunks
# Structured KB — store as records, retrieve by ID + assemble at query time

# 15) Modern alternatives
# - Semantic Chunking (LlamaIndex, langchain experimental): use embeddings to find natural boundaries
# - Late chunking: embed long text in one pass, then split the embeddings
# - Parent Document Retriever: small chunks for search, big chunks for context

# Late chunking — newer approach for top retrieval quality
# 1. Embed the FULL document
# 2. Extract per-token embeddings for each chunk's tokens
# 3. Mean-pool per chunk → embeddings with full-doc context
# Available via the jinaai/jina-embeddings-v3 model and similar.

# 16) Best practices summary
#   • Default: RecursiveCharacterTextSplitter.from_tiktoken_encoder(500, 100)
#   • Markdown/HTML: split by headings first, then enforce max size
#   • Code: language-aware splitter with bigger chunks
#   • Preserve + enrich metadata (source, section, last_modified)
#   • Evaluate on a real eval set; tune chunk size + overlap together

from langchain_text_splitters import RecursiveCharacterTextSplitter
splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=150)
chunks = splitter.split_documents(docs)