fix(rag): allow special tokens in document text during chunk measurement (#26210)
Documents (especially AI/LLM documentation) legitimately contain special token strings like <|endoftext|> as literal text. The tiktoken encoder raises a ValueError when encountering these during chunk size measurement in merge_docs_to_target_size(), preventing the entire file from being indexed. Pass disallowed_special=() to encoding.encode() to treat all text as normal content.
This commit is contained in:
1 file changed
+1
-1
@@ -1541,7 +1541,7 @@ def merge_docs_to_target_size(
|
|||||||
measure: Callable[[str], int] = len
|
measure: Callable[[str], int] = len
|
||||||
if config.TEXT_SPLITTER == 'token':
|
if config.TEXT_SPLITTER == 'token':
|
||||||
encoding = tiktoken.get_encoding(str(config.TIKTOKEN_ENCODING_NAME))
|
encoding = tiktoken.get_encoding(str(config.TIKTOKEN_ENCODING_NAME))
|
||||||
measure = lambda text: len(encoding.encode(text))
|
measure = lambda text: len(encoding.encode(text, disallowed_special=()))
|
||||||
|
|
||||||
def _merge_backward(result: list[Document], content: str, chunk: Document) -> bool:
|
def _merge_backward(result: list[Document], content: str, chunk: Document) -> bool:
|
||||||
"""Try to append content into the last emitted chunk. Returns True on success."""
|
"""Try to append content into the last emitted chunk. Returns True on success."""
|
||||||
|
|||||||
Reference in new issue
Block a user