refac

backend/open_webui/retrieval/loaders/main.py

@@ -233,34 +233,7 @@ class Loader:
 
     def load(self, filename: str, file_content_type: str, file_path: str) -> list[Document]:
         loader = self._get_loader(filename, file_content_type, file_path)
-
+        docs = loader.load()
-        try:
-            docs = loader.load()
-        except (UnicodeDecodeError, RuntimeError, TimeoutError) as e:
-            # Only retry with latin-1 for text-encoding loaders (TextLoader,
-            # CSVLoader).  Binary format loaders (PDF, DOCX, PPTX, …) can
-            # raise RuntimeError for unrelated reasons; retrying those as
-            # plain text would produce garbage.
-            if not isinstance(loader, (TextLoader, CSVLoader)):
-                raise
-
-            # TextLoader/CSVLoader with autodetect_encoding=True can fail when:
-            # - chardet times out (5 s hardcoded in langchain)
-            # - chardet returns a wrong/low-confidence encoding
-            # - the file contains non-UTF-8 bytes (Latin-1, Windows-1252, …)
-            #
-            # Latin-1 is a safe last resort: every byte 0x00–0xFF is valid,
-            # and ftfy.fix_text() (applied below) repairs most mojibake that
-            # results from treating Windows-1252 content as Latin-1.
-            log.warning(
-                "Primary loader failed for %s (%s), retrying with latin-1 encoding: %s",
-                filename,
-                type(e).__name__,
-                e,
-            )
-            fallback_loader = TextLoader(file_path, encoding="latin-1")
-            docs = fallback_loader.load()
-
         return [Document(page_content=ftfy.fix_text(doc.page_content), metadata=doc.metadata) for doc in docs]
 
     async def aload(self, filename: str, file_content_type: str, file_path: str) -> list[Document]:
@@ -283,6 +256,140 @@ class Loader:
             and not file_content_type.find('html') >= 0
         )
 
+    def _detect_text_encoding(self, file_path: str) -> str:
+        """Detect the encoding of a text file with CJK-aware fallbacks.
+
+        Langchain's ``TextLoader`` uses chardet internally when
+        ``autodetect_encoding=True``, but chardet frequently misidentifies
+        CJK encodings (e.g. GB18030 detected as GB2312 or even Cyrillic).
+        This method replaces that by:
+
+        1. Trying UTF-8 first (fast path for the vast majority of files).
+        2. Using chardet as a *hint* to prioritise the right CJK codec
+           family, but mapping subset names to their superset
+           (e.g. GB2312 → gb18030).
+        3. Validating that decoded text actually contains CJK characters,
+           guarding against codecs that "succeed" but produce garbage.
+        4. Falling back to latin-1 (always valid, ftfy fixes mojibake later).
+        """
+        try:
+            with open(file_path, "rb") as f:
+                raw = f.read()
+        except OSError:
+            return "utf-8"
+
+        if not raw:
+            return "utf-8"
+
+        # Fast path: most files are UTF-8
+        try:
+            raw.decode("utf-8")
+            return "utf-8"
+        except UnicodeDecodeError:
+            pass
+
+        # Use chardet as a hint, not as ground truth
+        import chardet
+
+        detected = chardet.detect(raw)
+        detected_enc = (detected.get("encoding") or "").lower().replace("-", "").replace("_", "")
+
+        # Map chardet's detected encoding to the correct superset codec.
+        # chardet often reports GB2312 for content that is actually GB18030;
+        # GB18030 is a strict superset of both GB2312 and GBK.
+        _ENC_FAMILY = {
+            "gb2312": "gb18030",
+            "gb18030": "gb18030",
+            "gbk": "gb18030",
+            "big5": "big5",
+            "euckr": "euc-kr",
+            "eucjp": "euc-jp",
+            "iso2022jp": "euc-jp",
+            "shiftjis": "shift_jis",
+        }
+
+        # Build priority list: chardet-hinted codec first, then remaining CJK
+        base_order = ["gb18030", "big5", "euc-kr", "euc-jp"]
+        hinted = _ENC_FAMILY.get(detected_enc)
+        if hinted and hinted in base_order:
+            ordered = [hinted] + [e for e in base_order if e != hinted]
+        else:
+            ordered = base_order
+
+        for enc in ordered:
+            try:
+                text = raw.decode(enc)
+                if text.strip() and self._has_cjk_characters(text):
+                    log.info(
+                        "Detected encoding %s for %s (chardet guessed %s)",
+                        enc,
+                        file_path,
+                        detected.get("encoding"),
+                    )
+                    return enc
+            except (UnicodeDecodeError, LookupError):
+                continue
+
+        # If chardet gave a non-CJK answer that isn't in our family map,
+        # try it directly — it might be a valid Western encoding.
+        chardet_encoding = detected.get("encoding")
+        if chardet_encoding:
+            try:
+                raw.decode(chardet_encoding)
+                log.info(
+                    "Using chardet-detected encoding %s for %s",
+                    chardet_encoding,
+                    file_path,
+                )
+                return chardet_encoding
+            except (UnicodeDecodeError, LookupError):
+                pass
+
+        # latin-1 is the ultimate fallback: every byte 0x00–0xFF is valid.
+        # ftfy.fix_text() (applied downstream) repairs most mojibake that
+        # results from treating Windows-1252 content as Latin-1.
+        log.info("Falling back to latin-1 encoding for %s", file_path)
+        return "latin-1"
+
+    @staticmethod
+    def _has_cjk_characters(text: str, threshold: float = 0.05) -> bool:
+        """Check if decoded text contains a meaningful proportion of CJK characters.
+
+        This guards against codecs that technically "succeed" but decode the
+        bytes into wrong Unicode codepoints (e.g. PUA chars, random symbols).
+        A genuine CJK document should have at least ``threshold`` fraction of
+        its non-whitespace characters in CJK Unicode blocks.
+        """
+        if not text:
+            return False
+
+        cjk_count = 0
+        total = 0
+        for ch in text:
+            if ch.isspace():
+                continue
+            total += 1
+            cp = ord(ch)
+            if (
+                0x4E00 <= cp <= 0x9FFF       # CJK Unified Ideographs
+                or 0x3400 <= cp <= 0x4DBF     # CJK Extension A
+                or 0x20000 <= cp <= 0x2A6DF   # CJK Extension B
+                or 0x2A700 <= cp <= 0x2B73F   # CJK Extension C
+                or 0x2B740 <= cp <= 0x2B81F   # CJK Extension D
+                or 0xF900 <= cp <= 0xFAFF     # CJK Compatibility Ideographs
+                or 0x3000 <= cp <= 0x303F     # CJK Symbols and Punctuation
+                or 0x3040 <= cp <= 0x309F     # Hiragana
+                or 0x30A0 <= cp <= 0x30FF     # Katakana
+                or 0xAC00 <= cp <= 0xD7AF     # Hangul Syllables
+                or 0xFF00 <= cp <= 0xFFEF     # Halfwidth and Fullwidth Forms
+            ):
+                cjk_count += 1
+
+        if total == 0:
+            return False
+
+        return (cjk_count / total) >= threshold
+
     def _get_loader(self, filename: str, file_content_type: str, file_path: str):
         file_ext = filename.split('.')[-1].lower()
 
@@ -300,7 +407,7 @@ class Loader:
             )
         elif self.engine == 'tika' and self.kwargs.get('TIKA_SERVER_URL'):
             if self._is_text_file(file_ext, file_content_type):
-                loader = TextLoader(file_path, autodetect_encoding=True)
+                loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path))
             else:
                 loader = TikaLoader(
                     url=self.kwargs.get('TIKA_SERVER_URL'),
@@ -352,7 +459,7 @@ class Loader:
             )
         elif self.engine == 'docling' and self.kwargs.get('DOCLING_SERVER_URL'):
             if self._is_text_file(file_ext, file_content_type):
-                loader = TextLoader(file_path, autodetect_encoding=True)
+                loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path))
             else:
                 # Build params for DoclingLoader
                 params = self.kwargs.get('DOCLING_PARAMS', {})
@@ -437,7 +544,7 @@ class Loader:
                     mode=self.kwargs.get('PDF_LOADER_MODE', 'page'),
                 )
             elif file_ext == 'csv':
-                loader = CSVLoader(file_path, autodetect_encoding=True)
+                loader = CSVLoader(file_path, encoding=self._detect_text_encoding(file_path))
             elif file_ext == 'rst':
                 try:
                     from langchain_community.document_loaders import UnstructuredRSTLoader
@@ -449,7 +556,7 @@ class Loader:
                         'Falling back to plain text loading for .rst file. '
                         'Install it with: pip install unstructured'
                     )
-                    loader = TextLoader(file_path, autodetect_encoding=True)
+                    loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path))
             elif file_ext == 'xml':
                 try:
                     from langchain_community.document_loaders import UnstructuredXMLLoader
@@ -461,11 +568,11 @@ class Loader:
                         'Falling back to plain text loading for .xml file. '
                         'Install it with: pip install unstructured'
                     )
-                    loader = TextLoader(file_path, autodetect_encoding=True)
+                    loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path))
             elif file_ext in ['htm', 'html']:
                 loader = BSHTMLLoader(file_path, open_encoding='unicode_escape')
             elif file_ext == 'md':
-                loader = TextLoader(file_path, autodetect_encoding=True)
+                loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path))
             elif file_content_type == 'application/epub+zip':
                 try:
                     from langchain_community.document_loaders import UnstructuredEPubLoader
@@ -534,8 +641,9 @@ class Loader:
                         'Install it with: pip install unstructured'
                     )
             elif self._is_text_file(file_ext, file_content_type):
-                loader = TextLoader(file_path, autodetect_encoding=True)
+                loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path))
             else:
-                loader = TextLoader(file_path, autodetect_encoding=True)
+                loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path))
 
         return loader
+