bhetherman/open-webui
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: format

Browse Source
This commit is contained in:
Timothy Jaeryang Baek committed 2026-06-01 13:56:55 -07:00
1 parent 1f49ebf431
commit 6fce92aa12
139 files changed
+92569 -3249
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/open_webui/retrieval/loaders/main.py
+32 -33
View File
@@ -273,18 +273,18 @@ class Loader:
4. Falling back to latin-1 (always valid, ftfy fixes mojibake later).
"""
try:
with open(file_path, "rb") as f:
with open(file_path, 'rb') as f:
raw = f.read()
except OSError:
return "utf-8"
return 'utf-8'
if not raw:
return "utf-8"
return 'utf-8'
# Fast path: most files are UTF-8
try:
raw.decode("utf-8")
return "utf-8"
raw.decode('utf-8')
return 'utf-8'
except UnicodeDecodeError:
pass
@@ -292,24 +292,24 @@ class Loader:
import chardet
detected = chardet.detect(raw)
detected_enc = (detected.get("encoding") or "").lower().replace("-", "").replace("_", "")
detected_enc = (detected.get('encoding') or '').lower().replace('-', '').replace('_', '')
# Map chardet's detected encoding to the correct superset codec.
# chardet often reports GB2312 for content that is actually GB18030;
# GB18030 is a strict superset of both GB2312 and GBK.
_ENC_FAMILY = {
"gb2312": "gb18030",
"gb18030": "gb18030",
"gbk": "gb18030",
"big5": "big5",
"euckr": "euc-kr",
"eucjp": "euc-jp",
"iso2022jp": "euc-jp",
"shiftjis": "shift_jis",
'gb2312': 'gb18030',
'gb18030': 'gb18030',
'gbk': 'gb18030',
'big5': 'big5',
'euckr': 'euc-kr',
'eucjp': 'euc-jp',
'iso2022jp': 'euc-jp',
'shiftjis': 'shift_jis',
}
# Build priority list: chardet-hinted codec first, then remaining CJK
base_order = ["gb18030", "big5", "euc-kr", "euc-jp"]
base_order = ['gb18030', 'big5', 'euc-kr', 'euc-jp']
hinted = _ENC_FAMILY.get(detected_enc)
if hinted and hinted in base_order:
ordered = [hinted] + [e for e in base_order if e != hinted]
@@ -321,10 +321,10 @@ class Loader:
text = raw.decode(enc)
if text.strip() and self._has_cjk_characters(text):
log.info(
"Detected encoding %s for %s (chardet guessed %s)",
'Detected encoding %s for %s (chardet guessed %s)',
enc,
file_path,
detected.get("encoding"),
detected.get('encoding'),
)
return enc
except (UnicodeDecodeError, LookupError):
@@ -332,12 +332,12 @@ class Loader:
# If chardet gave a non-CJK answer that isn't in our family map,
# try it directly — it might be a valid Western encoding.
chardet_encoding = detected.get("encoding")
chardet_encoding = detected.get('encoding')
if chardet_encoding:
try:
raw.decode(chardet_encoding)
log.info(
"Using chardet-detected encoding %s for %s",
'Using chardet-detected encoding %s for %s',
chardet_encoding,
file_path,
)
@@ -348,8 +348,8 @@ class Loader:
# latin-1 is the ultimate fallback: every byte 0x00–0xFF is valid.
# ftfy.fix_text() (applied downstream) repairs most mojibake that
# results from treating Windows-1252 content as Latin-1.
log.info("Falling back to latin-1 encoding for %s", file_path)
return "latin-1"
log.info('Falling back to latin-1 encoding for %s', file_path)
return 'latin-1'
@staticmethod
def _has_cjk_characters(text: str, threshold: float = 0.05) -> bool:
@@ -371,17 +371,17 @@ class Loader:
total += 1
cp = ord(ch)
if (
0x4E00 <= cp <= 0x9FFF # CJK Unified Ideographs
or 0x3400 <= cp <= 0x4DBF # CJK Extension A
or 0x20000 <= cp <= 0x2A6DF # CJK Extension B
or 0x2A700 <= cp <= 0x2B73F # CJK Extension C
or 0x2B740 <= cp <= 0x2B81F # CJK Extension D
or 0xF900 <= cp <= 0xFAFF # CJK Compatibility Ideographs
or 0x3000 <= cp <= 0x303F # CJK Symbols and Punctuation
or 0x3040 <= cp <= 0x309F # Hiragana
or 0x30A0 <= cp <= 0x30FF # Katakana
or 0xAC00 <= cp <= 0xD7AF # Hangul Syllables
or 0xFF00 <= cp <= 0xFFEF # Halfwidth and Fullwidth Forms
0x4E00 <= cp <= 0x9FFF # CJK Unified Ideographs
or 0x3400 <= cp <= 0x4DBF # CJK Extension A
or 0x20000 <= cp <= 0x2A6DF # CJK Extension B
or 0x2A700 <= cp <= 0x2B73F # CJK Extension C
or 0x2B740 <= cp <= 0x2B81F # CJK Extension D
or 0xF900 <= cp <= 0xFAFF # CJK Compatibility Ideographs
or 0x3000 <= cp <= 0x303F # CJK Symbols and Punctuation
or 0x3040 <= cp <= 0x309F # Hiragana
or 0x30A0 <= cp <= 0x30FF # Katakana
or 0xAC00 <= cp <= 0xD7AF # Hangul Syllables
or 0xFF00 <= cp <= 0xFFEF # Halfwidth and Fullwidth Forms
):
cjk_count += 1
@@ -646,4 +646,3 @@ class Loader:
loader = TextLoader(file_path, encoding=self._detect_text_encoding(file_path))
return loader
backend/open_webui/retrieval/utils.py
+5 -10
View File
@@ -933,6 +933,7 @@ def get_embedding_function(
'SentenceTransformer model name, or configure an external '
'RAG_EMBEDDING_ENGINE (ollama, openai, azure_openai).'
)
# Sentence transformers: CPU-bound sync operation
async def async_embedding_function(query, prefix=None, user=None):
return await asyncio.to_thread(
@@ -1108,8 +1109,7 @@ async def filter_accessible_collections(
rejected = collection_names - safe_names
if rejected:
log.warning(
'filter_accessible_collections: rejected %d collection name(s) '
'with unsafe characters (user_id=%s)',
'filter_accessible_collections: rejected %d collection name(s) with unsafe characters (user_id=%s)',
len(rejected),
getattr(user, 'id', '<unknown>'),
)
@@ -1363,10 +1363,7 @@ async def get_sources_from_items(
files = await Knowledges.get_files_by_id(knowledge_base.id)
owned_names = {f'file-{f.id}' for f in files}
owned_names.add(knowledge_base.id)
valid_names = [
n for n in (item.get('collection_names') or [])
if n in owned_names
]
valid_names = [n for n in (item.get('collection_names') or []) if n in owned_names]
collection_names = valid_names if valid_names else [knowledge_base.id]
else:
collection_names.append(item['id'])
@@ -1382,8 +1379,7 @@ async def get_sources_from_items(
collection_names.append(item['collection_name'])
else:
log.debug(
"get_sources_from_items: ignoring untrusted direct "
"collection_name '%s' on item without type",
"get_sources_from_items: ignoring untrusted direct collection_name '%s' on item without type",
item.get('collection_name'),
)
elif item.get('collection_names'):
@@ -1391,8 +1387,7 @@ async def get_sources_from_items(
collection_names.extend(item['collection_names'])
else:
log.debug(
"get_sources_from_items: ignoring untrusted direct "
"collection_names on item without type",
'get_sources_from_items: ignoring untrusted direct collection_names on item without type',
)
# If query_result is None
backend/open_webui/retrieval/vector/dbs/milvus_multitenancy.py
+1 -3
View File
@@ -284,9 +284,7 @@ class MilvusClient(VectorDBBase):
elif isinstance(value, (int, float)):
expr.append(f"metadata['{key}'] == {value}")
else:
raise TypeError(
f'Unsupported Milvus filter value type for key {key!r}: {type(value).__name__}'
)
raise TypeError(f'Unsupported Milvus filter value type for key {key!r}: {type(value).__name__}')
iterator = collection.query_iterator(
expr=' and '.join(expr),
backend/open_webui/retrieval/vector/dbs/valkey.py
+3 -4
View File
@@ -56,8 +56,7 @@ def _import_glide():
)
except ImportError as e:
raise ImportError(
'valkey-glide-sync is required when VECTOR_DB=valkey. '
'Install it with: pip install valkey-glide-sync==2.3.1'
'valkey-glide-sync is required when VECTOR_DB=valkey. Install it with: pip install valkey-glide-sync==2.3.1'
) from e
return {
'Batch': Batch,
@@ -193,7 +192,7 @@ class ValkeyClient(VectorDBBase):
addresses=[NodeAddress(host=host, port=port)],
database_id=db if db else None,
request_timeout=5000,
client_name="open_webui_vector_store_client",
client_name='open_webui_vector_store_client',
)
try:
self.client = GlideClient.create(config)
@@ -206,7 +205,7 @@ class ValkeyClient(VectorDBBase):
addresses=[NodeAddress(host=host, port=port)],
database_id=db if db else None,
request_timeout=10000, # 10s — HNSW indexing can take 1-4s per vector
client_name="open_webui_vector_store_batch_client",
client_name='open_webui_vector_store_batch_client',
)
try:
self.batch_client = GlideClient.create(batch_config)
backend/open_webui/retrieval/web/bing.py
+1 -3
View File
@@ -66,9 +66,7 @@ def main():
results = search_bing(
os.environ.get('BING_SEARCH_V7_SUBSCRIPTION_KEY', ''),
os.environ.get(
'BING_SEARCH_V7_ENDPOINT', 'https://api.bing.microsoft.com/v7.0/search'
),
os.environ.get('BING_SEARCH_V7_ENDPOINT', 'https://api.bing.microsoft.com/v7.0/search'),
args.locale,
args.query,
args.count,
backend/open_webui/retrieval/web/linkup.py
+1 -3
View File
@@ -51,9 +51,7 @@ def search_linkup(
output_type = merged.get('outputType', 'sourcedAnswer')
search_results = (
json_response.get('sources', [])
if output_type == 'sourcedAnswer'
else json_response.get('results', [])
json_response.get('sources', []) if output_type == 'sourcedAnswer' else json_response.get('results', [])
)
if filter_list: