refac
This commit is contained in:
1 parent
f3f7659da7
commit
2a0274a0a0
3 files changed
+51
-3
No files matched your search
@@ -3,6 +3,7 @@ import csv
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
import zipfile
|
||||||
|
|
||||||
import ftfy
|
import ftfy
|
||||||
import requests
|
import requests
|
||||||
@@ -14,7 +15,6 @@ from langchain_community.document_loaders import (
|
|||||||
Docx2txtLoader,
|
Docx2txtLoader,
|
||||||
PyPDFLoader,
|
PyPDFLoader,
|
||||||
TextLoader,
|
TextLoader,
|
||||||
YoutubeLoader,
|
|
||||||
)
|
)
|
||||||
from langchain_core.documents import Document
|
from langchain_core.documents import Document
|
||||||
from open_webui.env import (
|
from open_webui.env import (
|
||||||
@@ -90,6 +90,15 @@ known_source_ext = [
|
|||||||
'toml',
|
'toml',
|
||||||
]
|
]
|
||||||
|
|
||||||
|
known_archive_ext = {'docx', 'epub', 'odt', 'pptx', 'xlsx'}
|
||||||
|
known_archive_content_types = {
|
||||||
|
'application/epub+zip',
|
||||||
|
'application/vnd.oasis.opendocument.text',
|
||||||
|
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
||||||
|
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
||||||
|
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
class ExcelLoader:
|
class ExcelLoader:
|
||||||
"""Fallback Excel loader using pandas when unstructured is not installed."""
|
"""Fallback Excel loader using pandas when unstructured is not installed."""
|
||||||
@@ -477,6 +486,27 @@ class Loader:
|
|||||||
def _get_loader(self, filename: str, file_content_type: str, file_path: str):
|
def _get_loader(self, filename: str, file_content_type: str, file_path: str):
|
||||||
file_ext = filename.split('.')[-1].lower()
|
file_ext = filename.split('.')[-1].lower()
|
||||||
|
|
||||||
|
if file_ext in known_archive_ext or file_content_type in known_archive_content_types:
|
||||||
|
max_file_size = self.kwargs.get('FILE_MAX_SIZE')
|
||||||
|
try:
|
||||||
|
max_file_size_bytes = int(max_file_size) * 1024 * 1024 if max_file_size else 100 * 1024 * 1024
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
max_file_size_bytes = 100 * 1024 * 1024
|
||||||
|
|
||||||
|
if max_file_size_bytes > 0:
|
||||||
|
try:
|
||||||
|
with zipfile.ZipFile(file_path) as archive:
|
||||||
|
uncompressed_size = sum(entry.file_size for entry in archive.infolist())
|
||||||
|
except (zipfile.BadZipFile, OSError):
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
max_bytes = min(
|
||||||
|
max(10 * 1024 * 1024, os.path.getsize(file_path) * 100),
|
||||||
|
max_file_size_bytes,
|
||||||
|
)
|
||||||
|
if uncompressed_size > max_bytes:
|
||||||
|
raise ValueError('Document archive is too large after decompression')
|
||||||
|
|
||||||
if (
|
if (
|
||||||
self.engine == 'external'
|
self.engine == 'external'
|
||||||
and self.kwargs.get('EXTERNAL_DOCUMENT_LOADER_URL')
|
and self.kwargs.get('EXTERNAL_DOCUMENT_LOADER_URL')
|
||||||
|
|||||||
@@ -153,6 +153,7 @@ def build_loader_from_config(request, config: dict):
|
|||||||
from open_webui.retrieval.loaders.main import Loader
|
from open_webui.retrieval.loaders.main import Loader
|
||||||
|
|
||||||
loader_config = {key: config.get(key) for key in LOADER_CONFIG_KEYS if key.isupper()}
|
loader_config = {key: config.get(key) for key in LOADER_CONFIG_KEYS if key.isupper()}
|
||||||
|
loader_config['FILE_MAX_SIZE'] = config.get('file_max_size')
|
||||||
return Loader(
|
return Loader(
|
||||||
engine=loader_config['CONTENT_EXTRACTION_ENGINE'],
|
engine=loader_config['CONTENT_EXTRACTION_ENGINE'],
|
||||||
**{key: value for key, value in loader_config.items() if key != 'CONTENT_EXTRACTION_ENGINE'},
|
**{key: value for key, value in loader_config.items() if key != 'CONTENT_EXTRACTION_ENGINE'},
|
||||||
|
|||||||
@@ -458,7 +458,11 @@ function appendDelta(current: unknown, delta: unknown): unknown {
|
|||||||
return delta ?? current ?? '';
|
return delta ?? current ?? '';
|
||||||
}
|
}
|
||||||
|
|
||||||
function ensureItem(output: OutputItem[], outputIndex: number, fallback?: OutputItem): OutputItem {
|
function ensureOutputItem(
|
||||||
|
output: OutputItem[],
|
||||||
|
outputIndex: number,
|
||||||
|
fallback?: OutputItem
|
||||||
|
): OutputItem {
|
||||||
while (output.length <= outputIndex) {
|
while (output.length <= outputIndex) {
|
||||||
output.push(
|
output.push(
|
||||||
fallback ?? { type: 'message', status: 'in_progress', role: 'assistant', content: [] }
|
fallback ?? { type: 'message', status: 'in_progress', role: 'assistant', content: [] }
|
||||||
@@ -484,6 +488,15 @@ function findOutputItemIndex(output: OutputItem[], item: OutputItem): number {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function responseEventUpdatesOutputItem(eventType: string): boolean {
|
||||||
|
return (
|
||||||
|
eventType === 'response.content_part.added' ||
|
||||||
|
eventType === 'response.reasoning_summary_part.added' ||
|
||||||
|
eventType.endsWith('.delta') ||
|
||||||
|
eventType.endsWith('.done')
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export function applyResponseStreamEvent(
|
export function applyResponseStreamEvent(
|
||||||
output: OutputItem[] = [],
|
output: OutputItem[] = [],
|
||||||
event: ResponseStreamEvent
|
event: ResponseStreamEvent
|
||||||
@@ -530,7 +543,11 @@ export function applyResponseStreamEvent(
|
|||||||
return nextOutput;
|
return nextOutput;
|
||||||
}
|
}
|
||||||
|
|
||||||
const item = ensureItem(nextOutput, outputIndex, {
|
if (!responseEventUpdatesOutputItem(eventType)) {
|
||||||
|
return output;
|
||||||
|
}
|
||||||
|
|
||||||
|
const item = ensureOutputItem(nextOutput, outputIndex, {
|
||||||
id: event.item_id,
|
id: event.item_id,
|
||||||
type: eventType.includes('reasoning')
|
type: eventType.includes('reasoning')
|
||||||
? 'reasoning'
|
? 'reasoning'
|
||||||
|
|||||||
Reference in new issue
Block a user