This commit is contained in:
Timothy Jaeryang Baek committed 2026-07-27 03:53:54 -04:00
1 parent c4ae8c8678
commit b7394c63fa
1 file changed
+17 -7
+17 -7
View File
@@ -1579,12 +1579,21 @@ def get_transformers_tokenizer(request: Request, config: RetrievalConfig):
if not os.path.exists(tokenizer_model) and '/' not in tokenizer_model: if not os.path.exists(tokenizer_model) and '/' not in tokenizer_model:
tokenizer_model = f'sentence-transformers/{tokenizer_model}' tokenizer_model = f'sentence-transformers/{tokenizer_model}'
return AutoTokenizer.from_pretrained( cache_dir = os.getenv('SENTENCE_TRANSFORMERS_HOME') or os.getenv('HF_HUB_CACHE')
local_files_only = not RAG_EMBEDDING_MODEL_AUTO_UPDATE
tokenizer_key = (tokenizer_model, cache_dir, local_files_only)
cached_tokenizer = getattr(request.app.state, 'transformers_tokenizer', None)
if cached_tokenizer and cached_tokenizer[0] == tokenizer_key:
return cached_tokenizer[1]
tokenizer = AutoTokenizer.from_pretrained(
tokenizer_model, tokenizer_model,
cache_dir=os.getenv('SENTENCE_TRANSFORMERS_HOME') or os.getenv('HF_HUB_CACHE'), cache_dir=cache_dir,
trust_remote_code=RAG_EMBEDDING_MODEL_TRUST_REMOTE_CODE, trust_remote_code=RAG_EMBEDDING_MODEL_TRUST_REMOTE_CODE,
local_files_only=not RAG_EMBEDDING_MODEL_AUTO_UPDATE, local_files_only=local_files_only,
) )
request.app.state.transformers_tokenizer = (tokenizer_key, tokenizer)
return tokenizer
tokenizer = getattr(getattr(request.app.state, 'ef', None), 'tokenizer', None) tokenizer = getattr(getattr(request.app.state, 'ef', None), 'tokenizer', None)
if tokenizer is not None: if tokenizer is not None:
@@ -1952,7 +1961,7 @@ async def process_file(
] ]
text_content = ' '.join([doc.page_content for doc in docs]) text_content = ' '.join([doc.page_content for doc in docs])
log.debug(f'text_content: {text_content}') log.debug('text_content: %s', text_content)
await Files.update_file_data_by_id( await Files.update_file_data_by_id(
file.id, file.id,
{'content': text_content}, {'content': text_content},
@@ -2094,7 +2103,7 @@ async def process_text(
) )
] ]
text_content = form_data.content text_content = form_data.content
log.debug(f'text_content: {text_content}') log.debug('text_content: %s', text_content)
config = await get_retrieval_config() config = await get_retrieval_config()
result = await run_in_threadpool(save_docs_to_vector_db, request, docs, collection_name, config, user=user) result = await run_in_threadpool(save_docs_to_vector_db, request, docs, collection_name, config, user=user)
@@ -2131,7 +2140,7 @@ async def process_web(
config = await get_retrieval_config() config = await get_retrieval_config()
try: try:
content, docs = await get_content_from_url(request, form_data.url) content, docs = await get_content_from_url(request, form_data.url)
log.debug(f'text_content: {content}') log.debug('text_content: %s', content)
if process: if process:
collection_name = form_data.collection_name collection_name = form_data.collection_name
@@ -2645,8 +2654,9 @@ async def process_web_search(request: Request, form_data: SearchForm, user=Depen
urls = [ urls = [
doc.metadata.get('source') for doc in docs if doc.metadata.get('source') doc.metadata.get('source') for doc in docs if doc.metadata.get('source')
] # only keep the urls returned by the loader ] # only keep the urls returned by the loader
url_set = set(urls)
result_items = [ result_items = [
dict(item) for item in result_items if item.link in urls dict(item) for item in result_items if item.link in url_set
] # only keep the search results that have been loaded ] # only keep the search results that have been loaded
if config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL: if config.BYPASS_WEB_SEARCH_EMBEDDING_AND_RETRIEVAL: