refac
This commit is contained in:
1 parent
073cd65afe
commit
48625e657f
2 files changed
+70
-2
No files matched your search
@@ -3,7 +3,8 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from typing import Optional
|
from copy import deepcopy
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
from open_webui.internal.db import Base, JSONField, get_async_db_context
|
from open_webui.internal.db import Base, JSONField, get_async_db_context
|
||||||
from open_webui.models.access_grants import AccessGrantModel, AccessGrants
|
from open_webui.models.access_grants import AccessGrantModel, AccessGrants
|
||||||
@@ -22,6 +23,38 @@ log = logging.getLogger(__name__)
|
|||||||
_warned_profile_urls: set[str] = set()
|
_warned_profile_urls: set[str] = set()
|
||||||
|
|
||||||
|
|
||||||
|
def strip_extracted_content_from_model_knowledge(knowledge: Any) -> Any:
|
||||||
|
"""Drop duplicated extracted text from ModelMeta.knowledge."""
|
||||||
|
if not isinstance(knowledge, list):
|
||||||
|
return knowledge
|
||||||
|
|
||||||
|
sanitized = []
|
||||||
|
|
||||||
|
for item in knowledge:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
sanitized.append(item)
|
||||||
|
continue
|
||||||
|
|
||||||
|
next_item = item
|
||||||
|
data = item.get('data')
|
||||||
|
if isinstance(data, dict) and 'content' in data:
|
||||||
|
next_item = deepcopy(item)
|
||||||
|
next_item.get('data', {}).pop('content', None)
|
||||||
|
|
||||||
|
file = next_item.get('file')
|
||||||
|
file_data = file.get('data') if isinstance(file, dict) else None
|
||||||
|
if isinstance(file_data, dict) and 'content' in file_data:
|
||||||
|
if next_item is item:
|
||||||
|
next_item = deepcopy(item)
|
||||||
|
file = next_item.get('file')
|
||||||
|
file_data = file.get('data') if isinstance(file, dict) else None
|
||||||
|
file_data.pop('content', None)
|
||||||
|
|
||||||
|
sanitized.append(next_item)
|
||||||
|
|
||||||
|
return sanitized
|
||||||
|
|
||||||
|
|
||||||
# --- Models DB Schema ---
|
# --- Models DB Schema ---
|
||||||
|
|
||||||
|
|
||||||
@@ -37,6 +70,7 @@ class ModelMeta(BaseModel):
|
|||||||
profile_image_url: str | None = None
|
profile_image_url: str | None = None
|
||||||
description: str | None = Field(default=None, description='User-facing description of the model.')
|
description: str | None = Field(default=None, description='User-facing description of the model.')
|
||||||
capabilities: dict | None = None
|
capabilities: dict | None = None
|
||||||
|
knowledge: list[Any] | None = None
|
||||||
|
|
||||||
model_config = ConfigDict(extra='allow')
|
model_config = ConfigDict(extra='allow')
|
||||||
|
|
||||||
@@ -56,6 +90,11 @@ class ModelMeta(BaseModel):
|
|||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
@field_validator('knowledge', mode='before')
|
||||||
|
@classmethod
|
||||||
|
def strip_knowledge_content(cls, v):
|
||||||
|
return strip_extracted_content_from_model_knowledge(v)
|
||||||
|
|
||||||
@model_validator(mode='before')
|
@model_validator(mode='before')
|
||||||
@classmethod
|
@classmethod
|
||||||
def normalize_tags(cls, data):
|
def normalize_tags(cls, data):
|
||||||
@@ -152,6 +191,14 @@ class ModelsTable:
|
|||||||
access_grants: list[AccessGrantModel | None] = None,
|
access_grants: list[AccessGrantModel | None] = None,
|
||||||
db: AsyncSession | None = None,
|
db: AsyncSession | None = None,
|
||||||
) -> ModelModel:
|
) -> ModelModel:
|
||||||
|
if isinstance(model.meta, dict):
|
||||||
|
knowledge = model.meta.get('knowledge')
|
||||||
|
stripped_knowledge = strip_extracted_content_from_model_knowledge(knowledge)
|
||||||
|
if stripped_knowledge != knowledge:
|
||||||
|
model.meta = {**model.meta, 'knowledge': stripped_knowledge}
|
||||||
|
if db is not None:
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
model_data = ModelModel.model_validate(model).model_dump(exclude={'access_grants'})
|
model_data = ModelModel.model_validate(model).model_dump(exclude={'access_grants'})
|
||||||
model_data['access_grants'] = (
|
model_data['access_grants'] = (
|
||||||
access_grants if access_grants is not None else await self._get_access_grants(model_data['id'], db=db)
|
access_grants if access_grants is not None else await self._get_access_grants(model_data['id'], db=db)
|
||||||
|
|||||||
@@ -148,6 +148,27 @@
|
|||||||
voices = res?.voices ?? [];
|
voices = res?.voices ?? [];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const toModelKnowledgeReference = (item: any) => {
|
||||||
|
if (!item || typeof item !== 'object') {
|
||||||
|
return item;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Object.fromEntries(
|
||||||
|
[
|
||||||
|
'id',
|
||||||
|
'name',
|
||||||
|
'type',
|
||||||
|
'description',
|
||||||
|
'context',
|
||||||
|
'legacy',
|
||||||
|
'collection_name',
|
||||||
|
'collection_names'
|
||||||
|
]
|
||||||
|
.filter((key) => item[key] !== undefined && item[key] !== null && item[key] !== '')
|
||||||
|
.map((key) => [key, item[key]])
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
const submitHandler = async () => {
|
const submitHandler = async () => {
|
||||||
loading = true;
|
loading = true;
|
||||||
|
|
||||||
@@ -194,7 +215,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (knowledge.length > 0) {
|
if (knowledge.length > 0) {
|
||||||
info.meta.knowledge = knowledge;
|
info.meta.knowledge = knowledge.map(toModelKnowledgeReference);
|
||||||
} else {
|
} else {
|
||||||
if (info.meta.knowledge) {
|
if (info.meta.knowledge) {
|
||||||
delete info.meta.knowledge;
|
delete info.meta.knowledge;
|
||||||
|
|||||||
Reference in new issue
Block a user