refac: kb sync

2026-05-20 01:13:52 +04:00
parent d0b17f0569
commit 60c9db1cb8
6 changed files with 548 additions and 18 deletions
@@ -470,9 +470,83 @@ export const resetKnowledgeById = async (token: string, id: string) => {
 	return res;
 };

+export const syncKnowledgeDiff = async (
+	token: string,
+	id: string,
+	manifest: Array<{ filename: string; path: string; checksum: string; size: number }>
+) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/sync/diff`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({ manifest })
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.error(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
+export const syncKnowledgeCleanup = async (
+	token: string,
+	id: string,
+	fileIds: string[],
+	dirIds: string[] = []
+) => {
+	let error = null;
+
+	const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/sync/cleanup`, {
+		method: 'POST',
+		headers: {
+			Accept: 'application/json',
+			'Content-Type': 'application/json',
+			authorization: `Bearer ${token}`
+		},
+		body: JSON.stringify({ file_ids: fileIds, dir_ids: dirIds })
+	})
+		.then(async (res) => {
+			if (!res.ok) throw await res.json();
+			return res.json();
+		})
+		.then((json) => {
+			return json;
+		})
+		.catch((err) => {
+			error = err.detail;
+			console.error(err);
+			return null;
+		});
+
+	if (error) {
+		throw error;
+	}
+
+	return res;
+};
+
 export const deleteKnowledgeById = async (token: string, id: string) => {
 	let error = null;

+
 	const res = await fetch(`${WEBUI_API_BASE_URL}/knowledge/${id}/delete`, {
 		method: 'DELETE',
 		headers: {
@@ -37,11 +37,14 @@
 		createKnowledgeDirectory,
 		updateKnowledgeDirectory,
 		deleteKnowledgeDirectory,
-		moveFileInKnowledge
+		moveFileInKnowledge,
+		syncKnowledgeDiff,
+		syncKnowledgeCleanup
 	} from '$lib/apis/knowledge';
 	import { processWeb, processYoutubeVideo } from '$lib/apis/retrieval';

 	import { blobToFile, isYoutubeUrl, copyToClipboard } from '$lib/utils';
+	import { computeFileHash } from '$lib/utils/hash';

 	import Spinner from '$lib/components/common/Spinner.svelte';
 	import Tooltip from '$lib/components/common/Tooltip.svelte';
@@ -75,6 +78,7 @@
 	let showNewDirectoryModal = false;

 	let showSyncConfirmModal = false;
+	let pendingSyncFiles: Array<{path: string, filename: string, file: File}> | null = null;
 	let showAccessControlModal = false;

 	let minSize = 0;
@@ -537,25 +541,171 @@
 		}
 	};

-	// Helper function to maintain file paths within zip
-	const syncDirectoryHandler = async () => {
-		if (fileItems.length > 0) {
-			const res = await resetKnowledgeById(localStorage.token, id).catch((e) => {
-				toast.error(`${e}`);
-			});
+	// Collect files from a directory without uploading — returns {path, filename, file}[]
+	const collectDirectoryFiles = async (): Promise<Array<{path: string, filename: string, file: File}> | null> => {
+		const isFileSystemAccessSupported = 'showDirectoryPicker' in window;

-			if (res) {
-				fileItems = [];
-				toast.success($i18n.t('Knowledge reset successfully.'));
+		try {
+			if (isFileSystemAccessSupported) {
+				const dirHandle = await window.showDirectoryPicker();
+				const collected: Array<{path: string, filename: string, file: File}> = [];

-				// Upload directory
-				uploadDirectoryHandler();
+				async function traverse(handle: FileSystemDirectoryHandle, dirPath = '') {
+					for await (const entry of handle.values()) {
+						if (entry.name.startsWith('.')) continue;
+						const entryPath = dirPath ? `${dirPath}/${entry.name}` : entry.name;
+						if (hasHiddenFolder(entryPath)) continue;
+
+						if (entry.kind === 'file') {
+							const file = await entry.getFile();
+							collected.push({ path: dirPath, filename: entry.name, file });
+						} else if (entry.kind === 'directory') {
+							await traverse(entry, entryPath);
+						}
+					}
+				}
+
+				await traverse(dirHandle);
+				return collected;
+			} else {
+				// Firefox fallback
+				return new Promise((resolve, reject) => {
+					const input = document.createElement('input');
+					input.type = 'file';
+					input.webkitdirectory = true;
+					input.directory = true;
+					input.multiple = true;
+					input.style.display = 'none';
+					document.body.appendChild(input);
+
+					input.onchange = () => {
+						try {
+							const files = Array.from(input.files || [])
+								.filter((file) => !hasHiddenFolder(file.webkitRelativePath) && !file.name.startsWith('.'));
+
+							const collected = files.map((file) => {
+								const parts = file.webkitRelativePath.split('/');
+								// Remove root dir name, extract path and filename
+								const withoutRoot = parts.slice(1);
+								const filename = withoutRoot.pop() || file.name;
+								const path = withoutRoot.join('/');
+								return { path, filename, file };
+							});
+
+							document.body.removeChild(input);
+							resolve(collected);
+						} catch (error) {
+							document.body.removeChild(input);
+							reject(error);
+						}
+					};
+
+					input.onerror = (error) => {
+						document.body.removeChild(input);
+						reject(error);
+					};
+
+					input.click();
+				});
 			}
-		} else {
-			uploadDirectoryHandler();
+		} catch (error) {
+			handleUploadError(error);
+			return null;
 		}
 	};

+	// Incremental sync: hash locally → diff on server → upload only what changed
+	const syncDirectoryHandler = async () => {
+		if (!pendingSyncFiles?.length) return;
+
+		// ── 2. Compute checksums ──
+		toast.info($i18n.t('Computing checksums...'));
+		const manifest = await Promise.all(
+			pendingSyncFiles.map(async (entry) => ({
+				...entry,
+				checksum: await computeFileHash(entry.file),
+				size: entry.file.size
+			}))
+		);
+		pendingSyncFiles = null;
+
+		// ── 3. Diff against knowledge base ──
+		toast.info($i18n.t('Comparing with knowledge base...'));
+		const diff = await syncKnowledgeDiff(
+			localStorage.token,
+			id,
+			manifest.map(({ filename, path, checksum, size }) => ({ filename, path, checksum, size }))
+		);
+
+		if (!diff) {
+			toast.error($i18n.t('Failed to compare files.'));
+			return;
+		}
+
+		// ── 4. mkdir — create missing directories (parents first) ──
+		const createdDirectoryIds: Record<string, string> = {};
+		for (const dirPath of diff.mkdir) {
+			const segments = dirPath.split('/');
+			const name = segments.at(-1)!;
+			const parentPath = segments.slice(0, -1).join('/');
+			const parentId = parentPath ? createdDirectoryIds[parentPath] : null;
+
+			const directory = await createKnowledgeDirectory(
+				localStorage.token, knowledge.id, name, parentId
+			);
+			if (directory) {
+				createdDirectoryIds[dirPath] = directory.id;
+			}
+		}
+
+		// ── 5. Upload added + modified files ──
+		const filesToUpload = manifest.filter((entry) =>
+			diff.added.some((a: any) => a.filename === entry.filename && a.path === entry.path) ||
+			diff.modified.some((m: any) => m.filename === entry.filename && m.path === entry.path)
+		);
+
+		let uploadedCount = 0;
+		for (const entry of filesToUpload) {
+			const fileObject = new File([entry.file], entry.filename, { type: entry.file.type });
+			await uploadFile(localStorage.token, fileObject, {
+				knowledge_id: knowledge.id,
+				file_hash: entry.checksum,
+				directory_id: entry.path ? createdDirectoryIds[entry.path] : null
+			});
+			uploadedCount++;
+			toast.info(
+				$i18n.t('Uploading: {{current}}/{{total}}', {
+					current: uploadedCount,
+					total: filesToUpload.length
+				})
+			);
+		}
+
+		// ── 6. Cleanup — remove deleted files + rmdir orphaned directories ──
+		const staleFileIds = [
+			...diff.deleted.map((d: any) => d.file_id),
+			...diff.modified.map((m: any) => m.stale_file_id)
+		];
+
+		if (staleFileIds.length > 0 || diff.rmdir.length > 0) {
+			await syncKnowledgeCleanup(localStorage.token, id, staleFileIds, diff.rmdir);
+		}
+
+		// ── 7. Report ──
+		toast.success(
+			$i18n.t(
+				'Sync complete: {{added}} added, {{modified}} modified, {{deleted}} deleted, {{unmodified}} unmodified',
+				{
+					added: diff.added.length,
+					modified: diff.modified.length,
+					deleted: diff.deleted.length,
+					unmodified: diff.unmodified_count
+				}
+			)
+		);
+		init();
+	};
+
 	const addFileHandler = async (fileId) => {
 		const res = await addFileToKnowledgeById(
 			localStorage.token,
@@ -924,11 +1074,15 @@
 <SyncConfirmDialog
 	bind:show={showSyncConfirmModal}
 	message={$i18n.t(
-		'This will reset the knowledge base and sync all files. Do you wish to continue?'
+		'{{count}} files selected. Only new and modified files will be uploaded. Deleted files will be removed. The folder structure will be mirrored. Continue?',
+		{ count: pendingSyncFiles?.length ?? 0 }
 	)}
 	on:confirm={() => {
 		syncDirectoryHandler();
 	}}
+	on:cancel={() => {
+		pendingSyncFiles = null;
+	}}
 />

 <AttachWebpageModal
@@ -1113,8 +1267,11 @@
 										document.getElementById('files-input').click();
 									}
 								}}
-								onSync={() => {
-									showSyncConfirmModal = true;
+								onSync={async () => {
+									pendingSyncFiles = await collectDirectoryFiles();
+									if (pendingSyncFiles?.length) {
+										showSyncConfirmModal = true;
+									}
 								}}
 							/>
 						</div>
@@ -90,7 +90,7 @@

 			<Tooltip
 				content={$i18n.t(
-					'This option will delete all existing files in the collection and replace them with newly uploaded files.'
+					'Sync a local directory with this knowledge base. Only new and modified files will be uploaded. The directory structure will be mirrored.'
 				)}
 				className="w-full"
 			>
@@ -0,0 +1,115 @@
+/**
+ * Compute SHA-256 checksum of a File.
+ *
+ * Uses the Web Crypto API when available (HTTPS / localhost).
+ * Falls back to a pure-JS implementation for plain HTTP deployments
+ * where crypto.subtle is unavailable.
+ *
+ * @returns lowercase hex-encoded SHA-256 digest
+ */
+export async function computeFileHash(file: File): Promise<string> {
+	const buffer = await file.arrayBuffer();
+
+	if (globalThis.crypto?.subtle) {
+		const digest = await crypto.subtle.digest('SHA-256', buffer);
+		return hexEncode(new Uint8Array(digest));
+	}
+
+	return sha256Fallback(new Uint8Array(buffer));
+}
+
+function hexEncode(bytes: Uint8Array): string {
+	const hex: string[] = new Array(bytes.length);
+	for (let i = 0; i < bytes.length; i++) {
+		hex[i] = bytes[i].toString(16).padStart(2, '0');
+	}
+	return hex.join('');
+}
+
+// ── Pure-JS SHA-256 (RFC 6234) ──────────────────────────────────────────────
+// Used only when crypto.subtle is unavailable (HTTP deployments).
+
+const K: Uint32Array = new Uint32Array([
+	0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4,
+	0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe,
+	0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f,
+	0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
+	0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc,
+	0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b,
+	0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116,
+	0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
+	0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7,
+	0xc67178f2
+]);
+
+function rotr(n: number, x: number): number {
+	return (x >>> n) | (x << (32 - n));
+}
+
+function sha256Fallback(data: Uint8Array): string {
+	// Pre-processing: padding
+	const bitLen = data.length * 8;
+	const padLen = (((data.length + 8) >>> 6) + 1) << 6;
+	const padded = new Uint8Array(padLen);
+	padded.set(data);
+	padded[data.length] = 0x80;
+
+	// Append original length in bits as 64-bit big-endian
+	const view = new DataView(padded.buffer);
+	view.setUint32(padLen - 4, bitLen, false);
+
+	// Initial hash values
+	let h0 = 0x6a09e667;
+	let h1 = 0xbb67ae85;
+	let h2 = 0x3c6ef372;
+	let h3 = 0xa54ff53a;
+	let h4 = 0x510e527f;
+	let h5 = 0x9b05688c;
+	let h6 = 0x1f83d9ab;
+	let h7 = 0x5be0cd19;
+
+	const w = new Uint32Array(64);
+
+	for (let offset = 0; offset < padLen; offset += 64) {
+		// Prepare message schedule
+		for (let i = 0; i < 16; i++) {
+			w[i] = view.getUint32(offset + i * 4, false);
+		}
+		for (let i = 16; i < 64; i++) {
+			const s0 = rotr(7, w[i - 15]) ^ rotr(18, w[i - 15]) ^ (w[i - 15] >>> 3);
+			const s1 = rotr(17, w[i - 2]) ^ rotr(19, w[i - 2]) ^ (w[i - 2] >>> 10);
+			w[i] = (w[i - 16] + s0 + w[i - 7] + s1) | 0;
+		}
+
+		let a = h0, b = h1, c = h2, d = h3;
+		let e = h4, f = h5, g = h6, h = h7;
+
+		for (let i = 0; i < 64; i++) {
+			const S1 = rotr(6, e) ^ rotr(11, e) ^ rotr(25, e);
+			const ch = (e & f) ^ (~e & g);
+			const temp1 = (h + S1 + ch + K[i] + w[i]) | 0;
+			const S0 = rotr(2, a) ^ rotr(13, a) ^ rotr(22, a);
+			const maj = (a & b) ^ (a & c) ^ (b & c);
+			const temp2 = (S0 + maj) | 0;
+
+			h = g; g = f; f = e;
+			e = (d + temp1) | 0;
+			d = c; c = b; b = a;
+			a = (temp1 + temp2) | 0;
+		}
+
+		h0 = (h0 + a) | 0; h1 = (h1 + b) | 0;
+		h2 = (h2 + c) | 0; h3 = (h3 + d) | 0;
+		h4 = (h4 + e) | 0; h5 = (h5 + f) | 0;
+		h6 = (h6 + g) | 0; h7 = (h7 + h) | 0;
+	}
+
+	const result = new Uint8Array(32);
+	const out = new DataView(result.buffer);
+	out.setUint32(0, h0, false); out.setUint32(4, h1, false);
+	out.setUint32(8, h2, false); out.setUint32(12, h3, false);
+	out.setUint32(16, h4, false); out.setUint32(20, h5, false);
+	out.setUint32(24, h6, false); out.setUint32(28, h7, false);
+
+	return hexEncode(result);
+}