jialin
11 months ago
parent
ec45f23485
commit
f288c9a9ad
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,69 @@
|
||||
import _ from 'lodash';
|
||||
import { PCA } from 'ml-pca';
|
||||
|
||||
self.onmessage = (
|
||||
event: MessageEvent<{
|
||||
embeddings: any[];
|
||||
fileList: { text: string; name: string; uid: number | string }[];
|
||||
textList: { text: string; name: string; uid: number | string }[];
|
||||
}>
|
||||
) => {
|
||||
const { embeddings, fileList, textList } = event.data;
|
||||
|
||||
try {
|
||||
const dataList = embeddings.map((item) => {
|
||||
return item.embedding;
|
||||
});
|
||||
|
||||
console.log('list:', dataList);
|
||||
const pca = new PCA(dataList);
|
||||
console.log('list:', pca);
|
||||
const pcadata = pca.predict(dataList, { nComponents: 2 }).to2DArray();
|
||||
|
||||
const input = [
|
||||
...textList.map((item) => item.text).filter((item) => item),
|
||||
...fileList.map((item) => item.text).filter((item) => item)
|
||||
];
|
||||
|
||||
const list = pcadata.map((item: number[], index: number) => {
|
||||
return {
|
||||
value: item,
|
||||
name: index + 1,
|
||||
text: input[index]
|
||||
};
|
||||
});
|
||||
|
||||
const embeddingJson = embeddings.map((o, index) => {
|
||||
const item = _.cloneDeep(o);
|
||||
item.embedding = item.embedding.slice(0, 5);
|
||||
item.embedding.push(null);
|
||||
return item;
|
||||
});
|
||||
const embeddingData = {
|
||||
code: JSON.stringify(embeddingJson, null, 2).replace(/null/g, '...'),
|
||||
copyValue: JSON.stringify(embeddings, null, 2)
|
||||
};
|
||||
|
||||
self.postMessage({
|
||||
scatterData: list,
|
||||
embeddingData: embeddingData
|
||||
});
|
||||
} catch (e) {
|
||||
console.log('error:', e);
|
||||
self.postMessage({
|
||||
scatterData: [],
|
||||
embeddingData: {
|
||||
code: '',
|
||||
copyValue: ''
|
||||
}
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
self.onerror = (e) => {
|
||||
console.log('error:', e);
|
||||
};
|
||||
|
||||
self.onmessageerror = (e) => {
|
||||
console.log('message error:', e);
|
||||
};
|
||||
@ -0,0 +1,42 @@
|
||||
import { useEffect, useRef } from 'react';
|
||||
|
||||
export default function useEmbeddingWorker() {
|
||||
const workerRef = useRef<Worker | null>(null);
|
||||
|
||||
const createWorker = () => {
|
||||
if (workerRef.current) {
|
||||
workerRef.current.terminate();
|
||||
}
|
||||
workerRef.current = new Worker(
|
||||
new URL('../config/embedding-worker.worker.ts', import.meta.url),
|
||||
{
|
||||
type: 'module'
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
const postMessage = (params: {
|
||||
embeddings: any[];
|
||||
textList: { text: string; name: string; uid: number | string }[];
|
||||
fileList: { text: string; name: string; uid: number | string }[];
|
||||
}) => {
|
||||
if (workerRef.current) {
|
||||
workerRef.current.postMessage(params);
|
||||
}
|
||||
};
|
||||
|
||||
const terminateWorker = () => {
|
||||
if (workerRef.current) {
|
||||
workerRef.current.terminate();
|
||||
workerRef.current = null;
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
terminateWorker();
|
||||
};
|
||||
}, []);
|
||||
|
||||
return { workerRef, createWorker, postMessage, terminateWorker };
|
||||
}
|
||||
Loading…
Reference in new issue