From 91444e753231d71f330efc8c712ddb520d4d2349 Mon Sep 17 00:00:00 2001 From: Frieren <1692219062wang@gmail.com> Date: Mon, 13 May 2024 00:11:07 +0800 Subject: [PATCH 1/3] second commit --- .idea/misc.xml | 3 + .idea/workspace.xml | 106 ++++++++++++++++++++++++++---- requirements.txt | 9 ++- src/base/PdfLoader.py | 15 +++++ src/base/chinese_text_splitter.py | 48 ++++++++++++-- src/serve/embedding.py | 28 ++++++++ 6 files changed, 185 insertions(+), 24 deletions(-) create mode 100644 src/base/PdfLoader.py create mode 100644 src/serve/embedding.py diff --git a/.idea/misc.xml b/.idea/misc.xml index 7830771..7427ae4 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,4 +1,7 @@ + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml index d70da44..205572e 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -1,33 +1,75 @@ + + - + + + + + + + + + + + + + + - + { + "associatedIndex": 8 +} - + + + - { + "keyToString": { + "Python.RAG.executor": "Run", + "Python.embedding.executor": "Run", + "RunOnceActivity.OpenProjectViewOnStart": "true", + "RunOnceActivity.ShowReadmeOnStart": "true", + "git-widget-placeholder": "f7939674", + "last_opened_file_path": "G:/code/py/LLM", + "node.js.detected.package.eslint": "true", + "node.js.detected.package.tslint": "true", + "node.js.selected.package.eslint": "(autodetect)", + "node.js.selected.package.tslint": "(autodetect)", + "nodejs_package_manager_path": "npm", + "settings.editor.selected.configurable": "com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable", + "vue.rearranger.settings.migration": "true" } -}]]> +} + + + + + + @@ -43,8 +85,44 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ecdfcaa..f5aee2c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,7 @@ langchain~=0.1.19 -httpx-sse -langchainhub -pyjwt \ No newline at end of file +httpx +pyjwt +transformers +text2vec~=1.2.9 +chardet +pypdf~=4.2.0 \ No newline at end of file diff --git a/src/base/PdfLoader.py b/src/base/PdfLoader.py new file mode 100644 index 0000000..25bdd12 --- /dev/null +++ b/src/base/PdfLoader.py @@ -0,0 +1,15 @@ +from langchain_community.document_loaders import PyPDFLoader + + +class PDFLoader: + @staticmethod + def loader(file_path: str): + content = "" + loader = PyPDFLoader(file_path) + for page in loader.load(): + content += page.page_content + return content + + +if __name__ == '__main__': + print(PDFLoader.loader("C:\\Users\\16922\\Desktop\\文档1.pdf")) diff --git a/src/base/chinese_text_splitter.py b/src/base/chinese_text_splitter.py index 14914ec..414fd41 100644 --- a/src/base/chinese_text_splitter.py +++ b/src/base/chinese_text_splitter.py @@ -1,25 +1,59 @@ +from langchain.text_splitter import CharacterTextSplitter import re from typing import List -from langchain.text_splitter import CharacterTextSplitter - class ChineseTextSplitter(CharacterTextSplitter): - def __init__(self, pdf: bool = False, **kwargs): + def __init__(self, pdf: bool = False, sentence_size: int = 250, **kwargs): super().__init__(**kwargs) self.pdf = pdf + self.sentence_size = sentence_size - def split_text(self, text: str) -> List[str]: + def split_text1(self, text: str) -> List[str]: if self.pdf: text = re.sub(r"\n{3,}", "\n", text) text = re.sub('\s', ' ', text) text = text.replace("\n\n", "") - sent_sep_pattern = re.compile( - '([﹒﹔﹖﹗．。！？]["’”」』]{0,2}|(?=["‘“「『]{1,2}|$))') + sent_sep_pattern = re.compile('([﹒﹔﹖﹗．。！？]["’”」』]{0,2}|(?=["‘“「『]{1,2}|$))') # del ：； sent_list = [] for ele in sent_sep_pattern.split(text): if sent_sep_pattern.match(ele) and sent_list: sent_list[-1] += ele elif ele: sent_list.append(ele) - return sent_list \ No newline at end of file + return sent_list + + def split_text(self, text: str) -> List[str]: ##此处需要进一步优化逻辑 + if self.pdf: + text = re.sub(r"\n{3,}", r"\n", text) + text = re.sub('\s', " ", text) + text = re.sub("\n\n", "", text) + + text = re.sub(r'([;；.!?。！？\?])([^”’])', r"\1\n\2", text) # 单字符断句符 + text = re.sub(r'(\.{6})([^"’”」』])', r"\1\n\2", text) # 英文省略号 + text = re.sub(r'(\…{2})([^"’”」』])', r"\1\n\2", text) # 中文省略号 + text = re.sub(r'([;；!?。！？\?]["’”」』]{0,2})([^;；!?，。！？\?])', r'\1\n\2', text) + # 如果双引号前有终止符，那么双引号才是句子的终点，把分句符\n放到双引号后，注意前面的几句都小心保留了双引号 + text = text.rstrip() # 段尾如果有多余的\n就去掉它 + # 很多规则中会考虑分号;，但是这里我把它忽略不计，破折号、英文双引号等同样忽略，需要的再做些简单调整即可。 + ls = [i for i in text.split("\n") if i] + for ele in ls: + if len(ele) > self.sentence_size: + ele1 = re.sub(r'([,，.]["’”」』]{0,2})([^,，.])', r'\1\n\2', ele) + ele1_ls = ele1.split("\n") + for ele_ele1 in ele1_ls: + if len(ele_ele1) > self.sentence_size: + ele_ele2 = re.sub(r'([\n]{1,}| {2,}["’”」』]{0,2})([^\s])', r'\1\n\2', ele_ele1) + ele2_ls = ele_ele2.split("\n") + for ele_ele2 in ele2_ls: + if len(ele_ele2) > self.sentence_size: + ele_ele3 = re.sub('( ["’”」』]{0,2})([^ ])', r'\1\n\2', ele_ele2) + ele2_id = ele2_ls.index(ele_ele2) + ele2_ls = ele2_ls[:ele2_id] + [i for i in ele_ele3.split("\n") if i] + ele2_ls[ + ele2_id + 1:] + ele_id = ele1_ls.index(ele_ele1) + ele1_ls = ele1_ls[:ele_id] + [i for i in ele2_ls if i] + ele1_ls[ele_id + 1:] + + id = ls.index(ele) + ls = ls[:id] + [i for i in ele1_ls if i] + ls[id + 1:] + return ls \ No newline at end of file diff --git a/src/serve/embedding.py b/src/serve/embedding.py new file mode 100644 index 0000000..c7b71f4 --- /dev/null +++ b/src/serve/embedding.py @@ -0,0 +1,28 @@ +from langchain_community.document_loaders import PyPDFLoader +from langchain_community.vectorstores import Chroma +from text2vec import SentenceModel +from src.base.chinese_text_splitter import ChineseTextSplitter + + +class SentenceEmbedding: + __model = SentenceModel('shibing624/text2vec-base-chinese') + def __init__(self, file_path: str): + self.file_path = file_path + content = "" + loader = PyPDFLoader(file_path) + for page in loader.load(): + content += page.page_content + sentences = ChineseTextSplitter(True).split_text(content) + embeddings = SentenceEmbedding.__model.encode(sentences) + self.vectorstore = Chroma.add_texts(iter(sentences), embeddings) + + def get_vectorstore(self): + return self.vectorstore + + def search(self, query:str): + embeddings = SentenceEmbedding.__model.encode(query) + self.vectorstore + + +if __name__ == '__main__': + SentenceEmbedding("C:\\Users\\16922\\Desktop\\文档1.pdf"). From 2f9b99b50ace11603b5741a28fe683f24534f104 Mon Sep 17 00:00:00 2001 From: Frieren <1692219062wang@gmail.com> Date: Mon, 13 May 2024 00:12:43 +0800 Subject: [PATCH 2/3] second commit --- .idea/workspace.xml | 78 +++++++++++++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 28 deletions(-) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 205572e..47574ce 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -4,14 +4,7 @@