From 392e515c3fec6fbba9a98f0ae394ae803a6e0f50 Mon Sep 17 00:00:00 2001
From: KevinHuSh <kevinhu.sh@gmail.com>
Date: Wed, 3 Apr 2024 11:00:50 +0800
Subject: [PATCH] fix bug about reload knowledgebase configuration reloading
(#210)
### What problem does this PR solve?
_Briefly describe what this PR aims to solve. Include background context
that will help reviewers understand the purpose of the PR._
Issue link:#[[Link the issue
here](https://github.com/infiniflow/ragflow/issues/209)]
### Type of change
- [x] Bug Fix (non-breaking change which fixes an issue)
---
api/apps/chunk_app.py | 4 ++--
api/db/services/knowledgebase_service.py | 3 ++-
rag/app/manual.py | 9 ++++++---
rag/app/qa.py | 2 +-
rag/nlp/query.py | 2 +-
5 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/api/apps/chunk_app.py b/api/apps/chunk_app.py
index 247627a..0458160 100644
--- a/api/apps/chunk_app.py
+++ b/api/apps/chunk_app.py
@@ -79,7 +79,7 @@ def list():
return get_json_result(data=res)
except Exception as e:
if str(e).find("not_found") > 0:
- return get_json_result(data=False, retmsg=f'Index not found!',
+ return get_json_result(data=False, retmsg=f'No chunk found!',
retcode=RetCode.DATA_ERROR)
return server_error_response(e)
@@ -262,6 +262,6 @@ def retrieval_test():
return get_json_result(data=ranks)
except Exception as e:
if str(e).find("not_found") > 0:
- return get_json_result(data=False, retmsg=f'Index not found!',
+ return get_json_result(data=False, retmsg=f'No chunk found! Check the chunk status please!',
retcode=RetCode.DATA_ERROR)
return server_error_response(e)
diff --git a/api/db/services/knowledgebase_service.py b/api/db/services/knowledgebase_service.py
index ce34b72..3146cab 100644
--- a/api/db/services/knowledgebase_service.py
+++ b/api/db/services/knowledgebase_service.py
@@ -44,7 +44,8 @@ class KnowledgebaseService(CommonService):
def get_detail(cls, kb_id):
fields = [
cls.model.id,
- Tenant.embd_id,
+ #Tenant.embd_id,
+ cls.embd_id,
cls.model.avatar,
cls.model.name,
cls.model.language,
diff --git a/rag/app/manual.py b/rag/app/manual.py
index 284e3d6..bc8e0f1 100644
--- a/rag/app/manual.py
+++ b/rag/app/manual.py
@@ -85,7 +85,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
for t, lvl in pdf_parser.outlines:
tks = set([t[i] + t[i + 1] for i in range(len(t) - 1)])
tks_ = set([txt[i] + txt[i + 1]
- for i in range(min(len(t), len(txt) - 1))])
+ for i in range(min(len(t), len(txt) - 1))])
if len(set(tks & tks_)) / max([len(tks), len(tks_), 1]) > 0.8:
levels.append(lvl)
break
@@ -109,7 +109,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
sections = [(txt, sec_ids[i], poss)
for i, (txt, _, poss) in enumerate(sections)]
for (img, rows), poss in tbls:
- if not rows:continue
+ if not rows: continue
sections.append((rows if isinstance(rows, str) else rows[0], -1,
[(p[0] + 1 - from_page, p[1], p[2], p[3], p[4]) for p in poss]))
@@ -125,7 +125,7 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
for txt, sec_id, poss in sorted(sections, key=lambda x: (
x[-1][0][0], x[-1][0][3], x[-1][0][1])):
poss = "\t".join([tag(*pos) for pos in poss])
- if tk_cnt < 2048 and (sec_id == last_sid or sec_id == -1):
+ if tk_cnt < 32 or (tk_cnt < 1024 and (sec_id == last_sid or sec_id == -1)):
if chunks:
chunks[-1] += "\n" + txt + poss
tk_cnt += num_tokens_from_string(txt)
@@ -143,6 +143,9 @@ def chunk(filename, binary=None, from_page=0, to_page=100000,
if __name__ == "__main__":
import sys
+
def dummy(prog=None, msg=""):
pass
+
+
chunk(sys.argv[1], callback=dummy)
diff --git a/rag/app/qa.py b/rag/app/qa.py
index 1b42f1a..a4d49cd 100644
--- a/rag/app/qa.py
+++ b/rag/app/qa.py
@@ -133,7 +133,7 @@ def chunk(filename, binary=None, lang="Chinese", callback=None, **kwargs):
return res
raise NotImplementedError(
- "file type not supported yet(pptx, pdf supported)")
+ "Excel and csv(txt) format files are supported.")
if __name__ == "__main__":
diff --git a/rag/nlp/query.py b/rag/nlp/query.py
index 6a829a1..26753d7 100644
--- a/rag/nlp/query.py
+++ b/rag/nlp/query.py
@@ -73,7 +73,7 @@ class EsQueryer:
return True
qs, keywords = [], []
- for tt in self.tw.split(txt): # .split(" "):
+ for tt in self.tw.split(txt)[:256]: # .split(" "):
if not tt:
continue
twts = self.tw.weights([tt])
--
GitLab