diff --git a/api/apps/conversation_app.py b/api/apps/conversation_app.py
index 23db038e66fbd287547a906a5eae4ab715d85695..dbebc39ef8b5e6be85b9218b0b6cf8fda3a573d3 100644
--- a/api/apps/conversation_app.py
+++ b/api/apps/conversation_app.py
@@ -118,14 +118,13 @@ def message_fit_in(msg, max_length=4000):
c = count()
if c < max_length: return c, msg
- msg = [m for m in msg if m.role in ["system", "user"]]
- c = count()
- if c < max_length: return c, msg
+
msg_ = [m for m in msg[:-1] if m.role == "system"]
msg_.append(msg[-1])
msg = msg_
c = count()
if c < max_length: return c, msg
+
ll = num_tokens_from_string(msg_[0].content)
l = num_tokens_from_string(msg_[-1].content)
if ll / (ll + l) > 0.8:
diff --git a/api/apps/document_app.py b/api/apps/document_app.py
index 4a6d3f71bc028b345821d6086f1b1fc32a8292c9..04f721ac946af801733299b9ee16bb0ed6ad65f6 100644
--- a/api/apps/document_app.py
+++ b/api/apps/document_app.py
@@ -218,7 +218,7 @@ def rm():
ELASTICSEARCH.deleteByQuery(Q("match", doc_id=doc.id), idxnm=search.index_name(tenant_id))
DocumentService.increment_chunk_num(doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1, 0)
- if not DocumentService.delete_by_id(req["doc_id"]):
+ if not DocumentService.delete(doc):
return get_data_error_result(
retmsg="Database error (Document removal)!")
diff --git a/api/db/db_models.py b/api/db/db_models.py
index c32b4d56b5df9a372edb12e73d064e6e218ef2a0..5cec88a3df96067ccd08a08c529b832f61fa051e 100644
--- a/api/db/db_models.py
+++ b/api/db/db_models.py
@@ -353,7 +353,7 @@ class User(DataBaseModel, UserMixin):
email = CharField(max_length=255, null=False, help_text="email", index=True)
avatar = TextField(null=True, help_text="avatar base64 string")
language = CharField(max_length=32, null=True, help_text="English|Chinese", default="Chinese")
- color_schema = CharField(max_length=32, null=True, help_text="Bright|Dark", default="Dark")
+ color_schema = CharField(max_length=32, null=True, help_text="Bright|Dark", default="Bright")
timezone = CharField(max_length=64, null=True, help_text="Timezone", default="UTC+8\tAsia/Shanghai")
last_login_time = DateTimeField(null=True)
is_authenticated = CharField(max_length=1, null=False, default="1")
diff --git a/api/db/init_data.py b/api/db/init_data.py
index 3418bcfdcf820826d911d6f18daf813c296bcd3c..b4b63d36bb26a56e76e2ba14a9043ce3f144cc65 100644
--- a/api/db/init_data.py
+++ b/api/db/init_data.py
@@ -223,7 +223,7 @@ def init_llm_factory():
"fid": factory_infos[3]["name"],
"llm_name": "qwen-14B-chat",
"tags": "LLM,CHAT,",
- "max_tokens": 8191,
+ "max_tokens": 4096,
"model_type": LLMType.CHAT.value
}, {
"fid": factory_infos[3]["name"],
@@ -271,11 +271,15 @@ def init_llm_factory():
pass
"""
+ modify service_config
drop table llm;
- drop table factories;
+ drop table llm_factories;
update tenant_llm set llm_factory='Tongyi-Qianwen' where llm_factory='通义ĺŤé—®';
update tenant_llm set llm_factory='ZHIPU-AI' where llm_factory='智谱AI';
update tenant set parser_ids='naive:General,one:One,qa:Q&A,resume:Resume,table:Table,laws:Laws,manual:Manual,book:Book,paper:Paper,presentation:Presentation,picture:Picture';
+ alter table knowledgebase modify avatar longtext;
+ alter table user modify avatar longtext;
+ alter table dialog modify icon longtext;
"""
diff --git a/api/db/services/document_service.py b/api/db/services/document_service.py
index f58b0e1b8d498b7b91010b7405eb2e2d17934f28..b219861b9a663dd73ec0580d63ef595d431e4a5a 100644
--- a/api/db/services/document_service.py
+++ b/api/db/services/document_service.py
@@ -60,6 +60,15 @@ class DocumentService(CommonService):
raise RuntimeError("Database error (Knowledgebase)!")
return doc
+ @classmethod
+ @DB.connection_context()
+ def delete(cls, doc):
+ e, kb = KnowledgebaseService.get_by_id(doc.kb_id)
+ if not KnowledgebaseService.update_by_id(
+ kb.id, {"doc_num": kb.doc_num - 1}):
+ raise RuntimeError("Database error (Knowledgebase)!")
+ return cls.delete_by_id(doc.id)
+
@classmethod
@DB.connection_context()
def get_newly_uploaded(cls, tm, mod=0, comm=1, items_per_page=64):
diff --git a/deepdoc/parser/pdf_parser.py b/deepdoc/parser/pdf_parser.py
index 4d8e025616a219b5e47acb029a10681d635c09a1..65185280160d69c119384535d0927376a9b127d7 100644
--- a/deepdoc/parser/pdf_parser.py
+++ b/deepdoc/parser/pdf_parser.py
@@ -11,7 +11,7 @@ import logging
from PIL import Image, ImageDraw
import numpy as np
-from api.db import ParserType
+from PyPDF2 import PdfReader as pdf2_read
from deepdoc.vision import OCR, Recognizer, LayoutRecognizer, TableStructureRecognizer
from rag.nlp import huqie
from copy import deepcopy
@@ -288,9 +288,9 @@ class HuParser:
for b in bxs])
self.boxes.append(bxs)
- def _layouts_rec(self, ZM):
+ def _layouts_rec(self, ZM, drop=True):
assert len(self.page_images) == len(self.boxes)
- self.boxes, self.page_layout = self.layouter(self.page_images, self.boxes, ZM)
+ self.boxes, self.page_layout = self.layouter(self.page_images, self.boxes, ZM, drop=drop)
# cumlative Y
for i in range(len(self.boxes)):
self.boxes[i]["top"] += \
@@ -908,6 +908,23 @@ class HuParser:
self.page_images.append(img)
self.page_chars.append([])
+ self.outlines = []
+ try:
+ self.pdf = pdf2_read(fnm if isinstance(fnm, str) else BytesIO(fnm))
+ outlines = self.pdf.outline
+
+ def dfs(arr, depth):
+ for a in arr:
+ if isinstance(a, dict):
+ self.outlines.append((a["/Title"], depth))
+ continue
+ dfs(a, depth+1)
+ dfs(outlines, 0)
+ except Exception as e:
+ logging.warning(f"Outlines exception: {e}")
+ if not self.outlines:
+ logging.warning(f"Miss outlines")
+
logging.info("Images converted.")
self.is_english = [re.search(r"[a-zA-Z0-9,/¸;:'\[\]\(\)!@#$%^&*\"?<>._-]{30,}", "".join(
random.choices([c["text"] for c in self.page_chars[i]], k=min(100, len(self.page_chars[i]))))) for i in
diff --git a/deepdoc/vision/layout_recognizer.py b/deepdoc/vision/layout_recognizer.py
index f38a0f539aa909b45d49892e2d78cd23496f6dec..47067068e565051ec6cd6f78736f2f34bb508e49 100644
--- a/deepdoc/vision/layout_recognizer.py
+++ b/deepdoc/vision/layout_recognizer.py
@@ -39,7 +39,7 @@ class LayoutRecognizer(Recognizer):
super().__init__(self.labels, domain, os.path.join(get_project_base_directory(), "rag/res/deepdoc/"))
self.garbage_layouts = ["footer", "header", "reference"]
- def __call__(self, image_list, ocr_res, scale_factor=3, thr=0.2, batch_size=16):
+ def __call__(self, image_list, ocr_res, scale_factor=3, thr=0.2, batch_size=16, drop=True):
def __is_garbage(b):
patt = [r"^•+$", r"(ç‰ćťĺ˝’©|免责条款|地址[:：])", r"\.{3,}", "^[0-9]{1,2} / ?[0-9]{1,2}$",
r"^[0-9]{1,2} of [0-9]{1,2}$", "^http://[^ ]{12,}",
@@ -88,7 +88,11 @@ class LayoutRecognizer(Recognizer):
i += 1
continue
lts_[ii]["visited"] = True
- if lts_[ii]["type"] in self.garbage_layouts:
+ keep_feats = [
+ lts_[ii]["type"] == "footer" and bxs[i]["bottom"] < image_list[pn].size[1]*0.9/scale_factor,
+ lts_[ii]["type"] == "header" and bxs[i]["top"] > image_list[pn].size[1]*0.1/scale_factor,
+ ]
+ if drop and lts_[ii]["type"] in self.garbage_layouts and not any(keep_feats):
if lts_[ii]["type"] not in garbages:
garbages[lts_[ii]["type"]] = []
garbages[lts_[ii]["type"]].append(bxs[i]["text"])
diff --git a/rag/app/manual.py b/rag/app/manual.py
index 7ca5451971d896af54a1cdce79717dc98bbca3d6..e72c2ac30f746f13836266fd76706dc7289e20fc 100644
--- a/rag/app/manual.py
+++ b/rag/app/manual.py
@@ -51,15 +51,30 @@ class Pdf(PdfParser):
# set pivot using the most frequent type of title,
# then merge between 2 pivot
- bull = bullets_category([b["text"] for b in self.boxes])
- most_level, levels = title_frequency(bull, [(b["text"], b.get("layout_no","")) for b in self.boxes])
+ if len(self.boxes)>0 and len(self.outlines)/len(self.boxes) > 0.1:
+ max_lvl = max([lvl for _, lvl in self.outlines])
+ most_level = max(0, max_lvl-1)
+ levels = []
+ for b in self.boxes:
+ for t,lvl in self.outlines:
+ tks = set([t[i]+t[i+1] for i in range(len(t)-1)])
+ tks_ = set([b["text"][i]+b["text"][i+1] for i in range(min(len(t), len(b["text"])-1))])
+ if len(set(tks & tks_))/max([len(tks), len(tks_), 1]) > 0.8:
+ levels.append(lvl)
+ break
+ else:
+ levels.append(max_lvl + 1)
+ else:
+ bull = bullets_category([b["text"] for b in self.boxes])
+ most_level, levels = title_frequency(bull, [(b["text"], b.get("layout_no","")) for b in self.boxes])
+
assert len(self.boxes) == len(levels)
sec_ids = []
sid = 0
for i, lvl in enumerate(levels):
if lvl <= most_level and i > 0 and lvl != levels[i-1]: sid += 1
sec_ids.append(sid)
- #print(lvl, self.boxes[i]["text"], most_level)
+ #print(lvl, self.boxes[i]["text"], most_level, sid)
sections = [(b["text"], sec_ids[i], self.get_position(b, zoomin)) for i, b in enumerate(self.boxes)]
for (img, rows), poss in tbls:
@@ -67,13 +82,16 @@ class Pdf(PdfParser):
chunks = []
last_sid = -2
+ tk_cnt = 0
for txt, sec_id, poss in sorted(sections, key=lambda x: (x[-1][0][0], x[-1][0][3], x[-1][0][1])):
poss = "\t".join([tag(*pos) for pos in poss])
- if sec_id == last_sid or sec_id == -1:
+ if tk_cnt < 2048 and (sec_id == last_sid or sec_id == -1):
if chunks:
chunks[-1] += "\n" + txt + poss
+ tk_cnt += num_tokens_from_string(txt)
continue
chunks.append(txt + poss)
+ tk_cnt = num_tokens_from_string(txt)
if sec_id >-1: last_sid = sec_id
return chunks, tbls
@@ -97,37 +115,17 @@ def chunk(filename, binary=None, from_page=0, to_page=100000, lang="Chinese", ca
# is it English
eng = lang.lower() == "english"#pdf_parser.is_english
- i = 0
- chunk = []
- tk_cnt = 0
res = tokenize_table(tbls, doc, eng)
- def add_chunk():
- nonlocal chunk, res, doc, pdf_parser, tk_cnt
+ for ck in cks:
d = copy.deepcopy(doc)
- ck = "\n".join(chunk)
- tokenize(d, pdf_parser.remove_tag(ck), eng)
d["image"], poss = pdf_parser.crop(ck, need_position=True)
add_positions(d, poss)
+ tokenize(d, pdf_parser.remove_tag(ck), eng)
res.append(d)
- chunk = []
- tk_cnt = 0
-
- while i < len(cks):
- if tk_cnt > 256: add_chunk()
- txt = cks[i]
- txt_ = pdf_parser.remove_tag(txt)
- i += 1
- cnt = num_tokens_from_string(txt_)
- chunk.append(txt)
- tk_cnt += cnt
- if chunk: add_chunk()
-
- for i, d in enumerate(res):
- print(d)
- # d["image"].save(f"./logs/{i}.jpg")
return res
+
if __name__ == "__main__":
import sys
def dummy(prog=None, msg=""):
diff --git a/rag/app/one.py b/rag/app/one.py
index d43961a48718234615567cca9a746836cc3b8f4c..cce70277a7d1e8a9d8edf3421423f035453b8dd5 100644
--- a/rag/app/one.py
+++ b/rag/app/one.py
@@ -10,12 +10,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-import copy
import re
from rag.app import laws
-from rag.nlp import huqie, is_english, tokenize, naive_merge, tokenize_table, add_positions
+from rag.nlp import huqie, tokenize
from deepdoc.parser import PdfParser, ExcelParser
-from rag.settings import cron_logger
class Pdf(PdfParser):
@@ -33,7 +31,7 @@ class Pdf(PdfParser):
from timeit import default_timer as timer
start = timer()
- self._layouts_rec(zoomin)
+ self._layouts_rec(zoomin, drop=False)
callback(0.63, "Layout analysis finished.")
print("paddle layouts:", timer() - start)
self._table_transformer_job(zoomin)
diff --git a/rag/nlp/search.py b/rag/nlp/search.py
index 9f89cd5ab4369bf98ff5ca63438e39d732565634..3b6f20bbb4d7e6a61d9c4976d11f4476f24364f1 100644
--- a/rag/nlp/search.py
+++ b/rag/nlp/search.py
@@ -215,7 +215,7 @@ class Dealer:
else:
pieces = re.split(r"([^\|][；。？!ďĽ\n]|[a-z][.?;!][ \n])", answer)
for i in range(1, len(pieces)):
- if re.match(r"[a-z][.?;!][ \n]", pieces[i]):
+ if re.match(r"([^\|][；。？!ďĽ\n]|[a-z][.?;!][ \n])", pieces[i]):
pieces[i - 1] += pieces[i][0]
pieces[i] = pieces[i][1:]
idx = []
@@ -243,7 +243,8 @@ class Dealer:
chunks_tks,
tkweight, vtweight)
mx = np.max(sim) * 0.99
- if mx < 0.65:
+ es_logger.info("{} SIM: {}".format(pieces_[i], mx))
+ if mx < 0.63:
continue
cites[idx[i]] = list(
set([str(ii) for ii in range(len(chunk_v)) if sim[ii] > mx]))[:4]
diff --git a/rag/svr/task_broker.py b/rag/svr/task_broker.py
index 62f0d0767259c43a08d87fbc2450b79f2cd52377..9bcc8aa3120a8f89d9d03a9d1ab7c33c07603332 100644
--- a/rag/svr/task_broker.py
+++ b/rag/svr/task_broker.py
@@ -82,8 +82,8 @@ def dispatch():
tsks = []
if r["type"] == FileType.PDF.value:
pages = PdfParser.total_page_number(r["name"], MINIO.get(r["kb_id"], r["location"]))
- page_size = 5
- if r["parser_id"] == "paper": page_size = 12
+ page_size = 12
+ if r["parser_id"] == "paper": page_size = 22
if r["parser_id"] == "one": page_size = 1000000000
for s,e in r["parser_config"].get("pages", [(0,100000)]):
e = min(e, pages)