centersl/MinerU
1
0
Fork 0
mirror of https://github.com/opendatalab/MinerU.git synced 2026-03-27 11:08:32 +07:00
Code Issues Packages Projects Releases Wiki Activity

fix(merge_text): add ligature replacement functionality

Browse Source 
- Implement __replace_ligatures function to split ligature characters- Integrate ligature replacement into the merge_para_with_text function
- Handle common ligatures such as fi, fl, ff, ffi, and ffl
This commit is contained in:
myhloli
2024-11-04 18:44:58 +08:00
parent 24fb70417b
commit bd75596219
1 changed files with 12 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
magic_pdf/dict2md/ocr_mkcontent.py
View File

@@ -119,6 +119,16 @@ def detect_language(text):
return 'empty'
# 连写字符拆分
def __replace_ligatures(text: str):
text = re.sub(r'fi', 'fi', text) # 替换 fi 连写符
text = re.sub(r'fl', 'fl', text) # 替换 fl 连写符
text = re.sub(r'ff', 'ff', text) # 替换 ff 连写符
text = re.sub(r'ffi', 'ffi', text) # 替换 ffi 连写符
text = re.sub(r'ffl', 'ffl', text) # 替换 ffl 连写符
return text
def merge_para_with_text(para_block):
para_text = ''
for i, line in enumerate(para_block['lines']):
@@ -166,6 +176,8 @@ def merge_para_with_text(para_block):
para_text += content
else:
continue
# 连写字符拆分
para_text = __replace_ligatures(para_text)
return para_text