centersl/MinerU
1
0
Fork 0
mirror of https://github.com/opendatalab/MinerU.git synced 2026-03-27 11:08:32 +07:00
Code Issues Packages Projects Releases Wiki Activity

add pdf tools

Browse Source
This commit is contained in:
liusilu
2024-03-20 11:25:15 +08:00
parent d3e6853a2b
commit 2fb4b2efea
1 changed files with 11 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
tests/overall_indicator.py
View File

@@ -22,9 +22,9 @@ def indicator_cal(json_standard,json_test):
'''数据集总体指标'''
a=json_test[['id','mid_json']]
b=json_standard[['id','mid_json']]
b=json_standard[['id','mid_json','pass_label']]
outer_merge=pd.merge(a,b,on='id',how='outer')
outer_merge.columns=['id','standard_mid_json','test_mid_json']
outer_merge.columns=['id','standard_mid_json','test_mid_json','pass_label']
standard_exist=outer_merge.standard_mid_json.apply(lambda x: not isnull(x))
test_exist=outer_merge.test_mid_json.apply(lambda x: not isnull(x))
@@ -36,7 +36,7 @@ def indicator_cal(json_standard,json_test):
inner_merge=pd.merge(a,b,on='id',how='inner')
inner_merge.columns=['id','standard_mid_json','test_mid_json']
inner_merge.columns=['id','standard_mid_json','test_mid_json','pass_label']
json_standard = inner_merge['standard_mid_json']#check一下是否对齐
json_test = inner_merge['test_mid_json']
@@ -156,7 +156,14 @@ def indicator_cal(json_standard,json_test):
"""
'''计算pdf之间的总体编辑距离和bleu'''
'''
计算pdf之间的总体编辑距离和bleu
这里只计算正例的pdf
'''
test_para_text=np.asarray(test_para_text, dtype = object)[inner_merge['pass_label']=='yes']
standard_para_text=np.asarray(standard_para_text, dtype = object)[inner_merge['pass_label']=='yes']
pdf_dis=[]
pdf_bleu=[]
for a,b in zip(test_para_text,standard_para_text):