Update mineru/model/mfr/unimernet/Unimernet.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Merge pull request #3143 from myhloli/dev
2026-03-27 19:18:34 +07:00 · 2025-07-23 00:20:08 +08:00 · 2025-07-23 00:12:51 +08:00 · 2025-07-23 00:11:52 +08:00 · 2025-07-22 22:23:24 +08:00 · 2025-07-22 22:22:12 +08:00
10 changed files with 36 additions and 288 deletions
--- a/.github/workflows/cla.yml
+++ b/.github/workflows/cla.yml
@@ -18,9 +18,9 @@ jobs:
    steps:
      - name: "CLA Assistant"
        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
-        uses: contributor-assistant/github-action@v2.5.0
+        uses: contributor-assistant/github-action@v2.6.1
        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN  }}
          # the below token should have repo scope and must be manually added by you in the repository's secret
          # This token is required only if you have configured to store the signatures in a remote repository/organization
          PERSONAL_ACCESS_TOKEN: ${{ secrets.RELEASE_TOKEN }}
@@ -28,7 +28,7 @@ jobs:
          path-to-signatures: 'signatures/version1/cla.json'
          path-to-document: 'https://github.com/opendatalab/MinerU/blob/master/MinerU_CLA.md' # e.g. a CLA or a DCO document
          # branch should not be protected
-          branch: 'master'
+          branch: 'cla'
          allowlist: myhloli,dt-yy,Focusshang,renpengli01,icecraft,drunkpig,wangbinDL,qiangqiang199,GDDGCZ518,papayalove,conghui,quyuan,LollipopsAndWine,Sidney233

         # the followings are the optional inputs - If the optional inputs are not given, then default values will be taken
--- a/README.md
+++ b/README.md
@@ -43,7 +43,10 @@
 </div>

 # Changelog
-
+- 2025/07/22 2.1.3 Released
+  - Bug Fixes
+    - Fixed the issue of excessive memory consumption during the `MFR` step in the `pipeline` backend under certain scenarios #2771
+    - Fixed the inaccurate matching between `image`/`table` and `caption`/`footnote` under certain conditions #3129
 - 2025/07/16 2.1.1 Released
  - Bug fixes
    - Fixed text block content loss issue that could occur in certain `pipeline` scenarios #3005
--- a/README_zh-CN.md
+++ b/README_zh-CN.md
@@ -43,6 +43,10 @@
 </div>

 # 更新记录
+- 2025/07/22 2.1.3发布
+  - bug修复
+    - 修复`pipeline`后端中`MFR`步骤在某些情况下显存消耗过大的问题 #2771
+    - 修复某些情况下`image`/`table`与`caption`/`footnote`匹配不准确的问题 #3129
 - 2025/07/16 2.1.1发布
  - bug修复 
    - 修复`pipeline`在某些情况可能发生的文本块内容丢失问题 #3005
--- a/mineru/backend/pipeline/batch_analyze.py
+++ b/mineru/backend/pipeline/batch_analyze.py
@@ -12,6 +12,7 @@ from ...utils.ocr_utils import get_adjusted_mfdetrec_res, get_ocr_result_list, O
 YOLO_LAYOUT_BASE_BATCH_SIZE = 8
 MFD_BASE_BATCH_SIZE = 1
 MFR_BASE_BATCH_SIZE = 16
+OCR_DET_BASE_BATCH_SIZE = 16


 class BatchAnalyze:
@@ -170,9 +171,9 @@ class BatchAnalyze:
                        batch_images.append(padded_img)

                    # 批处理检测
-                    batch_size = min(len(batch_images), self.batch_ratio * 16)  # 增加批处理大小
-                    # logger.debug(f"OCR-det batch: {batch_size} images, target size: {target_h}x{target_w}")
-                    batch_results = ocr_model.text_detector.batch_predict(batch_images, batch_size)
+                    det_batch_size = min(len(batch_images), self.batch_ratio * OCR_DET_BASE_BATCH_SIZE)  # 增加批处理大小
+                    # logger.debug(f"OCR-det batch: {det_batch_size} images, target size: {target_h}x{target_w}")
+                    batch_results = ocr_model.text_detector.batch_predict(batch_images, det_batch_size)

                    # 处理批处理结果
                    for i, (crop_info, (dt_boxes, elapse)) in enumerate(zip(group_crops, batch_results)):
--- a/mineru/backend/pipeline/pipeline_analyze.py
+++ b/mineru/backend/pipeline/pipeline_analyze.py
@@ -74,10 +74,10 @@ def doc_analyze(
        table_enable=True,
 ):
    """
-    适当调大MIN_BATCH_INFERENCE_SIZE可以提高性能，可能会增加显存使用量，
-    可通过环境变量MINERU_MIN_BATCH_INFERENCE_SIZE设置，默认值为128。
+    适当调大MIN_BATCH_INFERENCE_SIZE可以提高性能，更大的 MIN_BATCH_INFERENCE_SIZE会消耗更多内存，
+    可通过环境变量MINERU_MIN_BATCH_INFERENCE_SIZE设置，默认值为384。
    """
-    min_batch_inference_size = int(os.environ.get('MINERU_MIN_BATCH_INFERENCE_SIZE', 128))
+    min_batch_inference_size = int(os.environ.get('MINERU_MIN_BATCH_INFERENCE_SIZE', 384))

    # 收集所有页面信息
    all_pages_info = []  # 存储(dataset_index, page_index, img, ocr, lang, width, height)
--- a/mineru/backend/pipeline/pipeline_magic_model.py
+++ b/mineru/backend/pipeline/pipeline_magic_model.py
@@ -275,7 +275,8 @@ class MagicModel:


            fst_idx, fst_kind, left_x, top_y = candidates[0]
-            candidates.sort(key=lambda x: (x[2] - left_x) ** 2 + (x[3] - top_y)**2)
+            fst_bbox = subjects[fst_idx]['bbox'] if fst_kind == SUB_BIT_KIND else objects[fst_idx - OBJ_IDX_OFFSET]['bbox']
+            candidates.sort(key=lambda x: bbox_distance(fst_bbox, subjects[x[0]]['bbox']) if x[1] == SUB_BIT_KIND else bbox_distance(fst_bbox, objects[x[0] - OBJ_IDX_OFFSET]['bbox']))
            nxt = None

            for i in range(1, len(candidates)):
@@ -294,7 +295,8 @@ class MagicModel:
            pair_dis = bbox_distance(subjects[sub_idx]['bbox'], objects[obj_idx]['bbox'])
            nearest_dis = float('inf')
            for i in range(N):
-                if i in seen_idx or i == sub_idx:continue
+                # 取消原先算法中 1对1 匹配的偏置
+                # if i in seen_idx or i == sub_idx:continue
                nearest_dis = min(nearest_dis, bbox_distance(subjects[i]['bbox'], objects[obj_idx]['bbox']))

            if pair_dis >= 3*nearest_dis:
--- a/mineru/model/mfr/unimernet/Unimernet.py
+++ b/mineru/model/mfr/unimernet/Unimernet.py
@@ -104,6 +104,10 @@ class UnimernetModel(object):

        # Create dataset with sorted images
        dataset = MathDataset(sorted_images, transform=self.model.transform)
+
+        # 如果batch_size> len(sorted_images)，则设置为不超过len(sorted_images)的2的幂
+        batch_size = min(batch_size, 2 ** (len(sorted_images).bit_length() - 1))
+
        dataloader = DataLoader(dataset, batch_size=batch_size, num_workers=0)

        # Process batches and store results
@@ -115,7 +119,7 @@ class UnimernetModel(object):
                mf_img = mf_img.to(dtype=self.model.dtype)
                mf_img = mf_img.to(self.device)
                with torch.no_grad():
-                    output = self.model.generate({"image": mf_img})
+                    output = self.model.generate({"image": mf_img}, batch_size=batch_size)
                mfr_res.extend(output["fixed_str"])

                # 更新进度条，每次增加batch_size，但要注意最后一个batch可能不足batch_size
--- a/mineru/model/mfr/unimernet/unimernet_hf/modeling_unimernet.py
+++ b/mineru/model/mfr/unimernet/unimernet_hf/modeling_unimernet.py
@@ -468,7 +468,7 @@ class UnimernetModel(VisionEncoderDecoderModel):
        ).loss
        return {"loss": loss}

-    def generate(self, samples, do_sample: bool = False, temperature: float = 0.2, top_p: float = 0.95):
+    def generate(self, samples, do_sample: bool = False, temperature: float = 0.2, top_p: float = 0.95, batch_size=64):
        pixel_values = samples["image"]
        num_channels = pixel_values.shape[1]
        if num_channels == 1:
@@ -478,7 +478,13 @@ class UnimernetModel(VisionEncoderDecoderModel):
        if do_sample:
            kwargs["temperature"] = temperature
            kwargs["top_p"] = top_p
-        
+
+        if self.tokenizer.tokenizer.model_max_length > 1152:
+            if batch_size <= 32:
+                self.tokenizer.tokenizer.model_max_length = 1152  # 6g
+            else:
+                self.tokenizer.tokenizer.model_max_length = 1344  # 8g
+
        outputs = super().generate(
            pixel_values=pixel_values,
            max_new_tokens=self.tokenizer.tokenizer.model_max_length, # required
--- a/mineru/model/ocr/paddleocr2pytorch/pytorch_paddle.py
+++ b/mineru/model/ocr/paddleocr2pytorch/pytorch_paddle.py
@@ -88,7 +88,7 @@ class PytorchPaddleOCR(TextSystem):
        kwargs['det_model_path'] = det_model_path
        kwargs['rec_model_path'] = rec_model_path
        kwargs['rec_char_dict_path'] = os.path.join(root_dir, 'pytorchocr', 'utils', 'resources', 'dict', dict_file)
-        # kwargs['rec_batch_num'] = 8
+        kwargs['rec_batch_num'] = 16

        kwargs['device'] = device

--- a/signatures/version1/cla.json
+++ b/signatures/version1/cla.json
@@ -391,278 +391,6 @@
      "created_at": "2025-07-16T08:53:24Z",
      "repoId": 765083837,
      "pullRequestNo": 3070
-    },
-    {
-      "name": "huazZeng",
-      "id": 125243371,
-      "comment_id": 3100630363,
-      "created_at": "2025-07-22T03:04:40Z",
-      "repoId": 765083837,
-      "pullRequestNo": 3129
-    },
-    {
-      "name": "jinghuan-Chen",
-      "id": 42742857,
-      "comment_id": 3114162786,
-      "created_at": "2025-07-24T16:49:20Z",
-      "repoId": 765083837,
-      "pullRequestNo": 3175
-    },
-    {
-      "name": "androllen",
-      "id": 5212108,
-      "comment_id": 3124534114,
-      "created_at": "2025-07-27T16:47:06Z",
-      "repoId": 765083837,
-      "pullRequestNo": 3190
-    },
-    {
-      "name": "SirlyDreamer",
-      "id": 45280500,
-      "comment_id": 3138729334,
-      "created_at": "2025-07-31T06:36:17Z",
-      "repoId": 765083837,
-      "pullRequestNo": 3222
-    },
-    {
-      "name": "yeahjack",
-      "id": 48432972,
-      "comment_id": 3155914249,
-      "created_at": "2025-08-05T16:59:05Z",
-      "repoId": 765083837,
-      "pullRequestNo": 3269
-    },
-    {
-      "name": "loveRhythm1990",
-      "id": 52204121,
-      "comment_id": 3166629630,
-      "created_at": "2025-08-08T05:32:33Z",
-      "repoId": 765083837,
-      "pullRequestNo": 3281
-    },
-    {
-      "name": "sleepyy-dog",
-      "id": 203856888,
-      "comment_id": 3209716785,
-      "created_at": "2025-08-21T09:23:23Z",
-      "repoId": 765083837,
-      "pullRequestNo": 3354
-    },
-    {
-      "name": "147phoenix",
-      "id": 136807090,
-      "comment_id": 3294702328,
-      "created_at": "2025-09-16T03:18:30Z",
-      "repoId": 765083837,
-      "pullRequestNo": 3477
-    },
-    {
-      "name": "e06084",
-      "id": 24450527,
-      "comment_id": 3302810047,
-      "created_at": "2025-09-17T12:42:23Z",
-      "repoId": 765083837,
-      "pullRequestNo": 3489
-    },
-    {
-      "name": "cjsdurj",
-      "id": 29703648,
-      "comment_id": 3385821350,
-      "created_at": "2025-10-09T13:13:49Z",
-      "repoId": 765083837,
-      "pullRequestNo": 3672
-    },
-    {
-      "name": "ye7love7",
-      "id": 72247118,
-      "comment_id": 3393172470,
-      "created_at": "2025-10-11T10:34:06Z",
-      "repoId": 765083837,
-      "pullRequestNo": 3699
-    },
-    {
-      "name": "magicyuan876",
-      "id": 72277650,
-      "comment_id": 3408782163,
-      "created_at": "2025-10-16T01:09:35Z",
-      "repoId": 765083837,
-      "pullRequestNo": 3739
-    },
-    {
-      "name": "yongtenglei",
-      "id": 65058560,
-      "comment_id": 3408954858,
-      "created_at": "2025-10-16T02:42:08Z",
-      "repoId": 765083837,
-      "pullRequestNo": 3740
-    },
-    {
-      "name": "xvlincaigou",
-      "id": 123342936,
-      "comment_id": 3446786700,
-      "created_at": "2025-10-25T14:38:54Z",
-      "repoId": 765083837,
-      "pullRequestNo": 3841
-    },
-    {
-      "name": "pzc163",
-      "id": 105222039,
-      "comment_id": 3449251055,
-      "created_at": "2025-10-27T02:17:34Z",
-      "repoId": 765083837,
-      "pullRequestNo": 3842
-    },
-    {
-      "name": "aopstudio",
-      "id": 32569353,
-      "comment_id": 3460341998,
-      "created_at": "2025-10-29T08:27:03Z",
-      "repoId": 765083837,
-      "pullRequestNo": 3870
-    },
-    {
-      "name": "Flynn-Zh",
-      "id": 23734538,
-      "comment_id": 3510150100,
-      "created_at": "2025-11-10T08:32:38Z",
-      "repoId": 765083837,
-      "pullRequestNo": 3966
-    },
-    {
-      "name": "eric-ozim",
-      "id": 233696103,
-      "comment_id": 3562698108,
-      "created_at": "2025-11-21T11:56:22Z",
-      "repoId": 765083837,
-      "pullRequestNo": 4040
-    },
-    {
-      "name": "zyileven",
-      "id": 40888939,
-      "comment_id": 3579403940,
-      "created_at": "2025-11-26T06:12:20Z",
-      "repoId": 765083837,
-      "pullRequestNo": 4066
-    },
-    {
-      "name": "borderlayout",
-      "id": 19464348,
-      "comment_id": 3626366462,
-      "created_at": "2025-12-08T11:07:40Z",
-      "repoId": 765083837,
-      "pullRequestNo": 4141
-    },
-    {
-      "name": "lc345",
-      "id": 67670904,
-      "comment_id": 3659976666,
-      "created_at": "2025-12-16T11:03:21Z",
-      "repoId": 765083837,
-      "pullRequestNo": 4196
-    },
-    {
-      "name": "RIORYO",
-      "id": 39397888,
-      "comment_id": 3701766690,
-      "created_at": "2025-12-31T08:54:17Z",
-      "repoId": 765083837,
-      "pullRequestNo": 4277
-    },
-    {
-      "name": "kingdomad",
-      "id": 34766852,
-      "comment_id": 3707777776,
-      "created_at": "2026-01-04T06:16:13Z",
-      "repoId": 765083837,
-      "pullRequestNo": 4283
-    },
-    {
-      "name": "tommygood",
-      "id": 96759292,
-      "comment_id": 3748312307,
-      "created_at": "2026-01-14T08:05:44Z",
-      "repoId": 765083837,
-      "pullRequestNo": 4365
-    },
-    {
-      "name": "pgoslatara",
-      "id": 14027534,
-      "comment_id": 3799210889,
-      "created_at": "2026-01-26T11:58:59Z",
-      "repoId": 765083837,
-      "pullRequestNo": 4421
-    },
-    {
-      "name": "guguducken",
-      "id": 22561920,
-      "comment_id": 3815970299,
-      "created_at": "2026-01-29T07:33:53Z",
-      "repoId": 765083837,
-      "pullRequestNo": 4435
-    },
-    {
-      "name": "Arrmsgt",
-      "id": 115080429,
-      "comment_id": 3851141607,
-      "created_at": "2026-02-05T05:17:32Z",
-      "repoId": 765083837,
-      "pullRequestNo": 4498
-    },
-    {
-      "name": "wzgrx",
-      "id": 39661556,
-      "comment_id": 3854380968,
-      "created_at": "2026-02-05T15:26:44Z",
-      "repoId": 765083837,
-      "pullRequestNo": 4504
-    },
-    {
-      "name": "marswen",
-      "id": 24496561,
-      "comment_id": 3971065560,
-      "created_at": "2026-02-27T06:26:29Z",
-      "repoId": 765083837,
-      "pullRequestNo": 4555
-    },
-    {
-      "name": "vanchy-z",
-      "id": 63965264,
-      "comment_id": 3976573836,
-      "created_at": "2026-02-28T07:14:07Z",
-      "repoId": 765083837,
-      "pullRequestNo": 4560
-    },
-    {
-      "name": "troyhantech",
-      "id": 92877246,
-      "comment_id": 4088446017,
-      "created_at": "2026-03-19T07:58:56Z",
-      "repoId": 765083837,
-      "pullRequestNo": 4631
-    },
-    {
-      "name": "vivekvar-dl",
-      "id": 69627205,
-      "comment_id": 4103182168,
-      "created_at": "2026-03-21T11:54:30Z",
-      "repoId": 765083837,
-      "pullRequestNo": 4636
-    },
-    {
-      "name": "vivekvar-dl",
-      "id": 69627205,
-      "comment_id": 4103186013,
-      "created_at": "2026-03-21T11:57:40Z",
-      "repoId": 765083837,
-      "pullRequestNo": 4636
-    },
-    {
-      "name": "UaRuairc",
-      "id": 77743840,
-      "comment_id": 4118931754,
-      "created_at": "2026-03-24T14:51:59Z",
-      "repoId": 765083837,
-      "pullRequestNo": 4654
    }
  ]
 }
Author	SHA1	Message	Date
Xiaomeng Zhao	fae9fb9820	Update mineru/model/mfr/unimernet/Unimernet.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>	2025-07-23 00:20:08 +08:00
Xiaomeng Zhao	58268df3ce	Merge pull request #3143 from myhloli/dev docs: update changelog for version 2.1.3 with bug fixes and improvements	2025-07-23 00:12:51 +08:00
myhloli	f1ae0afcd6	docs: update changelog for version 2.1.3 with bug fixes and improvements	2025-07-23 00:11:52 +08:00
Xiaomeng Zhao	4618ff6fea	Merge pull request #3140 from myhloli/dev docs: update changelog for version 2.1.2 with bug fixes	2025-07-22 22:23:24 +08:00
myhloli	e4639e1321	docs: update changelog for version 2.1.2 with bug fixes	2025-07-22 22:22:12 +08:00
Xiaomeng Zhao	1c2b233f5d	Merge pull request #3138 from myhloli/dev fix: update GITHUB_TOKEN usage in CLA action and change branch to 'cla'	2025-07-22 21:15:38 +08:00
Xiaomeng Zhao	38af25776f	Merge branch 'dev' into dev	2025-07-22 21:15:17 +08:00
myhloli	fffb68797c	fix: update GITHUB_TOKEN usage in CLA action and change branch to 'cla'	2025-07-22 21:12:42 +08:00
Xiaomeng Zhao	7e8cb108f9	Merge pull request #3129 from huazZeng/master improve the caption match algorithm	2025-07-22 21:08:06 +08:00
Xiaomeng Zhao	300b691713	Update mineru/backend/pipeline/pipeline_magic_model.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>	2025-07-22 21:06:21 +08:00
Xiaomeng Zhao	b25f42a2b3	Merge pull request #3137 from myhloli/dev fix: adjust batch sizes and improve performance settings in various modules	2025-07-22 21:04:41 +08:00
myhloli	2c0d971e24	fix: update CLA Assistant action version and adjust token usage	2025-07-22 21:03:22 +08:00
myhloli	a5e4179139	fix: adjust batch sizes and improve performance settings in various modules	2025-07-22 20:57:09 +08:00
huazZeng	725fdbff1c	improve the caption match algorithm	2025-07-22 10:58:10 +08:00