Merge pull request #2231 from myhloli/dev

build(docker): add torch and torchvision installation
2026-03-27 19:18:34 +07:00 · 2025-04-15 09:56:47 +08:00 · 2025-04-15 09:55:57 +08:00 · 2025-04-14 19:03:22 +08:00 · 2025-04-14 10:41:06 +00:00 · 2025-04-14 10:34:29 +00:00
6 changed files with 23 additions and 6 deletions
--- a/docker/ascend_npu/Dockerfile
+++ b/docker/ascend_npu/Dockerfile
@@ -35,6 +35,7 @@ RUN /bin/bash -c "wget https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/m
    cp magic-pdf.template.json /root/magic-pdf.json && \
    source /opt/mineru_venv/bin/activate && \
    pip3 install --upgrade pip -i https://mirrors.aliyun.com/pypi/simple && \
+    pip3 install torch==2.3.1 torchvision==0.18.1 -i https://mirrors.aliyun.com/pypi/simple && \
    pip3 install -U magic-pdf[full] -i https://mirrors.aliyun.com/pypi/simple && \
    wget https://gitee.com/ascend/pytorch/releases/download/v6.0.rc2-pytorch2.3.1/torch_npu-2.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl && \
    pip3 install torch_npu-2.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"
--- a/magic_pdf/libs/version.py
+++ b/magic_pdf/libs/version.py
@@ -1 +1 @@
-__version__ = "1.3.1"
+__version__ = "1.3.3"
--- a/magic_pdf/model/doc_analyze_by_custom_model.py
+++ b/magic_pdf/model/doc_analyze_by_custom_model.py
@@ -147,7 +147,7 @@ def doc_analyze(
            images.append(img_dict['img'])
            page_wh_list.append((img_dict['width'], img_dict['height']))

-    images_with_extra_info = [(images[index], ocr, dataset._lang) for index in range(len(dataset))]
+    images_with_extra_info = [(images[index], ocr, dataset._lang) for index in range(len(images))]

    if len(images) >= MIN_BATCH_INFERENCE_SIZE:
        batch_size = MIN_BATCH_INFERENCE_SIZE
--- a/magic_pdf/pre_proc/ocr_detect_all_bboxes.py
+++ b/magic_pdf/pre_proc/ocr_detect_all_bboxes.py
@@ -99,11 +99,11 @@ def ocr_prepare_bboxes_for_layout_split_v2(
    all_discarded_blocks = []
    add_bboxes(discarded_blocks, BlockType.Discarded, all_discarded_blocks)

-    """footnote识别：宽度超过1/3页面宽度的，高度超过10的，处于页面下半50%区域的"""
+    """footnote识别：宽度超过1/3页面宽度的，高度超过10的，处于页面下半30%区域的"""
    footnote_blocks = []
    for discarded in discarded_blocks:
        x0, y0, x1, y1 = discarded['bbox']
-        if (x1 - x0) > (page_w / 3) and (y1 - y0) > 10 and y0 > (page_h / 2):
+        if (x1 - x0) > (page_w / 3) and (y1 - y0) > 10 and y0 > (page_h * 0.7):
            footnote_blocks.append([x0, y0, x1, y1])

    """移除在footnote下面的任何框"""
--- a/setup.py
+++ b/setup.py
@@ -43,7 +43,7 @@ if __name__ == '__main__':
                     "matplotlib>=3.10,<4",
                     "ultralytics>=8.3.48,<9",  # yolov8,公式检测
                     "doclayout_yolo==0.0.2b1",  # doclayout_yolo
-                     "dill>=0.3.9,<1",  # doclayout_yolo
+                     "dill>=0.3.8,<1",  # doclayout_yolo
                     "rapid_table>=1.0.5,<2.0.0",  # rapid_table
                     "PyYAML>=6.0.2,<7",  # yaml
                     "ftfy>=6.3.1,<7",  # unimernet_hf
@@ -56,7 +56,7 @@ if __name__ == '__main__':
                    "matplotlib>=3.10,<=3.10.1",
                    "ultralytics>=8.3.48,<=8.3.104",  # yolov8,公式检测
                    "doclayout_yolo==0.0.2b1",  # doclayout_yolo
-                    "dill==0.3.9",  # doclayout_yolo
+                    "dill==0.3.8",  # doclayout_yolo
                    "PyYAML==6.0.2",  # yaml
                    "ftfy==6.3.1",  # unimernet_hf
                    "openai==1.71.0",  # openai SDK
--- a/signatures/version1/cla.json
+++ b/signatures/version1/cla.json
@@ -223,6 +223,22 @@
      "created_at": "2025-03-24T12:58:56Z",
      "repoId": 765083837,
      "pullRequestNo": 1982
+    },
+    {
+      "name": "zjx20",
+      "id": 2639200,
+      "comment_id": 2800714918,
+      "created_at": "2025-04-14T07:25:26Z",
+      "repoId": 765083837,
+      "pullRequestNo": 2215
+    },
+    {
+      "name": "Doge2077",
+      "id": 91442300,
+      "comment_id": 2801283257,
+      "created_at": "2025-04-14T10:40:54Z",
+      "repoId": 765083837,
+      "pullRequestNo": 2226
    }
  ]
 }
Author	SHA1	Message	Date
Xiaomeng Zhao	27883619f5	Merge pull request #2231 from myhloli/dev build(docker): add torch and torchvision installation	2025-04-15 09:56:47 +08:00
myhloli	5ddd6799aa	build(docker): add torch and torchvision installation - Add pip install command for torch and torchvision - Specify version2.3.1 for both packages - Use Aliyun mirror for faster download	2025-04-15 09:55:57 +08:00
Xiaomeng Zhao	410d0afc81	Merge pull request #2227 from opendatalab/master master->dev	2025-04-14 19:03:22 +08:00
github-actions[bot]	c774a4dde1	@Doge2077 has signed the CLA in opendatalab/MinerU#2226	2025-04-14 10:41:06 +00:00
myhloli	29b47466ff	Update version.py with new version	2025-04-14 10:34:29 +00:00
Xiaomeng Zhao	a1df670e34	Merge pull request #2225 from opendatalab/release-1.3.3 Release 1.3.3	2025-04-14 18:33:07 +08:00
Xiaomeng Zhao	a67de492b1	Merge pull request #2224 from opendatalab/dev build(deps): downgrade dill to 0.3.8 for doclayout_yolo compatibility	2025-04-14 18:31:49 +08:00
Xiaomeng Zhao	222af4f2f5	Merge pull request #2223 from myhloli/dev build(deps): downgrade dill to 0.3.8 for doclayout_yolo compatibility	2025-04-14 18:31:04 +08:00
myhloli	b9eed5d865	build(deps): downgrade dill to 0.3.8 for doclayout_yolo compatibility - Change dill dependency from >=0.3.9,<1 to >=0.3.8,<1 - Update dill version in both general and specific requirements	2025-04-14 18:29:47 +08:00
Xiaomeng Zhao	99ab04f588	Merge pull request #2220 from myhloli/refactor-pipeline fix(magic_pdf): correct range for images in document analysis	2025-04-14 17:30:45 +08:00
myhloli	67b31a78d0	fix(magic_pdf): correct range for images in document analysis - Update the range used to generate images_with_extra_info to match the number of images - This fixes a potential IndexError when the number of images differs from the dataset length	2025-04-14 17:24:58 +08:00
Xiaomeng Zhao	4f129a64aa	Merge pull request #7 from myhloli/dev refactor(footnote_detection): adjust footnote detection threshold	2025-04-14 16:30:32 +08:00
github-actions[bot]	47d287a2a0	@zjx20 has signed the CLA in opendatalab/MinerU#2215	2025-04-14 07:25:39 +00:00
Xiaomeng Zhao	bc51f9f75e	Merge pull request #2214 from myhloli/dev refactor(footnote_detection): adjust footnote detection threshold	2025-04-14 15:23:31 +08:00
myhloli	8caf59f7cb	refactor(footnote_detection): adjust footnote detection threshold - Change footnote detection threshold from 50% of page height to 30% - Improve accuracy of footnote identification in PDF processing	2025-04-14 15:16:33 +08:00
Xiaomeng Zhao	4df8523a31	Merge pull request #2208 from opendatalab/master master->dev	2025-04-13 21:53:37 +08:00
Xiaomeng Zhao	c7a609fa7a	Merge pull request #2207 from opendatalab/release-1.3.2 build(docker): remove requirements.txt and update package installation	2025-04-13 21:52:44 +08:00
myhloli	5957cb65f9	Update version.py with new version	2025-04-12 11:04:26 +00:00
Xiaomeng Zhao	d0ed731b9e	Merge pull request #2199 from opendatalab/release-1.3.2 Release 1.3.2	2025-04-12 18:58:15 +08:00
Xiaomeng Zhao	b60166a541	Merge pull request #2157 from opendatalab/release-1.3.1 Release 1.3.1	2025-04-08 18:16:33 +08:00
Xiaomeng Zhao	ccf2ea04cb	Merge pull request #2156 from opendatalab/dev Dev	2025-04-08 18:16:07 +08:00
Xiaomeng Zhao	cb9c2e7616	Merge pull request #2154 from opendatalab/release-1.3.2 Release 1.3.2	2025-04-08 18:11:26 +08:00