From 56f25a4e90f3c743dbae549097328036c119f292 Mon Sep 17 00:00:00 2001 From: myhloli Date: Thu, 24 Jul 2025 20:59:25 +0800 Subject: [PATCH 1/4] refactor: update imports and adapt to sglang version changes in processing logic --- mineru/model/vlm_sglang_model/__init__.py | 13 +- .../model/vlm_sglang_model/image_processor.py | 159 +++++++++--------- mineru/model/vlm_sglang_model/model.py | 62 +++---- pyproject.toml | 2 +- 4 files changed, 112 insertions(+), 124 deletions(-) diff --git a/mineru/model/vlm_sglang_model/__init__.py b/mineru/model/vlm_sglang_model/__init__.py index 914822cb..8b193cc9 100644 --- a/mineru/model/vlm_sglang_model/__init__.py +++ b/mineru/model/vlm_sglang_model/__init__.py @@ -1,16 +1,9 @@ from sglang.srt.configs.model_config import multimodal_model_archs from sglang.srt.models.registry import ModelRegistry -try: - # sglang==0.4.5.post3 - from sglang.srt.managers.multimodal_processor import ( - PROCESSOR_MAPPING as PROCESSOR_MAPPING, - ) -except ImportError: - # sglang==0.4.4.post1 - from sglang.srt.managers.image_processor import ( - IMAGE_PROCESSOR_MAPPING as PROCESSOR_MAPPING, - ) +from sglang.srt.managers.multimodal_processor import ( + PROCESSOR_MAPPING as PROCESSOR_MAPPING, +) from .. import vlm_hf_model as _ from .image_processor import Mineru2ImageProcessor diff --git a/mineru/model/vlm_sglang_model/image_processor.py b/mineru/model/vlm_sglang_model/image_processor.py index 400154f0..7fe7e99d 100644 --- a/mineru/model/vlm_sglang_model/image_processor.py +++ b/mineru/model/vlm_sglang_model/image_processor.py @@ -5,21 +5,21 @@ from typing import List, Optional, Union import numpy as np -try: - # sglang==0.4.5.post3 +from sglang.version import __version__ as sglang_version +if sglang_version >= "0.4.9": + # sglang >= 0.4.9 + from sglang.srt.multimodal.processors.base_processor import ( + BaseMultimodalProcessor as BaseProcessor, + ) + from sglang.srt.multimodal.mm_utils import divide_to_patches, expand2square, select_best_resolution +else: + # 0.4.7 <= sglang < 0.4.9 from sglang.srt.managers.multimodal_processors.base_processor import ( BaseMultimodalProcessor as BaseProcessor, ) + from sglang.srt.mm_utils import divide_to_patches, expand2square, select_best_resolution - get_global_processor = None -except ImportError: - # sglang==0.4.4.post1 - from sglang.srt.managers.image_processors.base_image_processor import ( - BaseImageProcessor as BaseProcessor, - get_global_processor, - ) - -from sglang.srt.mm_utils import divide_to_patches, expand2square, select_best_resolution +get_global_processor = None from sglang.srt.utils import load_image, logger from sglang.utils import get_exception_traceback @@ -123,64 +123,6 @@ class Mineru2ImageProcessor(BaseProcessor): image_processor, ) - # sglang==0.4.4.post1 - async def process_images_async( - self, - image_data: List[Union[str, bytes]], - input_text, - request_obj, - *args, - **kwargs, - ): - if not image_data: - return None - - modalities = request_obj.modalities or ["image"] - aspect_ratio = getattr(self.hf_config, "image_aspect_ratio", "") - - grid_pinpoints = ( - self.hf_config.image_grid_pinpoints - if hasattr(self.hf_config, "image_grid_pinpoints") and "anyres" in aspect_ratio - else None - ) - - if isinstance(image_data, str): - image_data = [image_data] - - if isinstance(image_data, list) and len(image_data) > 0: - if "multi-images" in modalities or "video" in modalities: - # Multiple images - aspect_ratio = "pad" # LLaVA OneVision Handling: more than one image --> interleaved image mode or video mode. We do not use anyres - pixel_values, image_hashes, image_sizes = [], [], [] - res = [] - for img_data in image_data: - res.append(self._process_single_image(img_data, aspect_ratio, grid_pinpoints)) - res = await asyncio.gather(*res) - for pixel_v, image_h, image_s in res: - pixel_values.append(pixel_v) - image_hashes.append(image_h) - image_sizes.append(image_s) - - if isinstance(pixel_values[0], np.ndarray): - pixel_values = np.stack(pixel_values, axis=0) - else: - # A single image - pixel_values, image_hash, image_size = await self._process_single_image( - image_data[0], aspect_ratio, grid_pinpoints - ) - image_hashes = [image_hash] - image_sizes = [image_size] - else: - raise ValueError(f"Invalid image data: {image_data}") - - return { - "pixel_values": pixel_values, - "image_hashes": image_hashes, - "image_sizes": image_sizes, - "modalities": request_obj.modalities or ["image"], - } - - # sglang==0.4.5.post3 async def process_mm_data_async( self, image_data: List[Union[str, bytes]], @@ -191,11 +133,50 @@ class Mineru2ImageProcessor(BaseProcessor): ): from sglang.srt.managers.schedule_batch import Modality, MultimodalDataItem - result = await self.process_images_async(image_data, input_text, request_obj, *args, **kwargs) - - if result is None: + if not image_data: return None + modalities = request_obj.modalities or ["image"] + aspect_ratio = getattr(self.hf_config, "image_aspect_ratio", None) + grid_pinpoints = ( + self.hf_config.image_grid_pinpoints + if hasattr(self.hf_config, "image_grid_pinpoints") + and "anyres" in aspect_ratio + else None + ) + + if isinstance(image_data, str): + image_data = [image_data] + + if isinstance(image_data, list) and len(image_data) > 0: + if "multi-images" in modalities or "video" in modalities: + # Multiple images + aspect_ratio = "pad" # LLaVA OneVision Handling: more than one image --> interleaved image mode or video mode. We do not use anyres + pixel_values, data_hashes, image_sizes = [], [], [] + res = [] + for img_data in image_data: + res.append( + self._process_single_image( + img_data, aspect_ratio, grid_pinpoints + ) + ) + + res = await asyncio.gather(*res) + for pixel_v, image_h, image_s in res: + pixel_values.append(pixel_v) + data_hashes.append(image_h) + image_sizes.append(image_s) + + if isinstance(pixel_values[0], np.ndarray): + pixel_values = np.stack(pixel_values, axis=0) + else: + # A single image + pixel_values, image_hash, image_size = await self._process_single_image( + image_data[0], aspect_ratio, grid_pinpoints + ) + image_sizes = [image_size] + else: + raise ValueError(f"Invalid image data: {image_data}") modality = Modality.IMAGE if isinstance(request_obj.modalities, list): if request_obj.modalities[0] == "multi-images": @@ -203,15 +184,29 @@ class Mineru2ImageProcessor(BaseProcessor): elif request_obj.modalities[0] == "video": modality = Modality.VIDEO - return { - "mm_items": [ - MultimodalDataItem( - pixel_values=result["pixel_values"], - image_sizes=result["image_sizes"], - modality=modality, - ) - ], - } - + if sglang_version >= "0.4.9.post3": + # sglang >= 0.4.9.post3 + return { + "mm_items": [ + MultimodalDataItem( + feature=pixel_values, + model_specific_data={ + "image_sizes": image_sizes, + }, + modality=modality, + ) + ], + } + else: + # 0.4.7 <= sglang <= 0.4.9.post2 + return { + "mm_items": [ + MultimodalDataItem( + pixel_values=pixel_values, + image_sizes=image_sizes, + modality=modality, + ) + ], + } ImageProcessorMapping = {Mineru2QwenForCausalLM: Mineru2ImageProcessor} diff --git a/mineru/model/vlm_sglang_model/model.py b/mineru/model/vlm_sglang_model/model.py index e06fec27..1220f5c9 100644 --- a/mineru/model/vlm_sglang_model/model.py +++ b/mineru/model/vlm_sglang_model/model.py @@ -5,9 +5,19 @@ from typing import Iterable, List, Optional, Tuple import numpy as np import torch from sglang.srt.layers.quantization.base_config import QuantizationConfig -from sglang.srt.mm_utils import ( - get_anyres_image_grid_shape, # unpad_image, unpad_image_shape -) + +from sglang.version import __version__ as sglang_version +if sglang_version >= "0.4.9": + # sglang >= 0.4.9 + from sglang.srt.multimodal.mm_utils import ( + get_anyres_image_grid_shape, + ) +else: + # 0.4.7 <= sglang < 0.4.9 + from sglang.srt.mm_utils import ( + get_anyres_image_grid_shape, + ) + from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.qwen2 import Qwen2ForCausalLM @@ -111,14 +121,9 @@ class Mineru2QwenForCausalLM(nn.Module): raise ValueError(f"Unexpected select feature: {self.select_feature}") def pad_input_ids(self, input_ids: List[int], image_inputs): - if hasattr(image_inputs, "mm_items"): # MultimodalInputs - # sglang==0.4.5.post3 - image_sizes = flatten_nested_list([item.image_sizes for item in image_inputs.mm_items]) - pad_values = [item.pad_value for item in image_inputs.mm_items] - else: # ImageInputs - # sglang==0.4.4.post1 - image_sizes = image_inputs.image_sizes - pad_values = image_inputs.pad_values + + image_sizes = flatten_nested_list([item.image_sizes for item in image_inputs.mm_items]) + pad_values = [item.pad_value for item in image_inputs.mm_items] # hardcode for spatial_unpad + anyres # if image_inputs.modalities is not None and ( @@ -196,14 +201,8 @@ class Mineru2QwenForCausalLM(nn.Module): positions: torch.Tensor, forward_batch: ForwardBatch, ) -> torch.Tensor: - if hasattr(forward_batch, "mm_inputs"): - # sglang==0.4.5.post3 - image_inputs = forward_batch.mm_inputs - is_sglang_mm_inputs = True - else: - # sglang==0.4.4.post1 - image_inputs = forward_batch.image_inputs - is_sglang_mm_inputs = False + + image_inputs = forward_batch.mm_inputs if image_inputs is None: image_inputs = [] @@ -223,12 +222,7 @@ class Mineru2QwenForCausalLM(nn.Module): max_image_offset = [] for im in image_inputs: if im: - if hasattr(im, "mm_items"): - # sglang==0.4.5.post3 - modalities_list.extend([downgrade_modality(item.modality) for item in im.mm_items]) - elif im.modalities is not None: - # sglang==0.4.4.post1 - modalities_list.extend(im.modalities) + modalities_list.extend([downgrade_modality(item.modality) for item in im.mm_items]) if im and im.image_offsets: max_image_offset.append(np.max(np.array(im.image_offsets) + np.array(im.image_pad_len))) else: @@ -240,8 +234,18 @@ class Mineru2QwenForCausalLM(nn.Module): if need_vision.any(): bs = forward_batch.batch_size - if is_sglang_mm_inputs: - # sglang==0.4.5.post3 + if sglang_version >= "0.4.9.post3": + # sglang >= 0.4.9.post3 + pixel_values = flatten_nested_list( + [[item.feature for item in image_inputs[i].mm_items] for i in range(bs) if need_vision[i]] + ) # image_inputs[batch_idx].mm_items[item_idx].pixel_values is Tensor + image_sizes = [ + flatten_nested_list([item.model_specific_data["image_sizes"] for item in image_inputs[i].mm_items]) + for i in range(bs) + if need_vision[i] + ] # image_inputs[batch_idx].mm_items[item_idx].image_sizes should be tuple, but is list of tuple for now. + else: + # 0.4.7 <= sglang <= 0.4.9.post2 pixel_values = flatten_nested_list( [[item.pixel_values for item in image_inputs[i].mm_items] for i in range(bs) if need_vision[i]] ) # image_inputs[batch_idx].mm_items[item_idx].pixel_values is Tensor @@ -250,10 +254,6 @@ class Mineru2QwenForCausalLM(nn.Module): for i in range(bs) if need_vision[i] ] # image_inputs[batch_idx].mm_items[item_idx].image_sizes should be tuple, but is list of tuple for now. - else: - # sglang==0.4.4.post1 - pixel_values = [image_inputs[i].pixel_values for i in range(bs) if need_vision[i]] - image_sizes = [image_inputs[i].image_sizes for i in range(bs) if need_vision[i]] ########## Encode Image ######## diff --git a/pyproject.toml b/pyproject.toml index 3742759a..94942a72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ vlm = [ "pydantic", ] sglang = [ - "sglang[all]>=0.4.8,<0.4.9", + "sglang[all]>=0.4.7,<0.4.10", ] pipeline = [ "matplotlib>=3.10,<4", From 6d9380323b6ef0f5d1da3839dfc223ac138ea41a Mon Sep 17 00:00:00 2001 From: myhloli Date: Thu, 24 Jul 2025 21:14:25 +0800 Subject: [PATCH 2/4] chore: update Dockerfile and documentation to use sglang v0.4.9 --- README.md | 2 ++ README_zh-CN.md | 2 ++ docker/china/Dockerfile | 4 +++- docker/global/Dockerfile | 4 +++- docs/en/quick_start/docker_deployment.md | 4 ++-- docs/zh/quick_start/docker_deployment.md | 4 ++-- 6 files changed, 14 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7f83c7e1..decb7867 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,8 @@ # Changelog +- 2025/07/24 2.1.5 Released + - `sglang` 0.4.9 version adaptation, synchronously upgrading the dockerfile base image to sglang 0.4.9.post3 - 2025/07/23 2.1.4 Released - Bug Fixes - Fixed the issue of excessive memory consumption during the `MFR` step in the `pipeline` backend under certain scenarios #2771 diff --git a/README_zh-CN.md b/README_zh-CN.md index 5986b354..f73560e8 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -43,6 +43,8 @@ # 更新记录 +- 2025/07/24 2.1.5发布 + - `sglang` 0.4.9 版本适配，同步升级dockerfile基础镜像为sglang 0.4.9.post3 - 2025/07/23 2.1.4发布 - bug修复 - 修复`pipeline`后端中`MFR`步骤在某些情况下显存消耗过大的问题 #2771 diff --git a/docker/china/Dockerfile b/docker/china/Dockerfile index 8aac3606..e192d450 100644 --- a/docker/china/Dockerfile +++ b/docker/china/Dockerfile @@ -1,5 +1,7 @@ # Use the official sglang image -FROM lmsysorg/sglang:v0.4.8.post1-cu126 +FROM lmsysorg/sglang:v0.4.9.post3-cu126 +# For blackwell GPU, use the following line instead: +# FROM lmsysorg/sglang:v0.4.9.post3-cu128-b200 # Install libgl for opencv support & Noto fonts for Chinese characters RUN apt-get update && \ diff --git a/docker/global/Dockerfile b/docker/global/Dockerfile index 6a4f72df..4986130a 100644 --- a/docker/global/Dockerfile +++ b/docker/global/Dockerfile @@ -1,5 +1,7 @@ # Use the official sglang image -FROM lmsysorg/sglang:v0.4.8.post1-cu126 +FROM lmsysorg/sglang:v0.4.9.post3-cu126 +# For blackwell GPU, use the following line instead: +# FROM lmsysorg/sglang:v0.4.9.post3-cu128-b200 # Install libgl for opencv support & Noto fonts for Chinese characters RUN apt-get update && \ diff --git a/docs/en/quick_start/docker_deployment.md b/docs/en/quick_start/docker_deployment.md index bb0a2cd2..fdea014b 100644 --- a/docs/en/quick_start/docker_deployment.md +++ b/docs/en/quick_start/docker_deployment.md @@ -10,8 +10,8 @@ docker build -t mineru-sglang:latest -f Dockerfile . ``` > [!TIP] -> The [Dockerfile](https://github.com/opendatalab/MinerU/blob/master/docker/global/Dockerfile) uses `lmsysorg/sglang:v0.4.8.post1-cu126` as the base image by default, supporting Turing/Ampere/Ada Lovelace/Hopper platforms. -> If you are using the newer `Blackwell` platform, please modify the base image to `lmsysorg/sglang:v0.4.8.post1-cu128-b200` before executing the build operation. +> The [Dockerfile](https://github.com/opendatalab/MinerU/blob/master/docker/global/Dockerfile) uses `lmsysorg/sglang:v0.4.9.post3-cu126` as the base image by default, supporting Turing/Ampere/Ada Lovelace/Hopper platforms. +> If you are using the newer `Blackwell` platform, please modify the base image to `lmsysorg/sglang:v0.4.9.post3-cu128-b200` before executing the build operation. ## Docker Description diff --git a/docs/zh/quick_start/docker_deployment.md b/docs/zh/quick_start/docker_deployment.md index 8cf088e6..1f110ab6 100644 --- a/docs/zh/quick_start/docker_deployment.md +++ b/docs/zh/quick_start/docker_deployment.md @@ -10,8 +10,8 @@ docker build -t mineru-sglang:latest -f Dockerfile . ``` > [!TIP] -> [Dockerfile](https://github.com/opendatalab/MinerU/blob/master/docker/china/Dockerfile)默认使用`lmsysorg/sglang:v0.4.8.post1-cu126`作为基础镜像，支持Turing/Ampere/Ada Lovelace/Hopper平台， -> 如您使用较新的`Blackwell`平台，请将基础镜像修改为`lmsysorg/sglang:v0.4.8.post1-cu128-b200` 再执行build操作。 +> [Dockerfile](https://github.com/opendatalab/MinerU/blob/master/docker/china/Dockerfile)默认使用`lmsysorg/sglang:v0.4.9.post3-cu126`作为基础镜像，支持Turing/Ampere/Ada Lovelace/Hopper平台， +> 如您使用较新的`Blackwell`平台，请将基础镜像修改为`lmsysorg/sglang:v0.4.9.post3-cu128-b200` 再执行build操作。 ## Docker说明 From 62d1ef184ed98e7be4b2c31394413c922440cdfa Mon Sep 17 00:00:00 2001 From: myhloli Date: Thu, 24 Jul 2025 21:29:54 +0800 Subject: [PATCH 3/4] refactor: update sglang version checks to use packaging.version for comparison --- mineru/model/vlm_sglang_model/image_processor.py | 5 +++-- mineru/model/vlm_sglang_model/model.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mineru/model/vlm_sglang_model/image_processor.py b/mineru/model/vlm_sglang_model/image_processor.py index 7fe7e99d..70297d2b 100644 --- a/mineru/model/vlm_sglang_model/image_processor.py +++ b/mineru/model/vlm_sglang_model/image_processor.py @@ -6,7 +6,8 @@ from typing import List, Optional, Union import numpy as np from sglang.version import __version__ as sglang_version -if sglang_version >= "0.4.9": +from packaging import version +if version.parse(sglang_version) >= version.parse("0.4.9"): # sglang >= 0.4.9 from sglang.srt.multimodal.processors.base_processor import ( BaseMultimodalProcessor as BaseProcessor, @@ -184,7 +185,7 @@ class Mineru2ImageProcessor(BaseProcessor): elif request_obj.modalities[0] == "video": modality = Modality.VIDEO - if sglang_version >= "0.4.9.post3": + if version.parse(sglang_version) >= version.parse("0.4.9.post3"): # sglang >= 0.4.9.post3 return { "mm_items": [ diff --git a/mineru/model/vlm_sglang_model/model.py b/mineru/model/vlm_sglang_model/model.py index 1220f5c9..37a69364 100644 --- a/mineru/model/vlm_sglang_model/model.py +++ b/mineru/model/vlm_sglang_model/model.py @@ -7,7 +7,8 @@ import torch from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.version import __version__ as sglang_version -if sglang_version >= "0.4.9": +from packaging import version +if version.parse(sglang_version) >= version.parse("0.4.9"): # sglang >= 0.4.9 from sglang.srt.multimodal.mm_utils import ( get_anyres_image_grid_shape, @@ -234,7 +235,7 @@ class Mineru2QwenForCausalLM(nn.Module): if need_vision.any(): bs = forward_batch.batch_size - if sglang_version >= "0.4.9.post3": + if version.parse(sglang_version) >= version.parse("0.4.9.post3"): # sglang >= 0.4.9.post3 pixel_values = flatten_nested_list( [[item.feature for item in image_inputs[i].mm_items] for i in range(bs) if need_vision[i]] From 86690409ab404d9508c6a2f9e00f3bacbacff474 Mon Sep 17 00:00:00 2001 From: Xiaomeng Zhao Date: Thu, 24 Jul 2025 21:33:16 +0800 Subject: [PATCH 4/4] Update mineru/model/vlm_sglang_model/model.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- mineru/model/vlm_sglang_model/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mineru/model/vlm_sglang_model/model.py b/mineru/model/vlm_sglang_model/model.py index 37a69364..2019980a 100644 --- a/mineru/model/vlm_sglang_model/model.py +++ b/mineru/model/vlm_sglang_model/model.py @@ -14,7 +14,7 @@ if version.parse(sglang_version) >= version.parse("0.4.9"): get_anyres_image_grid_shape, ) else: - # 0.4.7 <= sglang < 0.4.9 + # 0.4.7 <= sglang < 0.4.9 from sglang.srt.mm_utils import ( get_anyres_image_grid_shape, )