mirror of
https://github.com/opendatalab/MinerU.git
synced 2026-03-27 11:08:32 +07:00
feat: remove unused functions and clean up client and fast_api modules
This commit is contained in:
@@ -1,6 +1,5 @@
|
|||||||
# Copyright (c) Opendatalab. All rights reserved.
|
# Copyright (c) Opendatalab. All rights reserved.
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import threading
|
import threading
|
||||||
@@ -186,10 +185,6 @@ class LiveTaskStatusRenderer:
|
|||||||
self._task_states.clear()
|
self._task_states.clear()
|
||||||
self.clear_locked()
|
self.clear_locked()
|
||||||
|
|
||||||
def snapshot_lines(self) -> list[str]:
|
|
||||||
with self.sink.lock:
|
|
||||||
return self._build_render_lines_locked()
|
|
||||||
|
|
||||||
def clear_locked(self) -> None:
|
def clear_locked(self) -> None:
|
||||||
if self._rendered_line_count <= 0:
|
if self._rendered_line_count <= 0:
|
||||||
return
|
return
|
||||||
@@ -664,29 +659,6 @@ async def submit_task(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def submit_task_sync(
|
|
||||||
base_url: str,
|
|
||||||
planned_task: PlannedTask,
|
|
||||||
form_data: dict[str, str | list[str]],
|
|
||||||
) -> SubmitResponse:
|
|
||||||
try:
|
|
||||||
return _api_client.submit_parse_task_sync(
|
|
||||||
base_url=base_url,
|
|
||||||
upload_assets=[
|
|
||||||
_api_client.UploadAsset(
|
|
||||||
path=document.path,
|
|
||||||
upload_name=f"{document.stem}{document.path.suffix}",
|
|
||||||
)
|
|
||||||
for document in planned_task.documents
|
|
||||||
],
|
|
||||||
form_data=form_data,
|
|
||||||
)
|
|
||||||
except click.ClickException as exc:
|
|
||||||
raise click.ClickException(
|
|
||||||
f"Failed to submit {format_task_label(planned_task)}: {exc}"
|
|
||||||
) from exc
|
|
||||||
|
|
||||||
|
|
||||||
async def wait_for_task_result(
|
async def wait_for_task_result(
|
||||||
client: httpx.AsyncClient,
|
client: httpx.AsyncClient,
|
||||||
submit_response: SubmitResponse,
|
submit_response: SubmitResponse,
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import mimetypes
|
import mimetypes
|
||||||
import os
|
import os
|
||||||
import re
|
|
||||||
import shutil
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
@@ -269,19 +268,6 @@ def validate_parse_method(parse_method: str) -> str:
|
|||||||
return parse_method
|
return parse_method
|
||||||
|
|
||||||
|
|
||||||
def sanitize_filename(filename: str) -> str:
|
|
||||||
"""
|
|
||||||
格式化压缩文件的文件名
|
|
||||||
移除路径遍历字符, 保留 Unicode 字母、数字、._-
|
|
||||||
禁止隐藏文件
|
|
||||||
"""
|
|
||||||
sanitized = re.sub(r"[/\\.]{2,}|[/\\]", "", filename)
|
|
||||||
sanitized = re.sub(r"[^\w.-]", "_", sanitized, flags=re.UNICODE)
|
|
||||||
if sanitized.startswith("."):
|
|
||||||
sanitized = "_" + sanitized[1:]
|
|
||||||
return sanitized or "unnamed"
|
|
||||||
|
|
||||||
|
|
||||||
def cleanup_file(file_path: str) -> None:
|
def cleanup_file(file_path: str) -> None:
|
||||||
"""清理临时文件或目录"""
|
"""清理临时文件或目录"""
|
||||||
try:
|
try:
|
||||||
|
|||||||
Reference in New Issue
Block a user