fix: 并发上传5文件刷新后丢失 + 分片写入阻塞事件循环
根因: 1. pump() 仅在 addFiles() 调用一次,上传完成/失败后不重新触发, 导致 CONCURRENCY=3 之后的文件(4、5)永远不启动 → 无 DB 会话 → 刷新后消失 2. write_chunk 的 uploaded_chunks 是 read-modify-write,并发分片写入 后者覆盖前者 → 分片记录丢失 → complete 报 409 3. upload_chunk 是 async 但同步调 write_chunk(fsync+DB commit), 阻塞 uvicorn 事件循环 → 所有 web 请求被串行化 修复: - _shared.py: pump() 加 finally 块,上传完成/失败后都触发下一文件; 文件并发(FILE_CONCURRENCY=5)与分片并发(CHUNK_CONCURRENCY=3)分离 - upload_service.py: 按 upload_id 的进程级锁串行化 uploaded_chunks 更新, 持锁后 db.refresh 重读最新值再 append,杜绝丢失更新;complete 后清理锁 - upload_router.py: upload_chunk 的 write_chunk 调用改用 run_in_threadpool, 阻塞 I/O 移出事件循环,web 请求不再被分片写入阻塞 验证:5 文件并发上传后刷新全部可见;4 任务并发处理 4/4 成功(19.2s)
This commit is contained in:
@@ -16,6 +16,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
@@ -36,6 +37,20 @@ from ..schemas.task import (
|
||||
|
||||
logger = logging.getLogger("audio2text.upload")
|
||||
|
||||
# 按 upload_id 串行化 uploaded_chunks 的读-改-写,避免并发分片写入丢失更新。
|
||||
# SQLite 无行锁,JSON 列的 append 操作不是原子的,必须进程内加锁。
|
||||
_chunk_locks: dict[str, threading.Lock] = {}
|
||||
_chunk_locks_guard = threading.Lock()
|
||||
|
||||
|
||||
def _get_chunk_lock(upload_id: str) -> threading.Lock:
|
||||
with _chunk_locks_guard:
|
||||
lock = _chunk_locks.get(upload_id)
|
||||
if lock is None:
|
||||
lock = threading.Lock()
|
||||
_chunk_locks[upload_id] = lock
|
||||
return lock
|
||||
|
||||
|
||||
class UploadService:
|
||||
def __init__(self, db: Session) -> None:
|
||||
@@ -93,6 +108,7 @@ class UploadService:
|
||||
session_dir.mkdir(parents=True, exist_ok=True)
|
||||
chunk_path = session_dir / f"{index}.part"
|
||||
|
||||
# 先落盘(无锁,IO 可并行),再持锁更新 DB 计数
|
||||
try:
|
||||
with chunk_path.open("wb") as out:
|
||||
out.write(data)
|
||||
@@ -102,12 +118,16 @@ class UploadService:
|
||||
chunk_path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
uploaded = list(session.uploaded_chunks or [])
|
||||
if index not in uploaded:
|
||||
uploaded.append(index)
|
||||
session.uploaded_chunks = uploaded
|
||||
session.updated_at = datetime.now(timezone.utc)
|
||||
self.db.commit()
|
||||
# 持锁重读+更新:避免并发分片各自 read old list → append → write,后者覆盖前者
|
||||
lock = _get_chunk_lock(upload_id)
|
||||
with lock:
|
||||
self.db.refresh(session) # 拿最新 uploaded_chunks,不被其他并发请求的旧值覆盖
|
||||
uploaded = list(session.uploaded_chunks or [])
|
||||
if index not in uploaded:
|
||||
uploaded.append(index)
|
||||
session.uploaded_chunks = uploaded
|
||||
session.updated_at = datetime.now(timezone.utc)
|
||||
self.db.commit()
|
||||
return sorted(uploaded)
|
||||
|
||||
# ---------------- 拼接 + 创建任务 ----------------
|
||||
@@ -163,6 +183,9 @@ class UploadService:
|
||||
|
||||
# 清理分片暂存(commit 后,即使清理失败也不影响已建任务)
|
||||
self._cleanup_session_dir(upload_id)
|
||||
# 清理进程内锁,避免长期运行后 _chunk_locks 无限增长
|
||||
with _chunk_locks_guard:
|
||||
_chunk_locks.pop(upload_id, None)
|
||||
|
||||
logger.info("上传完成 task_id=%s file=%s size=%d", task.id, session.filename, session.size_bytes)
|
||||
return CompleteResponse(
|
||||
|
||||
Reference in New Issue
Block a user