fix: 并发上传5文件刷新后丢失 + 分片写入阻塞事件循环
根因: 1. pump() 仅在 addFiles() 调用一次,上传完成/失败后不重新触发, 导致 CONCURRENCY=3 之后的文件(4、5)永远不启动 → 无 DB 会话 → 刷新后消失 2. write_chunk 的 uploaded_chunks 是 read-modify-write,并发分片写入 后者覆盖前者 → 分片记录丢失 → complete 报 409 3. upload_chunk 是 async 但同步调 write_chunk(fsync+DB commit), 阻塞 uvicorn 事件循环 → 所有 web 请求被串行化 修复: - _shared.py: pump() 加 finally 块,上传完成/失败后都触发下一文件; 文件并发(FILE_CONCURRENCY=5)与分片并发(CHUNK_CONCURRENCY=3)分离 - upload_service.py: 按 upload_id 的进程级锁串行化 uploaded_chunks 更新, 持锁后 db.refresh 重读最新值再 append,杜绝丢失更新;complete 后清理锁 - upload_router.py: upload_chunk 的 write_chunk 调用改用 run_in_threadpool, 阻塞 I/O 移出事件循环,web 请求不再被分片写入阻塞 验证:5 文件并发上传后刷新全部可见;4 任务并发处理 4/4 成功(19.2s)
This commit is contained in:
@@ -10,7 +10,8 @@ from __future__ import annotations
|
||||
|
||||
# 分片上传参数(home / upload 共用)
|
||||
DEFAULT_CHUNK_SIZE = 4 * 1024 * 1024
|
||||
DEFAULT_CONCURRENCY = 3
|
||||
DEFAULT_CONCURRENCY = 3 # 单文件内分片并发数
|
||||
DEFAULT_FILE_CONCURRENCY = 5 # 同时上传的文件数(不阻塞 web 请求/ffmpeg/gpu)
|
||||
MAX_RETRY = 2
|
||||
POLL_INTERVAL = 2000
|
||||
|
||||
@@ -259,7 +260,8 @@ def render_upload_js(on_complete: str) -> str:
|
||||
"""
|
||||
return f"""
|
||||
const CHUNK_SIZE = {DEFAULT_CHUNK_SIZE};
|
||||
const CONCURRENCY = {DEFAULT_CONCURRENCY};
|
||||
const CHUNK_CONCURRENCY = {DEFAULT_CONCURRENCY}; // 单文件内分片并发
|
||||
const FILE_CONCURRENCY = {DEFAULT_FILE_CONCURRENCY}; // 同时上传的文件数
|
||||
const MAX_RETRY = {MAX_RETRY};
|
||||
const UPLOAD_API = "/api/tasks/chunk-uploads";
|
||||
|
||||
@@ -315,13 +317,13 @@ function setUploadProgress(t, pct) {{
|
||||
}}
|
||||
|
||||
function pump() {{
|
||||
// 统计正在上传的文件数,启动等待中的文件直到达到 FILE_CONCURRENCY
|
||||
const active = pending.filter(t => t.state === "running").length;
|
||||
for (const t of pending) {{
|
||||
if (active >= CONCURRENCY) break;
|
||||
if (active >= FILE_CONCURRENCY) break;
|
||||
if (t.state === "pending") {{
|
||||
t.state = "running";
|
||||
setUploadState(t, "running");
|
||||
startUpload(t);
|
||||
active++;
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
@@ -342,7 +344,7 @@ async function startUpload(t) {{
|
||||
|
||||
const need = [];
|
||||
for (let i = 0; i < t.totalChunks; i++) if (!t.uploaded.has(i)) need.push(i);
|
||||
await runPool(need, CONCURRENCY, i => uploadChunk(t, i));
|
||||
await runPool(need, CHUNK_CONCURRENCY, i => uploadChunk(t, i));
|
||||
if (t.uploaded.size < t.totalChunks) throw new Error("部分分片未能上传");
|
||||
|
||||
setUploadState(t, "hashing");
|
||||
@@ -359,6 +361,9 @@ async function startUpload(t) {{
|
||||
meta.className = "task-meta fail-msg";
|
||||
meta.textContent = String(e.message || e);
|
||||
t.el.appendChild(meta);
|
||||
}} finally {{
|
||||
// 无论成功还是失败,都触发 pump 让队列中下一个文件开始上传
|
||||
pump();
|
||||
}}
|
||||
}}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user