feat: 调度器+并发管线+GPU优化+日志分级+前端修复
- scheduler: ffmpeg 异步线程 + GPU 串行调度 + 模型复用(2N→2 次加载) - pipeline: 阶段拆分(extract/asr/translate),中间数据存 Task 字段 - translate_service: 长度排序批处理,padding 浪费减少 91% - model_manager: ASR/翻译不共驻,BatchedInferencePipeline 批量解码 - 日志分级: INFO=任务流转里程碑,DEBUG=进度详情;默认 INFO - 前端: 日志最新在上+滚动感知+退避轮询;24h 时间;上传中状态显示 - /health: 返回完整 Whisper/NLLB 配置 - upload_service: 单事务 complete + 扩展名白名单 - task_router: 合并 UploadSession 虚拟任务到列表 - Dockerfile: CPU/GPU 独立构建链,deps 缓存稳定 - prefetch_models: 安装时预下载模型权重
This commit is contained in:
@@ -1,9 +1,9 @@
|
||||
"""日志路由:查询内存日志缓冲,供 /logs 页面消费。
|
||||
|
||||
分层语义:
|
||||
- debug:详细(ffmpeg 命令、模型加载/卸载、转写逐段、翻译逐批进度)
|
||||
- info:简略(仅任务阶段转换,如 "任务 N [transcribing 55%]")
|
||||
- warning / error:更详细,error 含完整 traceback
|
||||
- debug:进度详情(任务 [status pct%]、转写逐段统计、翻译逐批进度、ffmpeg 命令)
|
||||
- info:任务流转里程碑(音频提取/ASR/翻译 的开始与完成、模型加载与卸载、批次汇总)
|
||||
- warning / error:异常与失败,error 含完整 traceback
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
@@ -10,7 +11,8 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from ..config import get_settings
|
||||
from ..database import get_db
|
||||
from ..models.task import Task
|
||||
from ..models.task import Task, STATUS_UPLOADING
|
||||
from ..models.upload_session import UploadSession
|
||||
from ..schemas.task import TaskListResponse, TaskResponse
|
||||
|
||||
router = APIRouter(prefix="/api/tasks", tags=["task"])
|
||||
@@ -28,6 +30,31 @@ def _to_resp(task: Task) -> TaskResponse:
|
||||
)
|
||||
|
||||
|
||||
def _upload_to_resp(session: UploadSession) -> TaskResponse:
|
||||
"""把上传中的 UploadSession 映射为虚拟 TaskResponse。
|
||||
|
||||
is_upload=true 让前端走上传状态轮询而非任务轮询。
|
||||
progress = 已传分片数 / 总分片数 × 100(映射到 0-5 区间,与 extract 阶段衔接)。
|
||||
"""
|
||||
uploaded = len(session.uploaded_chunks or [])
|
||||
total = session.total_chunks or 1
|
||||
# 上传进度映射到 0-4%(extract 从 5% 开始,留 1% 给 complete 拼接)
|
||||
progress = min(4.0, uploaded / total * 4.0)
|
||||
now = datetime.now(timezone.utc)
|
||||
return TaskResponse(
|
||||
id=0, # 虚拟 id,前端用 upload_id 轮询
|
||||
filename=session.filename,
|
||||
status=STATUS_UPLOADING,
|
||||
progress=progress,
|
||||
error=None,
|
||||
created_at=session.created_at,
|
||||
updated_at=session.updated_at or now,
|
||||
size_bytes=session.size_bytes,
|
||||
is_upload=True,
|
||||
upload_id=session.upload_id,
|
||||
)
|
||||
|
||||
|
||||
@router.get("", response_model=TaskListResponse, summary="任务列表")
|
||||
def list_tasks(
|
||||
limit: int = Query(100, ge=1, le=500),
|
||||
@@ -35,14 +62,30 @@ def list_tasks(
|
||||
q: str = Query("", description="按文件名模糊搜索(大小写不敏感,匹配子串)"),
|
||||
db: Session = Depends(get_db),
|
||||
) -> TaskListResponse:
|
||||
q_base = db.query(Task).order_by(Task.id.desc())
|
||||
# 上传中的会话(pending 状态)也作为虚拟任务返回,让前端能看到上传进度
|
||||
upload_q = db.query(UploadSession).filter(UploadSession.status == "pending")
|
||||
if q.strip():
|
||||
# SQLite 的 LIKE 默认大小写不敏感(ASCII),ilike 等价于 LIKE
|
||||
like = f"%{q.strip()}%"
|
||||
q_base = q_base.filter(Task.filename.ilike(like))
|
||||
total = q_base.count()
|
||||
tasks = q_base.offset(offset).limit(limit).all()
|
||||
return TaskListResponse(tasks=[_to_resp(t) for t in tasks], total=total)
|
||||
upload_q = upload_q.filter(UploadSession.filename.ilike(like))
|
||||
uploads = upload_q.order_by(UploadSession.created_at.desc()).all()
|
||||
|
||||
# 已创建的 Task(含 extracting/transcribing/.../done/failed)
|
||||
task_q = db.query(Task).order_by(Task.created_at.desc(), Task.id.desc())
|
||||
if q.strip():
|
||||
like = f"%{q.strip()}%"
|
||||
task_q = task_q.filter(Task.filename.ilike(like))
|
||||
total_tasks = task_q.count()
|
||||
tasks = task_q.offset(offset).limit(limit).all()
|
||||
|
||||
# 合并:上传会话 + Task,按 created_at 倒序
|
||||
upload_resps = [_upload_to_resp(s) for s in uploads]
|
||||
task_resps = [_to_resp(t) for t in tasks]
|
||||
all_resps = upload_resps + task_resps
|
||||
all_resps.sort(key=lambda r: r.created_at, reverse=True)
|
||||
|
||||
# 分页:offset/limit 作用于合并后列表(上传会话通常很少,主要影响首页前几条)
|
||||
paged = all_resps[offset:offset + limit]
|
||||
return TaskListResponse(tasks=paged, total=len(all_resps))
|
||||
|
||||
|
||||
@router.get("/{task_id}", response_model=TaskResponse, summary="任务状态")
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""分片上传路由:建会话 / 查状态 / 传分片 / complete。
|
||||
|
||||
协议与 server 完全一致,区别仅在 complete 后创建的是转写 Task 而非 UploadedFile。
|
||||
complete 成功后创建转写 Task 并交由 scheduler 入队。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -60,10 +60,11 @@ def complete_session(
|
||||
"""拼接分片 + 创建转写任务 + 入队管线。
|
||||
|
||||
controller 负责编排:service.complete 只管存储(拼接 + 建 Task),
|
||||
管线触发由 controller 调用,service 不依赖 pipeline(避免循环依赖)。
|
||||
管线触发由 controller 调用 scheduler(ffmpeg 异步 + GPU 串行调度),service 不依赖
|
||||
scheduler(避免循环依赖)。
|
||||
"""
|
||||
resp = service.complete(upload_id)
|
||||
# 仅新建任务时入队(幂等 complete 返回的也是同一 task_id,enqueue 幂等无副作用)
|
||||
from ..services.pipeline import enqueue_task
|
||||
from ..services.scheduler import enqueue_task
|
||||
enqueue_task(resp.task_id)
|
||||
return resp
|
||||
|
||||
Reference in New Issue
Block a user