feat: 调度器+并发管线+GPU优化+日志分级+前端修复

- scheduler: ffmpeg 异步线程 + GPU 串行调度 + 模型复用(2N→2 次加载)
- pipeline: 阶段拆分(extract/asr/translate),中间数据存 Task 字段
- translate_service: 长度排序批处理,padding 浪费减少 91%
- model_manager: ASR/翻译不共驻,BatchedInferencePipeline 批量解码
- 日志分级: INFO=任务流转里程碑,DEBUG=进度详情;默认 INFO
- 前端: 日志最新在上+滚动感知+退避轮询;24h 时间;上传中状态显示
- /health: 返回完整 Whisper/NLLB 配置
- upload_service: 单事务 complete + 扩展名白名单
- task_router: 合并 UploadSession 虚拟任务到列表
- Dockerfile: CPU/GPU 独立构建链,deps 缓存稳定
- prefetch_models: 安装时预下载模型权重
This commit is contained in:
audio2text dev
2026-07-06 21:59:59 +08:00
parent 00e2a95fb7
commit 73110848f4
30 changed files with 1238 additions and 259 deletions

View File

@@ -1,9 +1,9 @@
"""日志路由:查询内存日志缓冲,供 /logs 页面消费。
分层语义:
- debug详细ffmpeg 命令、模型加载/卸载、转写逐段、翻译逐批进度)
- info简略(仅任务阶段转换,如 "任务 N [transcribing 55%]"
- warning / error更详细error 含完整 traceback
- debug进度详情(任务 [status pct%]、转写逐段统计、翻译逐批进度、ffmpeg 命令
- info任务流转里程碑(音频提取/ASR/翻译 的开始与完成、模型加载与卸载、批次汇总
- warning / error异常与失败error 含完整 traceback
"""
from __future__ import annotations

View File

@@ -2,6 +2,7 @@
from __future__ import annotations
from datetime import datetime, timezone
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException, Query
@@ -10,7 +11,8 @@ from sqlalchemy.orm import Session
from ..config import get_settings
from ..database import get_db
from ..models.task import Task
from ..models.task import Task, STATUS_UPLOADING
from ..models.upload_session import UploadSession
from ..schemas.task import TaskListResponse, TaskResponse
router = APIRouter(prefix="/api/tasks", tags=["task"])
@@ -28,6 +30,31 @@ def _to_resp(task: Task) -> TaskResponse:
)
def _upload_to_resp(session: UploadSession) -> TaskResponse:
"""把上传中的 UploadSession 映射为虚拟 TaskResponse。
is_upload=true 让前端走上传状态轮询而非任务轮询。
progress = 已传分片数 / 总分片数 × 100映射到 0-5 区间,与 extract 阶段衔接)。
"""
uploaded = len(session.uploaded_chunks or [])
total = session.total_chunks or 1
# 上传进度映射到 0-4%extract 从 5% 开始,留 1% 给 complete 拼接)
progress = min(4.0, uploaded / total * 4.0)
now = datetime.now(timezone.utc)
return TaskResponse(
id=0, # 虚拟 id前端用 upload_id 轮询
filename=session.filename,
status=STATUS_UPLOADING,
progress=progress,
error=None,
created_at=session.created_at,
updated_at=session.updated_at or now,
size_bytes=session.size_bytes,
is_upload=True,
upload_id=session.upload_id,
)
@router.get("", response_model=TaskListResponse, summary="任务列表")
def list_tasks(
limit: int = Query(100, ge=1, le=500),
@@ -35,14 +62,30 @@ def list_tasks(
q: str = Query("", description="按文件名模糊搜索(大小写不敏感,匹配子串)"),
db: Session = Depends(get_db),
) -> TaskListResponse:
q_base = db.query(Task).order_by(Task.id.desc())
# 上传中的会话pending 状态)也作为虚拟任务返回,让前端能看到上传进度
upload_q = db.query(UploadSession).filter(UploadSession.status == "pending")
if q.strip():
# SQLite 的 LIKE 默认大小写不敏感ASCIIilike 等价于 LIKE
like = f"%{q.strip()}%"
q_base = q_base.filter(Task.filename.ilike(like))
total = q_base.count()
tasks = q_base.offset(offset).limit(limit).all()
return TaskListResponse(tasks=[_to_resp(t) for t in tasks], total=total)
upload_q = upload_q.filter(UploadSession.filename.ilike(like))
uploads = upload_q.order_by(UploadSession.created_at.desc()).all()
# 已创建的 Task含 extracting/transcribing/.../done/failed
task_q = db.query(Task).order_by(Task.created_at.desc(), Task.id.desc())
if q.strip():
like = f"%{q.strip()}%"
task_q = task_q.filter(Task.filename.ilike(like))
total_tasks = task_q.count()
tasks = task_q.offset(offset).limit(limit).all()
# 合并:上传会话 + Task按 created_at 倒序
upload_resps = [_upload_to_resp(s) for s in uploads]
task_resps = [_to_resp(t) for t in tasks]
all_resps = upload_resps + task_resps
all_resps.sort(key=lambda r: r.created_at, reverse=True)
# 分页offset/limit 作用于合并后列表(上传会话通常很少,主要影响首页前几条)
paged = all_resps[offset:offset + limit]
return TaskListResponse(tasks=paged, total=len(all_resps))
@router.get("/{task_id}", response_model=TaskResponse, summary="任务状态")

View File

@@ -1,6 +1,6 @@
"""分片上传路由:建会话 / 查状态 / 传分片 / complete。
协议与 server 完全一致,区别仅在 complete 后创建的是转写 Task 而非 UploadedFile
complete 成功后创建转写 Task 并交由 scheduler 入队
"""
from __future__ import annotations
@@ -60,10 +60,11 @@ def complete_session(
"""拼接分片 + 创建转写任务 + 入队管线。
controller 负责编排service.complete 只管存储(拼接 + 建 Task
管线触发由 controller 调用service 不依赖 pipeline避免循环依赖
管线触发由 controller 调用 schedulerffmpeg 异步 + GPU 串行调度)service 不依赖
scheduler避免循环依赖
"""
resp = service.complete(upload_id)
# 仅新建任务时入队(幂等 complete 返回的也是同一 task_idenqueue 幂等无副作用)
from ..services.pipeline import enqueue_task
from ..services.scheduler import enqueue_task
enqueue_task(resp.task_id)
return resp