feat: 调度器+并发管线+GPU优化+日志分级+前端修复
- scheduler: ffmpeg 异步线程 + GPU 串行调度 + 模型复用(2N→2 次加载) - pipeline: 阶段拆分(extract/asr/translate),中间数据存 Task 字段 - translate_service: 长度排序批处理,padding 浪费减少 91% - model_manager: ASR/翻译不共驻,BatchedInferencePipeline 批量解码 - 日志分级: INFO=任务流转里程碑,DEBUG=进度详情;默认 INFO - 前端: 日志最新在上+滚动感知+退避轮询;24h 时间;上传中状态显示 - /health: 返回完整 Whisper/NLLB 配置 - upload_service: 单事务 complete + 扩展名白名单 - task_router: 合并 UploadSession 虚拟任务到列表 - Dockerfile: CPU/GPU 独立构建链,deps 缓存稳定 - prefetch_models: 安装时预下载模型权重
This commit is contained in:
@@ -7,8 +7,8 @@
|
||||
...
|
||||
<upload_dir>/<yyyy>/<mm>/<uuid>.<ext> complete 后的正式视频
|
||||
|
||||
与 server 的区别:视频无需 sha256 去重(每个视频都转写),complete 直接创建 Task。
|
||||
管线触发由 controller 调用 pipeline.enqueue_task,本服务不依赖 pipeline。
|
||||
complete 创建转写 Task,管线触发由 controller 调用 scheduler.enqueue_task,
|
||||
本服务不依赖 scheduler(避免循环依赖)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -112,6 +112,14 @@ class UploadService:
|
||||
|
||||
# ---------------- 拼接 + 创建任务 ----------------
|
||||
|
||||
# 允许的音视频扩展名白名单(防可执行文件落盘到上传目录)
|
||||
_ALLOWED_EXTS = frozenset({
|
||||
".mp4", ".mkv", ".avi", ".mov", ".webm", ".flv",
|
||||
".mp3", ".wav", ".flac", ".aac", ".m4a", ".ogg", ".wma",
|
||||
})
|
||||
|
||||
# ---------------- 拼接 + 创建任务 ----------------
|
||||
|
||||
def complete(self, upload_id: str) -> CompleteResponse:
|
||||
session = self._require_session(upload_id)
|
||||
|
||||
@@ -135,6 +143,8 @@ class UploadService:
|
||||
final_path = self._assemble(session)
|
||||
rel = str(final_path.relative_to(self.upload_root))
|
||||
|
||||
# 单事务:建 Task + 更新 session 状态 + 关联 task_id 一次 commit
|
||||
# 避免双 commit 之间崩溃产生孤儿 Task(Task 已建但 session.task_id 为空)
|
||||
task = Task(
|
||||
filename=session.filename,
|
||||
source_path=rel,
|
||||
@@ -144,14 +154,14 @@ class UploadService:
|
||||
self.db.add(task)
|
||||
session.status = "completed"
|
||||
session.final_path = rel
|
||||
session.task_id = None # 占位,flush 后用 task.id 赋值
|
||||
session.updated_at = datetime.now(timezone.utc)
|
||||
self.db.commit()
|
||||
self.db.refresh(task)
|
||||
# 正向关联:session → task(替代旧的 source_path 反向查找)
|
||||
self.db.flush() # 拿到 task.id(不 commit,仍在事务内)
|
||||
session.task_id = task.id
|
||||
self.db.commit()
|
||||
self.db.refresh(task)
|
||||
|
||||
# 清理分片暂存
|
||||
# 清理分片暂存(commit 后,即使清理失败也不影响已建任务)
|
||||
self._cleanup_session_dir(upload_id)
|
||||
|
||||
logger.info("上传完成 task_id=%s file=%s size=%d", task.id, session.filename, session.size_bytes)
|
||||
@@ -215,7 +225,10 @@ class UploadService:
|
||||
|
||||
def _assemble(self, session: UploadSession) -> Path:
|
||||
"""按 index 顺序拼接全部分片为正式视频文件。"""
|
||||
ext = Path(session.filename).suffix or ".mp4"
|
||||
# 扩展名取自客户端 filename,但做白名单净化:不在允许列表内则回退 .bin
|
||||
ext = Path(session.filename).suffix.lower()
|
||||
if ext not in self._ALLOWED_EXTS:
|
||||
ext = ".bin"
|
||||
now = datetime.now(timezone.utc)
|
||||
sub = self.upload_root / f"{now:%Y}" / f"{now:%m}"
|
||||
sub.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
Reference in New Issue
Block a user