feat: 调度器+并发管线+GPU优化+日志分级+前端修复
- scheduler: ffmpeg 异步线程 + GPU 串行调度 + 模型复用(2N→2 次加载) - pipeline: 阶段拆分(extract/asr/translate),中间数据存 Task 字段 - translate_service: 长度排序批处理,padding 浪费减少 91% - model_manager: ASR/翻译不共驻,BatchedInferencePipeline 批量解码 - 日志分级: INFO=任务流转里程碑,DEBUG=进度详情;默认 INFO - 前端: 日志最新在上+滚动感知+退避轮询;24h 时间;上传中状态显示 - /health: 返回完整 Whisper/NLLB 配置 - upload_service: 单事务 complete + 扩展名白名单 - task_router: 合并 UploadSession 虚拟任务到列表 - Dockerfile: CPU/GPU 独立构建链,deps 缓存稳定 - prefetch_models: 安装时预下载模型权重
This commit is contained in:
@@ -14,6 +14,17 @@ def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
# 任务状态枚举(单一真源,供 scheduler / cache_cleaner / views 引用)
|
||||
STATUS_UPLOADING = "uploading" # 虚拟状态:分片上传中(不在 Task 表,由 UploadSession 映射)
|
||||
STATUS_QUEUED = "queued"
|
||||
STATUS_EXTRACTING = "extracting"
|
||||
STATUS_TRANSCRIBING = "transcribing"
|
||||
STATUS_SEGMENTING = "segmenting"
|
||||
STATUS_TRANSLATING = "translating"
|
||||
STATUS_DONE = "done"
|
||||
STATUS_FAILED = "failed"
|
||||
|
||||
|
||||
class Task(Base):
|
||||
__tablename__ = "task"
|
||||
|
||||
@@ -23,7 +34,7 @@ class Task(Base):
|
||||
# 视频在 upload_dir 下的相对路径(提取音频前后可能被删)
|
||||
source_path: Mapped[str] = mapped_column(String(1024), nullable=False)
|
||||
# 状态机:queued → extracting → transcribing → segmenting → translating → done | failed
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued")
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default=STATUS_QUEUED, index=True)
|
||||
# 0-100 进度
|
||||
progress: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
||||
# 失败原因
|
||||
@@ -33,7 +44,13 @@ class Task(Base):
|
||||
en_srt_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
zh_srt_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=_now)
|
||||
# ---- 流水线阶段间传递的中间数据(scheduler 调度用)----
|
||||
# ffmpeg 提取出的 wav 绝对路径(extract 阶段写,asr 阶段读)
|
||||
wav_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
# ASR + 断句结果(asr 阶段写 JSON,translate 阶段读)。JSON 序列化的 list[Subtitle]
|
||||
segments_json: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=_now, index=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime, default=_now, onupdate=_now)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
"""分片上传会话 ORM:支撑断点续传。对齐 server 的 UploadSession 形态(SQLite 版)。"""
|
||||
"""分片上传会话 ORM:支撑断点续传。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
@@ -11,6 +12,8 @@ from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from ..database import Base
|
||||
|
||||
logger = logging.getLogger("audio2text.models")
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
@@ -26,7 +29,14 @@ class _IntList(TypeDecorator):
|
||||
return json.dumps(value) if value is not None else None
|
||||
|
||||
def process_result_value(self, value: Any, dialect) -> list[int]:
|
||||
return json.loads(value) if value else []
|
||||
if not value:
|
||||
return []
|
||||
try:
|
||||
return json.loads(value)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
# DB 脏数据(手工修改/并发写截断)不应让整行读取失败
|
||||
logger.warning("uploaded_chunks JSON 解析失败,回退空列表:%r", value[:80])
|
||||
return []
|
||||
|
||||
|
||||
class UploadSession(Base):
|
||||
@@ -39,11 +49,13 @@ class UploadSession(Base):
|
||||
total_chunks: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
uploaded_chunks: Mapped[list[int]] = mapped_column(_IntList, default=list)
|
||||
# pending → completed(complete 成功)| abandoned(reaper 清理)
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="pending")
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="pending", index=True)
|
||||
# 拼接完成后的视频相对 upload_dir 路径
|
||||
final_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
# complete 后关联的 Task.id(直接引用,避免反向查找 source_path)
|
||||
task_id: Mapped[int | None] = mapped_column(ForeignKey("task.id"), nullable=True)
|
||||
# complete 后关联的 Task.id。ondelete=SET NULL:Task 被删时 session 保留(task_id 置空)
|
||||
task_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("task.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=_now)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, default=_now, index=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime, default=_now, onupdate=_now)
|
||||
|
||||
Reference in New Issue
Block a user