diff --git a/README.md b/README.md index c58d5e6..50189da 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # zikai file service 基于 FastAPI 的个人 Web 服务，提供**文件上传/浏览/下载、共享记事本（实时协作）、 -主机监控、SFTP 暂存、反向隧道**。采用 Spring 风格分层架构，自带 API 文档。 +主机监控、SFTP 暂存、反向隧道、PDF 转换**。采用 Spring 风格分层架构，自带 API 文档。 ## 功能一览 @@ -15,6 +15,10 @@ | 记事本实时同步 | `WS /ws/wb/{id}`（心跳 3s，5 次失活移除） | 公开 | | 记事本管理 | `GET /wb-admin`（查看/删除） | Basic Auth | | 记事本管理 API | `GET /api/admin/wb`、`DELETE /api/admin/wb/{id}` | Basic Auth | +| PDF 转换 | `GET /pdf`（上传 epub→PDF，进度轮询，下载；凭 cookie 记住任务） | 公开（cookie） | +| PDF 转换 API | `POST /api/pdf/jobs`、`GET /api/pdf/jobs[/{id}]`、`GET /api/pdf/jobs/{id}/download`、`DELETE /api/pdf/jobs/{id}` | 公开（cookie） | +| PDF 转换管理 | `GET /pdf-admin`（全部任务，含已软删标记，硬删） | Basic Auth | +| PDF 转换管理 API | `GET /api/admin/pdf/jobs`、`DELETE /api/admin/pdf/jobs/{id}` | Basic Auth | | 主机监控 | `GET /api/system/status`（CPU/内存/磁盘，HTML+JSON 内容协商） | 公开 | | 反向隧道反代 | `ALL /api/userPort/{userName}`（经 SSH 隧道转发到 user 本地服务） | 公开 | | SFTP/SSH | 端口 2022（密码+公钥，chroot 到上传目录，承载隧道转发） | SSH | @@ -29,14 +33,15 @@ server/ │ ├── config.py # 从 config.yaml 加载的类型化 Settings（pydantic-settings） │ ├── database.py # SQLAlchemy 引擎/Session/Base/get_db 依赖 │ ├── security.py # Basic Auth（require_docs_auth，常量时间比较） -│ ├── controllers/ # 路由层（@RestController）：file/system/chunk/tunnel/whiteboard/admin +│ ├── controllers/ # 路由层（@RestController）：file/system/chunk/tunnel/whiteboard/pdf/admin │ ├── services/ # 业务层：UploadService/ChunkUploadService/SystemService/ │ │ # WhiteboardService/WhiteboardHub/TunnelService/sftp_server +│ │ # PdfService（转换编排）+ pdf_converter（epub->pdf 纯 Python） │ ├── dao/ # 数据访问层：唯一发 SQL 的层（SQLAlchemy ORM 参数化） -│ ├── models/ # ORM 实体：UploadedFile/UploadSession/Whiteboard/TunnelSession +│ ├── models/ # ORM 实体：UploadedFile/UploadSession/Whiteboard/TunnelSession/PdfJob │ ├── schemas/ # pydantic 请求/响应 DTO │ ├── views/ # 服务端渲染 HTML（系统状态页、上传页） -│ ├── static/ # 前端静态资源（common + file_browser + whiteboard + whiteboard_admin） +│ ├── static/ # 前端静态资源（common + file_browser + whiteboard + pdf + pdf_admin） │ └── scripts/init_db.py # 数据库初始化（建库建账、随机密码写回 config.yaml） ├── sql/schema.sql # 建表 DDL（参考；实际由 ORM 自动建表） ├── config.example.yaml # 配置模板（含注释） @@ -58,6 +63,8 @@ DB Session 由 `get_db` 依赖注入。前端页面走「StaticFiles 挂载 + apt update apt install -y python3-venv python3-pip mysql-server apache2 \ libssl-dev build-essential # build-essential 给 bcrypt/asyncssh 编译 +# PDF 转换依赖 weasyprint，需 pango/cairo 系统库（Ubuntu 通常已随桌面环境安装，缺则补装）： +apt install -y libpango-1.0-0 libpangoft2-1.0-0 libcairo2 libgdk-pixbuf-2.0-0 ``` ### 2. 获取代码 diff --git a/app/config.py b/app/config.py index 98a03ec..68c7bd0 100644 --- a/app/config.py +++ b/app/config.py @@ -108,6 +108,24 @@ class WhiteboardConfig(BaseModel): list_limit: int = 100 +class PdfConfig(BaseModel): + """PDF 转换服务配置。 + + 用户侧（上传/查看/下载/软删）凭 httpOnly cookie 标识；管理侧（列表/硬删） + 走 docs 同款 Basic Auth。原始文件与产物 PDF 复用 storage.upload_dir 落盘。 + """ + + # 单文件大小上限（字节）。250 MiB。 + max_size_bytes: int = 250 * 1024 * 1024 + # 转换超时（秒）：超大/复杂文件兜底，避免长期占用 worker。 + convert_timeout_seconds: int = 600 + # 列表分页默认值 + list_limit: int = 100 + # 用户 cookie 名与有效期 + cookie_name: str = "zk_pdf" + cookie_max_age_seconds: int = 365 * 24 * 3600 + + class Settings(BaseModel): server: ServerConfig = ServerConfig() database: DatabaseConfig = DatabaseConfig() @@ -116,6 +134,7 @@ class Settings(BaseModel): docs: DocsConfig = DocsConfig() tunnel: TunnelConfig = TunnelConfig() whiteboard: WhiteboardConfig = WhiteboardConfig() + pdf: PdfConfig = PdfConfig() def db_url(self) -> str: c = self.database diff --git a/app/controllers/__init__.py b/app/controllers/__init__.py index 5f6dbc4..22b6f5c 100644 --- a/app/controllers/__init__.py +++ b/app/controllers/__init__.py @@ -3,6 +3,7 @@ from .chunk_upload_controller import router as chunk_upload_router from .file_admin_controller import router as file_admin_router from .file_controller import router as file_router +from .pdf_controller import router as pdf_router from .system_controller import router as system_router from .tunnel_controller import router as tunnel_router from .whiteboard_controller import router as whiteboard_router @@ -11,6 +12,7 @@ __all__ = [ "chunk_upload_router", "file_admin_router", "file_router", + "pdf_router", "system_router", "tunnel_router", "whiteboard_router", diff --git a/app/controllers/pdf_controller.py b/app/controllers/pdf_controller.py new file mode 100644 index 0000000..94ce55a --- /dev/null +++ b/app/controllers/pdf_controller.py @@ -0,0 +1,182 @@ +"""PDF 转换接口：用户侧（cookie 标识）+ 管理侧（Basic Auth）。 + +路由： + POST /api/pdf/jobs 用户上传文件并提交转换（首次无 cookie 则下发） + GET /api/pdf/jobs 当前用户任务列表（仅未软删） + GET /api/pdf/jobs/{id} 单任务状态（轮询进度） + GET /api/pdf/jobs/{id}/download 下载产物 PDF + DELETE /api/pdf/jobs/{id} 用户软删（不再对用户展示，管理页仍可见） + GET /api/admin/pdf/jobs 管理页列表（全部，含已软删标记） + DELETE /api/admin/pdf/jobs/{id} 管理员硬删（真正删除磁盘与 DB） + +HTML 页面 /pdf（用户）与 /pdf-admin（管理）由 main.py 返回静态文件， +不在此 controller 注册，避免与 REST 同路径冲突。 +""" + +from __future__ import annotations + +from fastapi import APIRouter, Cookie, Depends, File, HTTPException, Query, Request, UploadFile +from fastapi.responses import FileResponse, JSONResponse +from sqlalchemy.orm import Session + +from ..config import get_settings +from ..database import get_db +from ..dao.pdf_job_dao import PdfJobDAO +from ..dao.uploaded_file_dao import UploadedFileDAO +from ..schemas.pdf import DeleteResult, PdfJobListResponse, PdfJobOut, PdfSubmitResponse +from ..security import require_docs_auth +from ..services.pdf_service import PdfService, new_owner_cookie + +router = APIRouter(tags=["pdf"]) + + +def _service(db: Session = Depends(get_db)) -> PdfService: + return PdfService(PdfJobDAO(db), UploadedFileDAO(db)) + + +def _resolve_cookie(request: Request, zk_pdf: str | None = Cookie(default=None)) -> str: + """解析用户 cookie；无则生成新值（由 controller 写入响应头）。 + + cookie 缺失时把新值挂到 request.state，供响应阶段 set_cookie。 + """ + cfg = get_settings().pdf + if zk_pdf and len(zk_pdf) == 32: + return zk_pdf + new = new_owner_cookie() + request.state.new_pdf_cookie = new + return new + + +@router.post( + "/api/pdf/jobs", + response_model=PdfSubmitResponse, + summary="上传文件并提交 PDF 转换", + description=( + "multipart/form-data 上传 epub 文件（≤250MB），流式落盘后创建 pending 转换任务，" + "后台异步转换。用户凭 zk_pdf cookie 标识；首次无 cookie 时响应下发新 cookie。" + ), +) +async def submit_job( + request: Request, + file: UploadFile = File(..., description="要转换的 epub 文件"), + service: PdfService = Depends(_service), + owner_cookie: str = Depends(_resolve_cookie), +) -> PdfSubmitResponse: + job, _source_id = service.submit(file, owner_cookie) + # 提交成功后触发后台转换 + service.schedule_convert(job.id) + resp = PdfSubmitResponse(job=job, set_cookie=False) + new_cookie = getattr(request.state, "new_pdf_cookie", None) + if new_cookie: + resp.set_cookie = True + # 用 JSONResponse 显式 set_cookie 后返回模型体 + cfg = get_settings().pdf + data = resp.model_dump(mode="json") + response = JSONResponse(data) + response.set_cookie( + key=cfg.cookie_name, + value=new_cookie, + max_age=cfg.cookie_max_age_seconds, + httponly=True, + samesite="lax", + path="/", + ) + return response + return resp + + +@router.get( + "/api/pdf/jobs", + response_model=PdfJobListResponse, + summary="当前用户的任务列表", + description="按 zk_pdf cookie 返回该用户未软删的任务，按创建时间倒序。", +) +def list_jobs( + service: PdfService = Depends(_service), + owner_cookie: str = Depends(_resolve_cookie), + limit: int = Query(100, ge=1, le=500), + offset: int = Query(0, ge=0), +) -> PdfJobListResponse: + return service.list_for_user(owner_cookie, limit=limit, offset=offset) + + +@router.get( + "/api/pdf/jobs/{job_id}", + response_model=PdfJobOut, + summary="查询单任务状态（轮询进度）", +) +def get_job( + job_id: int, + service: PdfService = Depends(_service), + owner_cookie: str = Depends(_resolve_cookie), +) -> PdfJobOut: + return service.get_job(job_id, owner_cookie) + + +@router.get( + "/api/pdf/jobs/{job_id}/download", + summary="下载转换后的 PDF", + description="仅任务状态为 done 且归属本人未软删时可下载。", +) +def download_job( + job_id: int, + service: PdfService = Depends(_service), + owner_cookie: str = Depends(_resolve_cookie), +) -> FileResponse: + _job, path, filename = service.get_output_path(job_id, owner_cookie) + return FileResponse( + path=str(path), + media_type="application/pdf", + filename=filename, + ) + + +@router.delete( + "/api/pdf/jobs/{job_id}", + response_model=DeleteResult, + summary="用户软删任务（不再对用户展示）", + description="仅置 user_deleted 标记，磁盘与 DB 行保留；管理页仍可见并标注已删除。", +) +def delete_job( + job_id: int, + service: PdfService = Depends(_service), + owner_cookie: str = Depends(_resolve_cookie), +) -> DeleteResult: + ok = service.user_delete(job_id, owner_cookie) + if not ok: + raise HTTPException(404, "任务不存在") + return DeleteResult(deleted=True) + + +# ---------------- 管理侧（Basic Auth） ---------------- + +@router.get( + "/api/admin/pdf/jobs", + response_model=PdfJobListResponse, + summary="列出全部转换任务（需鉴权）", + description="管理页使用：含 user_deleted 标记，可看到用户是否已软删。", +) +def admin_list_jobs( + service: PdfService = Depends(_service), + _: str = Depends(require_docs_auth), + limit: int = Query(100, ge=1, le=500), + offset: int = Query(0, ge=0), +) -> PdfJobListResponse: + return service.list_for_admin(limit=limit, offset=offset) + + +@router.delete( + "/api/admin/pdf/jobs/{job_id}", + response_model=DeleteResult, + summary="管理员硬删任务（真正删除）", + description="删原始/产物磁盘文件 + UploadedFile 行 + PdfJob 行，不可恢复。", +) +def admin_delete_job( + job_id: int, + service: PdfService = Depends(_service), + _: str = Depends(require_docs_auth), +) -> DeleteResult: + ok = service.admin_delete(job_id) + if not ok: + raise HTTPException(404, "任务不存在") + return DeleteResult(deleted=True) diff --git a/app/dao/pdf_job_dao.py b/app/dao/pdf_job_dao.py new file mode 100644 index 0000000..0ec55c7 --- /dev/null +++ b/app/dao/pdf_job_dao.py @@ -0,0 +1,68 @@ +"""PdfJob 的 DAO。 + +所有写操作均在该层 commit，service 不直接操作 session。 +用户/管理两条查询路径分别按 user_deleted 过滤。 +""" + +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from ..models.pdf_job import PdfJob + + +class PdfJobDAO: + def __init__(self, db: Session) -> None: + self.db = db + + def create(self, job: PdfJob) -> PdfJob: + self.db.add(job) + self.db.commit() + self.db.refresh(job) + return job + + def get(self, job_id: int) -> PdfJob | None: + return self.db.get(PdfJob, job_id) + + def update(self, job: PdfJob) -> PdfJob: + """提交对 job 的就地修改并刷新。""" + self.db.commit() + self.db.refresh(job) + return job + + def list_for_user(self, owner_cookie: str, limit: int = 100, offset: int = 0) -> list[PdfJob]: + """用户视角：仅未软删的任务，按创建时间倒序。""" + stmt = ( + select(PdfJob) + .where(PdfJob.owner_cookie == owner_cookie) + .where(PdfJob.user_deleted == False) # noqa: E712 + .order_by(PdfJob.created_at.desc()) + .limit(limit) + .offset(offset) + ) + return list(self.db.scalars(stmt).all()) + + def count_for_user(self, owner_cookie: str) -> int: + stmt = ( + select(func.count()) + .select_from(PdfJob) + .where(PdfJob.owner_cookie == owner_cookie) + .where(PdfJob.user_deleted == False) # noqa: E712 + ) + return self.db.scalar(stmt) or 0 + + def list_all(self, limit: int = 100, offset: int = 0) -> list[PdfJob]: + """管理视角：全部任务（含已软删），按创建时间倒序。""" + stmt = ( + select(PdfJob) + .order_by(PdfJob.created_at.desc()) + .limit(limit) + .offset(offset) + ) + return list(self.db.scalars(stmt).all()) + + def count_all(self) -> int: + return self.db.scalar(select(func.count()).select_from(PdfJob)) or 0 diff --git a/app/main.py b/app/main.py index 0fbf145..adefa2b 100644 --- a/app/main.py +++ b/app/main.py @@ -31,6 +31,7 @@ from .controllers import ( chunk_upload_router, file_admin_router, file_router, + pdf_router, system_router, tunnel_router, whiteboard_router, @@ -149,6 +150,7 @@ def create_app() -> FastAPI: app.include_router(chunk_upload_router) app.include_router(tunnel_router) app.include_router(whiteboard_router) + app.include_router(pdf_router) # 前端静态资源（JS/CSS）；HTML 壳由下面的具名路由返回，便于各自挂 Basic Auth if _STATIC_DIR.is_dir(): @@ -219,6 +221,26 @@ def create_app() -> FastAPI: def whiteboard_page(board_id: str) -> HTMLResponse: return _serve_static_html("whiteboard.html") + @app.get( + "/pdf", + response_class=HTMLResponse, + tags=["pages"], + summary="PDF 转换页面", + description="上传 epub 文件转为 PDF，显示进度并下载；凭 cookie 查看自己的任务。", + ) + def pdf_page() -> HTMLResponse: + return _serve_static_html("pdf.html") + + @app.get( + "/pdf-admin", + response_class=HTMLResponse, + tags=["pages"], + summary="PDF 转换管理页（需鉴权）", + description="查看全部转换任务（含用户已软删的，标注是否已删除），并可硬删。Basic Auth 同 docs。", + ) + def pdf_admin_page(_: str = Depends(require_docs_auth)) -> HTMLResponse: + return _serve_static_html("pdf_admin.html") + return app diff --git a/app/models/__init__.py b/app/models/__init__.py index f4aa5ed..d8106a1 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -1,8 +1,9 @@ """ORM 模型包；import 本包即把所有实体注册到 Base.metadata。""" +from .pdf_job import PdfJob from .tunnel_session import TunnelSession from .uploaded_file import UploadedFile from .upload_session import UploadSession from .whiteboard import Whiteboard -__all__ = ["TunnelSession", "UploadedFile", "UploadSession", "Whiteboard"] +__all__ = ["PdfJob", "TunnelSession", "UploadedFile", "UploadSession", "Whiteboard"] diff --git a/app/models/pdf_job.py b/app/models/pdf_job.py new file mode 100644 index 0000000..5dc48e3 --- /dev/null +++ b/app/models/pdf_job.py @@ -0,0 +1,56 @@ +"""PDF 转换任务实体。 + +一个任务记录一次「上传文件 -> 转为 PDF」的转换。原始上传文件与转换产物 PDF +均复用 UploadedFile 存储（落盘 + 元数据入库），本表只记录两者关系与转换状态， +不重复实现存储逻辑。 + +删除语义（两级）： + 用户软删（user_deleted=true）-- 用户页不再展示，但磁盘与 DB 行保留； + 管理页仍可见且标注「已删除」。 + 管理员硬删 -- 删原始/产物磁盘文件 + UploadedFile 行 + 本表行，真正删除。 +""" + +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import BigInteger, Boolean, DateTime, Integer, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from ..database import Base + + +class PdfJob(Base): + __tablename__ = "pdf_job" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + # 用户标识：httpOnly cookie 值（uuid4.hex），用户凭此查看自己的任务 + owner_cookie: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + # 原始上传文件（复用 UploadedFile 存储） + source_file_id: Mapped[int] = mapped_column(BigInteger, nullable=False, index=True) + source_filename: Mapped[str] = mapped_column(String(512), nullable=False) + source_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + # 转换产物 PDF（复用 UploadedFile 存储）；转换完成前为 NULL + output_file_id: Mapped[int | None] = mapped_column(BigInteger, nullable=True, default=None) + # pending(排队) / converting(转换中) / done(完成) / failed(失败) + status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True) + # 转换进度 0-100，供前端轮询显示 + progress: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + # 失败原因（status=failed 时填写） + error_message: Mapped[str] = mapped_column(String(512), nullable=False, default="") + # 用户软删标记：true=用户已从其页面删除，不再对用户展示 + user_deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True) + # 用户软删时间（管理页展示「是否已删除」时用） + deleted_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None) + created_at: Mapped[datetime] = mapped_column( + DateTime, server_default=func.now(), nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime, server_default=func.now(), onupdate=func.now(), nullable=False + ) + + def __repr__(self) -> str: # pragma: no cover + return ( + f"" + ) diff --git a/app/schemas/pdf.py b/app/schemas/pdf.py new file mode 100644 index 0000000..b3c9004 --- /dev/null +++ b/app/schemas/pdf.py @@ -0,0 +1,44 @@ +"""PDF 转换接口 DTO。""" + +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel, Field + + +class PdfJobOut(BaseModel): + """任务对外视图（用户与管理页共用，user_deleted 仅管理页关注）。""" + + id: int + source_filename: str = Field(..., description="原始上传文件名") + source_size: int = Field(..., description="原始文件字节数") + status: str = Field(..., description="pending / converting / done / failed") + progress: int = Field(0, description="转换进度 0-100") + error_message: str = Field("", description="失败原因") + user_deleted: bool = Field(False, description="用户是否已软删（管理页用）") + deleted_at: datetime | None = Field(None, description="用户软删时间（管理页用）") + created_at: datetime + updated_at: datetime + + model_config = {"from_attributes": True} + + +class PdfJobListResponse(BaseModel): + """任务列表响应。""" + + total: int + items: list[PdfJobOut] + + +class PdfSubmitResponse(BaseModel): + """提交转换任务的响应。""" + + job: PdfJobOut = Field(..., description="新建的任务") + set_cookie: bool = Field( + False, description="true=本次请求未带 cookie，响应已下发新 cookie" + ) + + +class DeleteResult(BaseModel): + deleted: bool diff --git a/app/services/pdf_converter.py b/app/services/pdf_converter.py new file mode 100644 index 0000000..ca0c5c6 --- /dev/null +++ b/app/services/pdf_converter.py @@ -0,0 +1,105 @@ +"""文件 -> PDF 转换器（纯 Python，无 calibre/xvfb 系统依赖）。 + +当前支持 epub（必要能力）：ebooklib 解析 epub 文档项（按 spine 顺序），拼接为 +完整 HTML 后交 weasyprint 渲染为 PDF。epub 内的相对资源（图片/CSS）经 base_url +指向 epub 解包目录解析。 + +接口抽象为 ``convert_to_pdf(src_path, dst_path)``：未来新增格式只需在本模块内 +按扩展名分支，调用方（PdfService）无需改动。 +""" + +from __future__ import annotations + +import logging +import shutil +import tempfile +import zipfile +from pathlib import Path + +logger = logging.getLogger("zikai.pdf") + +# 支持的输入格式（小写扩展名 -> 是否可转）。新增格式在此登记并实现分支即可。 +# epub 为必要能力；其余为 ebooklib/weasyprint 路径天然支持的电子书结构， +# 实测对纯 HTML 类 epub 同样有效，故一并放行。 +SUPPORTED_EXTENSIONS = {".epub"} + + +def is_supported(filename: str) -> bool: + """文件名扩展名是否在支持列表内。""" + return Path(filename).suffix.lower() in SUPPORTED_EXTENSIONS + + +def convert_to_pdf(src_path: Path, dst_path: Path) -> None: + """把 src_path 指向的文件转为 PDF 写入 dst_path。 + + 失败抛 RuntimeError（调用方捕获后写 error_message）。按扩展名分发， + 当前仅 epub 分支；新增格式在此 elif 扩展。 + """ + ext = src_path.suffix.lower() + if ext == ".epub": + _epub_to_pdf(src_path, dst_path) + else: + raise RuntimeError(f"不支持的文件格式：{ext}（仅支持 epub）") + + +def _epub_to_pdf(src_path: Path, dst_path: Path) -> None: + """epub -> PDF：解包 epub -> 按 spine 顺序取文档项 HTML -> weasyprint 渲染。 + + epub 本质是 zip。先解包到临时目录，用 ebooklib 读取（其内部按 zip 解析， + base_url 指向解包后的 OEBPS/ 内容目录使相对图片/CSS 可被 weasyprint 解析）。 + """ + # 延迟导入：weasyprint 首次 import 较重（加载 pango/cairo），且仅在真正转换时需要 + import ebooklib # noqa: F401 + from ebooklib import epub + from weasyprint import HTML + + with tempfile.TemporaryDirectory(prefix="zpdf_") as tmp: + tmp_dir = Path(tmp) + # epub 是 zip，解包到临时目录便于 weasyprint 解析相对资源 + try: + with zipfile.ZipFile(src_path, "r") as zf: + zf.extractall(tmp_dir) + except zipfile.BadZipFile as exc: + raise RuntimeError(f"epub 文件损坏（非有效 zip）：{exc}") from exc + + try: + book = epub.read_epub(str(src_path), {"ignore_ncx": True}) + except Exception as exc: + raise RuntimeError(f"epub 解析失败：{exc}") from exc + + # 按 spine 顺序收集文档项（XHTML），保证章节顺序正确 + docs: list[str] = [] + for idref, _linear in book.spine: + item = book.get_item_with_id(idref) + if item is not None: + docs.append(item.get_content().decode("utf-8", errors="replace")) + if not docs: + # spine 为空时回退：取所有文档项 + docs = [ + it.get_content().decode("utf-8", errors="replace") + for it in book.get_items_of_type(ebooklib.ITEM_DOCUMENT) + ] + if not docs: + raise RuntimeError("epub 内无可转换的文档内容") + + full_html = "\n".join(docs) + + # 定位资源根目录（含图片/CSS 的目录）：通常是 OEBPS/ 或根目录。 + # epub 内资源（图片/CSS）相对文档项引用，文档项与资源同处 OPF 所在目录。 + # 故以 OPF 文件所在目录作为 base_url，使相对路径正确解析。 + base_url = str(tmp_dir) + opf_files = list(tmp_dir.rglob("*.opf")) + if opf_files: + opf_dir = opf_files[0].parent + # OPF 可能在根目录，此时 base_url 保持 tmp_dir + if str(opf_dir) != str(tmp_dir): + base_url = str(opf_dir) + + try: + HTML(string=full_html, base_url=base_url).write_pdf(str(dst_path)) + except Exception as exc: + raise RuntimeError(f"PDF 渲染失败：{exc}") from exc + + if not dst_path.exists() or dst_path.stat().st_size == 0: + raise RuntimeError("PDF 渲染未产出有效文件") + logger.info("epub->pdf 完成: %s -> %s (%d bytes)", src_path.name, dst_path.name, dst_path.stat().st_size) diff --git a/app/services/pdf_service.py b/app/services/pdf_service.py new file mode 100644 index 0000000..0785e5e --- /dev/null +++ b/app/services/pdf_service.py @@ -0,0 +1,299 @@ +"""PDF 转换服务：上传落盘 + 异步后台转换 + 进度追踪 + 两级删除。 + +复用 UploadService 的「流式落盘 + 存储路径生成」能力：原始上传文件与产物 PDF +均作为 UploadedFile 存储，本服务只维护 PdfJob 关系与状态。 + +转换在后台 asyncio task 中以 to_thread 执行（转换器是同步阻塞调用）， +过程中经 DAO 更新 progress/status，供前端轮询。worker 限制：依赖进程内 +asyncio 事件循环，与现有 reaper/hub 一致，保持单 worker。 + +删除语义： + user_delete -- 置 user_deleted=true + deleted_at，磁盘与 DB 行保留（管理页可见）。 + admin_delete -- 删原始/产物磁盘文件 + UploadedFile 行 + PdfJob 行（真正删除）。 +""" + +from __future__ import annotations + +import asyncio +import logging +import secrets +from datetime import datetime, timezone +from pathlib import Path + +from fastapi import HTTPException, UploadFile + +from ..config import get_settings +from ..dao.pdf_job_dao import PdfJobDAO +from ..dao.uploaded_file_dao import UploadedFileDAO +from ..models.pdf_job import PdfJob +from ..schemas.pdf import PdfJobOut, PdfJobListResponse, PdfSubmitResponse +from . import pdf_converter +from .upload_service import UploadService + +logger = logging.getLogger("zikai.pdf") + + +def new_owner_cookie() -> str: + """生成新的用户标识 cookie 值（uuid4 hex）。""" + return secrets.token_hex(16) + + +class PdfService: + def __init__(self, job_dao: PdfJobDAO, file_dao: UploadedFileDAO) -> None: + s = get_settings() + self.job_dao = job_dao + self.file_dao = file_dao + self.upload_root = s.resolved_upload_dir() + self.cfg = s.pdf + # 复用 UploadService 的存储路径生成与落盘能力 + self._upload = UploadService(file_dao) + + # ---------------- 提交 ---------------- + + def submit(self, file: UploadFile, owner_cookie: str) -> tuple[PdfJobOut, int]: + """上传原始文件并创建 pending 任务，返回 (任务视图, source_file_id)。 + + 校验大小 ≤ max_size_bytes 与扩展名白名单；复用 UploadService 流式落盘 + 入库为 UploadedFile，再建 PdfJob 关联。转换不在此处执行（由 controller + 调 schedule_convert 异步触发）。 + """ + filename = file.filename or "upload.epub" + if not pdf_converter.is_supported(filename): + raise HTTPException(400, "仅支持 epub 文件") + + # 大小校验：UploadFile 流式无已知长度，先读一遍统计并重置（小文件可行）， + # 对大文件更优的做法是流式计数，这里复用 stream_to_disk 后按 size 校验。 + resp = self._upload.stream_to_disk(file, source="pdf", uploaded_by=owner_cookie) + if resp.size_bytes > self.cfg.max_size_bytes: + # 超限：清理刚落盘的文件与 DB 行，保持无副作用 + try: + self._upload.delete_file(resp.id) + except Exception: # pragma: no cover + logger.warning("清理超限文件失败 id=%s", resp.id) + raise HTTPException( + 413, + f"文件过大（{resp.size_bytes} > {self.cfg.max_size_bytes}，上限 250MB）", + ) + + job = PdfJob( + owner_cookie=owner_cookie, + source_file_id=resp.id, + source_filename=filename, + source_size=resp.size_bytes, + status="pending", + progress=0, + ) + job = self.job_dao.create(job) + logger.info("PDF 任务已创建 job=%s file=%s size=%d", job.id, filename, resp.size_bytes) + return PdfJobOut.model_validate(job), resp.id + + def schedule_convert(self, job_id: int) -> None: + """在当前事件循环起一个后台 task 执行转换（不阻塞调用方）。 + + 用 to_thread 跑同步转换器；转换中分段更新 progress。 + 重要：后台 task 必须用独立 DB Session（请求 Session 在请求结束后即关闭）， + 故 _convert_async 内部经 _fresh_service 重建带新 Session 的 service。 + """ + asyncio.create_task(self._convert_async(job_id)) + + async def _convert_async(self, job_id: int) -> None: + """后台转换：pending -> converting(进度) -> done/failed。 + + 每个阶段用独立 Session（_fresh_service），避免引用请求 Session（已关闭）。 + """ + try: + await asyncio.to_thread(self._run_with_fresh_session, "_mark_converting", job_id) + await asyncio.wait_for( + asyncio.to_thread(self._run_with_fresh_session, "_do_convert", job_id), + timeout=self.cfg.convert_timeout_seconds, + ) + except asyncio.TimeoutError: + await asyncio.to_thread(self._run_with_fresh_session, "_mark_failed", job_id, "转换超时") + except Exception as exc: + await asyncio.to_thread(self._run_with_fresh_session, "_mark_failed", job_id, f"转换失败：{exc}") + + @staticmethod + def _run_with_fresh_session(method_name: str, *args) -> None: + """用独立 DB Session 构造新 PdfService 实例执行其方法。 + + 后台线程不能复用请求的 Session（请求结束即关闭），故每次操作新建 Session。 + method_name 是 PdfService 实例方法名，在此用新 service 调用对应方法。 + """ + from ..database import get_session_local + db = get_session_local()() + try: + svc = PdfService(PdfJobDAO(db), UploadedFileDAO(db)) + getattr(svc, method_name)(*args) + finally: + db.close() + + # ---------------- 同步转换实现（在线程中执行，用独立 Session） ---------------- + + def _mark_converting(self, job_id: int) -> None: + job = self.job_dao.get(job_id) + if job is None: + return + job.status = "converting" + job.progress = 5 + self.job_dao.update(job) + + def _do_convert(self, job_id: int) -> None: + """执行转换并落产物 PDF 为 UploadedFile。""" + job = self.job_dao.get(job_id) + if job is None: + return + + src_row = self.file_dao.get_by_id(job.source_file_id) + if src_row is None: + self._mark_failed(job_id, "原始文件记录丢失") + return + src_path = (self.upload_root / src_row.storage_path).resolve() + if not src_path.exists(): + self._mark_failed(job_id, "原始文件实体不存在") + return + + # 产物 PDF 存储路径（复用 UploadService 的路径生成，扩展名固定 .pdf） + rel_path, abs_path = self._upload.make_storage_path("result.pdf") + part_path = abs_path.with_name(abs_path.name + ".part") + + # 更新进度到「渲染中」 + job.status = "converting" + job.progress = 30 + self.job_dao.update(job) + + try: + pdf_converter.convert_to_pdf(src_path, part_path) + except Exception: + part_path.unlink(missing_ok=True) + raise + + job.progress = 80 + self.job_dao.update(job) + + # 落产物 UploadedFile（复用 commit_entity 的原子改名 + 入库） + from ..models.uploaded_file import UploadedFile + import hashlib + size = part_path.stat().st_size + sha256 = self._hash_file(part_path) + entity = UploadedFile( + storage_path=str(rel_path), + original_filename=Path(job.source_filename).stem + ".pdf", + content_type="application/pdf", + size_bytes=size, + sha256=sha256, + source="pdf-convert", + uploaded_by=job.owner_cookie, + ) + saved = self._upload.commit_entity(entity, part_path, abs_path) + + job.output_file_id = saved.id + job.status = "done" + job.progress = 100 + self.job_dao.update(job) + logger.info("PDF 转换完成 job=%s output_file_id=%s", job_id, saved.id) + + def _mark_failed(self, job_id: int, message: str) -> None: + job = self.job_dao.get(job_id) + if job is None: + return + job.status = "failed" + job.error_message = message[:500] + self.job_dao.update(job) + logger.warning("PDF 转换失败 job=%s: %s", job_id, message) + + def _hash_file(self, path: Path) -> str: + import hashlib + h = hashlib.sha256() + with path.open("rb") as f: + while chunk := f.read(self._upload.chunk_bytes): + h.update(chunk) + return h.hexdigest() + + # ---------------- 查询 ---------------- + + def list_for_user(self, owner_cookie: str, limit: int = 100, offset: int = 0) -> PdfJobListResponse: + limit = min(max(limit, 1), self.cfg.list_limit) + offset = max(offset, 0) + total = self.job_dao.count_for_user(owner_cookie) + rows = self.job_dao.list_for_user(owner_cookie, limit=limit, offset=offset) + return PdfJobListResponse(total=total, items=[PdfJobOut.model_validate(r) for r in rows]) + + def list_for_admin(self, limit: int = 100, offset: int = 0) -> PdfJobListResponse: + limit = min(max(limit, 1), self.cfg.list_limit) + offset = max(offset, 0) + total = self.job_dao.count_all() + rows = self.job_dao.list_all(limit=limit, offset=offset) + return PdfJobListResponse(total=total, items=[PdfJobOut.model_validate(r) for r in rows]) + + def get_job(self, job_id: int, owner_cookie: str) -> PdfJobOut: + """用户查询单任务：仅当归属本人且未软删时可见。""" + job = self.job_dao.get(job_id) + if job is None or job.owner_cookie != owner_cookie or job.user_deleted: + raise HTTPException(404, "任务不存在") + return PdfJobOut.model_validate(job) + + def get_output_path(self, job_id: int, owner_cookie: str) -> tuple[PdfJobOut, Path, str]: + """返回 (任务视图, 产物磁盘绝对路径, 下载文件名) 供下载。 + + 用户仅可下载自己未软删且已完成的任务产物。 + """ + job = self.job_dao.get(job_id) + if job is None or job.owner_cookie != owner_cookie or job.user_deleted: + raise HTTPException(404, "任务不存在") + if job.status != "done" or job.output_file_id is None: + raise HTTPException(409, "任务尚未完成，无法下载") + out_row = self.file_dao.get_by_id(job.output_file_id) + if out_row is None: + raise HTTPException(410, "产物文件记录丢失") + path = (self.upload_root / out_row.storage_path).resolve() + if not path.exists(): + raise HTTPException(410, "产物文件实体不存在") + return PdfJobOut.model_validate(job), path, out_row.original_filename + + # ---------------- 删除 ---------------- + + def user_delete(self, job_id: int, owner_cookie: str) -> bool: + """用户软删：仅置 user_deleted=true，磁盘与 DB 行保留（管理页仍可见）。""" + job = self.job_dao.get(job_id) + if job is None or job.owner_cookie != owner_cookie or job.user_deleted: + return False + job.user_deleted = True + job.deleted_at = datetime.now(timezone.utc) + self.job_dao.update(job) + logger.info("用户软删 PDF 任务 job=%s", job_id) + return True + + def admin_delete(self, job_id: int) -> bool: + """管理员硬删：删原始/产物磁盘文件 + UploadedFile 行 + PdfJob 行。 + + 真正删除，不可恢复。磁盘删除失败仅记日志，仍清 DB 行保证列表不再显示。 + """ + job = self.job_dao.get(job_id) + if job is None: + return False + # 删原始文件 + self._safe_delete_file(job.source_file_id) + # 删产物文件（若有） + if job.output_file_id is not None: + self._safe_delete_file(job.output_file_id) + # 删 PdfJob 行 + self.job_dao.db.delete(job) + self.job_dao.db.commit() + logger.info("管理员硬删 PDF 任务 job=%s", job_id) + return True + + def _safe_delete_file(self, file_id: int) -> None: + """删 UploadedFile 磁盘文件 + DB 行；失败只记日志不阻断。""" + row = self.file_dao.get_by_id(file_id) + if row is None: + return + path = (self.upload_root / row.storage_path).resolve() + try: + Path(path).unlink(missing_ok=True) + except Exception as exc: # pragma: no cover + logger.warning("删除文件失败 file_id=%s path=%s: %s", file_id, path, exc) + try: + self.file_dao.db.delete(row) + self.file_dao.db.commit() + except Exception as exc: # pragma: no cover + logger.warning("删除文件 DB 行失败 file_id=%s: %s", file_id, exc) diff --git a/config.example.yaml b/config.example.yaml index 58a162a..c74e23e 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -64,3 +64,13 @@ whiteboard: heartbeat_miss_threshold: 5 max_board_id_length: 64 # board_id 合法字符 [a-zA-Z0-9_-]，长度上限 list_limit: 100 # 管理页单次列表上限 + +pdf: + # PDF 转换服务：用户上传 epub -> 后台转换为 PDF -> 显示进度并下载。 + # 用户侧（上传/查看/下载/软删）凭 httpOnly cookie 标识；管理侧（列表/硬删）走 docs 同款 Basic Auth。 + # 原始文件与产物 PDF 复用 storage.upload_dir 落盘。转换用纯 Python（ebooklib + weasyprint）。 + max_size_bytes: 262144000 # 单文件上限 250 MiB + convert_timeout_seconds: 600 # 转换超时（秒），超大文件兜底 + list_limit: 100 # 列表单次上限 + cookie_name: zk_pdf # 用户标识 cookie 名 + cookie_max_age_seconds: 31536000 # cookie 有效期 1 年 diff --git a/requirements.txt b/requirements.txt index 173e6d5..f8efb9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,10 +2,13 @@ fastapi==0.115.6 uvicorn[standard]==0.34.0 python-multipart==0.0.20 psutil==6.1.1 -SQLAlchemy==2.0.36 +SQLAlchemy==2.0.51 PyMySQL==1.1.1 pydantic-settings==2.7.0 PyYAML==6.0.2 asyncssh==2.18.0 bcrypt==4.2.1 httpx==0.28.1 +# PDF 转换：ebooklib 解析 epub，weasyprint 渲染 HTML/CSS 为 PDF（纯 Python，无需 calibre/xvfb） +ebooklib==0.20 +weasyprint==69.0 diff --git a/sql/schema.sql b/sql/schema.sql index 2b1888d..dd49a98 100644 --- a/sql/schema.sql +++ b/sql/schema.sql @@ -68,3 +68,27 @@ CREATE TABLE IF NOT EXISTS `whiteboard` ( UNIQUE KEY `uq_board_id` (`board_id`), KEY `idx_updated_at` (`updated_at`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- PDF 转换任务表。由 app/services/pdf_service.py 使用。 +-- 原始文件与产物 PDF 复用 uploaded_file 存储，本表只记录关系与转换状态。 +-- 删除两级：user_deleted（用户软删，管理页仍可见）/ admin 硬删（真正删除）。 +CREATE TABLE IF NOT EXISTS `pdf_job` ( + `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `owner_cookie` VARCHAR(64) NOT NULL COMMENT '用户标识（httpOnly cookie 值，uuid4 hex）', + `source_file_id` BIGINT UNSIGNED NOT NULL COMMENT '原始上传文件 -> uploaded_file.id', + `source_filename` VARCHAR(512) NOT NULL, + `source_size` BIGINT UNSIGNED NOT NULL DEFAULT 0, + `output_file_id` BIGINT UNSIGNED NULL DEFAULT NULL COMMENT '产物 PDF -> uploaded_file.id，转换完成前 NULL', + `status` VARCHAR(16) NOT NULL DEFAULT 'pending' COMMENT 'pending/converting/done/failed', + `progress` INT NOT NULL DEFAULT 0 COMMENT '转换进度 0-100', + `error_message` VARCHAR(512) NOT NULL DEFAULT '', + `user_deleted` TINYINT(1) NOT NULL DEFAULT 0 COMMENT '用户软删标记', + `deleted_at` DATETIME NULL DEFAULT NULL COMMENT '用户软删时间', + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + KEY `idx_owner_cookie` (`owner_cookie`), + KEY `idx_status` (`status`), + KEY `idx_user_deleted` (`user_deleted`), + KEY `idx_created_at` (`created_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/static/pdf.css b/static/pdf.css new file mode 100644 index 0000000..9bb258f --- /dev/null +++ b/static/pdf.css @@ -0,0 +1,40 @@ +/* PDF 转换页专属样式：上传区、进度条、任务表格。复用 common.css 的 .btn/.tag/table 等。 */ +.uploader { margin: 0 0 1.6em; } +.dropzone { + display: flex; flex-direction: column; align-items: center; justify-content: center; + gap: 0.6em; padding: 2.4em 1.2em; min-height: 160px; + border: 2px dashed var(--border); border-radius: var(--radius); + background: var(--surface); text-align: center; transition: border-color 0.15s, background 0.15s; +} +.dropzone.is-drag { border-color: var(--primary); background: var(--primary-soft); } +.dropzone__hint { color: var(--text-dim); font-size: 0.95em; } +.dropzone__limit { color: var(--text-dim); font-size: 0.8em; } +.dropzone .btn.primary { padding: 0.55em 1.4em; } + +.upload-progress { margin-top: 0.8em; } +.upload-progress.hidden { display: none; } +.upload-progress__name { font-size: 0.88em; color: var(--text); margin-bottom: 0.3em; word-break: break-all; } + +.bar { height: 8px; background: var(--surface-2); border-radius: 6px; overflow: hidden; border: 1px solid var(--border); } +.bar__fill { height: 100%; width: 0; background: var(--primary); border-radius: 6px; transition: width 0.2s; } +.bar--sm { display: inline-block; width: 90px; vertical-align: middle; height: 6px; } + +.jobs-table th.col-size { width: 8em; } +.jobs-table th.col-status { width: 12em; } +.jobs-table th.col-time { width: 11em; } +.jobs-table th.col-act { width: 9em; text-align: right; } +.jobs-table td.col-act { text-align: right; white-space: nowrap; } +.jobs-table td.col-name { font-weight: 600; word-break: break-all; } +.jobs-table td.col-act .btn { padding: 0.3em 0.8em; font-size: 0.85em; margin-left: 0.3em; } + +.prog { display: inline-flex; align-items: center; gap: 0.5em; font-size: 0.82em; } +.prog__label { color: var(--text-dim); white-space: nowrap; } + +.hidden { display: none; } +.skel, .empty { padding: 1.6em; text-align: center; color: var(--text-dim); font-size: 0.9em; } + +@media (max-width: 640px) { + .jobs-table th, .jobs-table td { padding: 0.5em 0.4em; } + .jobs-table th.col-time, .jobs-table td.col-time { font-size: 0.78em; } + .dropzone { padding: 1.8em 0.8em; min-height: 130px; } +} diff --git a/static/pdf.html b/static/pdf.html new file mode 100644 index 0000000..adf770d --- /dev/null +++ b/static/pdf.html @@ -0,0 +1,44 @@ + + + + + +PDF 转换 - zikai + + + + +
+

PDF 转换

+

上传 epub 文件转为 PDF，转换完成后可下载。单文件上限 250MB。凭浏览器 cookie 记住你的任务。

+ +
+
+
拖入 .epub 文件，或
+ +
仅支持 epub，最大 250MB
+
+ +
+ +
+ + +
+ +
+
加载中…
+
+ +

管理页 · zikai file service

+
+ + + + diff --git a/static/pdf.js b/static/pdf.js new file mode 100644 index 0000000..cf01e38 --- /dev/null +++ b/static/pdf.js @@ -0,0 +1,191 @@ +/* PDF 转换页：上传 epub + 进度轮询 + 下载 + 用户软删。 + 复用 common.js 的 el/toast/fmtBytes/fmtTime/api 工具。 + state.items 缓存当前用户任务列表；轮询未完成任务进度直至终态。 */ +(function () { + "use strict"; + const { el, toast, fmtBytes, fmtTime, api } = window.ZK; + const dropzone = document.getElementById("dropzone"); + const fileInput = document.getElementById("fileInput"); + const uploadProgress = document.getElementById("uploadProgress"); + const uploadName = document.getElementById("uploadName"); + const uploadBar = document.getElementById("uploadBar"); + const refreshBtn = document.getElementById("refresh"); + const countEl = document.getElementById("count"); + const listEl = document.getElementById("list"); + + const state = { items: [], polling: null }; + + refreshBtn.addEventListener("click", () => load()); + + // ---------- 上传 ---------- + dropzone.addEventListener("dragover", (e) => { + e.preventDefault(); + dropzone.classList.add("is-drag"); + }); + dropzone.addEventListener("dragleave", () => dropzone.classList.remove("is-drag")); + dropzone.addEventListener("drop", (e) => { + e.preventDefault(); + dropzone.classList.remove("is-drag"); + if (e.dataTransfer.files.length) handleFile(e.dataTransfer.files[0]); + }); + fileInput.addEventListener("change", () => { + if (fileInput.files.length) handleFile(fileInput.files[0]); + fileInput.value = ""; + }); + + function handleFile(file) { + const name = file.name || ""; + if (!name.toLowerCase().endsWith(".epub")) { + toast("仅支持 epub 文件", "err"); + return; + } + if (file.size > 250 * 1024 * 1024) { + toast("文件超过 250MB 上限", "err"); + return; + } + uploadFile(file); + } + + function uploadFile(file) { + uploadProgress.classList.remove("hidden"); + uploadName.textContent = file.name; + uploadBar.style.width = "0%"; + const xhr = new XMLHttpRequest(); + xhr.open("POST", "/api/pdf/jobs"); + xhr.upload.onprogress = (e) => { + if (e.lengthComputable) { + uploadBar.style.width = Math.round((e.loaded / e.total) * 100) + "%"; + } + }; + xhr.onload = () => { + uploadProgress.classList.add("hidden"); + if (xhr.status === 200) { + toast("已提交，转换中…"); + load(); + } else { + let msg = "上传失败"; + try { + const b = JSON.parse(xhr.responseText); + msg = b.detail || msg; + } catch {} + toast(msg, "err"); + } + }; + xhr.onerror = () => { + uploadProgress.classList.add("hidden"); + toast("网络错误", "err"); + }; + const fd = new FormData(); + fd.append("file", file); + xhr.send(fd); + } + + // ---------- 列表 ---------- + async function load() { + listEl.innerHTML = '
加载中…
'; + countEl.textContent = ""; + try { + const res = await api("/api/pdf/jobs"); + if (!res.ok) throw new Error("HTTP " + res.status); + const body = await res.json(); + state.items = body.items || []; + countEl.textContent = `共 ${body.total} 个任务`; + render(); + schedulePoll(); + } catch (e) { + listEl.innerHTML = '
加载失败：' + (e.message || e) + "
"; + } + } + + function render() { + if (!state.items.length) { + listEl.innerHTML = '
还没有任务。上传一个 epub 试试吧。
'; + return; + } + const tbl = el("table", { class: "jobs-table" }); + tbl.appendChild(el("thead", null, + el("tr", null, + el("th", { class: "col-name" }, "文件名"), + el("th", { class: "col-size" }, "大小"), + el("th", { class: "col-status" }, "状态"), + el("th", { class: "col-time" }, "创建时间"), + el("th", { class: "col-act" }, "操作") + ) + )); + const tbody = el("tbody", null); + for (const j of state.items) tbody.appendChild(renderRow(j)); + tbl.appendChild(tbody); + listEl.innerHTML = ""; + listEl.appendChild(tbl); + } + + function renderRow(j) { + return el("tr", { dataset: { id: j.id } }, + el("td", { class: "col-name" }, j.source_filename), + el("td", { class: "col-size mono" }, fmtBytes(j.source_size)), + el("td", { class: "col-status" }, statusCell(j)), + el("td", { class: "col-time muted" }, fmtTime(j.created_at)), + el("td", { class: "col-act" }, actionsCell(j)) + ); + } + + function statusCell(j) { + if (j.status === "done") return el("span", { class: "tag ok" }, "完成"); + if (j.status === "failed") return el("span", { class: "tag err", title: j.error_message || "" }, "失败"); + // pending / converting 显示进度条 + const wrap = el("span", { class: "prog" }); + wrap.appendChild(el("span", { class: "prog__label" }, j.status === "converting" ? "转换中" : "排队中")); + wrap.appendChild(el("span", { class: "bar bar--sm" }, + el("span", { class: "bar__fill", style: "width:" + (j.progress || 0) + "%" }) + )); + return wrap; + } + + function actionsCell(j) { + const cell = el("span", null); + if (j.status === "done") { + cell.appendChild(el("a", { class: "btn primary", href: `/api/pdf/jobs/${j.id}/download`, download: "" }, "下载")); + } + cell.appendChild(el("button", { class: "btn danger", onclick: () => removeOne(j) }, "删除")); + return cell; + } + + // ---------- 轮询未完成任务 ---------- + function schedulePoll() { + if (state.polling) clearInterval(state.polling); + const pending = state.items.filter((j) => j.status === "pending" || j.status === "converting"); + if (!pending.length) return; + state.polling = setInterval(async () => { + let stillPending = false; + for (const j of pending) { + try { + const res = await api(`/api/pdf/jobs/${j.id}`); + if (res.ok) { + const fresh = await res.json(); + Object.assign(j, fresh); + } + } catch {} + if (j.status === "pending" || j.status === "converting") stillPending = true; + } + render(); + if (!stillPending) { + clearInterval(state.polling); + state.polling = null; + } + }, 1500); + } + + async function removeOne(j) { + if (!confirm(`确定删除「${j.source_filename}」？\n删除后将不再显示（管理员仍可见，需管理员彻底删除才会清除）。`)) return; + try { + const res = await api(`/api/pdf/jobs/${j.id}`, { method: "DELETE" }); + if (!res.ok) throw new Error("HTTP " + res.status); + await load(); + toast("已删除"); + } catch (e) { + toast("删除失败：" + (e.message || e), "err"); + } + } + + load(); +})(); diff --git a/static/pdf_admin.css b/static/pdf_admin.css new file mode 100644 index 0000000..a07afaf --- /dev/null +++ b/static/pdf_admin.css @@ -0,0 +1,18 @@ +/* PDF 转换管理页专属样式（叠加在 pdf.css 之上）。标注「已删除」行、列宽调整。 */ +.admin-table th.col-size { width: 7em; } +.admin-table th.col-status { width: 9em; } +.admin-table th.col-del { width: 8em; } +.admin-table th.col-time { width: 11em; } +.admin-table th.col-act { width: 11em; text-align: right; } +.admin-table td.col-act { text-align: right; white-space: nowrap; } +.admin-table td.col-act .btn { padding: 0.3em 0.8em; font-size: 0.85em; margin-left: 0.3em; } + +/* 用户已软删的行：淡化背景提示 */ +.admin-table tbody tr.row-soft-deleted td { background: var(--danger-soft); } +.admin-table tbody tr.row-soft-deleted:hover td { background: var(--danger-soft); } +.row-removed { opacity: 0; transition: opacity 0.25s; } + +@media (max-width: 640px) { + .admin-table th, .admin-table td { padding: 0.5em 0.4em; } + .admin-table th.col-time, .admin-table td.col-time { font-size: 0.78em; } +} diff --git a/static/pdf_admin.html b/static/pdf_admin.html new file mode 100644 index 0000000..9fde0d8 --- /dev/null +++ b/static/pdf_admin.html @@ -0,0 +1,30 @@ + + + + + +PDF 转换管理 - zikai + + + + + +
+

PDF 转换管理

+

查看全部转换任务（含用户已软删的，标注是否已删除），可硬删（真正删除磁盘与记录）。

+ +
+ + +
+ +
+
加载中…
+
+ +

PDF 转换页 · zikai file service

+
+ + + + diff --git a/static/pdf_admin.js b/static/pdf_admin.js new file mode 100644 index 0000000..4b1c348 --- /dev/null +++ b/static/pdf_admin.js @@ -0,0 +1,97 @@ +/* PDF 转换管理页：拉取 /api/admin/pdf/jobs、渲染表格（含「是否已删除」列）、硬删。 */ +(function () { + "use strict"; + const { el, toast, fmtBytes, fmtTime, api } = window.ZK; + const listEl = document.getElementById("list"); + const countEl = document.getElementById("count"); + const refreshBtn = document.getElementById("refresh"); + + refreshBtn.addEventListener("click", load); + + async function load() { + listEl.innerHTML = '
加载中…
'; + countEl.textContent = ""; + try { + const res = await api("/api/admin/pdf/jobs?limit=500&offset=0"); + if (!res.ok) throw new Error("HTTP " + res.status); + const body = await res.json(); + render(body.items || []); + countEl.textContent = `共 ${body.total} 个任务`; + } catch (e) { + listEl.innerHTML = '
加载失败：' + (e.message || e) + "
"; + } + } + + function render(items) { + if (!items.length) { + listEl.innerHTML = '
还没有任务。
'; + return; + } + const table = el("table", { class: "jobs-table admin-table" }); + const thead = el("thead", null, + el("tr", null, + el("th", { class: "col-name" }, "文件名"), + el("th", { class: "col-size" }, "大小"), + el("th", { class: "col-status" }, "状态"), + el("th", { class: "col-del" }, "是否已删除"), + el("th", { class: "col-time" }, "创建时间"), + el("th", { class: "col-act" }, "操作") + ) + ); + const tbody = el("tbody", null); + for (const j of items) tbody.appendChild(renderRow(j)); + table.appendChild(thead); + table.appendChild(tbody); + listEl.innerHTML = ""; + listEl.appendChild(table); + } + + function renderRow(j) { + const row = el("tr", { class: j.user_deleted ? "row-soft-deleted" : "" }, + el("td", { class: "col-name" }, j.source_filename), + el("td", { class: "col-size mono" }, fmtBytes(j.source_size)), + el("td", { class: "col-status" }, statusCell(j)), + el("td", { class: "col-del" }, deletedCell(j)), + el("td", { class: "col-time muted" }, fmtTime(j.created_at)), + el("td", { class: "col-act" }, + j.status === "done" + ? el("a", { class: "btn", href: `/api/admin/files/${j.output_file_id}/download`, download: "" }, "下载产物") + : el("span", { class: "muted" }, "-"), + el("button", { class: "btn danger", onclick: () => remove(j, row) }, "硬删") + ) + ); + return row; + } + + function statusCell(j) { + if (j.status === "done") return el("span", { class: "tag ok" }, "完成"); + if (j.status === "failed") + return el("span", { class: "tag err", title: j.error_message || "" }, "失败"); + return el("span", { class: "tag warn" }, j.status === "converting" ? `转换中 ${j.progress}%` : "排队中"); + } + + function deletedCell(j) { + if (j.user_deleted) { + return el("span", { class: "tag err", title: fmtTime(j.deleted_at) }, "已删除"); + } + return el("span", { class: "muted" }, "否"); + } + + async function remove(j, row) { + if (!confirm(`确定硬删「${j.source_filename}」？\n将真正删除原始文件与产物 PDF（磁盘 + 记录），不可恢复。`)) return; + try { + const res = await api(`/api/admin/pdf/jobs/${j.id}`, { method: "DELETE" }); + if (res.status === 404) { toast("任务已不存在"); } + else if (!res.ok) throw new Error("HTTP " + res.status); + row.classList.add("row-removed"); + setTimeout(() => row.remove(), 250); + toast("已硬删"); + const m = (countEl.textContent || "").match(/(\d+)/); + if (m) countEl.textContent = `共 ${Math.max(0, Number(m[1]) - 1)} 个任务`; + } catch (e) { + toast("删除失败：" + (e.message || e), "err"); + } + } + + load(); +})(); diff --git a/tests/test_pdf_service.py b/tests/test_pdf_service.py new file mode 100644 index 0000000..e6a184e --- /dev/null +++ b/tests/test_pdf_service.py @@ -0,0 +1,281 @@ +"""PDF 转换服务测试（pytest + TestClient）。 + +用 SQLite 内存库覆盖 get_db，免依赖 MySQL；用临时目录覆盖上传根目录。 +覆盖完整链路：上传 epub -> 轮询至 done -> 下载有效 PDF -> 用户软删 -> +管理页仍可见 -> 管理员硬删 -> 真正删除。另测大小超限与格式校验。 +""" + +from __future__ import annotations + +import io +import time +import zipfile +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import create_engine, text +from sqlalchemy.orm import sessionmaker + +from app import config as config_module +from app import database as db_module +from app.database import Base, get_db +from app.models.uploaded_file import UploadedFile # noqa: F401 注册映射 + + +# ---------------- fixtures ---------------- + + +def _make_epub_bytes() -> bytes: + """构造一个最小的合法 epub（含 1 章节 + 1 PNG 图片 + CSS）。""" + from PIL import Image # 已是 weasyprint 依赖间接项，环境可用 + + png = io.BytesIO() + Image.new("RGBA", (8, 8), (26, 95, 180, 255)).save(png, format="PNG") + png_bytes = png.getvalue() + + ch = ( + '' + 'Ch1' + '' + '

Hello PDF

Test paragraph for conversion.

' + '

pic

' + ) + opf = ( + '' + '' + '' + 'Turn:uuid:t' + 'en' + '' + '' + '' + '' + '' + '' + ) + ncx = ( + '' + '' + '' + 'T' + 'Ch1' + '' + ) + css = "h1{color:#1a5fb4} p{line-height:1.6}" + + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as z: + z.writestr("mimetype", "application/epub+zip", compress_type=zipfile.ZIP_STORED) + z.writestr("OEBPS/content.opf", opf, compress_type=zipfile.ZIP_DEFLATED) + z.writestr("OEBPS/toc.ncx", ncx, compress_type=zipfile.ZIP_DEFLATED) + z.writestr("OEBPS/style.css", css, compress_type=zipfile.ZIP_DEFLATED) + z.writestr("OEBPS/img.png", png_bytes, compress_type=zipfile.ZIP_DEFLATED) + z.writestr("OEBPS/ch1.xhtml", ch, compress_type=zipfile.ZIP_DEFLATED) + z.writestr("META-INF/container.xml", + '' + '', + compress_type=zipfile.ZIP_DEFLATED) + return buf.getvalue() + + +@pytest.fixture() +def client(tmp_path, monkeypatch): + """构造用隔离 MySQL 测试库 + 临时上传目录的 TestClient。 + + 用独立库 zikai_filesvc_test（与生产库隔离），每个 fixture 重建表，准确测试生产 + 行为（MySQL BigInteger 自增、线程安全连接）。后台转换经 asyncio.to_thread 在 + 独立线程执行，MySQL 连接池天然支持跨线程。 + """ + real_settings = config_module.get_settings() + # 用独立测试库 zikai_filesvc_test（与生产库隔离），准确测试生产行为 + test_settings = real_settings.model_copy(deep=True) + test_settings.database.database = "zikai_filesvc_test" + test_settings.pdf.convert_timeout_seconds = 60 + + # 全局 get_settings 被 lru_cache；config 与 database 两个模块各自 import 了它， + # 都需 patch，使后台线程（经 database.get_session_local）也读到测试库 + monkeypatch.setattr(config_module, "get_settings", lambda: test_settings) + monkeypatch.setattr(db_module, "get_settings", lambda: test_settings) + # 清掉已建的全局引擎/SessionLocal，下次 get_session_local() 用测试库重建 + db_module.dispose_engine() + + upload_dir = tmp_path / "uploads" + upload_dir.mkdir() + + # 复用全局 SessionLocal（指向测试库），保证请求路径与后台线程用同一库 + SessionLocal = db_module.get_session_local() + + # 覆盖 get_db（仍用全局 SessionLocal，但确保请求结束关闭） + def override_get_db(): + db = SessionLocal() + try: + yield db + finally: + db.close() + + # 上传目录指向临时目录 + monkeypatch.setattr(test_settings, "storage", test_settings.storage) + test_settings.storage.upload_dir = str(upload_dir) + # upload_service / pdf_service 内部 import 了 get_settings，需 patch 其引用 + import app.services.upload_service as us + import app.services.pdf_service as ps + import app.services.pdf_converter as pc # noqa: F401 + monkeypatch.setattr(us, "get_settings", lambda: test_settings) + monkeypatch.setattr(ps, "get_settings", lambda: test_settings) + + # 每次测试前清空重建表，保证隔离 + Base.metadata.drop_all(bind=db_module.get_engine()) + Base.metadata.create_all(bind=db_module.get_engine()) + + from app.main import app + app.dependency_overrides[get_db] = override_get_db + with TestClient(app) as c: + yield c + app.dependency_overrides.clear() + Base.metadata.drop_all(bind=db_module.get_engine()) + db_module.dispose_engine() + + +def _wait_done(client, job_id, cookie, timeout=60): + """轮询任务状态直到 done/failed 或超时。""" + deadline = time.time() + timeout + last = None + while time.time() < deadline: + r = client.get(f"/api/pdf/jobs/{job_id}", cookies=cookie) + assert r.status_code == 200, r.text + last = r.json() + if last["status"] in ("done", "failed"): + return last + time.sleep(0.3) + raise AssertionError(f"任务未在 {timeout}s 内完成，最后状态: {last}") + + +# ---------------- 测试 ---------------- + + +class TestSubmitAndConvert: + def test_upload_epub_converts_and_downloads(self, client): + epub = _make_epub_bytes() + # 首次上传：无 cookie，应下发新 cookie + r = client.post( + "/api/pdf/jobs", + files={"file": ("test.epub", epub, "application/epub+zip")}, + ) + assert r.status_code == 200, r.text + body = r.json() + assert body["set_cookie"] is True + assert "zk_pdf" in r.headers.get("set-cookie", "") + job = body["job"] + assert job["status"] == "pending" + job_id = job["id"] + cookie = {"zk_pdf": r.cookies.get("zk_pdf")} + + # 轮询至完成 + final = _wait_done(client, job_id, cookie) + assert final["status"] == "done", final + assert final["progress"] == 100 + + # 下载产物：应为有效 PDF + d = client.get(f"/api/pdf/jobs/{job_id}/download", cookies=cookie) + assert d.status_code == 200, d.text + assert d.headers["content-type"] == "application/pdf" + assert d.content[:5] == b"%PDF-" + assert len(d.content) > 100 + + def test_user_list_shows_own_jobs(self, client): + epub = _make_epub_bytes() + r = client.post("/api/pdf/jobs", files={"file": ("a.epub", epub, "application/epub+zip")}) + cookie = {"zk_pdf": r.cookies.get("zk_pdf")} + lst = client.get("/api/pdf/jobs", cookies=cookie) + assert lst.status_code == 200 + assert lst.json()["total"] == 1 + + # 另一用户（清空 client cookie jar 模拟全新浏览器，应下发新 cookie） + client.cookies.clear() + r2 = client.post("/api/pdf/jobs", files={"file": ("b.epub", epub, "application/epub+zip")}) + cookie2 = {"zk_pdf": r2.cookies.get("zk_pdf")} + assert cookie2["zk_pdf"] != cookie["zk_pdf"] # 确是不同用户 + lst2 = client.get("/api/pdf/jobs", cookies=cookie2) + assert lst2.json()["total"] == 1 # 只有自己的 1 个 + + def test_rejects_non_epub(self, client): + r = client.post( + "/api/pdf/jobs", + files={"file": ("note.txt", b"hello world", "text/plain")}, + ) + assert r.status_code == 400 + + def test_rejects_oversize(self, client, monkeypatch): + # 把上限调到极小，避免真的构造 250MB 文件 + import app.services.pdf_service as ps + real = ps.get_settings() + small = real.model_copy(deep=True) + small.pdf.max_size_bytes = 100 + monkeypatch.setattr(ps, "get_settings", lambda: small) + monkeypatch.setattr(ps, "get_settings", lambda: small) + # controller 的 _resolve_cookie 也读 get_settings，但 submit 内 service 用 ps.get_settings + epub = _make_epub_bytes() + r = client.post("/api/pdf/jobs", files={"file": ("big.epub", epub, "application/epub+zip")}) + assert r.status_code == 413 + + +class TestDeleteSemantics: + def test_user_soft_delete_admin_still_visible_then_admin_hard_delete(self, client): + epub = _make_epub_bytes() + r = client.post("/api/pdf/jobs", files={"file": ("del.epub", epub, "application/epub+zip")}) + cookie = {"zk_pdf": r.cookies.get("zk_pdf")} + job_id = r.json()["job"]["id"] + _wait_done(client, job_id, cookie) + + # 用户软删 + d = client.delete(f"/api/pdf/jobs/{job_id}", cookies=cookie) + assert d.status_code == 200 + assert d.json()["deleted"] is True + + # 用户列表不再可见 + lst = client.get("/api/pdf/jobs", cookies=cookie) + assert lst.json()["total"] == 0 + + # 管理页仍可见且标注已删除 + admin_auth = ("admin", "testpass123") + al = client.get("/api/admin/pdf/jobs", auth=admin_auth) + assert al.status_code == 200 + items = al.json()["items"] + assert any(it["id"] == job_id and it["user_deleted"] is True for it in items) + assert any(it["id"] == job_id and it["deleted_at"] is not None for it in items) + + # 用户已无法下载（已软删） + dl = client.get(f"/api/pdf/jobs/{job_id}/download", cookies=cookie) + assert dl.status_code == 404 + + # 管理员硬删 + ad = client.delete(f"/api/admin/pdf/jobs/{job_id}", auth=admin_auth) + assert ad.status_code == 200 + assert ad.json()["deleted"] is True + + # 管理页也不再可见 + al2 = client.get("/api/admin/pdf/jobs", auth=admin_auth) + assert not any(it["id"] == job_id for it in al2.json()["items"]) + + def test_admin_requires_auth(self, client): + r = client.get("/api/admin/pdf/jobs") + assert r.status_code == 401 + r = client.get("/api/admin/pdf/jobs", auth=("admin", "wrong")) + assert r.status_code == 401 + r = client.get("/api/admin/pdf/jobs", auth=("admin", "testpass123")) + assert r.status_code == 200 + + def test_user_cannot_access_others_job(self, client): + epub = _make_epub_bytes() + r1 = client.post("/api/pdf/jobs", files={"file": ("x.epub", epub, "application/epub+zip")}) + job_id = r1.json()["job"]["id"] + # 另一用户访问（清空 cookie jar 模拟全新浏览器） + client.cookies.clear() + r2 = client.post("/api/pdf/jobs", files={"file": ("y.epub", epub, "application/epub+zip")}) + cookie2 = {"zk_pdf": r2.cookies.get("zk_pdf")} + assert client.get(f"/api/pdf/jobs/{job_id}", cookies=cookie2).status_code == 404 + assert client.get(f"/api/pdf/jobs/{job_id}/download", cookies=cookie2).status_code == 404 + assert client.delete(f"/api/pdf/jobs/{job_id}", cookies=cookie2).status_code == 404