feat: 文件浏览页 + 共享白板 + 白板管理页（含补登记分片上传/隧道历史改动）

本次提交包含两批改动（7月2日遗留未入库 + 本次新功能），分述如下：【补登记：7月2日已上线但未提交的功能】 - 分片上传：chunk_upload_controller/service/dao + UploadSession model/schema，支持 4MiB 分片、乱序、断点续传、去重、幂等 complete；后台 reaper 清理过期会话。 - 反向隧道：tunnel_controller/service/dao + TunnelSession model/schema， SSH remote forward 经 /api/userPort/{userName} 反代到 user 本地服务。 - 上传页：views/upload_html.py（拖拽/多文件/分片/断点续传 UI）。 - config.py：StorageConfig.chunk_session_dir/ttl、TunnelConfig； requirements.txt 加 httpx；start.sh 清理 .work/ 残留； schema.sql 加 upload_session/tunnel_session 表；sftp_server 承载隧道转发。【本次新功能】 - 文件浏览页：GET /files（Basic Auth 同 docs）+ /api/admin/files（list/get/download/DELETE）。硬删除（DB 行 + 磁盘文件），删除后列表不再显示。前端 static/file_browser.*。 - 共享白板：GET /whiteboard/{id}（公开，不存在则新建）+ WS /ws/whiteboard/{id}。 MySQL 持久化（whiteboard 表），Canvas 实时同步，心跳 3s/5 次失活移除，清空/复制按钮，移动端兼容。WhiteboardHub 管理 {board_id: set[Connection]}， disconnect 幂等 + 空 set 清理防泄漏，broadcast 失败连接自动移除。 - 白板管理页：GET /whiteboard-admin（Basic Auth）+ /api/admin/whiteboards（list/DELETE）。删除时 hub.close_board 踢出在线连接。 - 清理：合并 UploadService.get_out_with_disk_path（下载/删除复用，消除重复 DB 读），移除无用 resolve_disk_path。 - config.py：WhiteboardConfig（heartbeat/threshold/board_id 长度/list_limit）； schema.sql 加 whiteboard 表；README 补新接口与心跳/内存说明。 - 验证：tests/manual_whiteboard_hub.py / _ws.py / _kick.py 全部通过。
2026-07-21 14:28:13 +00:00
parent e5a725fc91
commit fffba79022
48 changed files with 3569 additions and 216 deletions
--- a/app/services/upload_service.py
+++ b/app/services/upload_service.py
@@ -1,22 +1,35 @@
-"""上传服务：流式落盘 + 原子改名 + 元数据入库。"""
+"""上传服务：流式落盘 + 原子改名 + 元数据入库 + sha256 去重。
+
+两条上传路径（HTTP 整文件 / 分片拼接）共享本类的「存储路径生成 + 落库提交 +
+sha256 去重」逻辑，避免重复实现。SFTP 服务器仅作为文件暂存通道（chroot 到
+upload_root），不再有 HTTP 登记接口。
+"""

 from __future__ import annotations

 import hashlib
 import os
 import uuid
+from collections.abc import Iterator
 from datetime import datetime, timezone
 from pathlib import Path

-from fastapi import HTTPException, UploadFile
+from fastapi import UploadFile

 from ..config import get_settings
 from ..dao.uploaded_file_dao import UploadedFileDAO
 from ..models.uploaded_file import UploadedFile
 from ..schemas.file import FileUploadResponse, UploadedFileOut

-# SFTP 客户端登记前必须把文件先落到此目录（chroot 内）
-SFTP_INCOMING_DIR = "incoming"
+
+def hash_stream(chunks: Iterator[bytes]) -> tuple[int, str]:
+    """对一段字节块序列流式计算 (size, sha256)。供多条上传路径复用。"""
+    h = hashlib.sha256()
+    size = 0
+    for chunk in chunks:
+        size += len(chunk)
+        h.update(chunk)
+    return size, h.hexdigest()


 class UploadService:
@@ -36,7 +49,7 @@ class UploadService:

        失败仅会留下可识别的 ``.part`` 文件，由 start.sh 启动时统一清理。
        """
-        rel_path, abs_path = self._make_storage_path(file.filename or "")
+        rel_path, abs_path = self.make_storage_path(file.filename or "")
        part_path = abs_path.with_name(abs_path.name + ".part")
        size, digest = self._write_part(file, part_path)

@@ -49,34 +62,7 @@ class UploadService:
            source=source,
            uploaded_by=uploaded_by,
        )
-        return self._commit(entity, part_path, abs_path)
-
-    def register_sftp(
-        self, filename: str, original_filename: str, uploaded_by: str = "sftp",
-    ) -> FileUploadResponse:
-        """登记一个已通过 SFTP 落到 ``incoming/`` 下的文件。
-
-        相同 sha256 已存在时直接返回旧行并删掉新副本（去重）。
-        """
-        src_abs = self._validate_incoming_path(filename)
-        size, digest = self._hash_disk_file(src_abs)
-
-        existing = self.dao.get_by_sha256(digest)
-        if existing is not None:
-            src_abs.unlink(missing_ok=True)
-            return self._to_response(existing)
-
-        rel_path, abs_path = self._make_storage_path(original_filename or filename)
-        entity = UploadedFile(
-            storage_path=str(rel_path),
-            original_filename=os.path.basename(original_filename or src_abs.name),
-            content_type="",
-            size_bytes=size,
-            sha256=digest,
-            source="sftp",
-            uploaded_by=uploaded_by,
-        )
-        return self._commit(entity, src_abs, abs_path)
+        return self.commit_entity(entity, part_path, abs_path)

    # ---------------- 查询 ----------------

@@ -94,18 +80,26 @@ class UploadService:
        row = self.dao.get_by_sha256(sha256)
        return UploadedFileOut.model_validate(row) if row else None

-    def resolve_disk_path(self, file_id: int) -> Path | None:
-        row = self.dao.get_by_id(file_id)
-        return (self.upload_root / row.storage_path).resolve() if row else None
+    def get_out_with_disk_path(self, file_id: int) -> tuple[UploadedFileOut | None, Path | None]:
+        """合并查询：一次 DB 读取同时返回 (元数据, 磁盘绝对路径)。

-    # ---------------- 内部 ----------------
+        供下载/删除路径复用，避免原先 get_out + resolve_disk_path 各查一次的重复读。
+        """
+        row = self.dao.get_by_id(file_id)
+        if row is None:
+            return None, None
+        out = UploadedFileOut.model_validate(row)
+        path = (self.upload_root / row.storage_path).resolve()
+        return out, path
+
+    # ---------------- 共享 helper（供本类与 ChunkUploadService 复用） ----------------

    @staticmethod
    def _safe_ext(filename: str) -> str:
        return os.path.splitext(os.path.basename(filename))[1]

    @staticmethod
-    def _to_response(row: UploadedFile) -> FileUploadResponse:
+    def to_response(row: UploadedFile, *, deduplicated: bool = False) -> FileUploadResponse:
        return FileUploadResponse(
            id=row.id,
            filename=row.original_filename,
@@ -113,9 +107,10 @@ class UploadService:
            sha256=row.sha256,
            storage_path=row.storage_path,
            uploaded_at=row.uploaded_at,
+            deduplicated=deduplicated,
        )

-    def _make_storage_path(self, original_filename: str) -> tuple[Path, Path]:
+    def make_storage_path(self, original_filename: str) -> tuple[Path, Path]:
        """生成 ``(rel_path, abs_path)``；abs_path 的父目录已创建。"""
        now = datetime.now(timezone.utc)
        rel_dir = Path(f"{now:%Y}/{now:%m}")
@@ -123,21 +118,7 @@ class UploadService:
        rel_path = rel_dir / f"{uuid.uuid4().hex}{self._safe_ext(original_filename)}"
        return rel_path, self.upload_root / rel_path

-    def _validate_incoming_path(self, filename: str) -> Path:
-        """校验客户端给出的相对路径必须落在 upload_root/incoming/ 之下且是文件。"""
-        if not filename or filename.startswith(("/", "\\")):
-            raise HTTPException(400, "filename 必须是 incoming/ 下的相对路径")
-        incoming_root = (self.upload_root / SFTP_INCOMING_DIR).resolve()
-        try:
-            abs_path = (self.upload_root / filename).resolve()
-            abs_path.relative_to(incoming_root)
-        except ValueError:
-            raise HTTPException(400, f"filename 必须落在 {SFTP_INCOMING_DIR}/ 之下")
-        if not abs_path.is_file():
-            raise HTTPException(404, f"文件不存在或不是普通文件：{filename}")
-        return abs_path
-
-    def _commit(
+    def commit_entity(
        self, entity: UploadedFile, src_path: Path, dest_path: Path,
    ) -> FileUploadResponse:
        """落 DB 行后把 src_path 原子改名到 dest_path；任一失败回滚已生成的副作用。"""
@@ -154,17 +135,25 @@ class UploadService:
            finally:
                src_path.unlink(missing_ok=True)
            raise
-        return self._to_response(saved)
+        return self.to_response(saved)

-    def _hash_disk_file(self, path: Path) -> tuple[int, str]:
-        """流式读取磁盘文件，返回 (size, sha256)。"""
-        h = hashlib.sha256()
-        size = 0
-        with path.open("rb") as fh:
-            while chunk := fh.read(self.chunk_bytes):
-                size += len(chunk)
-                h.update(chunk)
-        return size, h.hexdigest()
+    def dedup_or_commit(
+        self, entity: UploadedFile, src_path: Path,
+    ) -> FileUploadResponse:
+        """公共尾部：按 entity.sha256 去重，命中则删 src 返回旧行；否则 commit。
+
+        供分片拼接等「先算出 sha256 再决定落盘」的路径复用，与 stream_to_disk
+        （边写边算、无独立 src）的区别在于这里 sha256 已在 entity 上。
+        """
+        existing = self.dao.get_by_sha256(entity.sha256)
+        if existing is not None:
+            src_path.unlink(missing_ok=True)
+            return self.to_response(existing, deduplicated=True)
+        rel_path, abs_path = self.make_storage_path(entity.original_filename)
+        entity.storage_path = str(rel_path)
+        return self.commit_entity(entity, src_path, abs_path)
+
+    # ---------------- 内部 ----------------

    def _write_part(self, file: UploadFile, part_path: Path) -> tuple[int, str]:
        """把上传流写到 part_path 并 fsync；返回 (size, sha256)。失败时清理残品。"""