feat: 文件浏览页 + 共享白板 + 白板管理页(含补登记分片上传/隧道历史改动)

本次提交包含两批改动(7月2日遗留未入库 + 本次新功能),分述如下:

【补登记:7月2日已上线但未提交的功能】
- 分片上传:chunk_upload_controller/service/dao + UploadSession model/schema,
  支持 4MiB 分片、乱序、断点续传、去重、幂等 complete;后台 reaper 清理过期会话。
- 反向隧道:tunnel_controller/service/dao + TunnelSession model/schema,
  SSH remote forward 经 /api/userPort/{userName} 反代到 user 本地服务。
- 上传页:views/upload_html.py(拖拽/多文件/分片/断点续传 UI)。
- config.py:StorageConfig.chunk_session_dir/ttl、TunnelConfig;
  requirements.txt 加 httpx;start.sh 清理 .work/ 残留;
  schema.sql 加 upload_session/tunnel_session 表;sftp_server 承载隧道转发。

【本次新功能】
- 文件浏览页:GET /files(Basic Auth 同 docs)+ /api/admin/files(list/get/download/DELETE)。
  硬删除(DB 行 + 磁盘文件),删除后列表不再显示。前端 static/file_browser.*。
- 共享白板:GET /whiteboard/{id}(公开,不存在则新建)+ WS /ws/whiteboard/{id}。
  MySQL 持久化(whiteboard 表),Canvas 实时同步,心跳 3s/5 次失活移除,
  清空/复制按钮,移动端兼容。WhiteboardHub 管理 {board_id: set[Connection]},
  disconnect 幂等 + 空 set 清理防泄漏,broadcast 失败连接自动移除。
- 白板管理页:GET /whiteboard-admin(Basic Auth)+ /api/admin/whiteboards(list/DELETE)。
  删除时 hub.close_board 踢出在线连接。
- 清理:合并 UploadService.get_out_with_disk_path(下载/删除复用,消除重复 DB 读),
  移除无用 resolve_disk_path。
- config.py:WhiteboardConfig(heartbeat/threshold/board_id 长度/list_limit);
  schema.sql 加 whiteboard 表;README 补新接口与心跳/内存说明。
- 验证:tests/manual_whiteboard_hub.py / _ws.py / _kick.py 全部通过。
This commit is contained in:
zikai
2026-07-21 14:28:13 +00:00
parent e5a725fc91
commit fffba79022
48 changed files with 3569 additions and 216 deletions

View File

@@ -1,6 +1,7 @@
"""业务逻辑层Service"""
from .chunk_upload_service import ChunkUploadService
from .system_service import SystemService
from .upload_service import UploadService
__all__ = ["SystemService", "UploadService"]
__all__ = ["ChunkUploadService", "SystemService", "UploadService"]

View File

@@ -0,0 +1,247 @@
"""分片上传服务:会话管理 + 分片落盘 + 拼接 + sha256 去重 + 原子入库。
存储布局::
uploads/.work/<upload_id>/0.part 分片暂存
uploads/.work/<upload_id>/1.part
...
uploads/2026/07/<uuid>.<ext> complete 后的正式文件
complete 流程:
1. 校验分片齐全;
2. 顺序拼接为 <final>.part流式算 sha256
3. 复用 UploadService.dedup_or_commit按 sha256 去重命中则删会话返回旧行,
否则写 DB 行 + os.replace 原子改名到正式路径;
4. 清理 .work/<upload_id>/。
"""
from __future__ import annotations
import io
import logging
import os
import shutil
import uuid
from collections.abc import Iterator
from pathlib import Path
from fastapi import HTTPException
from ..config import get_settings
from ..dao.upload_session_dao import UploadSessionDAO
from ..dao.uploaded_file_dao import UploadedFileDAO
from ..models.upload_session import UploadSession
from ..models.uploaded_file import UploadedFile
from ..schemas.chunk import (
CreateSessionRequest,
CreateSessionResponse,
SessionStatusResponse,
)
from ..schemas.file import FileUploadResponse
from .upload_service import UploadService, hash_stream
logger = logging.getLogger("zikai.chunk")
# 会话暂存目录名(位于 upload_root 下)
SESSION_WORK_DIR = ".work"
class ChunkUploadService:
def __init__(
self,
session_dao: UploadSessionDAO,
file_dao: UploadedFileDAO,
) -> None:
s = get_settings()
self.session_dao = session_dao
self.file_dao = file_dao
self.upload_root = s.resolved_upload_dir()
self.chunk_bytes = s.storage.chunk_bytes
# 会话暂存目录:配置给出的是相对 upload_root 的子目录名
session_sub = os.path.basename(s.storage.chunk_session_dir) or SESSION_WORK_DIR
self.work_root = (self.upload_root / session_sub).resolve()
self.work_root.mkdir(parents=True, exist_ok=True)
self.session_ttl = s.storage.chunk_session_ttl_seconds
# 复用 UploadService 的存储路径生成 / 落库提交 / sha256 去重逻辑
self._upload = UploadService(file_dao)
# ---------------- 会话生命周期 ----------------
def create_session(self, body: CreateSessionRequest) -> CreateSessionResponse:
upload_id = uuid.uuid4().hex
session = UploadSession(
upload_id=upload_id,
filename=body.filename,
size_bytes=body.size_bytes,
chunk_size=body.chunk_size,
total_chunks=body.total_chunks,
uploaded_chunks=[],
status="pending",
)
self.session_dao.create(session)
self._session_dir(upload_id).mkdir(parents=True, exist_ok=True)
return CreateSessionResponse(
upload_id=upload_id,
filename=body.filename,
size_bytes=body.size_bytes,
chunk_size=body.chunk_size,
total_chunks=body.total_chunks,
)
def get_status(self, upload_id: str) -> SessionStatusResponse:
session = self._require_session(upload_id)
return SessionStatusResponse(
upload_id=session.upload_id,
filename=session.filename,
size_bytes=session.size_bytes,
chunk_size=session.chunk_size,
total_chunks=session.total_chunks,
uploaded_chunks=list(session.uploaded_chunks or []),
completed=(session.status == "completed"),
file_id=session.file_id,
)
# ---------------- 分片写入 ----------------
def write_chunk(
self, upload_id: str, index: int, data: bytes,
) -> list[int]:
session = self._require_session(upload_id)
self._validate_index(session, index)
session_dir = self._session_dir(upload_id)
session_dir.mkdir(parents=True, exist_ok=True)
chunk_path = session_dir / f"{index}.part"
# 幂等覆盖:同一分片重传时直接覆盖旧文件
try:
with chunk_path.open("wb") as out:
out.write(data)
out.flush()
os.fsync(out.fileno())
except Exception:
chunk_path.unlink(missing_ok=True)
raise
uploaded = list(session.uploaded_chunks or [])
if index not in uploaded:
uploaded.append(index)
self.session_dao.mark_uploaded(session, uploaded)
return sorted(uploaded)
# ---------------- 拼接入库 ----------------
def complete(self, upload_id: str) -> FileUploadResponse:
session = self._require_session(upload_id)
# 幂等:已 complete 直接复用结果
if session.status == "completed" and session.file_id is not None:
row = self.file_dao.get_by_id(session.file_id)
if row is not None:
return UploadService.to_response(row, deduplicated=True)
uploaded = set(session.uploaded_chunks or [])
missing = [i for i in range(session.total_chunks) if i not in uploaded]
if missing:
raise HTTPException(
409,
f"分片不齐全:缺失 {len(missing)} 个,例如 {sorted(missing)[:10]}",
)
session_dir = self._session_dir(upload_id)
part_path = session_dir / "_assembled.part"
size, digest = self._assemble_and_hash(session, session_dir, part_path)
# 复用 UploadService 的「sha256 去重 + 落库 + 原子改名」公共尾部
entity = UploadedFile(
storage_path="", # 由 dedup_or_commit 内部生成
original_filename=os.path.basename(session.filename),
content_type="",
size_bytes=size,
sha256=digest,
source="chunk",
uploaded_by="web",
)
resp = self._upload.dedup_or_commit(entity, part_path)
self.session_dao.mark_completed(session, resp.id)
# 会话目录里的分片已拼接走,清理残留
self._cleanup_session_dir(upload_id)
return resp
# ---------------- 过期会话清理 ----------------
def reap_stale_sessions(self) -> int:
"""清理被放弃的会话:删 .work/<upload_id>/ 目录 + DB 记录。
判定标准status=pending 且 updated_at 距今超过 session_ttl 秒。
返回清理的会话数。不依赖 start.sh由后台任务周期调用。
"""
stale = self.session_dao.list_stale(self.session_ttl)
for session in stale:
self._cleanup_session_dir(session.upload_id)
self.session_dao.delete(session)
logger.info("清理过期分片会话 upload_id=%s file=%s", session.upload_id, session.filename)
return len(stale)
def _cleanup_session_dir(self, upload_id: str) -> None:
"""删除 .work/<upload_id>/ 目录;失败只记日志,不抛异常。"""
session_dir = self._session_dir(upload_id)
try:
if session_dir.exists():
shutil.rmtree(session_dir, ignore_errors=True)
except Exception as exc: # pragma: no cover
logger.warning("清理会话目录失败 upload_id=%s: %s", upload_id, exc)
# ---------------- 内部 ----------------
def _require_session(self, upload_id: str) -> UploadSession:
if not upload_id:
raise HTTPException(400, "upload_id 不能为空")
session = self.session_dao.get_by_upload_id(upload_id)
if session is None:
raise HTTPException(404, f"会话不存在或已过期:{upload_id}")
return session
@staticmethod
def _validate_index(session: UploadSession, index: int) -> None:
if index < 0 or index >= session.total_chunks:
raise HTTPException(
400,
f"分片下标越界:{index} 不在 [0, {session.total_chunks})",
)
def _session_dir(self, upload_id: str) -> Path:
return self.work_root / upload_id
def _assemble_and_hash(
self,
session: UploadSession,
session_dir: Path,
out_path: Path,
) -> tuple[int, str]:
"""按 index 顺序拼接全部分片为 out_path流式计算 (size, sha256)。"""
try:
with out_path.open("wb") as out:
size, digest = hash_stream(
self._iter_assembled_bytes(session, session_dir, out)
)
out.flush()
os.fsync(out.fileno())
except Exception:
out_path.unlink(missing_ok=True)
raise
return size, digest
def _iter_assembled_bytes(
self, session: UploadSession, session_dir: Path, out: io.BufferedWriter,
) -> Iterator[bytes]:
"""按 index 顺序读各分片,写入 out 同时 yield 每个字节块(供 hash_stream"""
for index in range(session.total_chunks):
chunk_path = session_dir / f"{index}.part"
if not chunk_path.is_file():
raise HTTPException(409, f"拼接时发现分片缺失:{index}.part")
with chunk_path.open("rb") as src:
while buf := src.read(self.chunk_bytes):
out.write(buf)
yield buf

View File

@@ -1,4 +1,4 @@
"""嵌入式 SFTP 服务,与 HTTP API 共用 uploads/ 目录。
"""嵌入式 SFTP 服务,与 HTTP API 共用 uploads/ 目录;同时承载反向隧道的 SSH 转发
启动方式:
python -m app.services.sftp_server
@@ -35,16 +35,34 @@ class ZikaiSFTPServer(asyncssh.SFTPServer):
logger.info("SFTP 会话结束 user=%s", self._username)
def _tunnel_dao():
"""惰性构造一个 TunnelSessionDAO避免 import 时的副作用)。"""
from ..database import get_session_local
from ..dao.tunnel_session_dao import TunnelSessionDAO
return TunnelSessionDAO(get_session_local()())
def _close_tunnel_dao(dao) -> None:
try:
dao.db.close()
except Exception: # pragma: no cover
pass
class ZikaiSSHServer(asyncssh.SSHServer):
"""支持密码bcrypt与公钥两种鉴权。"""
"""支持密码bcrypt与公钥两种鉴权;允许反向隧道 user 的 remote forwarding"""
def __init__(self, settings, authorized_keys: asyncssh.SSHAuthorizedKeys | None) -> None:
self._settings = settings
self._authorized_keys = authorized_keys
self._conn: asyncssh.SSHServerConnection | None = None
self._username: str | None = None # 认证成功后填入
self._peer_ip: str = ""
def connection_made(self, conn: asyncssh.SSHServerConnection) -> None: # type: ignore[override]
self._conn = conn
peer = conn.get_extra_info("peername")
self._peer_ip = peer[0] if isinstance(peer, tuple) and peer else ""
def begin_auth(self, username: str) -> bool:
# 始终要求鉴权;用户不存在时所有方法会失败 → 干净的 permission denied
@@ -56,14 +74,17 @@ class ZikaiSSHServer(asyncssh.SSHServer):
return True
def validate_password(self, username: str, password: str) -> bool:
user = next((u for u in self._settings.sftp.users if u.username == username), None)
if user is None or not user.password_hash or user.password_hash == "CHANGE_ME_BCRYPT_HASH":
user = self._find_sftp_user(username) or self._find_tunnel_user(username)
if user is None or not getattr(user, "password_hash", "") \
or user.password_hash == "CHANGE_ME_BCRYPT_HASH":
return False
try:
ok = bcrypt.checkpw(password.encode(), user.password_hash.encode())
except (ValueError, TypeError):
ok = False
logger.info("SFTP 密码鉴权 %s user=%s", "OK" if ok else "FAIL", username)
if ok:
self._username = username
logger.info("SSH 密码鉴权 %s user=%s", "OK" if ok else "FAIL", username)
return ok
# 公钥
@@ -74,21 +95,88 @@ class ZikaiSSHServer(asyncssh.SSHServer):
def validate_public_key(self, username: str, key: asyncssh.SSHKey) -> bool:
if self._authorized_keys is None:
return False
if not any(u.username == username for u in self._settings.sftp.users):
if self._find_sftp_user(username) is None and self._find_tunnel_user(username) is None:
return False
addr = ""
if self._conn:
peer = self._conn.get_extra_info("peername")
addr = peer[0] if isinstance(peer, tuple) and peer else ""
addr = self._peer_ip
try:
# asyncssh 命中返回 dict可能为空未命中返回 None
result = self._authorized_keys.validate(key, client_host=addr, client_addr=addr)
except Exception:
result = None
ok = result is not None
logger.info("SFTP 公钥鉴权 %s user=%s", "OK" if ok else "FAIL", username)
if ok:
self._username = username
logger.info("SSH 公钥鉴权 %s user=%s", "OK" if ok else "FAIL", username)
return ok
# 反向隧道remote port-forward 请求
def server_requested(self, listen_host: str, listen_port: int) -> bool:
"""客户端请求在本地server 侧)绑端口做反向转发时回调。
仅允许已认证的 tunnel user 绑定其配置中预定的 tunnel_port记一条 active
隧道会话到 DB。其他情况拒绝。
"""
if not self._settings.tunnel.enabled:
logger.warning("拒绝 remote forwardtunnel 未启用 user=%s", self._username)
return False
if not self._username:
logger.warning("拒绝 remote forward未认证")
return False
tunnel_user = self._find_tunnel_user(self._username)
if tunnel_user is None:
logger.warning("拒绝 remote forward%s 不是 tunnel user", self._username)
return False
if listen_port != tunnel_user.tunnel_port:
logger.warning(
"拒绝 remote forwarduser=%s 端口 %d 不等于配置 %d",
self._username, listen_port, tunnel_user.tunnel_port,
)
return False
dao = _tunnel_dao()
try:
from ..services.tunnel_service import TunnelService
TunnelService(dao).register(
user_name=self._username,
user_ip=self._peer_ip,
tunnel_port=listen_port,
local_port=tunnel_user.local_port,
)
except Exception as exc: # pragma: no cover
_close_tunnel_dao(dao)
logger.error("登记隧道会话失败:%s", exc)
return False
_close_tunnel_dao(dao)
logger.info(
"允许 remote forward user=%s listen=%s:%d -> local_port=%d",
self._username, listen_host, listen_port, tunnel_user.local_port,
)
return True
def connection_lost(self, exc: Exception | None) -> None: # type: ignore[override]
"""SSH 连接断开时清理该 user 的活跃隧道会话。"""
if self._username and self._settings.tunnel.enabled:
dao = _tunnel_dao()
try:
from ..services.tunnel_service import TunnelService
TunnelService(dao).close(self._username)
except Exception as e: # pragma: no cover
logger.warning("清理隧道会话失败 user=%s: %s", self._username, e)
_close_tunnel_dao(dao)
if exc:
logger.info("SSH 连接异常断开 user=%s: %s", self._username, exc)
else:
logger.info("SSH 连接关闭 user=%s", self._username)
# 内部
def _find_sftp_user(self, username: str):
return next((u for u in self._settings.sftp.users if u.username == username), None)
def _find_tunnel_user(self, username: str):
return self._settings.tunnel.find_user(username)
def _load_authorized_keys(path: Path) -> asyncssh.SSHAuthorizedKeys | None:
if not path.exists() or path.stat().st_size == 0:
@@ -119,7 +207,7 @@ async def _run() -> None:
upload_root = settings.resolved_upload_dir()
upload_root.mkdir(parents=True, exist_ok=True)
# SFTP 客户端登记前的暂存目录register-sftp 只接受此目录下的路径
# SFTP 客户端的暂存目录chroot 内)
(upload_root / "incoming").mkdir(parents=True, exist_ok=True)
host_key_path = (PROJECT_ROOT / settings.sftp.host_key_path).resolve()

View File

@@ -0,0 +1,70 @@
"""反向隧道业务逻辑:会话注册 / 注销 / 查询。
SSH 服务收到 remote port-forward 请求时调 register 记一条 active 会话;
user 断开时调 close 标记 ended_atHTTP 路由调 get_active 查隧道端口做反代。
"""
from __future__ import annotations
import logging
from ..config import get_settings
from ..dao.tunnel_session_dao import TunnelSessionDAO
from ..models.tunnel_session import TunnelSession
logger = logging.getLogger("zikai.tunnel")
class TunnelService:
def __init__(self, dao: TunnelSessionDAO) -> None:
self.dao = dao
self.settings = get_settings().tunnel
def register(
self, user_name: str, user_ip: str, tunnel_port: int, local_port: int,
) -> TunnelSession:
"""登记一条活跃隧道会话。
若该 user 已有活跃会话先关闭旧的(同一 user 同一时刻只保留一条)。
"""
self.dao.close_active_by_user(user_name)
session = TunnelSession(
user_name=user_name,
user_ip=user_ip,
local_port=local_port,
tunnel_port=tunnel_port,
status="active",
)
saved = self.dao.create(session)
logger.info(
"隧道建立 user=%s ip=%s tunnel_port=%d local_port=%d",
user_name, user_ip, tunnel_port, local_port,
)
return saved
def close(self, user_name: str) -> int:
"""关闭该 user 的活跃会话SSH 断开时调用),返回关闭条数。"""
n = self.dao.close_active_by_user(user_name)
if n:
logger.info("隧道关闭 user=%s 条数=%d", user_name, n)
return n
def get_active(self, user_name: str) -> TunnelSession | None:
return self.dao.get_active_by_user(user_name)
def is_port_allowed(self, user_name: str, tunnel_port: int) -> bool:
"""校验该 user 是否被允许绑定该隧道端口(防 user 乱绑端口)。"""
user = self.settings.find_user(user_name)
return user is not None and user.tunnel_port == tunnel_port
def reap_orphans(self) -> int:
"""兜底清理:关闭所有 active 会话(进程重启时 DB 里残留的孤儿记录)。
由后台 reaper 在启动后调用一次。SSH 实际断开时已有 close() 处理,
这里只兜底进程异常退出后 DB 与实际状态不一致的情况。
"""
active = self.dao.list_active()
for session in active:
self.dao.close(session)
logger.info("reaper 清理孤儿隧道 id=%d user=%s", session.id, session.user_name)
return len(active)

View File

@@ -1,22 +1,35 @@
"""上传服务:流式落盘 + 原子改名 + 元数据入库。"""
"""上传服务:流式落盘 + 原子改名 + 元数据入库 + sha256 去重
两条上传路径HTTP 整文件 / 分片拼接)共享本类的「存储路径生成 + 落库提交 +
sha256 去重」逻辑避免重复实现。SFTP 服务器仅作为文件暂存通道chroot 到
upload_root不再有 HTTP 登记接口。
"""
from __future__ import annotations
import hashlib
import os
import uuid
from collections.abc import Iterator
from datetime import datetime, timezone
from pathlib import Path
from fastapi import HTTPException, UploadFile
from fastapi import UploadFile
from ..config import get_settings
from ..dao.uploaded_file_dao import UploadedFileDAO
from ..models.uploaded_file import UploadedFile
from ..schemas.file import FileUploadResponse, UploadedFileOut
# SFTP 客户端登记前必须把文件先落到此目录chroot 内)
SFTP_INCOMING_DIR = "incoming"
def hash_stream(chunks: Iterator[bytes]) -> tuple[int, str]:
"""对一段字节块序列流式计算 (size, sha256)。供多条上传路径复用。"""
h = hashlib.sha256()
size = 0
for chunk in chunks:
size += len(chunk)
h.update(chunk)
return size, h.hexdigest()
class UploadService:
@@ -36,7 +49,7 @@ class UploadService:
失败仅会留下可识别的 ``.part`` 文件,由 start.sh 启动时统一清理。
"""
rel_path, abs_path = self._make_storage_path(file.filename or "")
rel_path, abs_path = self.make_storage_path(file.filename or "")
part_path = abs_path.with_name(abs_path.name + ".part")
size, digest = self._write_part(file, part_path)
@@ -49,34 +62,7 @@ class UploadService:
source=source,
uploaded_by=uploaded_by,
)
return self._commit(entity, part_path, abs_path)
def register_sftp(
self, filename: str, original_filename: str, uploaded_by: str = "sftp",
) -> FileUploadResponse:
"""登记一个已通过 SFTP 落到 ``incoming/`` 下的文件。
相同 sha256 已存在时直接返回旧行并删掉新副本(去重)。
"""
src_abs = self._validate_incoming_path(filename)
size, digest = self._hash_disk_file(src_abs)
existing = self.dao.get_by_sha256(digest)
if existing is not None:
src_abs.unlink(missing_ok=True)
return self._to_response(existing)
rel_path, abs_path = self._make_storage_path(original_filename or filename)
entity = UploadedFile(
storage_path=str(rel_path),
original_filename=os.path.basename(original_filename or src_abs.name),
content_type="",
size_bytes=size,
sha256=digest,
source="sftp",
uploaded_by=uploaded_by,
)
return self._commit(entity, src_abs, abs_path)
return self.commit_entity(entity, part_path, abs_path)
# ---------------- 查询 ----------------
@@ -94,18 +80,26 @@ class UploadService:
row = self.dao.get_by_sha256(sha256)
return UploadedFileOut.model_validate(row) if row else None
def resolve_disk_path(self, file_id: int) -> Path | None:
row = self.dao.get_by_id(file_id)
return (self.upload_root / row.storage_path).resolve() if row else None
def get_out_with_disk_path(self, file_id: int) -> tuple[UploadedFileOut | None, Path | None]:
"""合并查询:一次 DB 读取同时返回 (元数据, 磁盘绝对路径)。
# ---------------- 内部 ----------------
供下载/删除路径复用,避免原先 get_out + resolve_disk_path 各查一次的重复读。
"""
row = self.dao.get_by_id(file_id)
if row is None:
return None, None
out = UploadedFileOut.model_validate(row)
path = (self.upload_root / row.storage_path).resolve()
return out, path
# ---------------- 共享 helper供本类与 ChunkUploadService 复用) ----------------
@staticmethod
def _safe_ext(filename: str) -> str:
return os.path.splitext(os.path.basename(filename))[1]
@staticmethod
def _to_response(row: UploadedFile) -> FileUploadResponse:
def to_response(row: UploadedFile, *, deduplicated: bool = False) -> FileUploadResponse:
return FileUploadResponse(
id=row.id,
filename=row.original_filename,
@@ -113,9 +107,10 @@ class UploadService:
sha256=row.sha256,
storage_path=row.storage_path,
uploaded_at=row.uploaded_at,
deduplicated=deduplicated,
)
def _make_storage_path(self, original_filename: str) -> tuple[Path, Path]:
def make_storage_path(self, original_filename: str) -> tuple[Path, Path]:
"""生成 ``(rel_path, abs_path)``abs_path 的父目录已创建。"""
now = datetime.now(timezone.utc)
rel_dir = Path(f"{now:%Y}/{now:%m}")
@@ -123,21 +118,7 @@ class UploadService:
rel_path = rel_dir / f"{uuid.uuid4().hex}{self._safe_ext(original_filename)}"
return rel_path, self.upload_root / rel_path
def _validate_incoming_path(self, filename: str) -> Path:
"""校验客户端给出的相对路径必须落在 upload_root/incoming/ 之下且是文件。"""
if not filename or filename.startswith(("/", "\\")):
raise HTTPException(400, "filename 必须是 incoming/ 下的相对路径")
incoming_root = (self.upload_root / SFTP_INCOMING_DIR).resolve()
try:
abs_path = (self.upload_root / filename).resolve()
abs_path.relative_to(incoming_root)
except ValueError:
raise HTTPException(400, f"filename 必须落在 {SFTP_INCOMING_DIR}/ 之下")
if not abs_path.is_file():
raise HTTPException(404, f"文件不存在或不是普通文件:{filename}")
return abs_path
def _commit(
def commit_entity(
self, entity: UploadedFile, src_path: Path, dest_path: Path,
) -> FileUploadResponse:
"""落 DB 行后把 src_path 原子改名到 dest_path任一失败回滚已生成的副作用。"""
@@ -154,17 +135,25 @@ class UploadService:
finally:
src_path.unlink(missing_ok=True)
raise
return self._to_response(saved)
return self.to_response(saved)
def _hash_disk_file(self, path: Path) -> tuple[int, str]:
"""流式读取磁盘文件,返回 (size, sha256)。"""
h = hashlib.sha256()
size = 0
with path.open("rb") as fh:
while chunk := fh.read(self.chunk_bytes):
size += len(chunk)
h.update(chunk)
return size, h.hexdigest()
def dedup_or_commit(
self, entity: UploadedFile, src_path: Path,
) -> FileUploadResponse:
"""公共尾部:按 entity.sha256 去重,命中则删 src 返回旧行;否则 commit。
供分片拼接等「先算出 sha256 再决定落盘」的路径复用,与 stream_to_disk
(边写边算、无独立 src的区别在于这里 sha256 已在 entity 上。
"""
existing = self.dao.get_by_sha256(entity.sha256)
if existing is not None:
src_path.unlink(missing_ok=True)
return self.to_response(existing, deduplicated=True)
rel_path, abs_path = self.make_storage_path(entity.original_filename)
entity.storage_path = str(rel_path)
return self.commit_entity(entity, src_path, abs_path)
# ---------------- 内部 ----------------
def _write_part(self, file: UploadFile, part_path: Path) -> tuple[int, str]:
"""把上传流写到 part_path 并 fsync返回 (size, sha256)。失败时清理残品。"""

View File

@@ -0,0 +1,175 @@
"""白板 WebSocket 连接管理器(实时同步 + 心跳 + 失活清理)。
设计要点:
- 进程内单例 ``WhiteboardHub``,维护 ``{board_id: set[Connection]}``。
- 每个 Connection 封装 websocket / board_id / client_id / last_heartbeat。
- 心跳:客户端每 ``heartbeat_interval_seconds``(默认 3s发一次 ping服务端回 pong 并
刷新 last_heartbeat。reaper 每秒扫描,超过 ``interval * threshold``(默认 15s未心跳
的连接判为失活,关闭并从 hub 移除。
- 内存安全disconnect 幂等;空 set 从 dict 删除broadcast 对单连接异常立即 disconnect
close_board 关闭并清理整个 board 的连接集合。
- 并发:用一个 asyncio.Lock 保护 ``_boards`` 的结构变更add/remove board key
集合内的连接增删用 set 原子操作Python 单线程事件循环下安全)。
注意hub 是进程内存,多 worker 下不互通。生产部署需单 worker 或后续接 Redis pub/sub。
"""
from __future__ import annotations
import asyncio
import logging
import time
from dataclasses import dataclass, field
from fastapi import WebSocket
from ..config import get_settings
logger = logging.getLogger("zikai.whiteboard")
@dataclass(eq=False)
class Connection:
"""一个白板在线连接。eq=False 使其按对象 identity 哈希/比较,可放入 set。"""
websocket: WebSocket
board_id: str
client_id: str
last_heartbeat: float = field(default_factory=time.monotonic)
async def send_json(self, msg: dict) -> bool:
"""发送一条消息;失败返回 False调用方据此 disconnect"""
try:
await self.websocket.send_json(msg)
return True
except Exception as exc: # WebSocketDisconnect / 已关闭 / 编码失败
logger.debug("发送失败 board=%s client=%s: %s", self.board_id, self.client_id, exc)
return False
def touch(self) -> None:
self.last_heartbeat = time.monotonic()
class WhiteboardHub:
"""白板连接管理器单例。"""
def __init__(self) -> None:
cfg = get_settings().whiteboard
self.heartbeat_interval = cfg.heartbeat_interval_seconds
self.heartbeat_miss_threshold = cfg.heartbeat_miss_threshold
self.timeout_seconds = self.heartbeat_interval * self.heartbeat_miss_threshold
# {board_id: set[Connection]}
self._boards: dict[str, set[Connection]] = {}
self._lock = asyncio.Lock()
# ---------------- 连接生命周期 ----------------
async def register(self, conn: Connection) -> None:
"""把已 accept 的连接加入 board 集合WebSocket accept 由 controller 负责)。"""
async with self._lock:
conns = self._boards.setdefault(conn.board_id, set())
conns.add(conn)
logger.info("连接接入 board=%s client=%s(当前 %d 人)",
conn.board_id, conn.client_id, self.connection_count(conn.board_id))
async def disconnect(self, conn: Connection) -> None:
"""幂等移除连接;空 set 从 dict 删除以防内存泄漏。"""
async with self._lock:
conns = self._boards.get(conn.board_id)
if conns is None:
return
conns.discard(conn)
if not conns:
self._boards.pop(conn.board_id, None)
# 尽力关闭 websocket可能已关闭
try:
await conn.websocket.close()
except Exception: # pragma: no cover
pass
logger.info("连接移除 board=%s client=%s(剩余 %d 人)",
conn.board_id, conn.client_id, self.connection_count(conn.board_id))
def connection_count(self, board_id: str) -> int:
"""调试/监控用:某 board 当前在线人数。"""
return len(self._boards.get(board_id, ()))
# ---------------- 广播 ----------------
async def broadcast(self, board_id: str, msg: dict, exclude: Connection | None = None) -> None:
"""把 msg 发给 board 内所有在线连接(可排除发送者)。单连接失败不影响其他。"""
async with self._lock:
conns = list(self._boards.get(board_id, ()))
dead: list[Connection] = []
for conn in conns:
if exclude is not None and conn is exclude:
continue
ok = await conn.send_json(msg)
if not ok:
dead.append(conn)
# 发送失败的连接统一清理
for conn in dead:
await self.disconnect(conn)
# ---------------- 心跳 reaper ----------------
async def reap_loop(self, stop: asyncio.Event) -> None:
"""后台循环:扫描失活连接。每秒一次,粒度细于 timeout。"""
logger.info("白板心跳 reaper 已启动(间隔 1s超时 %ds", self.timeout_seconds)
while not stop.is_set():
try:
await self._reap_once()
except Exception as exc: # pragma: no cover
logger.warning("reaper 循环异常:%s", exc)
try:
await asyncio.wait_for(stop.wait(), timeout=1.0)
except asyncio.TimeoutError:
pass
async def _reap_once(self) -> None:
now = time.monotonic()
async with self._lock:
# 快照待检查连接,避免持锁时 await
stale: list[Connection] = []
for board_id, conns in self._boards.items():
for conn in conns:
if now - conn.last_heartbeat > self.timeout_seconds:
stale.append(conn)
for conn in stale:
logger.warning("心跳失活,移除 board=%s client=%s(静默 %ds",
conn.board_id, conn.client_id,
int(now - conn.last_heartbeat))
await self.disconnect(conn)
async def close_board(self, board_id: str) -> None:
"""关闭并清理某 board 的所有连接(删除白板时调用)。"""
async with self._lock:
conns = self._boards.pop(board_id, None)
if not conns:
return
await asyncio.gather(
*(c.send_json({"type": "error", "msg": "白板已被删除"}) for c in conns),
return_exceptions=True,
)
for conn in conns:
try:
await conn.websocket.close()
except Exception: # pragma: no cover
pass
logger.info("关闭白板 board=%s,踢出 %d 个连接", board_id, len(conns))
# 进程内单例(由 main.py lifespan / controller 共享)
_hub: WhiteboardHub | None = None
def get_hub() -> WhiteboardHub:
global _hub
if _hub is None:
_hub = WhiteboardHub()
return _hub
def reset_hub() -> None:
"""测试用:重置单例。"""
global _hub
_hub = None

View File

@@ -0,0 +1,92 @@
"""白板服务CRUD + 笔画操作。
不持有 WebSocket 连接状态(那是 hub 的职责);删除白板时通过可选的 hub 回调
通知 hub 踢出该 board 的所有在线连接,由 controller 在装配时注入,避免 service -> hub
的硬依赖(保持低耦合)。
"""
from __future__ import annotations
import re
from typing import TYPE_CHECKING, Any
from fastapi import HTTPException
from ..config import get_settings
from ..dao.whiteboard_dao import WhiteboardDAO
from ..schemas.whiteboard import WhiteboardListItem, WhiteboardOut
if TYPE_CHECKING: # 避免运行时循环导入
from .whiteboard_hub import WhiteboardHub
# board_id 合法字符集:字母数字下划线短横线
_BOARD_ID_RE = re.compile(r"^[a-zA-Z0-9_-]+$")
class WhiteboardService:
def __init__(self, dao: WhiteboardDAO, hub: "WhiteboardHub | None" = None) -> None:
self.dao = dao
self.hub = hub
cfg = get_settings().whiteboard
self.max_board_id_length = cfg.max_board_id_length
self.list_limit = cfg.list_limit
# ---------------- 校验 ----------------
def validate_board_id(self, board_id: str) -> None:
"""非法 board_id 直接 400防路径穿越/注入)。"""
if (
not board_id
or len(board_id) > self.max_board_id_length
or not _BOARD_ID_RE.match(board_id)
):
raise HTTPException(400, "board_id 非法仅允许字母数字下划线短横线1-64 字符)")
# ---------------- 读 ----------------
def get_or_create(self, board_id: str) -> WhiteboardOut:
self.validate_board_id(board_id)
board = self.dao.get_or_create(board_id)
return WhiteboardOut.model_validate(board)
def list_all(self, limit: int = 100, offset: int = 0) -> tuple[int, list[WhiteboardListItem]]:
limit = min(max(limit, 0), self.list_limit) or self.list_limit
offset = max(offset, 0)
total = self.dao.count()
rows = self.dao.list_all(limit=limit, offset=offset)
items = [WhiteboardListItem.model_validate(r) for r in rows]
return total, items
# ---------------- 写 ----------------
def append_stroke(self, board_id: str, stroke: dict[str, Any]) -> WhiteboardOut:
"""追加一条笔画并返回最新状态。"""
self.validate_board_id(board_id)
board = self.dao.append_strokes(board_id, [stroke])
if board is None:
raise HTTPException(404, "白板不存在")
return WhiteboardOut.model_validate(board)
def clear(self, board_id: str) -> WhiteboardOut:
"""清空白板stroke_count 仍自增以记录这次修改。"""
self.validate_board_id(board_id)
board = self.dao.replace_strokes(board_id, [])
if board is None:
raise HTTPException(404, "白板不存在")
return WhiteboardOut.model_validate(board)
def delete(self, board_id: str) -> bool:
"""删除白板;同时通知 hub 踢出该 board 的所有在线连接。"""
self.validate_board_id(board_id)
ok = self.dao.delete(board_id)
if ok and self.hub is not None:
# hub.close_board 是 async但删除走 REST 同步路径;安排到事件循环里执行
import asyncio
try:
loop = asyncio.get_running_loop()
loop.create_task(self.hub.close_board(board_id))
except RuntimeError:
# 无运行中事件循环(如脚本调用):同步调用会报错,忽略即可
pass
return ok