feat: 文件浏览页 + 共享白板 + 白板管理页(含补登记分片上传/隧道历史改动)
本次提交包含两批改动(7月2日遗留未入库 + 本次新功能),分述如下:
【补登记:7月2日已上线但未提交的功能】
- 分片上传:chunk_upload_controller/service/dao + UploadSession model/schema,
支持 4MiB 分片、乱序、断点续传、去重、幂等 complete;后台 reaper 清理过期会话。
- 反向隧道:tunnel_controller/service/dao + TunnelSession model/schema,
SSH remote forward 经 /api/userPort/{userName} 反代到 user 本地服务。
- 上传页:views/upload_html.py(拖拽/多文件/分片/断点续传 UI)。
- config.py:StorageConfig.chunk_session_dir/ttl、TunnelConfig;
requirements.txt 加 httpx;start.sh 清理 .work/ 残留;
schema.sql 加 upload_session/tunnel_session 表;sftp_server 承载隧道转发。
【本次新功能】
- 文件浏览页:GET /files(Basic Auth 同 docs)+ /api/admin/files(list/get/download/DELETE)。
硬删除(DB 行 + 磁盘文件),删除后列表不再显示。前端 static/file_browser.*。
- 共享白板:GET /whiteboard/{id}(公开,不存在则新建)+ WS /ws/whiteboard/{id}。
MySQL 持久化(whiteboard 表),Canvas 实时同步,心跳 3s/5 次失活移除,
清空/复制按钮,移动端兼容。WhiteboardHub 管理 {board_id: set[Connection]},
disconnect 幂等 + 空 set 清理防泄漏,broadcast 失败连接自动移除。
- 白板管理页:GET /whiteboard-admin(Basic Auth)+ /api/admin/whiteboards(list/DELETE)。
删除时 hub.close_board 踢出在线连接。
- 清理:合并 UploadService.get_out_with_disk_path(下载/删除复用,消除重复 DB 读),
移除无用 resolve_disk_path。
- config.py:WhiteboardConfig(heartbeat/threshold/board_id 长度/list_limit);
schema.sql 加 whiteboard 表;README 补新接口与心跳/内存说明。
- 验证:tests/manual_whiteboard_hub.py / _ws.py / _kick.py 全部通过。
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
"""业务逻辑层(Service)。"""
|
||||
|
||||
from .chunk_upload_service import ChunkUploadService
|
||||
from .system_service import SystemService
|
||||
from .upload_service import UploadService
|
||||
|
||||
__all__ = ["SystemService", "UploadService"]
|
||||
__all__ = ["ChunkUploadService", "SystemService", "UploadService"]
|
||||
|
||||
247
app/services/chunk_upload_service.py
Normal file
247
app/services/chunk_upload_service.py
Normal file
@@ -0,0 +1,247 @@
|
||||
"""分片上传服务:会话管理 + 分片落盘 + 拼接 + sha256 去重 + 原子入库。
|
||||
|
||||
存储布局::
|
||||
|
||||
uploads/.work/<upload_id>/0.part 分片暂存
|
||||
uploads/.work/<upload_id>/1.part
|
||||
...
|
||||
uploads/2026/07/<uuid>.<ext> complete 后的正式文件
|
||||
|
||||
complete 流程:
|
||||
1. 校验分片齐全;
|
||||
2. 顺序拼接为 <final>.part,流式算 sha256;
|
||||
3. 复用 UploadService.dedup_or_commit:按 sha256 去重命中则删会话返回旧行,
|
||||
否则写 DB 行 + os.replace 原子改名到正式路径;
|
||||
4. 清理 .work/<upload_id>/。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from ..config import get_settings
|
||||
from ..dao.upload_session_dao import UploadSessionDAO
|
||||
from ..dao.uploaded_file_dao import UploadedFileDAO
|
||||
from ..models.upload_session import UploadSession
|
||||
from ..models.uploaded_file import UploadedFile
|
||||
from ..schemas.chunk import (
|
||||
CreateSessionRequest,
|
||||
CreateSessionResponse,
|
||||
SessionStatusResponse,
|
||||
)
|
||||
from ..schemas.file import FileUploadResponse
|
||||
from .upload_service import UploadService, hash_stream
|
||||
|
||||
logger = logging.getLogger("zikai.chunk")
|
||||
|
||||
# 会话暂存目录名(位于 upload_root 下)
|
||||
SESSION_WORK_DIR = ".work"
|
||||
|
||||
|
||||
class ChunkUploadService:
|
||||
def __init__(
|
||||
self,
|
||||
session_dao: UploadSessionDAO,
|
||||
file_dao: UploadedFileDAO,
|
||||
) -> None:
|
||||
s = get_settings()
|
||||
self.session_dao = session_dao
|
||||
self.file_dao = file_dao
|
||||
self.upload_root = s.resolved_upload_dir()
|
||||
self.chunk_bytes = s.storage.chunk_bytes
|
||||
# 会话暂存目录:配置给出的是相对 upload_root 的子目录名
|
||||
session_sub = os.path.basename(s.storage.chunk_session_dir) or SESSION_WORK_DIR
|
||||
self.work_root = (self.upload_root / session_sub).resolve()
|
||||
self.work_root.mkdir(parents=True, exist_ok=True)
|
||||
self.session_ttl = s.storage.chunk_session_ttl_seconds
|
||||
# 复用 UploadService 的存储路径生成 / 落库提交 / sha256 去重逻辑
|
||||
self._upload = UploadService(file_dao)
|
||||
|
||||
# ---------------- 会话生命周期 ----------------
|
||||
|
||||
def create_session(self, body: CreateSessionRequest) -> CreateSessionResponse:
|
||||
upload_id = uuid.uuid4().hex
|
||||
session = UploadSession(
|
||||
upload_id=upload_id,
|
||||
filename=body.filename,
|
||||
size_bytes=body.size_bytes,
|
||||
chunk_size=body.chunk_size,
|
||||
total_chunks=body.total_chunks,
|
||||
uploaded_chunks=[],
|
||||
status="pending",
|
||||
)
|
||||
self.session_dao.create(session)
|
||||
self._session_dir(upload_id).mkdir(parents=True, exist_ok=True)
|
||||
return CreateSessionResponse(
|
||||
upload_id=upload_id,
|
||||
filename=body.filename,
|
||||
size_bytes=body.size_bytes,
|
||||
chunk_size=body.chunk_size,
|
||||
total_chunks=body.total_chunks,
|
||||
)
|
||||
|
||||
def get_status(self, upload_id: str) -> SessionStatusResponse:
|
||||
session = self._require_session(upload_id)
|
||||
return SessionStatusResponse(
|
||||
upload_id=session.upload_id,
|
||||
filename=session.filename,
|
||||
size_bytes=session.size_bytes,
|
||||
chunk_size=session.chunk_size,
|
||||
total_chunks=session.total_chunks,
|
||||
uploaded_chunks=list(session.uploaded_chunks or []),
|
||||
completed=(session.status == "completed"),
|
||||
file_id=session.file_id,
|
||||
)
|
||||
|
||||
# ---------------- 分片写入 ----------------
|
||||
|
||||
def write_chunk(
|
||||
self, upload_id: str, index: int, data: bytes,
|
||||
) -> list[int]:
|
||||
session = self._require_session(upload_id)
|
||||
self._validate_index(session, index)
|
||||
|
||||
session_dir = self._session_dir(upload_id)
|
||||
session_dir.mkdir(parents=True, exist_ok=True)
|
||||
chunk_path = session_dir / f"{index}.part"
|
||||
|
||||
# 幂等覆盖:同一分片重传时直接覆盖旧文件
|
||||
try:
|
||||
with chunk_path.open("wb") as out:
|
||||
out.write(data)
|
||||
out.flush()
|
||||
os.fsync(out.fileno())
|
||||
except Exception:
|
||||
chunk_path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
uploaded = list(session.uploaded_chunks or [])
|
||||
if index not in uploaded:
|
||||
uploaded.append(index)
|
||||
self.session_dao.mark_uploaded(session, uploaded)
|
||||
return sorted(uploaded)
|
||||
|
||||
# ---------------- 拼接入库 ----------------
|
||||
|
||||
def complete(self, upload_id: str) -> FileUploadResponse:
|
||||
session = self._require_session(upload_id)
|
||||
|
||||
# 幂等:已 complete 直接复用结果
|
||||
if session.status == "completed" and session.file_id is not None:
|
||||
row = self.file_dao.get_by_id(session.file_id)
|
||||
if row is not None:
|
||||
return UploadService.to_response(row, deduplicated=True)
|
||||
|
||||
uploaded = set(session.uploaded_chunks or [])
|
||||
missing = [i for i in range(session.total_chunks) if i not in uploaded]
|
||||
if missing:
|
||||
raise HTTPException(
|
||||
409,
|
||||
f"分片不齐全:缺失 {len(missing)} 个,例如 {sorted(missing)[:10]}",
|
||||
)
|
||||
|
||||
session_dir = self._session_dir(upload_id)
|
||||
part_path = session_dir / "_assembled.part"
|
||||
|
||||
size, digest = self._assemble_and_hash(session, session_dir, part_path)
|
||||
|
||||
# 复用 UploadService 的「sha256 去重 + 落库 + 原子改名」公共尾部
|
||||
entity = UploadedFile(
|
||||
storage_path="", # 由 dedup_or_commit 内部生成
|
||||
original_filename=os.path.basename(session.filename),
|
||||
content_type="",
|
||||
size_bytes=size,
|
||||
sha256=digest,
|
||||
source="chunk",
|
||||
uploaded_by="web",
|
||||
)
|
||||
resp = self._upload.dedup_or_commit(entity, part_path)
|
||||
self.session_dao.mark_completed(session, resp.id)
|
||||
# 会话目录里的分片已拼接走,清理残留
|
||||
self._cleanup_session_dir(upload_id)
|
||||
return resp
|
||||
|
||||
# ---------------- 过期会话清理 ----------------
|
||||
|
||||
def reap_stale_sessions(self) -> int:
|
||||
"""清理被放弃的会话:删 .work/<upload_id>/ 目录 + DB 记录。
|
||||
|
||||
判定标准:status=pending 且 updated_at 距今超过 session_ttl 秒。
|
||||
返回清理的会话数。不依赖 start.sh,由后台任务周期调用。
|
||||
"""
|
||||
stale = self.session_dao.list_stale(self.session_ttl)
|
||||
for session in stale:
|
||||
self._cleanup_session_dir(session.upload_id)
|
||||
self.session_dao.delete(session)
|
||||
logger.info("清理过期分片会话 upload_id=%s file=%s", session.upload_id, session.filename)
|
||||
return len(stale)
|
||||
|
||||
def _cleanup_session_dir(self, upload_id: str) -> None:
|
||||
"""删除 .work/<upload_id>/ 目录;失败只记日志,不抛异常。"""
|
||||
session_dir = self._session_dir(upload_id)
|
||||
try:
|
||||
if session_dir.exists():
|
||||
shutil.rmtree(session_dir, ignore_errors=True)
|
||||
except Exception as exc: # pragma: no cover
|
||||
logger.warning("清理会话目录失败 upload_id=%s: %s", upload_id, exc)
|
||||
|
||||
# ---------------- 内部 ----------------
|
||||
|
||||
def _require_session(self, upload_id: str) -> UploadSession:
|
||||
if not upload_id:
|
||||
raise HTTPException(400, "upload_id 不能为空")
|
||||
session = self.session_dao.get_by_upload_id(upload_id)
|
||||
if session is None:
|
||||
raise HTTPException(404, f"会话不存在或已过期:{upload_id}")
|
||||
return session
|
||||
|
||||
@staticmethod
|
||||
def _validate_index(session: UploadSession, index: int) -> None:
|
||||
if index < 0 or index >= session.total_chunks:
|
||||
raise HTTPException(
|
||||
400,
|
||||
f"分片下标越界:{index} 不在 [0, {session.total_chunks})",
|
||||
)
|
||||
|
||||
def _session_dir(self, upload_id: str) -> Path:
|
||||
return self.work_root / upload_id
|
||||
|
||||
def _assemble_and_hash(
|
||||
self,
|
||||
session: UploadSession,
|
||||
session_dir: Path,
|
||||
out_path: Path,
|
||||
) -> tuple[int, str]:
|
||||
"""按 index 顺序拼接全部分片为 out_path,流式计算 (size, sha256)。"""
|
||||
try:
|
||||
with out_path.open("wb") as out:
|
||||
size, digest = hash_stream(
|
||||
self._iter_assembled_bytes(session, session_dir, out)
|
||||
)
|
||||
out.flush()
|
||||
os.fsync(out.fileno())
|
||||
except Exception:
|
||||
out_path.unlink(missing_ok=True)
|
||||
raise
|
||||
return size, digest
|
||||
|
||||
def _iter_assembled_bytes(
|
||||
self, session: UploadSession, session_dir: Path, out: io.BufferedWriter,
|
||||
) -> Iterator[bytes]:
|
||||
"""按 index 顺序读各分片,写入 out 同时 yield 每个字节块(供 hash_stream)。"""
|
||||
for index in range(session.total_chunks):
|
||||
chunk_path = session_dir / f"{index}.part"
|
||||
if not chunk_path.is_file():
|
||||
raise HTTPException(409, f"拼接时发现分片缺失:{index}.part")
|
||||
with chunk_path.open("rb") as src:
|
||||
while buf := src.read(self.chunk_bytes):
|
||||
out.write(buf)
|
||||
yield buf
|
||||
@@ -1,4 +1,4 @@
|
||||
"""嵌入式 SFTP 服务,与 HTTP API 共用 uploads/ 目录。
|
||||
"""嵌入式 SFTP 服务,与 HTTP API 共用 uploads/ 目录;同时承载反向隧道的 SSH 转发。
|
||||
|
||||
启动方式:
|
||||
python -m app.services.sftp_server
|
||||
@@ -35,16 +35,34 @@ class ZikaiSFTPServer(asyncssh.SFTPServer):
|
||||
logger.info("SFTP 会话结束 user=%s", self._username)
|
||||
|
||||
|
||||
def _tunnel_dao():
|
||||
"""惰性构造一个 TunnelSessionDAO(避免 import 时的副作用)。"""
|
||||
from ..database import get_session_local
|
||||
from ..dao.tunnel_session_dao import TunnelSessionDAO
|
||||
return TunnelSessionDAO(get_session_local()())
|
||||
|
||||
|
||||
def _close_tunnel_dao(dao) -> None:
|
||||
try:
|
||||
dao.db.close()
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
|
||||
|
||||
class ZikaiSSHServer(asyncssh.SSHServer):
|
||||
"""支持密码(bcrypt)与公钥两种鉴权。"""
|
||||
"""支持密码(bcrypt)与公钥两种鉴权;允许反向隧道 user 的 remote forwarding。"""
|
||||
|
||||
def __init__(self, settings, authorized_keys: asyncssh.SSHAuthorizedKeys | None) -> None:
|
||||
self._settings = settings
|
||||
self._authorized_keys = authorized_keys
|
||||
self._conn: asyncssh.SSHServerConnection | None = None
|
||||
self._username: str | None = None # 认证成功后填入
|
||||
self._peer_ip: str = ""
|
||||
|
||||
def connection_made(self, conn: asyncssh.SSHServerConnection) -> None: # type: ignore[override]
|
||||
self._conn = conn
|
||||
peer = conn.get_extra_info("peername")
|
||||
self._peer_ip = peer[0] if isinstance(peer, tuple) and peer else ""
|
||||
|
||||
def begin_auth(self, username: str) -> bool:
|
||||
# 始终要求鉴权;用户不存在时所有方法会失败 → 干净的 permission denied
|
||||
@@ -56,14 +74,17 @@ class ZikaiSSHServer(asyncssh.SSHServer):
|
||||
return True
|
||||
|
||||
def validate_password(self, username: str, password: str) -> bool:
|
||||
user = next((u for u in self._settings.sftp.users if u.username == username), None)
|
||||
if user is None or not user.password_hash or user.password_hash == "CHANGE_ME_BCRYPT_HASH":
|
||||
user = self._find_sftp_user(username) or self._find_tunnel_user(username)
|
||||
if user is None or not getattr(user, "password_hash", "") \
|
||||
or user.password_hash == "CHANGE_ME_BCRYPT_HASH":
|
||||
return False
|
||||
try:
|
||||
ok = bcrypt.checkpw(password.encode(), user.password_hash.encode())
|
||||
except (ValueError, TypeError):
|
||||
ok = False
|
||||
logger.info("SFTP 密码鉴权 %s user=%s", "OK" if ok else "FAIL", username)
|
||||
if ok:
|
||||
self._username = username
|
||||
logger.info("SSH 密码鉴权 %s user=%s", "OK" if ok else "FAIL", username)
|
||||
return ok
|
||||
|
||||
# 公钥
|
||||
@@ -74,21 +95,88 @@ class ZikaiSSHServer(asyncssh.SSHServer):
|
||||
def validate_public_key(self, username: str, key: asyncssh.SSHKey) -> bool:
|
||||
if self._authorized_keys is None:
|
||||
return False
|
||||
if not any(u.username == username for u in self._settings.sftp.users):
|
||||
if self._find_sftp_user(username) is None and self._find_tunnel_user(username) is None:
|
||||
return False
|
||||
addr = ""
|
||||
if self._conn:
|
||||
peer = self._conn.get_extra_info("peername")
|
||||
addr = peer[0] if isinstance(peer, tuple) and peer else ""
|
||||
addr = self._peer_ip
|
||||
try:
|
||||
# asyncssh 命中返回 dict(可能为空),未命中返回 None
|
||||
result = self._authorized_keys.validate(key, client_host=addr, client_addr=addr)
|
||||
except Exception:
|
||||
result = None
|
||||
ok = result is not None
|
||||
logger.info("SFTP 公钥鉴权 %s user=%s", "OK" if ok else "FAIL", username)
|
||||
if ok:
|
||||
self._username = username
|
||||
logger.info("SSH 公钥鉴权 %s user=%s", "OK" if ok else "FAIL", username)
|
||||
return ok
|
||||
|
||||
# 反向隧道:remote port-forward 请求
|
||||
|
||||
def server_requested(self, listen_host: str, listen_port: int) -> bool:
|
||||
"""客户端请求在本地(server 侧)绑端口做反向转发时回调。
|
||||
|
||||
仅允许已认证的 tunnel user 绑定其配置中预定的 tunnel_port;记一条 active
|
||||
隧道会话到 DB。其他情况拒绝。
|
||||
"""
|
||||
if not self._settings.tunnel.enabled:
|
||||
logger.warning("拒绝 remote forward:tunnel 未启用 user=%s", self._username)
|
||||
return False
|
||||
if not self._username:
|
||||
logger.warning("拒绝 remote forward:未认证")
|
||||
return False
|
||||
tunnel_user = self._find_tunnel_user(self._username)
|
||||
if tunnel_user is None:
|
||||
logger.warning("拒绝 remote forward:%s 不是 tunnel user", self._username)
|
||||
return False
|
||||
if listen_port != tunnel_user.tunnel_port:
|
||||
logger.warning(
|
||||
"拒绝 remote forward:user=%s 端口 %d 不等于配置 %d",
|
||||
self._username, listen_port, tunnel_user.tunnel_port,
|
||||
)
|
||||
return False
|
||||
|
||||
dao = _tunnel_dao()
|
||||
try:
|
||||
from ..services.tunnel_service import TunnelService
|
||||
TunnelService(dao).register(
|
||||
user_name=self._username,
|
||||
user_ip=self._peer_ip,
|
||||
tunnel_port=listen_port,
|
||||
local_port=tunnel_user.local_port,
|
||||
)
|
||||
except Exception as exc: # pragma: no cover
|
||||
_close_tunnel_dao(dao)
|
||||
logger.error("登记隧道会话失败:%s", exc)
|
||||
return False
|
||||
_close_tunnel_dao(dao)
|
||||
logger.info(
|
||||
"允许 remote forward user=%s listen=%s:%d -> local_port=%d",
|
||||
self._username, listen_host, listen_port, tunnel_user.local_port,
|
||||
)
|
||||
return True
|
||||
|
||||
def connection_lost(self, exc: Exception | None) -> None: # type: ignore[override]
|
||||
"""SSH 连接断开时清理该 user 的活跃隧道会话。"""
|
||||
if self._username and self._settings.tunnel.enabled:
|
||||
dao = _tunnel_dao()
|
||||
try:
|
||||
from ..services.tunnel_service import TunnelService
|
||||
TunnelService(dao).close(self._username)
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.warning("清理隧道会话失败 user=%s: %s", self._username, e)
|
||||
_close_tunnel_dao(dao)
|
||||
if exc:
|
||||
logger.info("SSH 连接异常断开 user=%s: %s", self._username, exc)
|
||||
else:
|
||||
logger.info("SSH 连接关闭 user=%s", self._username)
|
||||
|
||||
# 内部
|
||||
|
||||
def _find_sftp_user(self, username: str):
|
||||
return next((u for u in self._settings.sftp.users if u.username == username), None)
|
||||
|
||||
def _find_tunnel_user(self, username: str):
|
||||
return self._settings.tunnel.find_user(username)
|
||||
|
||||
|
||||
def _load_authorized_keys(path: Path) -> asyncssh.SSHAuthorizedKeys | None:
|
||||
if not path.exists() or path.stat().st_size == 0:
|
||||
@@ -119,7 +207,7 @@ async def _run() -> None:
|
||||
|
||||
upload_root = settings.resolved_upload_dir()
|
||||
upload_root.mkdir(parents=True, exist_ok=True)
|
||||
# SFTP 客户端登记前的暂存目录;register-sftp 只接受此目录下的路径。
|
||||
# SFTP 客户端的暂存目录(chroot 内)。
|
||||
(upload_root / "incoming").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
host_key_path = (PROJECT_ROOT / settings.sftp.host_key_path).resolve()
|
||||
|
||||
70
app/services/tunnel_service.py
Normal file
70
app/services/tunnel_service.py
Normal file
@@ -0,0 +1,70 @@
|
||||
"""反向隧道业务逻辑:会话注册 / 注销 / 查询。
|
||||
|
||||
SSH 服务收到 remote port-forward 请求时调 register 记一条 active 会话;
|
||||
user 断开时调 close 标记 ended_at;HTTP 路由调 get_active 查隧道端口做反代。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from ..config import get_settings
|
||||
from ..dao.tunnel_session_dao import TunnelSessionDAO
|
||||
from ..models.tunnel_session import TunnelSession
|
||||
|
||||
logger = logging.getLogger("zikai.tunnel")
|
||||
|
||||
|
||||
class TunnelService:
|
||||
def __init__(self, dao: TunnelSessionDAO) -> None:
|
||||
self.dao = dao
|
||||
self.settings = get_settings().tunnel
|
||||
|
||||
def register(
|
||||
self, user_name: str, user_ip: str, tunnel_port: int, local_port: int,
|
||||
) -> TunnelSession:
|
||||
"""登记一条活跃隧道会话。
|
||||
|
||||
若该 user 已有活跃会话先关闭旧的(同一 user 同一时刻只保留一条)。
|
||||
"""
|
||||
self.dao.close_active_by_user(user_name)
|
||||
session = TunnelSession(
|
||||
user_name=user_name,
|
||||
user_ip=user_ip,
|
||||
local_port=local_port,
|
||||
tunnel_port=tunnel_port,
|
||||
status="active",
|
||||
)
|
||||
saved = self.dao.create(session)
|
||||
logger.info(
|
||||
"隧道建立 user=%s ip=%s tunnel_port=%d local_port=%d",
|
||||
user_name, user_ip, tunnel_port, local_port,
|
||||
)
|
||||
return saved
|
||||
|
||||
def close(self, user_name: str) -> int:
|
||||
"""关闭该 user 的活跃会话(SSH 断开时调用),返回关闭条数。"""
|
||||
n = self.dao.close_active_by_user(user_name)
|
||||
if n:
|
||||
logger.info("隧道关闭 user=%s 条数=%d", user_name, n)
|
||||
return n
|
||||
|
||||
def get_active(self, user_name: str) -> TunnelSession | None:
|
||||
return self.dao.get_active_by_user(user_name)
|
||||
|
||||
def is_port_allowed(self, user_name: str, tunnel_port: int) -> bool:
|
||||
"""校验该 user 是否被允许绑定该隧道端口(防 user 乱绑端口)。"""
|
||||
user = self.settings.find_user(user_name)
|
||||
return user is not None and user.tunnel_port == tunnel_port
|
||||
|
||||
def reap_orphans(self) -> int:
|
||||
"""兜底清理:关闭所有 active 会话(进程重启时 DB 里残留的孤儿记录)。
|
||||
|
||||
由后台 reaper 在启动后调用一次。SSH 实际断开时已有 close() 处理,
|
||||
这里只兜底进程异常退出后 DB 与实际状态不一致的情况。
|
||||
"""
|
||||
active = self.dao.list_active()
|
||||
for session in active:
|
||||
self.dao.close(session)
|
||||
logger.info("reaper 清理孤儿隧道 id=%d user=%s", session.id, session.user_name)
|
||||
return len(active)
|
||||
@@ -1,22 +1,35 @@
|
||||
"""上传服务:流式落盘 + 原子改名 + 元数据入库。"""
|
||||
"""上传服务:流式落盘 + 原子改名 + 元数据入库 + sha256 去重。
|
||||
|
||||
两条上传路径(HTTP 整文件 / 分片拼接)共享本类的「存储路径生成 + 落库提交 +
|
||||
sha256 去重」逻辑,避免重复实现。SFTP 服务器仅作为文件暂存通道(chroot 到
|
||||
upload_root),不再有 HTTP 登记接口。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import HTTPException, UploadFile
|
||||
from fastapi import UploadFile
|
||||
|
||||
from ..config import get_settings
|
||||
from ..dao.uploaded_file_dao import UploadedFileDAO
|
||||
from ..models.uploaded_file import UploadedFile
|
||||
from ..schemas.file import FileUploadResponse, UploadedFileOut
|
||||
|
||||
# SFTP 客户端登记前必须把文件先落到此目录(chroot 内)
|
||||
SFTP_INCOMING_DIR = "incoming"
|
||||
|
||||
def hash_stream(chunks: Iterator[bytes]) -> tuple[int, str]:
|
||||
"""对一段字节块序列流式计算 (size, sha256)。供多条上传路径复用。"""
|
||||
h = hashlib.sha256()
|
||||
size = 0
|
||||
for chunk in chunks:
|
||||
size += len(chunk)
|
||||
h.update(chunk)
|
||||
return size, h.hexdigest()
|
||||
|
||||
|
||||
class UploadService:
|
||||
@@ -36,7 +49,7 @@ class UploadService:
|
||||
|
||||
失败仅会留下可识别的 ``.part`` 文件,由 start.sh 启动时统一清理。
|
||||
"""
|
||||
rel_path, abs_path = self._make_storage_path(file.filename or "")
|
||||
rel_path, abs_path = self.make_storage_path(file.filename or "")
|
||||
part_path = abs_path.with_name(abs_path.name + ".part")
|
||||
size, digest = self._write_part(file, part_path)
|
||||
|
||||
@@ -49,34 +62,7 @@ class UploadService:
|
||||
source=source,
|
||||
uploaded_by=uploaded_by,
|
||||
)
|
||||
return self._commit(entity, part_path, abs_path)
|
||||
|
||||
def register_sftp(
|
||||
self, filename: str, original_filename: str, uploaded_by: str = "sftp",
|
||||
) -> FileUploadResponse:
|
||||
"""登记一个已通过 SFTP 落到 ``incoming/`` 下的文件。
|
||||
|
||||
相同 sha256 已存在时直接返回旧行并删掉新副本(去重)。
|
||||
"""
|
||||
src_abs = self._validate_incoming_path(filename)
|
||||
size, digest = self._hash_disk_file(src_abs)
|
||||
|
||||
existing = self.dao.get_by_sha256(digest)
|
||||
if existing is not None:
|
||||
src_abs.unlink(missing_ok=True)
|
||||
return self._to_response(existing)
|
||||
|
||||
rel_path, abs_path = self._make_storage_path(original_filename or filename)
|
||||
entity = UploadedFile(
|
||||
storage_path=str(rel_path),
|
||||
original_filename=os.path.basename(original_filename or src_abs.name),
|
||||
content_type="",
|
||||
size_bytes=size,
|
||||
sha256=digest,
|
||||
source="sftp",
|
||||
uploaded_by=uploaded_by,
|
||||
)
|
||||
return self._commit(entity, src_abs, abs_path)
|
||||
return self.commit_entity(entity, part_path, abs_path)
|
||||
|
||||
# ---------------- 查询 ----------------
|
||||
|
||||
@@ -94,18 +80,26 @@ class UploadService:
|
||||
row = self.dao.get_by_sha256(sha256)
|
||||
return UploadedFileOut.model_validate(row) if row else None
|
||||
|
||||
def resolve_disk_path(self, file_id: int) -> Path | None:
|
||||
row = self.dao.get_by_id(file_id)
|
||||
return (self.upload_root / row.storage_path).resolve() if row else None
|
||||
def get_out_with_disk_path(self, file_id: int) -> tuple[UploadedFileOut | None, Path | None]:
|
||||
"""合并查询:一次 DB 读取同时返回 (元数据, 磁盘绝对路径)。
|
||||
|
||||
# ---------------- 内部 ----------------
|
||||
供下载/删除路径复用,避免原先 get_out + resolve_disk_path 各查一次的重复读。
|
||||
"""
|
||||
row = self.dao.get_by_id(file_id)
|
||||
if row is None:
|
||||
return None, None
|
||||
out = UploadedFileOut.model_validate(row)
|
||||
path = (self.upload_root / row.storage_path).resolve()
|
||||
return out, path
|
||||
|
||||
# ---------------- 共享 helper(供本类与 ChunkUploadService 复用) ----------------
|
||||
|
||||
@staticmethod
|
||||
def _safe_ext(filename: str) -> str:
|
||||
return os.path.splitext(os.path.basename(filename))[1]
|
||||
|
||||
@staticmethod
|
||||
def _to_response(row: UploadedFile) -> FileUploadResponse:
|
||||
def to_response(row: UploadedFile, *, deduplicated: bool = False) -> FileUploadResponse:
|
||||
return FileUploadResponse(
|
||||
id=row.id,
|
||||
filename=row.original_filename,
|
||||
@@ -113,9 +107,10 @@ class UploadService:
|
||||
sha256=row.sha256,
|
||||
storage_path=row.storage_path,
|
||||
uploaded_at=row.uploaded_at,
|
||||
deduplicated=deduplicated,
|
||||
)
|
||||
|
||||
def _make_storage_path(self, original_filename: str) -> tuple[Path, Path]:
|
||||
def make_storage_path(self, original_filename: str) -> tuple[Path, Path]:
|
||||
"""生成 ``(rel_path, abs_path)``;abs_path 的父目录已创建。"""
|
||||
now = datetime.now(timezone.utc)
|
||||
rel_dir = Path(f"{now:%Y}/{now:%m}")
|
||||
@@ -123,21 +118,7 @@ class UploadService:
|
||||
rel_path = rel_dir / f"{uuid.uuid4().hex}{self._safe_ext(original_filename)}"
|
||||
return rel_path, self.upload_root / rel_path
|
||||
|
||||
def _validate_incoming_path(self, filename: str) -> Path:
|
||||
"""校验客户端给出的相对路径必须落在 upload_root/incoming/ 之下且是文件。"""
|
||||
if not filename or filename.startswith(("/", "\\")):
|
||||
raise HTTPException(400, "filename 必须是 incoming/ 下的相对路径")
|
||||
incoming_root = (self.upload_root / SFTP_INCOMING_DIR).resolve()
|
||||
try:
|
||||
abs_path = (self.upload_root / filename).resolve()
|
||||
abs_path.relative_to(incoming_root)
|
||||
except ValueError:
|
||||
raise HTTPException(400, f"filename 必须落在 {SFTP_INCOMING_DIR}/ 之下")
|
||||
if not abs_path.is_file():
|
||||
raise HTTPException(404, f"文件不存在或不是普通文件:{filename}")
|
||||
return abs_path
|
||||
|
||||
def _commit(
|
||||
def commit_entity(
|
||||
self, entity: UploadedFile, src_path: Path, dest_path: Path,
|
||||
) -> FileUploadResponse:
|
||||
"""落 DB 行后把 src_path 原子改名到 dest_path;任一失败回滚已生成的副作用。"""
|
||||
@@ -154,17 +135,25 @@ class UploadService:
|
||||
finally:
|
||||
src_path.unlink(missing_ok=True)
|
||||
raise
|
||||
return self._to_response(saved)
|
||||
return self.to_response(saved)
|
||||
|
||||
def _hash_disk_file(self, path: Path) -> tuple[int, str]:
|
||||
"""流式读取磁盘文件,返回 (size, sha256)。"""
|
||||
h = hashlib.sha256()
|
||||
size = 0
|
||||
with path.open("rb") as fh:
|
||||
while chunk := fh.read(self.chunk_bytes):
|
||||
size += len(chunk)
|
||||
h.update(chunk)
|
||||
return size, h.hexdigest()
|
||||
def dedup_or_commit(
|
||||
self, entity: UploadedFile, src_path: Path,
|
||||
) -> FileUploadResponse:
|
||||
"""公共尾部:按 entity.sha256 去重,命中则删 src 返回旧行;否则 commit。
|
||||
|
||||
供分片拼接等「先算出 sha256 再决定落盘」的路径复用,与 stream_to_disk
|
||||
(边写边算、无独立 src)的区别在于这里 sha256 已在 entity 上。
|
||||
"""
|
||||
existing = self.dao.get_by_sha256(entity.sha256)
|
||||
if existing is not None:
|
||||
src_path.unlink(missing_ok=True)
|
||||
return self.to_response(existing, deduplicated=True)
|
||||
rel_path, abs_path = self.make_storage_path(entity.original_filename)
|
||||
entity.storage_path = str(rel_path)
|
||||
return self.commit_entity(entity, src_path, abs_path)
|
||||
|
||||
# ---------------- 内部 ----------------
|
||||
|
||||
def _write_part(self, file: UploadFile, part_path: Path) -> tuple[int, str]:
|
||||
"""把上传流写到 part_path 并 fsync;返回 (size, sha256)。失败时清理残品。"""
|
||||
|
||||
175
app/services/whiteboard_hub.py
Normal file
175
app/services/whiteboard_hub.py
Normal file
@@ -0,0 +1,175 @@
|
||||
"""白板 WebSocket 连接管理器(实时同步 + 心跳 + 失活清理)。
|
||||
|
||||
设计要点:
|
||||
- 进程内单例 ``WhiteboardHub``,维护 ``{board_id: set[Connection]}``。
|
||||
- 每个 Connection 封装 websocket / board_id / client_id / last_heartbeat。
|
||||
- 心跳:客户端每 ``heartbeat_interval_seconds``(默认 3s)发一次 ping,服务端回 pong 并
|
||||
刷新 last_heartbeat。reaper 每秒扫描,超过 ``interval * threshold``(默认 15s)未心跳
|
||||
的连接判为失活,关闭并从 hub 移除。
|
||||
- 内存安全:disconnect 幂等;空 set 从 dict 删除;broadcast 对单连接异常立即 disconnect;
|
||||
close_board 关闭并清理整个 board 的连接集合。
|
||||
- 并发:用一个 asyncio.Lock 保护 ``_boards`` 的结构变更(add/remove board key),
|
||||
集合内的连接增删用 set 原子操作(Python 单线程事件循环下安全)。
|
||||
|
||||
注意:hub 是进程内存,多 worker 下不互通。生产部署需单 worker 或后续接 Redis pub/sub。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from fastapi import WebSocket
|
||||
|
||||
from ..config import get_settings
|
||||
|
||||
logger = logging.getLogger("zikai.whiteboard")
|
||||
|
||||
|
||||
@dataclass(eq=False)
|
||||
class Connection:
|
||||
"""一个白板在线连接。eq=False 使其按对象 identity 哈希/比较,可放入 set。"""
|
||||
|
||||
websocket: WebSocket
|
||||
board_id: str
|
||||
client_id: str
|
||||
last_heartbeat: float = field(default_factory=time.monotonic)
|
||||
|
||||
async def send_json(self, msg: dict) -> bool:
|
||||
"""发送一条消息;失败返回 False(调用方据此 disconnect)。"""
|
||||
try:
|
||||
await self.websocket.send_json(msg)
|
||||
return True
|
||||
except Exception as exc: # WebSocketDisconnect / 已关闭 / 编码失败
|
||||
logger.debug("发送失败 board=%s client=%s: %s", self.board_id, self.client_id, exc)
|
||||
return False
|
||||
|
||||
def touch(self) -> None:
|
||||
self.last_heartbeat = time.monotonic()
|
||||
|
||||
|
||||
class WhiteboardHub:
|
||||
"""白板连接管理器单例。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
cfg = get_settings().whiteboard
|
||||
self.heartbeat_interval = cfg.heartbeat_interval_seconds
|
||||
self.heartbeat_miss_threshold = cfg.heartbeat_miss_threshold
|
||||
self.timeout_seconds = self.heartbeat_interval * self.heartbeat_miss_threshold
|
||||
# {board_id: set[Connection]}
|
||||
self._boards: dict[str, set[Connection]] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
# ---------------- 连接生命周期 ----------------
|
||||
|
||||
async def register(self, conn: Connection) -> None:
|
||||
"""把已 accept 的连接加入 board 集合(WebSocket accept 由 controller 负责)。"""
|
||||
async with self._lock:
|
||||
conns = self._boards.setdefault(conn.board_id, set())
|
||||
conns.add(conn)
|
||||
logger.info("连接接入 board=%s client=%s(当前 %d 人)",
|
||||
conn.board_id, conn.client_id, self.connection_count(conn.board_id))
|
||||
|
||||
async def disconnect(self, conn: Connection) -> None:
|
||||
"""幂等移除连接;空 set 从 dict 删除以防内存泄漏。"""
|
||||
async with self._lock:
|
||||
conns = self._boards.get(conn.board_id)
|
||||
if conns is None:
|
||||
return
|
||||
conns.discard(conn)
|
||||
if not conns:
|
||||
self._boards.pop(conn.board_id, None)
|
||||
# 尽力关闭 websocket(可能已关闭)
|
||||
try:
|
||||
await conn.websocket.close()
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
logger.info("连接移除 board=%s client=%s(剩余 %d 人)",
|
||||
conn.board_id, conn.client_id, self.connection_count(conn.board_id))
|
||||
|
||||
def connection_count(self, board_id: str) -> int:
|
||||
"""调试/监控用:某 board 当前在线人数。"""
|
||||
return len(self._boards.get(board_id, ()))
|
||||
|
||||
# ---------------- 广播 ----------------
|
||||
|
||||
async def broadcast(self, board_id: str, msg: dict, exclude: Connection | None = None) -> None:
|
||||
"""把 msg 发给 board 内所有在线连接(可排除发送者)。单连接失败不影响其他。"""
|
||||
async with self._lock:
|
||||
conns = list(self._boards.get(board_id, ()))
|
||||
dead: list[Connection] = []
|
||||
for conn in conns:
|
||||
if exclude is not None and conn is exclude:
|
||||
continue
|
||||
ok = await conn.send_json(msg)
|
||||
if not ok:
|
||||
dead.append(conn)
|
||||
# 发送失败的连接统一清理
|
||||
for conn in dead:
|
||||
await self.disconnect(conn)
|
||||
|
||||
# ---------------- 心跳 reaper ----------------
|
||||
|
||||
async def reap_loop(self, stop: asyncio.Event) -> None:
|
||||
"""后台循环:扫描失活连接。每秒一次,粒度细于 timeout。"""
|
||||
logger.info("白板心跳 reaper 已启动(间隔 1s,超时 %ds)", self.timeout_seconds)
|
||||
while not stop.is_set():
|
||||
try:
|
||||
await self._reap_once()
|
||||
except Exception as exc: # pragma: no cover
|
||||
logger.warning("reaper 循环异常:%s", exc)
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), timeout=1.0)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
async def _reap_once(self) -> None:
|
||||
now = time.monotonic()
|
||||
async with self._lock:
|
||||
# 快照待检查连接,避免持锁时 await
|
||||
stale: list[Connection] = []
|
||||
for board_id, conns in self._boards.items():
|
||||
for conn in conns:
|
||||
if now - conn.last_heartbeat > self.timeout_seconds:
|
||||
stale.append(conn)
|
||||
for conn in stale:
|
||||
logger.warning("心跳失活,移除 board=%s client=%s(静默 %ds)",
|
||||
conn.board_id, conn.client_id,
|
||||
int(now - conn.last_heartbeat))
|
||||
await self.disconnect(conn)
|
||||
|
||||
async def close_board(self, board_id: str) -> None:
|
||||
"""关闭并清理某 board 的所有连接(删除白板时调用)。"""
|
||||
async with self._lock:
|
||||
conns = self._boards.pop(board_id, None)
|
||||
if not conns:
|
||||
return
|
||||
await asyncio.gather(
|
||||
*(c.send_json({"type": "error", "msg": "白板已被删除"}) for c in conns),
|
||||
return_exceptions=True,
|
||||
)
|
||||
for conn in conns:
|
||||
try:
|
||||
await conn.websocket.close()
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
logger.info("关闭白板 board=%s,踢出 %d 个连接", board_id, len(conns))
|
||||
|
||||
|
||||
# 进程内单例(由 main.py lifespan / controller 共享)
|
||||
_hub: WhiteboardHub | None = None
|
||||
|
||||
|
||||
def get_hub() -> WhiteboardHub:
|
||||
global _hub
|
||||
if _hub is None:
|
||||
_hub = WhiteboardHub()
|
||||
return _hub
|
||||
|
||||
|
||||
def reset_hub() -> None:
|
||||
"""测试用:重置单例。"""
|
||||
global _hub
|
||||
_hub = None
|
||||
92
app/services/whiteboard_service.py
Normal file
92
app/services/whiteboard_service.py
Normal file
@@ -0,0 +1,92 @@
|
||||
"""白板服务:CRUD + 笔画操作。
|
||||
|
||||
不持有 WebSocket 连接状态(那是 hub 的职责);删除白板时通过可选的 hub 回调
|
||||
通知 hub 踢出该 board 的所有在线连接,由 controller 在装配时注入,避免 service -> hub
|
||||
的硬依赖(保持低耦合)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from ..config import get_settings
|
||||
from ..dao.whiteboard_dao import WhiteboardDAO
|
||||
from ..schemas.whiteboard import WhiteboardListItem, WhiteboardOut
|
||||
|
||||
if TYPE_CHECKING: # 避免运行时循环导入
|
||||
from .whiteboard_hub import WhiteboardHub
|
||||
|
||||
# board_id 合法字符集:字母数字下划线短横线
|
||||
_BOARD_ID_RE = re.compile(r"^[a-zA-Z0-9_-]+$")
|
||||
|
||||
|
||||
class WhiteboardService:
|
||||
def __init__(self, dao: WhiteboardDAO, hub: "WhiteboardHub | None" = None) -> None:
|
||||
self.dao = dao
|
||||
self.hub = hub
|
||||
cfg = get_settings().whiteboard
|
||||
self.max_board_id_length = cfg.max_board_id_length
|
||||
self.list_limit = cfg.list_limit
|
||||
|
||||
# ---------------- 校验 ----------------
|
||||
|
||||
def validate_board_id(self, board_id: str) -> None:
|
||||
"""非法 board_id 直接 400(防路径穿越/注入)。"""
|
||||
if (
|
||||
not board_id
|
||||
or len(board_id) > self.max_board_id_length
|
||||
or not _BOARD_ID_RE.match(board_id)
|
||||
):
|
||||
raise HTTPException(400, "board_id 非法(仅允许字母数字下划线短横线,1-64 字符)")
|
||||
|
||||
# ---------------- 读 ----------------
|
||||
|
||||
def get_or_create(self, board_id: str) -> WhiteboardOut:
|
||||
self.validate_board_id(board_id)
|
||||
board = self.dao.get_or_create(board_id)
|
||||
return WhiteboardOut.model_validate(board)
|
||||
|
||||
def list_all(self, limit: int = 100, offset: int = 0) -> tuple[int, list[WhiteboardListItem]]:
|
||||
limit = min(max(limit, 0), self.list_limit) or self.list_limit
|
||||
offset = max(offset, 0)
|
||||
total = self.dao.count()
|
||||
rows = self.dao.list_all(limit=limit, offset=offset)
|
||||
items = [WhiteboardListItem.model_validate(r) for r in rows]
|
||||
return total, items
|
||||
|
||||
# ---------------- 写 ----------------
|
||||
|
||||
def append_stroke(self, board_id: str, stroke: dict[str, Any]) -> WhiteboardOut:
|
||||
"""追加一条笔画并返回最新状态。"""
|
||||
self.validate_board_id(board_id)
|
||||
board = self.dao.append_strokes(board_id, [stroke])
|
||||
if board is None:
|
||||
raise HTTPException(404, "白板不存在")
|
||||
return WhiteboardOut.model_validate(board)
|
||||
|
||||
def clear(self, board_id: str) -> WhiteboardOut:
|
||||
"""清空白板;stroke_count 仍自增以记录这次修改。"""
|
||||
self.validate_board_id(board_id)
|
||||
board = self.dao.replace_strokes(board_id, [])
|
||||
if board is None:
|
||||
raise HTTPException(404, "白板不存在")
|
||||
return WhiteboardOut.model_validate(board)
|
||||
|
||||
def delete(self, board_id: str) -> bool:
|
||||
"""删除白板;同时通知 hub 踢出该 board 的所有在线连接。"""
|
||||
self.validate_board_id(board_id)
|
||||
ok = self.dao.delete(board_id)
|
||||
if ok and self.hub is not None:
|
||||
# hub.close_board 是 async,但删除走 REST 同步路径;安排到事件循环里执行
|
||||
import asyncio
|
||||
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
loop.create_task(self.hub.close_board(board_id))
|
||||
except RuntimeError:
|
||||
# 无运行中事件循环(如脚本调用):同步调用会报错,忽略即可
|
||||
pass
|
||||
return ok
|
||||
Reference in New Issue
Block a user