feat: 文件浏览页 + 共享白板 + 白板管理页(含补登记分片上传/隧道历史改动)

本次提交包含两批改动(7月2日遗留未入库 + 本次新功能),分述如下:

【补登记:7月2日已上线但未提交的功能】
- 分片上传:chunk_upload_controller/service/dao + UploadSession model/schema,
  支持 4MiB 分片、乱序、断点续传、去重、幂等 complete;后台 reaper 清理过期会话。
- 反向隧道:tunnel_controller/service/dao + TunnelSession model/schema,
  SSH remote forward 经 /api/userPort/{userName} 反代到 user 本地服务。
- 上传页:views/upload_html.py(拖拽/多文件/分片/断点续传 UI)。
- config.py:StorageConfig.chunk_session_dir/ttl、TunnelConfig;
  requirements.txt 加 httpx;start.sh 清理 .work/ 残留;
  schema.sql 加 upload_session/tunnel_session 表;sftp_server 承载隧道转发。

【本次新功能】
- 文件浏览页:GET /files(Basic Auth 同 docs)+ /api/admin/files(list/get/download/DELETE)。
  硬删除(DB 行 + 磁盘文件),删除后列表不再显示。前端 static/file_browser.*。
- 共享白板:GET /whiteboard/{id}(公开,不存在则新建)+ WS /ws/whiteboard/{id}。
  MySQL 持久化(whiteboard 表),Canvas 实时同步,心跳 3s/5 次失活移除,
  清空/复制按钮,移动端兼容。WhiteboardHub 管理 {board_id: set[Connection]},
  disconnect 幂等 + 空 set 清理防泄漏,broadcast 失败连接自动移除。
- 白板管理页:GET /whiteboard-admin(Basic Auth)+ /api/admin/whiteboards(list/DELETE)。
  删除时 hub.close_board 踢出在线连接。
- 清理:合并 UploadService.get_out_with_disk_path(下载/删除复用,消除重复 DB 读),
  移除无用 resolve_disk_path。
- config.py:WhiteboardConfig(heartbeat/threshold/board_id 长度/list_limit);
  schema.sql 加 whiteboard 表;README 补新接口与心跳/内存说明。
- 验证:tests/manual_whiteboard_hub.py / _ws.py / _kick.py 全部通过。
This commit is contained in:
zikai
2026-07-21 14:28:13 +00:00
parent e5a725fc91
commit fffba79022
48 changed files with 3569 additions and 216 deletions

View File

@@ -33,6 +33,10 @@ class StorageConfig(BaseModel):
upload_dir: str = "./uploads"
chunk_bytes: int = 1024 * 1024
sha256_on_upload: bool = True
# 分片上传会话的暂存目录(相对 upload_dir完整文件拼接在此完成
chunk_session_dir: str = "./.work"
# 分片上传会话的存活秒数:超过该时长无活动的 pending 会话由后台 reaper 清理
chunk_session_ttl_seconds: int = 300
class SftpUser(BaseModel):
@@ -65,12 +69,51 @@ class DocsConfig(BaseModel):
return "" if v is None else str(v)
class TunnelUser(BaseModel):
"""一个反向隧道用户:凭据 + 固定的隧道端口与本地端口。"""
username: str
password_hash: str = "" # bcrypt与 SFTP 用户同款
# 该 user 在 server 侧绑定的隧道端口SSH remote forward 的 listen port
tunnel_port: int = 0
# 该 user 要暴露的本地服务端口(仅用于记录,实际转发由 user 端完成)
local_port: int = 0
class TunnelConfig(BaseModel):
"""反向隧道总开关与用户列表。"""
enabled: bool = False
users: list[TunnelUser] = Field(default_factory=list)
def find_user(self, username: str) -> TunnelUser | None:
return next((u for u in self.users if u.username == username), None)
class WhiteboardConfig(BaseModel):
"""共享白板配置。
白板本身无鉴权(任何人凭 /whiteboard/{id} 即可访问并实时协作);
管理页(/whiteboard-admin、/api/admin/whiteboards走 docs 同款 Basic Auth。
心跳按 heartbeat_interval_seconds 发送,连续丢失 heartbeat_miss_threshold 次即判失活。
"""
heartbeat_interval_seconds: int = 3
heartbeat_miss_threshold: int = 5
# board_id 合法字符集与长度上限,防路径/注入
max_board_id_length: int = 64
# 列表/管理页分页默认值
list_limit: int = 100
class Settings(BaseModel):
server: ServerConfig = ServerConfig()
database: DatabaseConfig = DatabaseConfig()
storage: StorageConfig = StorageConfig()
sftp: SftpConfig = SftpConfig()
docs: DocsConfig = DocsConfig()
tunnel: TunnelConfig = TunnelConfig()
whiteboard: WhiteboardConfig = WhiteboardConfig()
def db_url(self) -> str:
c = self.database

View File

@@ -1,6 +1,17 @@
"""Controller 层API 路由。"""
from .chunk_upload_controller import router as chunk_upload_router
from .file_admin_controller import router as file_admin_router
from .file_controller import router as file_router
from .system_controller import router as system_router
from .tunnel_controller import router as tunnel_router
from .whiteboard_controller import router as whiteboard_router
__all__ = ["file_router", "system_router"]
__all__ = [
"chunk_upload_router",
"file_admin_router",
"file_router",
"system_router",
"tunnel_router",
"whiteboard_router",
]

View File

@@ -0,0 +1,93 @@
"""分片上传接口:创建会话 / 查状态 / 传分片 / 完成拼接。
支持大文件分片上传与断点续传。前端先创建会话拿 upload_id逐片上传
可随时查 status 获取已传分片以补传缺失部分,最后 complete 触发拼接入库。
"""
from __future__ import annotations
from fastapi import APIRouter, Depends, Request
from sqlalchemy.orm import Session
from ..database import get_db
from ..dao.upload_session_dao import UploadSessionDAO
from ..dao.uploaded_file_dao import UploadedFileDAO
from ..schemas.chunk import (
ChunkUploadResponse,
CreateSessionRequest,
CreateSessionResponse,
SessionStatusResponse,
)
from ..schemas.file import FileUploadResponse
from ..services.chunk_upload_service import ChunkUploadService
router = APIRouter(prefix="/api/files/chunk-uploads", tags=["chunk-upload"])
def _service(db: Session = Depends(get_db)) -> ChunkUploadService:
return ChunkUploadService(UploadSessionDAO(db), UploadedFileDAO(db))
@router.post(
"",
response_model=CreateSessionResponse,
summary="创建分片上传会话",
description=(
"客户端把文件切成固定大小的分片后,先调本接口创建会话。"
"服务端生成 upload_id 返回,后续上传分片、查状态、完成拼接都需要它。"
),
)
def create_session(
body: CreateSessionRequest,
service: ChunkUploadService = Depends(_service),
) -> CreateSessionResponse:
return service.create_session(body)
@router.get(
"/{upload_id}/status",
response_model=SessionStatusResponse,
summary="查询会话状态(断点续传)",
description="返回已上传分片下标集合,前端据此只补传缺失分片。",
)
def session_status(
upload_id: str,
service: ChunkUploadService = Depends(_service),
) -> SessionStatusResponse:
return service.get_status(upload_id)
@router.post(
"/{upload_id}/chunks/{index}",
response_model=ChunkUploadResponse,
summary="上传单个分片",
description=(
"请求体为单个分片的原始二进制。分片可乱序上传,重传同一分片会覆盖。"
"返回当前已上传的分片下标集合。"
),
)
async def upload_chunk(
upload_id: str,
index: int,
request: Request,
service: ChunkUploadService = Depends(_service),
) -> ChunkUploadResponse:
uploaded = service.write_chunk(upload_id, index, await request.body())
return ChunkUploadResponse(upload_id=upload_id, index=index, uploaded_chunks=uploaded)
@router.post(
"/{upload_id}/complete",
response_model=FileUploadResponse,
summary="完成拼接并入库",
description=(
"服务端校验分片齐全后,按顺序拼接为完整文件、流式计算 sha256"
"按 sha256 去重(命中则返回旧行不重复落盘),最后原子改名入库。"
"重复调用幂等,返回同一 file_id。"
),
)
def complete_session(
upload_id: str,
service: ChunkUploadService = Depends(_service),
) -> FileUploadResponse:
return service.complete(upload_id)

View File

@@ -0,0 +1,111 @@
"""文件管理接口Basic Auth鉴权同 docs
与公开的 /api/files 区分:本路由面向「文件浏览页」,提供列表 / 查询 / 下载 / 删除,
均需 docs 凭据。公开路由user.py 依赖的查重、查询、下载)保留不变。
硬删除策略:删 DB 行 + 删磁盘文件unlink missing_ok列表只展示仍存在的行。
"""
from __future__ import annotations
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import FileResponse
from pydantic import BaseModel
from sqlalchemy.orm import Session
from ..database import get_db
from ..dao.uploaded_file_dao import UploadedFileDAO
from ..schemas.file import FileListResponse, UploadedFileOut
from ..security import require_docs_auth
from ..services.upload_service import UploadService
router = APIRouter(prefix="/api/admin/files", tags=["files-admin"])
def _service(db: Session = Depends(get_db)) -> UploadService:
return UploadService(UploadedFileDAO(db))
class DeleteResult(BaseModel):
deleted: bool
@router.get(
"",
response_model=FileListResponse,
summary="列出已上传文件(需鉴权)",
description="供文件浏览页使用;字段同公开 /api/files但要求 docs Basic Auth。",
)
def list_files(
limit: int = 100,
offset: int = 0,
service: UploadService = Depends(_service),
_: str = Depends(require_docs_auth),
) -> FileListResponse:
total, items = service.list_files(limit=limit, offset=offset)
return FileListResponse(total=total, items=items)
@router.get(
"/{file_id}",
response_model=UploadedFileOut,
summary="查询单个文件元数据(需鉴权)",
)
def get_file(
file_id: int,
service: UploadService = Depends(_service),
_: str = Depends(require_docs_auth),
) -> UploadedFileOut:
out = service.get_out(file_id)
if out is None:
raise HTTPException(404, "文件不存在")
return out
@router.get(
"/{file_id}/download",
summary="下载文件(需鉴权,校验磁盘存在)",
description="文件实体不在磁盘上时返回 410 Gone。",
)
def download_file(
file_id: int,
service: UploadService = Depends(_service),
_: str = Depends(require_docs_auth),
) -> FileResponse:
out, path = service.get_out_with_disk_path(file_id)
if out is None:
raise HTTPException(404, "文件不存在")
if path is None or not path.exists():
raise HTTPException(410, "文件实体已不在磁盘上")
return FileResponse(
path=str(path),
media_type=out.content_type or "application/octet-stream",
filename=out.original_filename,
)
@router.delete(
"/{file_id}",
response_model=DeleteResult,
summary="硬删除文件(需鉴权)",
description="删除 DB 行与磁盘文件实体;不可恢复。列表随后不再显示。",
)
def delete_file(
file_id: int,
service: UploadService = Depends(_service),
_: str = Depends(require_docs_auth),
) -> DeleteResult:
out, path = service.get_out_with_disk_path(file_id)
if out is None:
return DeleteResult(deleted=False)
# 先删磁盘文件,再删 DB 行;磁盘文件缺失不阻断 DB 清理
if path is not None:
try:
Path(path).unlink(missing_ok=True)
except Exception:
# 即使磁盘删除失败也继续清 DB 行,保证列表不再显示
pass
service.dao.delete(file_id)
return DeleteResult(deleted=True)

View File

@@ -2,8 +2,6 @@
from __future__ import annotations
from pathlib import Path
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
from fastapi.responses import FileResponse
from sqlalchemy.orm import Session
@@ -13,7 +11,6 @@ from ..dao.uploaded_file_dao import UploadedFileDAO
from ..schemas.file import (
FileListResponse,
FileUploadResponse,
SftpRegisterRequest,
UploadedFileOut,
)
from ..services.upload_service import UploadService
@@ -32,7 +29,7 @@ def _service(db: Session = Depends(get_db)) -> UploadService:
description=(
"multipart/form-data 上传,按 1 MiB 分片流式落盘,内存占用恒定;"
"落盘过程中计算 SHA-256 并入库。\n\n"
"极大或极慢的传输建议改用 SFTP详见 READMEHTTP 链路受 Apache 代理 300s 超时限制"
"极大或极慢的传输建议改用分片上传接口(/api/files/chunk-uploads或 SFTP"
),
)
async def upload_file(
@@ -70,26 +67,6 @@ def file_exists(
return out
@router.post(
"/register-sftp",
response_model=FileUploadResponse,
summary="登记一个已通过 SFTP 落盘的文件",
description=(
"客户端先把文件 SFTP 到 ``incoming/<name>``,再用本接口登记入库。"
"服务端会计算 sha256已存在则去重、把文件原子改名到 ``YYYY/MM/<uuid>.<ext>``、写 DB 行。"
),
)
def register_sftp(
body: SftpRegisterRequest,
service: UploadService = Depends(_service),
) -> FileUploadResponse:
return service.register_sftp(
filename=body.filename,
original_filename=body.original_filename,
uploaded_by=body.uploaded_by,
)
@router.get("/{file_id}", response_model=UploadedFileOut, summary="查询单个文件元数据")
def get_file(file_id: int, service: UploadService = Depends(_service)) -> UploadedFileOut:
out = service.get_out(file_id)
@@ -100,10 +77,9 @@ def get_file(file_id: int, service: UploadService = Depends(_service)) -> Upload
@router.get("/{file_id}/download", summary="下载文件")
def download_file(file_id: int, service: UploadService = Depends(_service)) -> FileResponse:
out = service.get_out(file_id)
out, path = service.get_out_with_disk_path(file_id)
if out is None:
raise HTTPException(404, "文件不存在")
path: Path | None = service.resolve_disk_path(file_id)
if path is None or not path.exists():
raise HTTPException(410, "文件实体已不在磁盘上")
return FileResponse(

View File

@@ -0,0 +1,97 @@
"""反向隧道 HTTP 路由:/api/userPort/{userName}
把进来的 HTTP 请求反代到该 user 当前活跃隧道对应的本地端口
SSH remote forward 绑定的 127.0.0.1:tunnel_port经隧道回指 user 的本地服务。
"""
from __future__ import annotations
import httpx
from fastapi import APIRouter, Depends, HTTPException, Request, Response
from sqlalchemy.orm import Session
from ..database import get_db
from ..dao.tunnel_session_dao import TunnelSessionDAO
from ..services.tunnel_service import TunnelService
router = APIRouter(prefix="/api/userPort", tags=["tunnel"])
# 不应透传给上游的 hop-by-hop / 控制头
_HOP_BY_HOP = {
"connection", "keep-alive", "proxy-authenticate", "proxy-authorization",
"te", "trailers", "transfer-encoding", "upgrade", "host", "content-length",
}
def _service(db: Session = Depends(get_db)) -> TunnelService:
return TunnelService(TunnelSessionDAO(db))
async def _proxy(request: Request, session, prefix: str) -> Response:
"""把请求透传到该 user 当前活跃隧道对应的本地端口。
去掉 /api/userPort/{userName} 前缀后才是上游路径,根路径补 /。
"""
upstream_path = request.url.path.replace(prefix, "", 1) or "/"
url = f"http://127.0.0.1:{session.tunnel_port}{upstream_path}"
if request.url.query:
url += f"?{request.url.query}"
body = await request.body()
headers = {k: v for k, v in request.headers.items() if k.lower() not in _HOP_BY_HOP}
try:
async with httpx.AsyncClient(timeout=30.0) as client:
upstream = await client.request(
request.method, url, content=body, headers=headers,
)
except httpx.RequestError as exc:
raise HTTPException(502, f"隧道端口不可达:{exc}") from exc
resp_headers = {k: v for k, v in upstream.headers.items() if k.lower() not in _HOP_BY_HOP}
return Response(content=upstream.content, status_code=upstream.status_code,
headers=resp_headers)
# 根路径:/api/userPort/{userName}
@router.api_route(
"/{userName}",
methods=["GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"],
summary="反代到指定 user 的隧道端口(根路径)",
description=(
"查 DB 该 user 当前活跃的隧道端口,把请求透传到 127.0.0.1:tunnel_port"
"(经 SSH 反向隧道回指 user 的本地服务)。无活跃隧道返回 502。"
),
)
async def proxy_to_tunnel(
userName: str,
request: Request,
service: TunnelService = Depends(_service),
) -> Response:
session = service.get_active(userName)
if session is None:
raise HTTPException(502, f"无活跃隧道user={userName}")
return await _proxy(request, session, f"/api/userPort/{userName}")
# 子路径:/api/userPort/{userName}/... —— 反向代理必须能透传任意路径与查询串,
# 否则上游服务里所有非根路由都会 404。
@router.api_route(
"/{userName}/{upstream_path:path}",
methods=["GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"],
summary="反代到指定 user 的隧道端口(子路径透传)",
description=(
"把 /api/userPort/{userName}/<path> 透传到 127.0.0.1:tunnel_port/<path>。"
"无活跃隧道返回 502。"
),
)
async def proxy_to_tunnel_path(
userName: str,
upstream_path: str,
request: Request,
service: TunnelService = Depends(_service),
) -> Response:
session = service.get_active(userName)
if session is None:
raise HTTPException(502, f"无活跃隧道user={userName}")
return await _proxy(request, session, f"/api/userPort/{userName}")

View File

@@ -0,0 +1,220 @@
"""白板接口REST访问/管理)+ WebSocket实时同步
路由:
GET /whiteboard/{board_id} 公开:访问白板,不存在则新建
WS /ws/whiteboard/{board_id} 公开:实时协作 + 心跳
GET /api/admin/whiteboards Basic Auth管理页列表
DELETE /api/admin/whiteboards/{id} Basic Auth删除白板
"""
from __future__ import annotations
import json
import logging
import uuid
from fastapi import APIRouter, Depends, HTTPException, Query, WebSocket, WebSocketDisconnect
from sqlalchemy.orm import Session
from ..database import get_db
from ..dao.whiteboard_dao import WhiteboardDAO
from ..schemas.whiteboard import WhiteboardListResponse, WhiteboardOut
from ..security import require_docs_auth
from ..services.whiteboard_hub import Connection, get_hub
from ..services.whiteboard_service import WhiteboardService
logger = logging.getLogger("zikai.whiteboard")
router = APIRouter(tags=["whiteboard"])
def _service(db: Session = Depends(get_db)) -> WhiteboardService:
"""REST 路径的 service注入 hub 以便删除时踢出连接。"""
return WhiteboardService(WhiteboardDAO(db), hub=get_hub())
# ---------------- 公开 REST ----------------
@router.get(
"/whiteboard/{board_id}",
response_model=WhiteboardOut,
summary="访问白板(不存在则新建)",
description="任何人凭 board_id 即可访问;不存在时自动创建空板并返回。",
)
def get_whiteboard(board_id: str, service: WhiteboardService = Depends(_service)) -> WhiteboardOut:
return service.get_or_create(board_id)
# ---------------- 管理 RESTBasic Auth ----------------
@router.get(
"/api/admin/whiteboards",
response_model=WhiteboardListResponse,
summary="列出所有白板(需鉴权)",
description="供白板管理页使用board_id / 创建时间 / 修改次数 / 上次修改时间。",
)
def list_whiteboards(
limit: int = Query(100, ge=1, le=500),
offset: int = Query(0, ge=0),
service: WhiteboardService = Depends(_service),
_: str = Depends(require_docs_auth),
) -> WhiteboardListResponse:
total, items = service.list_all(limit=limit, offset=offset)
return WhiteboardListResponse(total=total, items=items)
@router.delete(
"/api/admin/whiteboards/{board_id}",
summary="删除白板(需鉴权)",
description="删 DB 行,并关闭该 board 的所有在线 WebSocket 连接。",
)
def delete_whiteboard(
board_id: str,
service: WhiteboardService = Depends(_service),
_: str = Depends(require_docs_auth),
) -> dict:
ok = service.delete(board_id)
if not ok:
raise HTTPException(404, "白板不存在")
return {"deleted": True}
# ---------------- WebSocket公开实时同步 + 心跳) ----------------
@router.websocket("/ws/whiteboard/{board_id}")
async def whiteboard_ws(websocket: WebSocket, board_id: str) -> None:
"""白板实时协作端点。
协议JSON 文本帧):
client -> server:
{"type":"hello","client_id":"..."} 首帧(可选,未带则服务端生成)
{"type":"ping"} 心跳,服务端回 pong 并刷新计时
{"type":"stroke","stroke":{...}} 新增笔画,持久化并广播给他人
{"type":"clear"} 清空,持久化并广播给所有人
server -> client:
{"type":"init","strokes":[...],"stroke_count":n}
{"type":"pong"}
{"type":"stroke","stroke":{...},"client_id":"..."}
{"type":"cleared","client_id":"..."}
{"type":"error","msg":"..."}
"""
# 路径层只做最基本校验,详细校验交给 serviceservice 会查表)
hub = get_hub()
# 先 accept便于对非法 board_id 也回一条 error 再关闭
await websocket.accept()
# 读取首帧 hello或任意帧拿 client_id
try:
first = await websocket.receive_text()
except WebSocketDisconnect:
return
client_id = _extract_client_id(first) or uuid.uuid4().hex[:12]
# 校验 board_id 并加载白板(不存在则新建)
from ..database import get_session_local
try:
board = _with_db(lambda db: WhiteboardService(WhiteboardDAO(db), hub=hub).get_or_create(board_id))
except HTTPException as exc:
await _safe_send(websocket, {"type": "error", "msg": exc.detail})
await _safe_close(websocket)
return
# 注册连接并下发 init
conn = Connection(websocket=websocket, board_id=board_id, client_id=client_id)
await hub.register(conn)
await _safe_send(websocket, {
"type": "init",
"strokes": board.strokes,
"stroke_count": board.stroke_count,
})
# 主循环:收消息 -> 处理 -> 广播
try:
while True:
raw = await websocket.receive_text()
msg = _parse(raw)
if msg is None:
continue
mtype = msg.get("type")
if mtype == "ping":
conn.touch()
await _safe_send(websocket, {"type": "pong"})
continue
# 任何有效业务帧都视为活性证据
conn.touch()
if mtype == "stroke":
stroke = msg.get("stroke") or {}
try:
_with_db(lambda db: WhiteboardService(WhiteboardDAO(db), hub=hub).append_stroke(board_id, stroke))
except HTTPException as exc:
await _safe_send(websocket, {"type": "error", "msg": exc.detail})
continue
# 广播给他人(发送者本地已画,不回推)
await hub.broadcast(
board_id,
{"type": "stroke", "stroke": stroke, "client_id": client_id},
exclude=conn,
)
elif mtype == "clear":
try:
_with_db(lambda db: WhiteboardService(WhiteboardDAO(db), hub=hub).clear(board_id))
except HTTPException as exc:
await _safe_send(websocket, {"type": "error", "msg": exc.detail})
continue
# clear 广播给所有人(含发送者,用于确认)
await hub.broadcast(
board_id, {"type": "cleared", "client_id": client_id}
)
else:
await _safe_send(websocket, {"type": "error", "msg": f"未知消息类型 {mtype}"})
except WebSocketDisconnect:
pass
except Exception as exc: # pragma: no cover
logger.warning("白板 WS 异常 board=%s client=%s: %s", board_id, client_id, exc)
finally:
await hub.disconnect(conn)
# ---------------- helpers ----------------
def _with_db(fn):
"""在独立 Session 中执行 fn 并返回结果;用完即关。供 WS 路径每帧独立事务使用。"""
from ..database import get_session_local
db = get_session_local()()
try:
return fn(db)
finally:
db.close()
def _extract_client_id(raw: str) -> str | None:
try:
data = json.loads(raw)
except (json.JSONDecodeError, TypeError):
return None
cid = data.get("client_id") if isinstance(data, dict) else None
if isinstance(cid, str) and cid:
return cid
return None
def _parse(raw: str) -> dict | None:
try:
data = json.loads(raw)
except (json.JSONDecodeError, TypeError):
return None
return data if isinstance(data, dict) else None
async def _safe_send(ws: WebSocket, msg: dict) -> None:
try:
await ws.send_json(msg)
except Exception: # pragma: no cover
pass
async def _safe_close(ws: WebSocket) -> None:
try:
await ws.close()
except Exception: # pragma: no cover
pass

View File

@@ -1,5 +1,7 @@
"""DAO 层:数据库访问的唯一入口。"""
from .tunnel_session_dao import TunnelSessionDAO
from .uploaded_file_dao import UploadedFileDAO
from .upload_session_dao import UploadSessionDAO
__all__ = ["UploadedFileDAO"]
__all__ = ["TunnelSessionDAO", "UploadedFileDAO", "UploadSessionDAO"]

View File

@@ -0,0 +1,63 @@
"""TunnelSession 的 DAO。"""
from __future__ import annotations
from datetime import datetime
from sqlalchemy import select, update
from sqlalchemy.orm import Session
from ..models.tunnel_session import TunnelSession
class TunnelSessionDAO:
def __init__(self, db: Session) -> None:
self.db = db
def create(self, session: TunnelSession) -> TunnelSession:
self.db.add(session)
self.db.commit()
self.db.refresh(session)
return session
def get_active_by_user(self, user_name: str) -> TunnelSession | None:
"""返回该 user 当前活跃的隧道会话(至多一条)。"""
stmt = (
select(TunnelSession)
.where(TunnelSession.user_name == user_name)
.where(TunnelSession.status == "active")
.order_by(TunnelSession.started_at.desc())
.limit(1)
)
return self.db.scalars(stmt).first()
def get_active_by_port(self, tunnel_port: int) -> TunnelSession | None:
stmt = (
select(TunnelSession)
.where(TunnelSession.tunnel_port == tunnel_port)
.where(TunnelSession.status == "active")
.limit(1)
)
return self.db.scalars(stmt).first()
def list_active(self) -> list[TunnelSession]:
stmt = select(TunnelSession).where(TunnelSession.status == "active")
return list(self.db.scalars(stmt).all())
def close(self, session: TunnelSession) -> None:
"""标记会话结束。"""
session.ended_at = datetime.now()
session.status = "closed"
self.db.commit()
def close_active_by_user(self, user_name: str) -> int:
"""关闭该 user 所有 active 会话(断开清理用),返回关闭条数。"""
stmt = (
update(TunnelSession)
.where(TunnelSession.user_name == user_name)
.where(TunnelSession.status == "active")
.values(status="closed", ended_at=datetime.now())
)
result = self.db.execute(stmt)
self.db.commit()
return result.rowcount or 0

View File

@@ -0,0 +1,57 @@
"""UploadSession 的 DAO。"""
from __future__ import annotations
from datetime import datetime, timedelta
from sqlalchemy import select
from sqlalchemy.orm import Session
from ..models.upload_session import UploadSession
class UploadSessionDAO:
def __init__(self, db: Session) -> None:
self.db = db
def create(self, session: UploadSession) -> UploadSession:
self.db.add(session)
self.db.commit()
self.db.refresh(session)
return session
def get_by_upload_id(self, upload_id: str) -> UploadSession | None:
stmt = (
select(UploadSession)
.where(UploadSession.upload_id == upload_id)
.limit(1)
)
return self.db.scalars(stmt).first()
def mark_uploaded(self, session: UploadSession, chunks: list[int]) -> UploadSession:
"""更新已上传分片集合(整体替换,避免并发追加丢失)。"""
session.uploaded_chunks = sorted(set(chunks))
self.db.commit()
self.db.refresh(session)
return session
def mark_completed(self, session: UploadSession, file_id: int) -> UploadSession:
session.file_id = file_id
session.status = "completed"
self.db.commit()
self.db.refresh(session)
return session
def list_stale(self, ttl_seconds: int) -> list[UploadSession]:
"""返回 pending 且 updated_at 早于 cutoff 的会话(被放弃的上传)。"""
cutoff = datetime.now() - timedelta(seconds=ttl_seconds)
stmt = (
select(UploadSession)
.where(UploadSession.status == "pending")
.where(UploadSession.updated_at < cutoff)
)
return list(self.db.scalars(stmt).all())
def delete(self, session: UploadSession) -> None:
self.db.delete(session)
self.db.commit()

84
app/dao/whiteboard_dao.py Normal file
View File

@@ -0,0 +1,84 @@
"""Whiteboard 的 DAO。
所有写操作均在该层 commitservice 不直接操作 session。
get_or_create 用于「访问即新建」语义(路由 GET /whiteboard/{id} 不存在则建)。
"""
from __future__ import annotations
from typing import Any
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from ..models.whiteboard import Whiteboard
class WhiteboardDAO:
def __init__(self, db: Session) -> None:
self.db = db
def create(self, board: Whiteboard) -> Whiteboard:
self.db.add(board)
self.db.commit()
self.db.refresh(board)
return board
def get(self, board_id: str) -> Whiteboard | None:
stmt = select(Whiteboard).where(Whiteboard.board_id == board_id).limit(1)
return self.db.scalars(stmt).first()
def get_or_create(self, board_id: str) -> Whiteboard:
"""存在则返回,否则新建空板。利用 unique 约束兜底并发首访。"""
board = self.get(board_id)
if board is not None:
return board
board = Whiteboard(board_id=board_id, strokes=[], stroke_count=0)
try:
return self.create(board)
except Exception:
# 并发下另一事务已插入:回滚后重新读
self.db.rollback()
return self.get(board_id) # type: ignore[return-value]
def append_strokes(self, board_id: str, new_strokes: list[Any]) -> Whiteboard | None:
"""把新笔画追加到 strokes 数组尾部stroke_count 自增。"""
board = self.get(board_id)
if board is None:
return None
board.strokes = [*board.strokes, *new_strokes]
board.stroke_count = (board.stroke_count or 0) + len(new_strokes)
self.db.commit()
self.db.refresh(board)
return board
def replace_strokes(self, board_id: str, strokes: list[Any]) -> Whiteboard | None:
"""整体替换 strokes清空时传 []stroke_count 自增 1。"""
board = self.get(board_id)
if board is None:
return None
board.strokes = list(strokes)
board.stroke_count = (board.stroke_count or 0) + 1
self.db.commit()
self.db.refresh(board)
return board
def list_all(self, limit: int = 100, offset: int = 0) -> list[Whiteboard]:
stmt = (
select(Whiteboard)
.order_by(Whiteboard.updated_at.desc())
.limit(limit)
.offset(offset)
)
return list(self.db.scalars(stmt).all())
def count(self) -> int:
return self.db.scalar(select(func.count()).select_from(Whiteboard)) or 0
def delete(self, board_id: str) -> bool:
board = self.get(board_id)
if board is None:
return False
self.db.delete(board)
self.db.commit()
return True

View File

@@ -1,29 +1,103 @@
"""FastAPI 应用工厂。
路由概览:
GET / -> 仅返回版本号
GET /docs -> Swagger UIBasic Auth
GET /redoc -> ReDoc Basic Auth
GET /openapi.json -> OpenAPI 文档Basic Auth
GET /health -> 存活探针(公开)
GET /api/... -> 业务接口
GET / -> 仅返回版本号
GET /docs -> Swagger UIBasic Auth
GET /redoc -> ReDoc Basic Auth
GET /openapi.json -> OpenAPI 文档Basic Auth
GET /health -> 存活探针(公开)
GET /upload -> 上传页面(公开 HTML
GET /files -> 文件浏览页Basic Auth同 docs
GET /whiteboard/{id} -> 白板页面(公开,不存在则新建)
GET /whiteboard-admin -> 白板管理页Basic Auth同 docs
GET /api/... -> 业务接口
WS /ws/whiteboard/{id} -> 白板实时同步(公开)
/static/... -> 前端静态资源JS/CSS
"""
from __future__ import annotations
import asyncio
import logging
from contextlib import asynccontextmanager
from pathlib import Path
from fastapi import Depends, FastAPI
from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html
from fastapi.responses import JSONResponse, PlainTextResponse
from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse
from fastapi.staticfiles import StaticFiles
from .controllers import file_router, system_router
from .controllers import (
chunk_upload_router,
file_admin_router,
file_router,
system_router,
tunnel_router,
whiteboard_router,
)
from .database import init_db_schema
from .security import require_docs_auth
from .services.chunk_upload_service import ChunkUploadService
from .services.whiteboard_hub import get_hub
from .views.upload_html import render as render_upload_html
logger = logging.getLogger("zikai")
# 后台 reaper 的扫描间隔(秒)。不依赖 start.sh进程存活期间持续清理过期会话。
_REAPER_INTERVAL_SECONDS = 60
_STATIC_DIR = Path(__file__).resolve().parent.parent / "static"
def _reap_once() -> None:
"""同步执行一次过期会话清理(在 worker 线程里跑,避免阻塞事件循环)。"""
try:
from .database import get_session_local
from .dao.upload_session_dao import UploadSessionDAO
from .dao.uploaded_file_dao import UploadedFileDAO
db = get_session_local()()
try:
n = ChunkUploadService(UploadSessionDAO(db), UploadedFileDAO(db)).reap_stale_sessions()
if n:
logger.info("reaper 清理了 %d 个过期分片会话", n)
finally:
db.close()
except Exception as exc: # pragma: no cover
logger.warning("reaper 执行失败:%s", exc)
async def _reaper_loop(stop: asyncio.Event) -> None:
"""周期性清理被放弃的分片上传会话。"""
while not stop.is_set():
try:
await asyncio.to_thread(_reap_once)
except Exception as exc: # pragma: no cover
logger.warning("reaper 循环异常:%s", exc)
# 用 wait_for 实现「可被 stop 提前唤醒的 sleep」
try:
await asyncio.wait_for(stop.wait(), timeout=_REAPER_INTERVAL_SECONDS)
except asyncio.TimeoutError:
pass
def _tunnel_reap_once() -> None:
"""启动时清理 DB 里残留的 active 隧道会话(进程异常重启后的孤儿记录)。"""
try:
from .database import get_session_local
from .dao.tunnel_session_dao import TunnelSessionDAO
from .services.tunnel_service import TunnelService
db = get_session_local()()
try:
n = TunnelService(TunnelSessionDAO(db)).reap_orphans()
if n:
logger.info("tunnel reaper 清理了 %d 个孤儿隧道会话", n)
finally:
db.close()
except Exception as exc: # pragma: no cover
logger.warning("tunnel reaper 执行失败:%s", exc)
@asynccontextmanager
async def lifespan(app: FastAPI):
@@ -32,7 +106,24 @@ async def lifespan(app: FastAPI):
logger.info("数据库表已就绪。")
except Exception as exc: # pragma: no cover
logger.error("初始化数据库失败:%s", exc)
yield
# 启动时清理残留的 active 隧道会话(进程重启后的孤儿记录)
await asyncio.to_thread(_tunnel_reap_once)
# 启动后台 reaper清理被放弃的分片上传会话与临时文件
stop = asyncio.Event()
reaper = asyncio.create_task(_reaper_loop(stop))
logger.info("分片会话 reaper 已启动(间隔 %ds", _REAPER_INTERVAL_SECONDS)
# 启动白板心跳 reaper清理失活的 WebSocket 连接
wb_reaper = asyncio.create_task(get_hub().reap_loop(stop))
logger.info("白板心跳 reaper 已启动。")
try:
yield
finally:
stop.set()
for task in (reaper, wb_reaper):
try:
await asyncio.wait_for(task, timeout=5)
except (asyncio.TimeoutError, asyncio.CancelledError): # pragma: no cover
task.cancel()
def create_app() -> FastAPI:
@@ -54,34 +145,69 @@ def create_app() -> FastAPI:
app.include_router(system_router)
app.include_router(file_router)
app.include_router(file_admin_router)
app.include_router(chunk_upload_router)
app.include_router(tunnel_router)
app.include_router(whiteboard_router)
# 前端静态资源JS/CSSHTML 壳由下面的具名路由返回,便于各自挂 Basic Auth
if _STATIC_DIR.is_dir():
app.mount("/static", StaticFiles(directory=str(_STATIC_DIR)), name="static")
# 受 Basic Auth 保护的文档接口
@app.get("/openapi.json", include_in_schema=False)
@app.get("/openapi.json")
def protected_openapi(_: str = Depends(require_docs_auth)) -> JSONResponse:
return JSONResponse(app.openapi())
@app.get("/docs", include_in_schema=False)
@app.get("/docs")
def protected_docs(_: str = Depends(require_docs_auth)):
return get_swagger_ui_html(
openapi_url="/openapi.json", title="zikai docs", swagger_favicon_url=""
)
@app.get("/redoc", include_in_schema=False)
@app.get("/redoc")
def protected_redoc(_: str = Depends(require_docs_auth)):
return get_redoc_html(
openapi_url="/openapi.json", title="zikai docs", redoc_favicon_url=""
)
# 公开元信息(不暴露接口列表)
@app.get("/", include_in_schema=False, response_class=PlainTextResponse)
@app.get("/", response_class=PlainTextResponse)
def root() -> PlainTextResponse:
return PlainTextResponse(f"zikai {app.version}\n")
@app.get("/health", include_in_schema=False)
@app.get("/health")
def health() -> dict:
return {"status": "ok"}
@app.get("/upload", response_class=HTMLResponse)
def upload_page() -> HTMLResponse:
"""拖拽 / 多文件 / 分片上传页面(公开,对齐 /api/files/upload"""
return HTMLResponse(render_upload_html())
@app.get("/files", response_class=HTMLResponse)
def files_page(_: str = Depends(require_docs_auth)) -> HTMLResponse:
"""文件浏览页Basic Auth同 docs列出/下载/删除已上传文件。"""
return _serve_static_html("file_browser.html")
@app.get("/whiteboard-admin", response_class=HTMLResponse)
def whiteboard_admin_page(_: str = Depends(require_docs_auth)) -> HTMLResponse:
"""白板管理页Basic Auth同 docs查看/删除白板。"""
return _serve_static_html("whiteboard_admin.html")
@app.get("/whiteboard/{board_id}", response_class=HTMLResponse)
def whiteboard_page(board_id: str) -> HTMLResponse:
"""白板页面(公开):访问即协作,不存在则前端拉取时自动新建。"""
return _serve_static_html("whiteboard.html")
return app
def _serve_static_html(filename: str) -> HTMLResponse:
"""读取 static/ 下的 HTML 文件并返回;缺失时返回 404 文本。"""
path = _STATIC_DIR / filename
if not path.is_file():
return HTMLResponse(f"<h1>未找到 {filename}</h1>", status_code=404)
return HTMLResponse(path.read_text(encoding="utf-8"))
app = create_app()

View File

@@ -1,5 +1,8 @@
"""ORM 模型包import 本包即把所有实体注册到 Base.metadata。"""
from .tunnel_session import TunnelSession
from .uploaded_file import UploadedFile
from .upload_session import UploadSession
from .whiteboard import Whiteboard
__all__ = ["UploadedFile"]
__all__ = ["TunnelSession", "UploadedFile", "UploadSession", "Whiteboard"]

View File

@@ -0,0 +1,41 @@
"""反向隧道会话实体。
一条记录对应一次 user → server 的 SSH 反向隧道:user 连上 server 的 SSH(2022),
请求 remote port forwarding,server 在本地绑一个隧道端口,该端口经隧道回指 user
的本地服务。HTTP 路由 /api/userPort/{userName} 反代到该隧道端口。
"""
from __future__ import annotations
from datetime import datetime
from sqlalchemy import BigInteger, DateTime, Integer, String, func
from sqlalchemy.orm import Mapped, mapped_column
from ..database import Base
class TunnelSession(Base):
__tablename__ = "tunnel_session"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
# 发起隧道的 user 名(对应 config.tunnel.users[].username)
user_name: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
# user 端的公网/内网 IP(SSH 连接的 peer 地址)
user_ip: Mapped[str] = mapped_column(String(64), nullable=False, default="")
# user 要暴露的本地服务端口
local_port: Mapped[int] = mapped_column(Integer, nullable=False)
# server 侧的隧道端口(SSH 反向转发绑定的本地端口)
tunnel_port: Mapped[int] = mapped_column(Integer, nullable=False, index=True)
started_at: Mapped[datetime] = mapped_column(
DateTime, server_default=func.now(), nullable=False
)
ended_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
# active / closed
status: Mapped[str] = mapped_column(String(32), nullable=False, default="active")
def __repr__(self) -> str: # pragma: no cover
return (
f"<TunnelSession id={self.id} user={self.user_name!r} "
f"tunnel_port={self.tunnel_port} status={self.status}>"
)

View File

@@ -0,0 +1,44 @@
"""分片上传会话实体。
一个会话对应一次「分片上传 + 拼接入库」流程:客户端创建会话拿到 upload_id
逐片上传,最后 complete 触发服务端拼接、算 sha256、去重并落 DB。
"""
from __future__ import annotations
from datetime import datetime
from sqlalchemy import BigInteger, DateTime, JSON, String, func
from sqlalchemy.orm import Mapped, mapped_column
from ..database import Base
class UploadSession(Base):
__tablename__ = "upload_session"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
# 客户端持有的会话标识uuid4().hex服务端生成防伪造路径
upload_id: Mapped[str] = mapped_column(String(64), nullable=False, unique=True, index=True)
filename: Mapped[str] = mapped_column(String(512), nullable=False)
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
chunk_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
total_chunks: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
# 已上传分片下标数组,例如 [0, 1, 3]JSON 列
uploaded_chunks: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
# complete 成功后指向 UploadedFile.id失败/未完成时为 None
file_id: Mapped[int | None] = mapped_column(BigInteger, nullable=True, default=None)
# pending / completed
status: Mapped[str] = mapped_column(String(32), nullable=False, default="pending")
created_at: Mapped[datetime] = mapped_column(
DateTime, server_default=func.now(), nullable=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime, server_default=func.now(), onupdate=func.now(), nullable=False
)
def __repr__(self) -> str: # pragma: no cover
return (
f"<UploadSession upload_id={self.upload_id!r} "
f"file={self.filename!r} status={self.status}>"
)

39
app/models/whiteboard.py Normal file
View File

@@ -0,0 +1,39 @@
"""共享白板实体。
一个白板由 board_id 唯一标识(用户可读的 url idstrokes 以 JSON 列保存全部笔画。
白板长期留存,进程重启后仍可恢复;实时协作由 WebSocket hub 在内存中维护在线连接,
笔画变更经 service 落库后由 hub 广播给同 board 的其它在线连接。
"""
from __future__ import annotations
from datetime import datetime
from sqlalchemy import BigInteger, DateTime, Integer, JSON, String, func
from sqlalchemy.orm import Mapped, mapped_column
from ..database import Base
class Whiteboard(Base):
__tablename__ = "whiteboard"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
# 用户可读的 url id[a-zA-Z0-9_-]{1,64}),全局唯一
board_id: Mapped[str] = mapped_column(String(64), nullable=False, unique=True, index=True)
# 笔画数组 [{points:[[x,y],...], color, width}, ...]
strokes: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
# 修改次数:每次新增笔画或清空 +1供管理页统计
stroke_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
created_at: Mapped[datetime] = mapped_column(
DateTime, server_default=func.now(), nullable=False
)
updated_at: Mapped[datetime] = mapped_column(
DateTime, server_default=func.now(), onupdate=func.now(), nullable=False
)
def __repr__(self) -> str: # pragma: no cover
return (
f"<Whiteboard board_id={self.board_id!r} "
f"strokes={len(self.strokes)} mods={self.stroke_count}>"
)

View File

@@ -1,13 +1,37 @@
"""请求 / 响应的 Pydantic DTO。"""
from .chunk import (
ChunkUploadResponse,
CreateSessionRequest,
CreateSessionResponse,
SessionStatusResponse,
)
from .file import FileListResponse, FileUploadResponse, UploadedFileOut
from .system import DiskUsage, MemoryUsage, SystemStatus
from .tunnel import TunnelStatusResponse
from .whiteboard import (
Stroke,
StrokeOp,
WhiteboardListItem,
WhiteboardListResponse,
WhiteboardOut,
)
__all__ = [
"ChunkUploadResponse",
"CreateSessionRequest",
"CreateSessionResponse",
"DiskUsage",
"FileListResponse",
"FileUploadResponse",
"MemoryUsage",
"SessionStatusResponse",
"Stroke",
"StrokeOp",
"SystemStatus",
"TunnelStatusResponse",
"UploadedFileOut",
"WhiteboardListItem",
"WhiteboardListResponse",
"WhiteboardOut",
]

43
app/schemas/chunk.py Normal file
View File

@@ -0,0 +1,43 @@
"""分片上传接口 DTO。"""
from __future__ import annotations
from pydantic import BaseModel, Field
class CreateSessionRequest(BaseModel):
"""创建一个分片上传会话。"""
filename: str = Field(..., description="客户端原始文件名")
size_bytes: int = Field(..., ge=0, description="文件总字节数")
chunk_size: int = Field(..., gt=0, description="分片大小(字节)")
total_chunks: int = Field(..., gt=0, description="分片总数")
class CreateSessionResponse(BaseModel):
upload_id: str = Field(..., description="服务端生成的会话标识,后续接口都需要它")
filename: str
size_bytes: int
chunk_size: int
total_chunks: int
class SessionStatusResponse(BaseModel):
"""会话状态:前端据此决定还需补传哪些分片(断点续传)。"""
upload_id: str
filename: str
size_bytes: int
chunk_size: int
total_chunks: int
uploaded_chunks: list[int] = Field(..., description="已上传的分片下标集合")
completed: bool = Field(..., description="是否已完成拼接入库")
file_id: int | None = Field(None, description="completed=true 时指向 UploadedFile.id")
class ChunkUploadResponse(BaseModel):
"""单个分片上传成功的回执。"""
upload_id: str
index: int
uploaded_chunks: list[int]

View File

@@ -28,19 +28,11 @@ class FileUploadResponse(BaseModel):
sha256: str
storage_path: str
uploaded_at: datetime
deduplicated: bool = Field(
False, description="true=服务端已有同 sha256 文件,直接返回旧行,未重复落盘"
)
class FileListResponse(BaseModel):
total: int
items: list[UploadedFileOut]
class SftpRegisterRequest(BaseModel):
"""登记一个通过 SFTP 上传到 ``incoming/`` 下的文件。"""
filename: str = Field(
...,
description="文件在 SFTP chroot 下的相对路径,必须落在 incoming/ 之下,例如 incoming/abc.bin",
)
original_filename: str = Field(..., description="客户端原始文件名")
uploaded_by: str = Field(default="sftp", description="登记者标识,写入 uploaded_by 字段")

19
app/schemas/tunnel.py Normal file
View File

@@ -0,0 +1,19 @@
"""反向隧道接口 DTO。"""
from __future__ import annotations
from datetime import datetime
from pydantic import BaseModel
class TunnelStatusResponse(BaseModel):
"""隧道的当前状态(查 /api/userPort/{userName} 前可用于探测)。"""
user_name: str
local_port: int
tunnel_port: int
started_at: datetime
status: str
model_config = {"from_attributes": True}

53
app/schemas/whiteboard.py Normal file
View File

@@ -0,0 +1,53 @@
"""白板接口 DTO。"""
from __future__ import annotations
from datetime import datetime
from typing import Any
from pydantic import BaseModel, Field
class Stroke(BaseModel):
"""一条笔画:点序列 + 样式。结构宽松Any以兼容前端扩展字段。"""
points: list[list[float]] = Field(default_factory=list, description="[[x,y],...]")
color: str = Field(default="#1565c0", description="笔画颜色")
width: float = Field(default=3, description="笔画宽度")
class WhiteboardOut(BaseModel):
"""白板完整内容GET /whiteboard/{id} 与 WS init 帧)。"""
board_id: str
strokes: list[Any] = Field(default_factory=list, description="笔画数组")
stroke_count: int = Field(0, description="累计修改次数")
created_at: datetime
updated_at: datetime
model_config = {"from_attributes": True}
class WhiteboardListItem(BaseModel):
"""管理页列表项。"""
board_id: str
stroke_count: int
created_at: datetime
updated_at: datetime
model_config = {"from_attributes": True}
class WhiteboardListResponse(BaseModel):
"""管理页列表响应。"""
total: int
items: list[WhiteboardListItem]
class StrokeOp(BaseModel):
"""WS 笔画操作type=stroke 时携带)。"""
type: str = Field(..., description="add / clear")
stroke: dict[str, Any] | None = Field(None, description="type=add 时携带的笔画对象")

View File

@@ -1,6 +1,7 @@
"""业务逻辑层Service"""
from .chunk_upload_service import ChunkUploadService
from .system_service import SystemService
from .upload_service import UploadService
__all__ = ["SystemService", "UploadService"]
__all__ = ["ChunkUploadService", "SystemService", "UploadService"]

View File

@@ -0,0 +1,247 @@
"""分片上传服务:会话管理 + 分片落盘 + 拼接 + sha256 去重 + 原子入库。
存储布局::
uploads/.work/<upload_id>/0.part 分片暂存
uploads/.work/<upload_id>/1.part
...
uploads/2026/07/<uuid>.<ext> complete 后的正式文件
complete 流程:
1. 校验分片齐全;
2. 顺序拼接为 <final>.part流式算 sha256
3. 复用 UploadService.dedup_or_commit按 sha256 去重命中则删会话返回旧行,
否则写 DB 行 + os.replace 原子改名到正式路径;
4. 清理 .work/<upload_id>/。
"""
from __future__ import annotations
import io
import logging
import os
import shutil
import uuid
from collections.abc import Iterator
from pathlib import Path
from fastapi import HTTPException
from ..config import get_settings
from ..dao.upload_session_dao import UploadSessionDAO
from ..dao.uploaded_file_dao import UploadedFileDAO
from ..models.upload_session import UploadSession
from ..models.uploaded_file import UploadedFile
from ..schemas.chunk import (
CreateSessionRequest,
CreateSessionResponse,
SessionStatusResponse,
)
from ..schemas.file import FileUploadResponse
from .upload_service import UploadService, hash_stream
logger = logging.getLogger("zikai.chunk")
# 会话暂存目录名(位于 upload_root 下)
SESSION_WORK_DIR = ".work"
class ChunkUploadService:
def __init__(
self,
session_dao: UploadSessionDAO,
file_dao: UploadedFileDAO,
) -> None:
s = get_settings()
self.session_dao = session_dao
self.file_dao = file_dao
self.upload_root = s.resolved_upload_dir()
self.chunk_bytes = s.storage.chunk_bytes
# 会话暂存目录:配置给出的是相对 upload_root 的子目录名
session_sub = os.path.basename(s.storage.chunk_session_dir) or SESSION_WORK_DIR
self.work_root = (self.upload_root / session_sub).resolve()
self.work_root.mkdir(parents=True, exist_ok=True)
self.session_ttl = s.storage.chunk_session_ttl_seconds
# 复用 UploadService 的存储路径生成 / 落库提交 / sha256 去重逻辑
self._upload = UploadService(file_dao)
# ---------------- 会话生命周期 ----------------
def create_session(self, body: CreateSessionRequest) -> CreateSessionResponse:
upload_id = uuid.uuid4().hex
session = UploadSession(
upload_id=upload_id,
filename=body.filename,
size_bytes=body.size_bytes,
chunk_size=body.chunk_size,
total_chunks=body.total_chunks,
uploaded_chunks=[],
status="pending",
)
self.session_dao.create(session)
self._session_dir(upload_id).mkdir(parents=True, exist_ok=True)
return CreateSessionResponse(
upload_id=upload_id,
filename=body.filename,
size_bytes=body.size_bytes,
chunk_size=body.chunk_size,
total_chunks=body.total_chunks,
)
def get_status(self, upload_id: str) -> SessionStatusResponse:
session = self._require_session(upload_id)
return SessionStatusResponse(
upload_id=session.upload_id,
filename=session.filename,
size_bytes=session.size_bytes,
chunk_size=session.chunk_size,
total_chunks=session.total_chunks,
uploaded_chunks=list(session.uploaded_chunks or []),
completed=(session.status == "completed"),
file_id=session.file_id,
)
# ---------------- 分片写入 ----------------
def write_chunk(
self, upload_id: str, index: int, data: bytes,
) -> list[int]:
session = self._require_session(upload_id)
self._validate_index(session, index)
session_dir = self._session_dir(upload_id)
session_dir.mkdir(parents=True, exist_ok=True)
chunk_path = session_dir / f"{index}.part"
# 幂等覆盖:同一分片重传时直接覆盖旧文件
try:
with chunk_path.open("wb") as out:
out.write(data)
out.flush()
os.fsync(out.fileno())
except Exception:
chunk_path.unlink(missing_ok=True)
raise
uploaded = list(session.uploaded_chunks or [])
if index not in uploaded:
uploaded.append(index)
self.session_dao.mark_uploaded(session, uploaded)
return sorted(uploaded)
# ---------------- 拼接入库 ----------------
def complete(self, upload_id: str) -> FileUploadResponse:
session = self._require_session(upload_id)
# 幂等:已 complete 直接复用结果
if session.status == "completed" and session.file_id is not None:
row = self.file_dao.get_by_id(session.file_id)
if row is not None:
return UploadService.to_response(row, deduplicated=True)
uploaded = set(session.uploaded_chunks or [])
missing = [i for i in range(session.total_chunks) if i not in uploaded]
if missing:
raise HTTPException(
409,
f"分片不齐全:缺失 {len(missing)} 个,例如 {sorted(missing)[:10]}",
)
session_dir = self._session_dir(upload_id)
part_path = session_dir / "_assembled.part"
size, digest = self._assemble_and_hash(session, session_dir, part_path)
# 复用 UploadService 的「sha256 去重 + 落库 + 原子改名」公共尾部
entity = UploadedFile(
storage_path="", # 由 dedup_or_commit 内部生成
original_filename=os.path.basename(session.filename),
content_type="",
size_bytes=size,
sha256=digest,
source="chunk",
uploaded_by="web",
)
resp = self._upload.dedup_or_commit(entity, part_path)
self.session_dao.mark_completed(session, resp.id)
# 会话目录里的分片已拼接走,清理残留
self._cleanup_session_dir(upload_id)
return resp
# ---------------- 过期会话清理 ----------------
def reap_stale_sessions(self) -> int:
"""清理被放弃的会话:删 .work/<upload_id>/ 目录 + DB 记录。
判定标准status=pending 且 updated_at 距今超过 session_ttl 秒。
返回清理的会话数。不依赖 start.sh由后台任务周期调用。
"""
stale = self.session_dao.list_stale(self.session_ttl)
for session in stale:
self._cleanup_session_dir(session.upload_id)
self.session_dao.delete(session)
logger.info("清理过期分片会话 upload_id=%s file=%s", session.upload_id, session.filename)
return len(stale)
def _cleanup_session_dir(self, upload_id: str) -> None:
"""删除 .work/<upload_id>/ 目录;失败只记日志,不抛异常。"""
session_dir = self._session_dir(upload_id)
try:
if session_dir.exists():
shutil.rmtree(session_dir, ignore_errors=True)
except Exception as exc: # pragma: no cover
logger.warning("清理会话目录失败 upload_id=%s: %s", upload_id, exc)
# ---------------- 内部 ----------------
def _require_session(self, upload_id: str) -> UploadSession:
if not upload_id:
raise HTTPException(400, "upload_id 不能为空")
session = self.session_dao.get_by_upload_id(upload_id)
if session is None:
raise HTTPException(404, f"会话不存在或已过期:{upload_id}")
return session
@staticmethod
def _validate_index(session: UploadSession, index: int) -> None:
if index < 0 or index >= session.total_chunks:
raise HTTPException(
400,
f"分片下标越界:{index} 不在 [0, {session.total_chunks})",
)
def _session_dir(self, upload_id: str) -> Path:
return self.work_root / upload_id
def _assemble_and_hash(
self,
session: UploadSession,
session_dir: Path,
out_path: Path,
) -> tuple[int, str]:
"""按 index 顺序拼接全部分片为 out_path流式计算 (size, sha256)。"""
try:
with out_path.open("wb") as out:
size, digest = hash_stream(
self._iter_assembled_bytes(session, session_dir, out)
)
out.flush()
os.fsync(out.fileno())
except Exception:
out_path.unlink(missing_ok=True)
raise
return size, digest
def _iter_assembled_bytes(
self, session: UploadSession, session_dir: Path, out: io.BufferedWriter,
) -> Iterator[bytes]:
"""按 index 顺序读各分片,写入 out 同时 yield 每个字节块(供 hash_stream"""
for index in range(session.total_chunks):
chunk_path = session_dir / f"{index}.part"
if not chunk_path.is_file():
raise HTTPException(409, f"拼接时发现分片缺失:{index}.part")
with chunk_path.open("rb") as src:
while buf := src.read(self.chunk_bytes):
out.write(buf)
yield buf

View File

@@ -1,4 +1,4 @@
"""嵌入式 SFTP 服务,与 HTTP API 共用 uploads/ 目录。
"""嵌入式 SFTP 服务,与 HTTP API 共用 uploads/ 目录;同时承载反向隧道的 SSH 转发
启动方式:
python -m app.services.sftp_server
@@ -35,16 +35,34 @@ class ZikaiSFTPServer(asyncssh.SFTPServer):
logger.info("SFTP 会话结束 user=%s", self._username)
def _tunnel_dao():
"""惰性构造一个 TunnelSessionDAO避免 import 时的副作用)。"""
from ..database import get_session_local
from ..dao.tunnel_session_dao import TunnelSessionDAO
return TunnelSessionDAO(get_session_local()())
def _close_tunnel_dao(dao) -> None:
try:
dao.db.close()
except Exception: # pragma: no cover
pass
class ZikaiSSHServer(asyncssh.SSHServer):
"""支持密码bcrypt与公钥两种鉴权。"""
"""支持密码bcrypt与公钥两种鉴权;允许反向隧道 user 的 remote forwarding"""
def __init__(self, settings, authorized_keys: asyncssh.SSHAuthorizedKeys | None) -> None:
self._settings = settings
self._authorized_keys = authorized_keys
self._conn: asyncssh.SSHServerConnection | None = None
self._username: str | None = None # 认证成功后填入
self._peer_ip: str = ""
def connection_made(self, conn: asyncssh.SSHServerConnection) -> None: # type: ignore[override]
self._conn = conn
peer = conn.get_extra_info("peername")
self._peer_ip = peer[0] if isinstance(peer, tuple) and peer else ""
def begin_auth(self, username: str) -> bool:
# 始终要求鉴权;用户不存在时所有方法会失败 → 干净的 permission denied
@@ -56,14 +74,17 @@ class ZikaiSSHServer(asyncssh.SSHServer):
return True
def validate_password(self, username: str, password: str) -> bool:
user = next((u for u in self._settings.sftp.users if u.username == username), None)
if user is None or not user.password_hash or user.password_hash == "CHANGE_ME_BCRYPT_HASH":
user = self._find_sftp_user(username) or self._find_tunnel_user(username)
if user is None or not getattr(user, "password_hash", "") \
or user.password_hash == "CHANGE_ME_BCRYPT_HASH":
return False
try:
ok = bcrypt.checkpw(password.encode(), user.password_hash.encode())
except (ValueError, TypeError):
ok = False
logger.info("SFTP 密码鉴权 %s user=%s", "OK" if ok else "FAIL", username)
if ok:
self._username = username
logger.info("SSH 密码鉴权 %s user=%s", "OK" if ok else "FAIL", username)
return ok
# 公钥
@@ -74,21 +95,88 @@ class ZikaiSSHServer(asyncssh.SSHServer):
def validate_public_key(self, username: str, key: asyncssh.SSHKey) -> bool:
if self._authorized_keys is None:
return False
if not any(u.username == username for u in self._settings.sftp.users):
if self._find_sftp_user(username) is None and self._find_tunnel_user(username) is None:
return False
addr = ""
if self._conn:
peer = self._conn.get_extra_info("peername")
addr = peer[0] if isinstance(peer, tuple) and peer else ""
addr = self._peer_ip
try:
# asyncssh 命中返回 dict可能为空未命中返回 None
result = self._authorized_keys.validate(key, client_host=addr, client_addr=addr)
except Exception:
result = None
ok = result is not None
logger.info("SFTP 公钥鉴权 %s user=%s", "OK" if ok else "FAIL", username)
if ok:
self._username = username
logger.info("SSH 公钥鉴权 %s user=%s", "OK" if ok else "FAIL", username)
return ok
# 反向隧道remote port-forward 请求
def server_requested(self, listen_host: str, listen_port: int) -> bool:
"""客户端请求在本地server 侧)绑端口做反向转发时回调。
仅允许已认证的 tunnel user 绑定其配置中预定的 tunnel_port记一条 active
隧道会话到 DB。其他情况拒绝。
"""
if not self._settings.tunnel.enabled:
logger.warning("拒绝 remote forwardtunnel 未启用 user=%s", self._username)
return False
if not self._username:
logger.warning("拒绝 remote forward未认证")
return False
tunnel_user = self._find_tunnel_user(self._username)
if tunnel_user is None:
logger.warning("拒绝 remote forward%s 不是 tunnel user", self._username)
return False
if listen_port != tunnel_user.tunnel_port:
logger.warning(
"拒绝 remote forwarduser=%s 端口 %d 不等于配置 %d",
self._username, listen_port, tunnel_user.tunnel_port,
)
return False
dao = _tunnel_dao()
try:
from ..services.tunnel_service import TunnelService
TunnelService(dao).register(
user_name=self._username,
user_ip=self._peer_ip,
tunnel_port=listen_port,
local_port=tunnel_user.local_port,
)
except Exception as exc: # pragma: no cover
_close_tunnel_dao(dao)
logger.error("登记隧道会话失败:%s", exc)
return False
_close_tunnel_dao(dao)
logger.info(
"允许 remote forward user=%s listen=%s:%d -> local_port=%d",
self._username, listen_host, listen_port, tunnel_user.local_port,
)
return True
def connection_lost(self, exc: Exception | None) -> None: # type: ignore[override]
"""SSH 连接断开时清理该 user 的活跃隧道会话。"""
if self._username and self._settings.tunnel.enabled:
dao = _tunnel_dao()
try:
from ..services.tunnel_service import TunnelService
TunnelService(dao).close(self._username)
except Exception as e: # pragma: no cover
logger.warning("清理隧道会话失败 user=%s: %s", self._username, e)
_close_tunnel_dao(dao)
if exc:
logger.info("SSH 连接异常断开 user=%s: %s", self._username, exc)
else:
logger.info("SSH 连接关闭 user=%s", self._username)
# 内部
def _find_sftp_user(self, username: str):
return next((u for u in self._settings.sftp.users if u.username == username), None)
def _find_tunnel_user(self, username: str):
return self._settings.tunnel.find_user(username)
def _load_authorized_keys(path: Path) -> asyncssh.SSHAuthorizedKeys | None:
if not path.exists() or path.stat().st_size == 0:
@@ -119,7 +207,7 @@ async def _run() -> None:
upload_root = settings.resolved_upload_dir()
upload_root.mkdir(parents=True, exist_ok=True)
# SFTP 客户端登记前的暂存目录register-sftp 只接受此目录下的路径
# SFTP 客户端的暂存目录chroot 内)
(upload_root / "incoming").mkdir(parents=True, exist_ok=True)
host_key_path = (PROJECT_ROOT / settings.sftp.host_key_path).resolve()

View File

@@ -0,0 +1,70 @@
"""反向隧道业务逻辑:会话注册 / 注销 / 查询。
SSH 服务收到 remote port-forward 请求时调 register 记一条 active 会话;
user 断开时调 close 标记 ended_atHTTP 路由调 get_active 查隧道端口做反代。
"""
from __future__ import annotations
import logging
from ..config import get_settings
from ..dao.tunnel_session_dao import TunnelSessionDAO
from ..models.tunnel_session import TunnelSession
logger = logging.getLogger("zikai.tunnel")
class TunnelService:
def __init__(self, dao: TunnelSessionDAO) -> None:
self.dao = dao
self.settings = get_settings().tunnel
def register(
self, user_name: str, user_ip: str, tunnel_port: int, local_port: int,
) -> TunnelSession:
"""登记一条活跃隧道会话。
若该 user 已有活跃会话先关闭旧的(同一 user 同一时刻只保留一条)。
"""
self.dao.close_active_by_user(user_name)
session = TunnelSession(
user_name=user_name,
user_ip=user_ip,
local_port=local_port,
tunnel_port=tunnel_port,
status="active",
)
saved = self.dao.create(session)
logger.info(
"隧道建立 user=%s ip=%s tunnel_port=%d local_port=%d",
user_name, user_ip, tunnel_port, local_port,
)
return saved
def close(self, user_name: str) -> int:
"""关闭该 user 的活跃会话SSH 断开时调用),返回关闭条数。"""
n = self.dao.close_active_by_user(user_name)
if n:
logger.info("隧道关闭 user=%s 条数=%d", user_name, n)
return n
def get_active(self, user_name: str) -> TunnelSession | None:
return self.dao.get_active_by_user(user_name)
def is_port_allowed(self, user_name: str, tunnel_port: int) -> bool:
"""校验该 user 是否被允许绑定该隧道端口(防 user 乱绑端口)。"""
user = self.settings.find_user(user_name)
return user is not None and user.tunnel_port == tunnel_port
def reap_orphans(self) -> int:
"""兜底清理:关闭所有 active 会话(进程重启时 DB 里残留的孤儿记录)。
由后台 reaper 在启动后调用一次。SSH 实际断开时已有 close() 处理,
这里只兜底进程异常退出后 DB 与实际状态不一致的情况。
"""
active = self.dao.list_active()
for session in active:
self.dao.close(session)
logger.info("reaper 清理孤儿隧道 id=%d user=%s", session.id, session.user_name)
return len(active)

View File

@@ -1,22 +1,35 @@
"""上传服务:流式落盘 + 原子改名 + 元数据入库。"""
"""上传服务:流式落盘 + 原子改名 + 元数据入库 + sha256 去重
两条上传路径HTTP 整文件 / 分片拼接)共享本类的「存储路径生成 + 落库提交 +
sha256 去重」逻辑避免重复实现。SFTP 服务器仅作为文件暂存通道chroot 到
upload_root不再有 HTTP 登记接口。
"""
from __future__ import annotations
import hashlib
import os
import uuid
from collections.abc import Iterator
from datetime import datetime, timezone
from pathlib import Path
from fastapi import HTTPException, UploadFile
from fastapi import UploadFile
from ..config import get_settings
from ..dao.uploaded_file_dao import UploadedFileDAO
from ..models.uploaded_file import UploadedFile
from ..schemas.file import FileUploadResponse, UploadedFileOut
# SFTP 客户端登记前必须把文件先落到此目录chroot 内)
SFTP_INCOMING_DIR = "incoming"
def hash_stream(chunks: Iterator[bytes]) -> tuple[int, str]:
"""对一段字节块序列流式计算 (size, sha256)。供多条上传路径复用。"""
h = hashlib.sha256()
size = 0
for chunk in chunks:
size += len(chunk)
h.update(chunk)
return size, h.hexdigest()
class UploadService:
@@ -36,7 +49,7 @@ class UploadService:
失败仅会留下可识别的 ``.part`` 文件,由 start.sh 启动时统一清理。
"""
rel_path, abs_path = self._make_storage_path(file.filename or "")
rel_path, abs_path = self.make_storage_path(file.filename or "")
part_path = abs_path.with_name(abs_path.name + ".part")
size, digest = self._write_part(file, part_path)
@@ -49,34 +62,7 @@ class UploadService:
source=source,
uploaded_by=uploaded_by,
)
return self._commit(entity, part_path, abs_path)
def register_sftp(
self, filename: str, original_filename: str, uploaded_by: str = "sftp",
) -> FileUploadResponse:
"""登记一个已通过 SFTP 落到 ``incoming/`` 下的文件。
相同 sha256 已存在时直接返回旧行并删掉新副本(去重)。
"""
src_abs = self._validate_incoming_path(filename)
size, digest = self._hash_disk_file(src_abs)
existing = self.dao.get_by_sha256(digest)
if existing is not None:
src_abs.unlink(missing_ok=True)
return self._to_response(existing)
rel_path, abs_path = self._make_storage_path(original_filename or filename)
entity = UploadedFile(
storage_path=str(rel_path),
original_filename=os.path.basename(original_filename or src_abs.name),
content_type="",
size_bytes=size,
sha256=digest,
source="sftp",
uploaded_by=uploaded_by,
)
return self._commit(entity, src_abs, abs_path)
return self.commit_entity(entity, part_path, abs_path)
# ---------------- 查询 ----------------
@@ -94,18 +80,26 @@ class UploadService:
row = self.dao.get_by_sha256(sha256)
return UploadedFileOut.model_validate(row) if row else None
def resolve_disk_path(self, file_id: int) -> Path | None:
row = self.dao.get_by_id(file_id)
return (self.upload_root / row.storage_path).resolve() if row else None
def get_out_with_disk_path(self, file_id: int) -> tuple[UploadedFileOut | None, Path | None]:
"""合并查询:一次 DB 读取同时返回 (元数据, 磁盘绝对路径)。
# ---------------- 内部 ----------------
供下载/删除路径复用,避免原先 get_out + resolve_disk_path 各查一次的重复读。
"""
row = self.dao.get_by_id(file_id)
if row is None:
return None, None
out = UploadedFileOut.model_validate(row)
path = (self.upload_root / row.storage_path).resolve()
return out, path
# ---------------- 共享 helper供本类与 ChunkUploadService 复用) ----------------
@staticmethod
def _safe_ext(filename: str) -> str:
return os.path.splitext(os.path.basename(filename))[1]
@staticmethod
def _to_response(row: UploadedFile) -> FileUploadResponse:
def to_response(row: UploadedFile, *, deduplicated: bool = False) -> FileUploadResponse:
return FileUploadResponse(
id=row.id,
filename=row.original_filename,
@@ -113,9 +107,10 @@ class UploadService:
sha256=row.sha256,
storage_path=row.storage_path,
uploaded_at=row.uploaded_at,
deduplicated=deduplicated,
)
def _make_storage_path(self, original_filename: str) -> tuple[Path, Path]:
def make_storage_path(self, original_filename: str) -> tuple[Path, Path]:
"""生成 ``(rel_path, abs_path)``abs_path 的父目录已创建。"""
now = datetime.now(timezone.utc)
rel_dir = Path(f"{now:%Y}/{now:%m}")
@@ -123,21 +118,7 @@ class UploadService:
rel_path = rel_dir / f"{uuid.uuid4().hex}{self._safe_ext(original_filename)}"
return rel_path, self.upload_root / rel_path
def _validate_incoming_path(self, filename: str) -> Path:
"""校验客户端给出的相对路径必须落在 upload_root/incoming/ 之下且是文件。"""
if not filename or filename.startswith(("/", "\\")):
raise HTTPException(400, "filename 必须是 incoming/ 下的相对路径")
incoming_root = (self.upload_root / SFTP_INCOMING_DIR).resolve()
try:
abs_path = (self.upload_root / filename).resolve()
abs_path.relative_to(incoming_root)
except ValueError:
raise HTTPException(400, f"filename 必须落在 {SFTP_INCOMING_DIR}/ 之下")
if not abs_path.is_file():
raise HTTPException(404, f"文件不存在或不是普通文件:{filename}")
return abs_path
def _commit(
def commit_entity(
self, entity: UploadedFile, src_path: Path, dest_path: Path,
) -> FileUploadResponse:
"""落 DB 行后把 src_path 原子改名到 dest_path任一失败回滚已生成的副作用。"""
@@ -154,17 +135,25 @@ class UploadService:
finally:
src_path.unlink(missing_ok=True)
raise
return self._to_response(saved)
return self.to_response(saved)
def _hash_disk_file(self, path: Path) -> tuple[int, str]:
"""流式读取磁盘文件,返回 (size, sha256)。"""
h = hashlib.sha256()
size = 0
with path.open("rb") as fh:
while chunk := fh.read(self.chunk_bytes):
size += len(chunk)
h.update(chunk)
return size, h.hexdigest()
def dedup_or_commit(
self, entity: UploadedFile, src_path: Path,
) -> FileUploadResponse:
"""公共尾部:按 entity.sha256 去重,命中则删 src 返回旧行;否则 commit。
供分片拼接等「先算出 sha256 再决定落盘」的路径复用,与 stream_to_disk
(边写边算、无独立 src的区别在于这里 sha256 已在 entity 上。
"""
existing = self.dao.get_by_sha256(entity.sha256)
if existing is not None:
src_path.unlink(missing_ok=True)
return self.to_response(existing, deduplicated=True)
rel_path, abs_path = self.make_storage_path(entity.original_filename)
entity.storage_path = str(rel_path)
return self.commit_entity(entity, src_path, abs_path)
# ---------------- 内部 ----------------
def _write_part(self, file: UploadFile, part_path: Path) -> tuple[int, str]:
"""把上传流写到 part_path 并 fsync返回 (size, sha256)。失败时清理残品。"""

View File

@@ -0,0 +1,175 @@
"""白板 WebSocket 连接管理器(实时同步 + 心跳 + 失活清理)。
设计要点:
- 进程内单例 ``WhiteboardHub``,维护 ``{board_id: set[Connection]}``。
- 每个 Connection 封装 websocket / board_id / client_id / last_heartbeat。
- 心跳:客户端每 ``heartbeat_interval_seconds``(默认 3s发一次 ping服务端回 pong 并
刷新 last_heartbeat。reaper 每秒扫描,超过 ``interval * threshold``(默认 15s未心跳
的连接判为失活,关闭并从 hub 移除。
- 内存安全disconnect 幂等;空 set 从 dict 删除broadcast 对单连接异常立即 disconnect
close_board 关闭并清理整个 board 的连接集合。
- 并发:用一个 asyncio.Lock 保护 ``_boards`` 的结构变更add/remove board key
集合内的连接增删用 set 原子操作Python 单线程事件循环下安全)。
注意hub 是进程内存,多 worker 下不互通。生产部署需单 worker 或后续接 Redis pub/sub。
"""
from __future__ import annotations
import asyncio
import logging
import time
from dataclasses import dataclass, field
from fastapi import WebSocket
from ..config import get_settings
logger = logging.getLogger("zikai.whiteboard")
@dataclass(eq=False)
class Connection:
"""一个白板在线连接。eq=False 使其按对象 identity 哈希/比较,可放入 set。"""
websocket: WebSocket
board_id: str
client_id: str
last_heartbeat: float = field(default_factory=time.monotonic)
async def send_json(self, msg: dict) -> bool:
"""发送一条消息;失败返回 False调用方据此 disconnect"""
try:
await self.websocket.send_json(msg)
return True
except Exception as exc: # WebSocketDisconnect / 已关闭 / 编码失败
logger.debug("发送失败 board=%s client=%s: %s", self.board_id, self.client_id, exc)
return False
def touch(self) -> None:
self.last_heartbeat = time.monotonic()
class WhiteboardHub:
"""白板连接管理器单例。"""
def __init__(self) -> None:
cfg = get_settings().whiteboard
self.heartbeat_interval = cfg.heartbeat_interval_seconds
self.heartbeat_miss_threshold = cfg.heartbeat_miss_threshold
self.timeout_seconds = self.heartbeat_interval * self.heartbeat_miss_threshold
# {board_id: set[Connection]}
self._boards: dict[str, set[Connection]] = {}
self._lock = asyncio.Lock()
# ---------------- 连接生命周期 ----------------
async def register(self, conn: Connection) -> None:
"""把已 accept 的连接加入 board 集合WebSocket accept 由 controller 负责)。"""
async with self._lock:
conns = self._boards.setdefault(conn.board_id, set())
conns.add(conn)
logger.info("连接接入 board=%s client=%s(当前 %d 人)",
conn.board_id, conn.client_id, self.connection_count(conn.board_id))
async def disconnect(self, conn: Connection) -> None:
"""幂等移除连接;空 set 从 dict 删除以防内存泄漏。"""
async with self._lock:
conns = self._boards.get(conn.board_id)
if conns is None:
return
conns.discard(conn)
if not conns:
self._boards.pop(conn.board_id, None)
# 尽力关闭 websocket可能已关闭
try:
await conn.websocket.close()
except Exception: # pragma: no cover
pass
logger.info("连接移除 board=%s client=%s(剩余 %d 人)",
conn.board_id, conn.client_id, self.connection_count(conn.board_id))
def connection_count(self, board_id: str) -> int:
"""调试/监控用:某 board 当前在线人数。"""
return len(self._boards.get(board_id, ()))
# ---------------- 广播 ----------------
async def broadcast(self, board_id: str, msg: dict, exclude: Connection | None = None) -> None:
"""把 msg 发给 board 内所有在线连接(可排除发送者)。单连接失败不影响其他。"""
async with self._lock:
conns = list(self._boards.get(board_id, ()))
dead: list[Connection] = []
for conn in conns:
if exclude is not None and conn is exclude:
continue
ok = await conn.send_json(msg)
if not ok:
dead.append(conn)
# 发送失败的连接统一清理
for conn in dead:
await self.disconnect(conn)
# ---------------- 心跳 reaper ----------------
async def reap_loop(self, stop: asyncio.Event) -> None:
"""后台循环:扫描失活连接。每秒一次,粒度细于 timeout。"""
logger.info("白板心跳 reaper 已启动(间隔 1s超时 %ds", self.timeout_seconds)
while not stop.is_set():
try:
await self._reap_once()
except Exception as exc: # pragma: no cover
logger.warning("reaper 循环异常:%s", exc)
try:
await asyncio.wait_for(stop.wait(), timeout=1.0)
except asyncio.TimeoutError:
pass
async def _reap_once(self) -> None:
now = time.monotonic()
async with self._lock:
# 快照待检查连接,避免持锁时 await
stale: list[Connection] = []
for board_id, conns in self._boards.items():
for conn in conns:
if now - conn.last_heartbeat > self.timeout_seconds:
stale.append(conn)
for conn in stale:
logger.warning("心跳失活,移除 board=%s client=%s(静默 %ds",
conn.board_id, conn.client_id,
int(now - conn.last_heartbeat))
await self.disconnect(conn)
async def close_board(self, board_id: str) -> None:
"""关闭并清理某 board 的所有连接(删除白板时调用)。"""
async with self._lock:
conns = self._boards.pop(board_id, None)
if not conns:
return
await asyncio.gather(
*(c.send_json({"type": "error", "msg": "白板已被删除"}) for c in conns),
return_exceptions=True,
)
for conn in conns:
try:
await conn.websocket.close()
except Exception: # pragma: no cover
pass
logger.info("关闭白板 board=%s,踢出 %d 个连接", board_id, len(conns))
# 进程内单例(由 main.py lifespan / controller 共享)
_hub: WhiteboardHub | None = None
def get_hub() -> WhiteboardHub:
global _hub
if _hub is None:
_hub = WhiteboardHub()
return _hub
def reset_hub() -> None:
"""测试用:重置单例。"""
global _hub
_hub = None

View File

@@ -0,0 +1,92 @@
"""白板服务CRUD + 笔画操作。
不持有 WebSocket 连接状态(那是 hub 的职责);删除白板时通过可选的 hub 回调
通知 hub 踢出该 board 的所有在线连接,由 controller 在装配时注入,避免 service -> hub
的硬依赖(保持低耦合)。
"""
from __future__ import annotations
import re
from typing import TYPE_CHECKING, Any
from fastapi import HTTPException
from ..config import get_settings
from ..dao.whiteboard_dao import WhiteboardDAO
from ..schemas.whiteboard import WhiteboardListItem, WhiteboardOut
if TYPE_CHECKING: # 避免运行时循环导入
from .whiteboard_hub import WhiteboardHub
# board_id 合法字符集:字母数字下划线短横线
_BOARD_ID_RE = re.compile(r"^[a-zA-Z0-9_-]+$")
class WhiteboardService:
def __init__(self, dao: WhiteboardDAO, hub: "WhiteboardHub | None" = None) -> None:
self.dao = dao
self.hub = hub
cfg = get_settings().whiteboard
self.max_board_id_length = cfg.max_board_id_length
self.list_limit = cfg.list_limit
# ---------------- 校验 ----------------
def validate_board_id(self, board_id: str) -> None:
"""非法 board_id 直接 400防路径穿越/注入)。"""
if (
not board_id
or len(board_id) > self.max_board_id_length
or not _BOARD_ID_RE.match(board_id)
):
raise HTTPException(400, "board_id 非法仅允许字母数字下划线短横线1-64 字符)")
# ---------------- 读 ----------------
def get_or_create(self, board_id: str) -> WhiteboardOut:
self.validate_board_id(board_id)
board = self.dao.get_or_create(board_id)
return WhiteboardOut.model_validate(board)
def list_all(self, limit: int = 100, offset: int = 0) -> tuple[int, list[WhiteboardListItem]]:
limit = min(max(limit, 0), self.list_limit) or self.list_limit
offset = max(offset, 0)
total = self.dao.count()
rows = self.dao.list_all(limit=limit, offset=offset)
items = [WhiteboardListItem.model_validate(r) for r in rows]
return total, items
# ---------------- 写 ----------------
def append_stroke(self, board_id: str, stroke: dict[str, Any]) -> WhiteboardOut:
"""追加一条笔画并返回最新状态。"""
self.validate_board_id(board_id)
board = self.dao.append_strokes(board_id, [stroke])
if board is None:
raise HTTPException(404, "白板不存在")
return WhiteboardOut.model_validate(board)
def clear(self, board_id: str) -> WhiteboardOut:
"""清空白板stroke_count 仍自增以记录这次修改。"""
self.validate_board_id(board_id)
board = self.dao.replace_strokes(board_id, [])
if board is None:
raise HTTPException(404, "白板不存在")
return WhiteboardOut.model_validate(board)
def delete(self, board_id: str) -> bool:
"""删除白板;同时通知 hub 踢出该 board 的所有在线连接。"""
self.validate_board_id(board_id)
ok = self.dao.delete(board_id)
if ok and self.hub is not None:
# hub.close_board 是 async但删除走 REST 同步路径;安排到事件循环里执行
import asyncio
try:
loop = asyncio.get_running_loop()
loop.create_task(self.hub.close_board(board_id))
except RuntimeError:
# 无运行中事件循环(如脚本调用):同步调用会报错,忽略即可
pass
return ok

288
app/views/upload_html.py Normal file
View File

@@ -0,0 +1,288 @@
"""上传页面 HTML 渲染(拖拽 + 多文件 + 分片 + 断点续传)。
单文件 server-rendered内联 CSS+JS风格对齐 system_status_html.py
深色模式自适应、卡片、进度条。无构建步骤、无外部依赖。
"""
from __future__ import annotations
from html import escape
# 默认分片大小 4 MiB大于 Apache 300s 限制下单片可数秒传完,小到内存恒定。
DEFAULT_CHUNK_SIZE = 4 * 1024 * 1024
# 同一文件分片并发数
DEFAULT_CONCURRENCY = 3
# 单分片失败重试次数
MAX_RETRY = 2
def render() -> str:
return f"""<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>上传文件 — zikai</title>
<style>{_CSS}</style>
</head>
<body>
<h1>上传文件</h1>
<p class="sub">拖拽文件到下方,或点击选择。支持多文件、大文件分片上传与断点续传。</p>
<div id="drop" class="drop">
<p class="drop-hint">把文件拖到这里,或</p>
<label class="btn">选择文件<input id="file-input" type="file" multiple hidden></label>
</div>
<div id="tasks" class="tasks"></div>
<div id="summary" class="foot"></div>
<p class="foot"><a class="json" href="/api/system/status">系统状态</a> · zikai file service</p>
<script>
const CHUNK_SIZE = {DEFAULT_CHUNK_SIZE};
const CONCURRENCY = {DEFAULT_CONCURRENCY};
const MAX_RETRY = {MAX_RETRY};
const API = "/api/files/chunk-uploads";
const tasksEl = document.getElementById("tasks");
const summaryEl = document.getElementById("summary");
const fileInput = document.getElementById("file-input");
const drop = document.getElementById("drop");
let pending = []; // 等待开始的文件
let done = 0, failed = 0;
drop.addEventListener("dragover", e => {{ e.preventDefault(); drop.classList.add("drag"); }});
drop.addEventListener("dragleave", () => drop.classList.remove("drag"));
drop.addEventListener("drop", e => {{
e.preventDefault();
drop.classList.remove("drag");
addFiles(e.dataTransfer.files);
}});
fileInput.addEventListener("change", () => addFiles(fileInput.files));
function addFiles(fileList) {{
for (const f of fileList) {{
const task = makeTask(f);
pending.push(task);
tasksEl.appendChild(task.el);
}}
fileInput.value = "";
pump();
}}
function makeTask(file) {{
const totalChunks = Math.max(1, Math.ceil(file.size / CHUNK_SIZE));
const el = document.createElement("div");
el.className = "card task";
el.innerHTML = `
<div class="task-head">
<span class="fname"></span>
<span class="fsize"></span>
<span class="fstate">等待中</span>
</div>
<div class="bar"><div class="fill" style="width:0%"></div><span class="pct">0%</span></div>
<div class="task-meta"></div>
`;
el.querySelector(".fname").textContent = file.name;
el.querySelector(".fsize").textContent = fmtBytes(file.size);
return {{
file, totalChunks, el,
uploadId: null,
uploaded: new Set(),
state: "pending",
cancel: false,
}};
}}
// 限制并发文件数(同时最多 CONCURRENCY 个文件在传)
function pump() {{
const active = pending.filter(t => t.state === "running").length;
for (const t of pending) {{
if (active >= CONCURRENCY) break;
if (t.state === "pending") {{
t.state = "running";
startTask(t).finally(() => {{
pending = pending.filter(x => x !== t);
pump();
renderSummary();
}});
active++;
}}
}}
}}
function setState(t, s) {{
t.state = s;
const map = {{pending:"等待中", running:"上传中", hashing:"拼接中", done:"完成", dedup:"秒传", fail:"失败"}};
t.el.querySelector(".fstate").textContent = map[s] || s;
t.el.querySelector(".fstate").className = "fstate state-" + s;
}}
function setProgress(t, pct) {{
t.el.querySelector(".fill").style.width = pct.toFixed(1) + "%";
t.el.querySelector(".pct").textContent = pct.toFixed(0) + "%";
}}
async function startTask(t) {{
try {{
// 1. 创建会话
const cre = await fetch(API, {{
method: "POST",
headers: {{"Content-Type": "application/json"}},
body: JSON.stringify({{
filename: t.file.name, size_bytes: t.file.size,
chunk_size: CHUNK_SIZE, total_chunks: t.totalChunks,
}}),
}});
if (!cre.ok) throw new Error("创建会话失败: " + await cre.text());
const sess = await cre.json();
t.uploadId = sess.upload_id;
// 2. 查状态(断点续传:补传缺失分片)
const st = await fetch(API + "/" + t.uploadId + "/status");
const status = await st.json();
if (status.completed && status.file_id != null) {{
// 之前已完成
setState(t, "dedup");
setProgress(t, 100);
t.el.querySelector(".task-meta").textContent = "file_id=" + status.file_id + "(已完成)";
done++;
return;
}}
(status.uploaded_chunks || []).forEach(i => t.uploaded.add(i));
// 3. 传缺失分片
const need = [];
for (let i = 0; i < t.totalChunks; i++) if (!t.uploaded.has(i)) need.push(i);
await runPool(need, i => uploadChunk(t, i));
if (t.cancel) return;
if (t.uploaded.size < t.totalChunks) throw new Error("部分分片未能上传");
// 4. complete
setState(t, "hashing");
setProgress(t, 100);
const cmp = await fetch(API + "/" + t.uploadId + "/complete", {{ method: "POST" }});
if (!cmp.ok) throw new Error("complete 失败: " + await cmp.text());
const res = await cmp.json();
setState(t, res.deduplicated ? "dedup" : "done");
t.el.querySelector(".task-meta").innerHTML =
"file_id=" + res.id + " · sha256=<code>" + escapeHtml(res.sha256.slice(0,16)) + "…</code>" +
(res.deduplicated ? " · <b>秒传</b>(服务端已有相同内容)" : "");
done++;
}} catch (e) {{
setState(t, "fail");
t.el.querySelector(".task-meta").textContent = String(e.message || e);
failed++;
}}
}}
async function uploadChunk(t, index) {{
const start = index * CHUNK_SIZE;
const end = Math.min(start + CHUNK_SIZE, t.file.size);
const blob = t.file.slice(start, end);
let lastErr;
for (let attempt = 0; attempt <= MAX_RETRY; attempt++) {{
if (t.cancel) return;
try {{
const r = await fetch(API + "/" + t.uploadId + "/chunks/" + index, {{
method: "POST",
body: blob,
}});
if (!r.ok) throw new Error("HTTP " + r.status);
t.uploaded.add(index);
const pct = (t.uploaded.size / t.totalChunks) * 100;
setProgress(t, pct);
return;
}} catch (e) {{
lastErr = e;
}}
}}
throw lastErr;
}}
// 简易并发池:对 indices 逐个跑,最多 CONCURRENCY 并发
async function runPool(indices, worker) {{
let cursor = 0;
const runners = [];
for (let n = 0; n < CONCURRENCY; n++) {{
runners.push((async () => {{
while (cursor < indices.length) {{
const i = indices[cursor++];
await worker(i);
}}
}})());
}}
await Promise.all(runners);
}}
function renderSummary() {{
if (done + failed === 0) {{ summaryEl.textContent = ""; return; }}
summaryEl.textContent = "完成 " + done + ",失败 " + failed;
}}
function fmtBytes(n) {{
let x = n, u = 0;
const units = ["B","KiB","MiB","GiB","TiB"];
while (x >= 1024 && u < units.length-1) {{ x /= 1024; u++; }}
return u === 0 ? x + " B" : x.toFixed(1) + " " + units[u];
}}
function escapeHtml(s) {{
return s.replace(/[&<>"]/g, c => ({{"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;"}}[c]));
}}
</script>
</body>
</html>
"""
_CSS = """
:root { color-scheme: light dark; }
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
max-width: 880px; margin: 2em auto; padding: 0 1em; line-height: 1.5; }
h1 { margin-bottom: 0.1em; }
.sub { color: #777; margin-top: 0; font-size: 0.95em; }
.drop { border: 2px dashed #bbb; border-radius: 12px; padding: 2.5em 1em;
text-align: center; margin: 1.5em 0; transition: background 0.15s, border-color 0.15s; }
.drop.drag { background: rgba(21,101,192,0.08); border-color: #1565c0; }
.drop-hint { margin: 0 0 1em; color: #888; }
.btn { display: inline-block; padding: 0.5em 1.2em; border-radius: 6px;
background: #1565c0; color: #fff; cursor: pointer; font-size: 0.95em; }
.btn:hover { background: #0d47a1; }
.btn input { display: none; }
.tasks { margin: 1em 0; }
.card { border: 1px solid #ddd; border-radius: 8px; padding: 0.9em 1.1em;
margin: 0.7em 0; background: rgba(0,0,0,0.02); }
.task-head { display: flex; align-items: center; gap: 0.6em; margin-bottom: 0.5em; }
.fname { font-weight: 600; word-break: break-all; flex: 1; }
.fsize { color: #888; font-size: 0.85em; white-space: nowrap; }
.fstate { font-size: 0.85em; padding: 0.1em 0.6em; border-radius: 10px;
background: #eee; white-space: nowrap; }
.state-running { background: #e3f2fd; color: #1565c0; }
.state-hashing { background: #fff3e0; color: #e65100; }
.state-done { background: #e8f5e9; color: #2e7d32; }
.state-dedup { background: #f3e5f5; color: #7b1fa2; }
.state-fail { background: #ffebee; color: #c62828; }
.bar { position: relative; background: #e6e6e6; border-radius: 4px;
height: 18px; width: 100%; overflow: hidden; }
.bar .fill { height: 100%; width: 0; background: #43a047; transition: width 0.2s; }
.bar .pct { position: absolute; top: 0; left: 0; width: 100%; height: 100%;
text-align: center; font-size: 12px; line-height: 18px;
color: #fff; mix-blend-mode: difference; }
.task-meta { margin-top: 0.5em; font-size: 0.82em; color: #666; word-break: break-all; }
.task-meta code { background: rgba(0,0,0,0.06); padding: 0 0.3em; border-radius: 3px; }
.foot { color: #888; font-size: 0.85em; margin-top: 1.5em; text-align: center; }
a.json { color: #1565c0; text-decoration: none; }
@media (prefers-color-scheme: dark) {{
body {{ background: #1a1a1a; color: #e0e0e0; }}
.card {{ background: rgba(255,255,255,0.04); border-color: #333; }}
.drop {{ border-color: #555; }}
.drop.drag {{ background: rgba(21,101,192,0.18); }}
.fstate {{ background: #333; }}
.bar {{ background: #333; }}
.task-meta code {{ background: rgba(255,255,255,0.08); }}
}}
"""