From fffba790222713d5d855c8d0eb9d98340bf5e938 Mon Sep 17 00:00:00 2001 From: zikai Date: Tue, 21 Jul 2026 14:28:13 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=87=E4=BB=B6=E6=B5=8F=E8=A7=88?= =?UTF-8?q?=E9=A1=B5=20+=20=E5=85=B1=E4=BA=AB=E7=99=BD=E6=9D=BF=20+=20?= =?UTF-8?q?=E7=99=BD=E6=9D=BF=E7=AE=A1=E7=90=86=E9=A1=B5=EF=BC=88=E5=90=AB?= =?UTF-8?q?=E8=A1=A5=E7=99=BB=E8=AE=B0=E5=88=86=E7=89=87=E4=B8=8A=E4=BC=A0?= =?UTF-8?q?/=E9=9A=A7=E9=81=93=E5=8E=86=E5=8F=B2=E6=94=B9=E5=8A=A8?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 本次提交包含两批改动（7月2日遗留未入库 + 本次新功能），分述如下： 【补登记：7月2日已上线但未提交的功能】 - 分片上传：chunk_upload_controller/service/dao + UploadSession model/schema， 支持 4MiB 分片、乱序、断点续传、去重、幂等 complete；后台 reaper 清理过期会话。 - 反向隧道：tunnel_controller/service/dao + TunnelSession model/schema， SSH remote forward 经 /api/userPort/{userName} 反代到 user 本地服务。 - 上传页：views/upload_html.py（拖拽/多文件/分片/断点续传 UI）。 - config.py：StorageConfig.chunk_session_dir/ttl、TunnelConfig； requirements.txt 加 httpx；start.sh 清理 .work/ 残留； schema.sql 加 upload_session/tunnel_session 表；sftp_server 承载隧道转发。 【本次新功能】 - 文件浏览页：GET /files（Basic Auth 同 docs）+ /api/admin/files（list/get/download/DELETE）。 硬删除（DB 行 + 磁盘文件），删除后列表不再显示。前端 static/file_browser.*。 - 共享白板：GET /whiteboard/{id}（公开，不存在则新建）+ WS /ws/whiteboard/{id}。 MySQL 持久化（whiteboard 表），Canvas 实时同步，心跳 3s/5 次失活移除， 清空/复制按钮，移动端兼容。WhiteboardHub 管理 {board_id: set[Connection]}， disconnect 幂等 + 空 set 清理防泄漏，broadcast 失败连接自动移除。 - 白板管理页：GET /whiteboard-admin（Basic Auth）+ /api/admin/whiteboards（list/DELETE）。 删除时 hub.close_board 踢出在线连接。 - 清理：合并 UploadService.get_out_with_disk_path（下载/删除复用，消除重复 DB 读）， 移除无用 resolve_disk_path。 - config.py：WhiteboardConfig（heartbeat/threshold/board_id 长度/list_limit）； schema.sql 加 whiteboard 表；README 补新接口与心跳/内存说明。 - 验证：tests/manual_whiteboard_hub.py / _ws.py / _kick.py 全部通过。 --- README.md | 244 +++++++++++------ app/config.py | 43 +++ app/controllers/__init__.py | 13 +- app/controllers/chunk_upload_controller.py | 93 +++++++ app/controllers/file_admin_controller.py | 111 ++++++++ app/controllers/file_controller.py | 28 +- app/controllers/tunnel_controller.py | 97 +++++++ app/controllers/whiteboard_controller.py | 220 ++++++++++++++++ app/dao/__init__.py | 4 +- app/dao/tunnel_session_dao.py | 63 +++++ app/dao/upload_session_dao.py | 57 ++++ app/dao/whiteboard_dao.py | 84 ++++++ app/main.py | 156 +++++++++-- app/models/__init__.py | 5 +- app/models/tunnel_session.py | 41 +++ app/models/upload_session.py | 44 ++++ app/models/whiteboard.py | 39 +++ app/schemas/__init__.py | 24 ++ app/schemas/chunk.py | 43 +++ app/schemas/file.py | 14 +- app/schemas/tunnel.py | 19 ++ app/schemas/whiteboard.py | 53 ++++ app/services/__init__.py | 3 +- app/services/chunk_upload_service.py | 247 ++++++++++++++++++ app/services/sftp_server.py | 112 +++++++- app/services/tunnel_service.py | 70 +++++ app/services/upload_service.py | 117 ++++----- app/services/whiteboard_hub.py | 175 +++++++++++++ app/services/whiteboard_service.py | 92 +++++++ app/views/upload_html.py | 288 +++++++++++++++++++++ config.example.yaml | 22 ++ requirements.txt | 1 + sql/schema.sql | 48 ++++ start.sh | 12 +- static/common.css | 112 ++++++++ static/common.js | 87 +++++++ static/file_browser.css | 18 ++ static/file_browser.html | 29 +++ static/file_browser.js | 94 +++++++ static/whiteboard.css | 50 ++++ static/whiteboard.html | 39 +++ static/whiteboard.js | 271 +++++++++++++++++++ static/whiteboard_admin.css | 15 ++ static/whiteboard_admin.html | 30 +++ static/whiteboard_admin.js | 84 ++++++ tests/manual_whiteboard_hub.py | 130 ++++++++++ tests/manual_whiteboard_kick.py | 52 ++++ tests/manual_whiteboard_ws.py | 92 +++++++ 48 files changed, 3569 insertions(+), 216 deletions(-) create mode 100644 app/controllers/chunk_upload_controller.py create mode 100644 app/controllers/file_admin_controller.py create mode 100644 app/controllers/tunnel_controller.py create mode 100644 app/controllers/whiteboard_controller.py create mode 100644 app/dao/tunnel_session_dao.py create mode 100644 app/dao/upload_session_dao.py create mode 100644 app/dao/whiteboard_dao.py create mode 100644 app/models/tunnel_session.py create mode 100644 app/models/upload_session.py create mode 100644 app/models/whiteboard.py create mode 100644 app/schemas/chunk.py create mode 100644 app/schemas/tunnel.py create mode 100644 app/schemas/whiteboard.py create mode 100644 app/services/chunk_upload_service.py create mode 100644 app/services/tunnel_service.py create mode 100644 app/services/whiteboard_hub.py create mode 100644 app/services/whiteboard_service.py create mode 100644 app/views/upload_html.py create mode 100644 static/common.css create mode 100644 static/common.js create mode 100644 static/file_browser.css create mode 100644 static/file_browser.html create mode 100644 static/file_browser.js create mode 100644 static/whiteboard.css create mode 100644 static/whiteboard.html create mode 100644 static/whiteboard.js create mode 100644 static/whiteboard_admin.css create mode 100644 static/whiteboard_admin.html create mode 100644 static/whiteboard_admin.js create mode 100644 tests/manual_whiteboard_hub.py create mode 100644 tests/manual_whiteboard_kick.py create mode 100644 tests/manual_whiteboard_ws.py diff --git a/README.md b/README.md index c12ddb0..24dc6d2 100644 --- a/README.md +++ b/README.md @@ -1,94 +1,108 @@ # zikai file service -A Python web service (FastAPI) for **f.zikai.wang** that provides host -monitoring and large-file upload over **both HTTP and SFTP**. It follows a -Spring-style layered architecture (`controllers` → `services` → `dao`, plus -`models` and `schemas`), ships with auto-generated API docs, and runs entirely -from a self-contained `.venv`. +`f.zikai.wang` 的 Python Web 服务（FastAPI），提供主机监控、大文件上传、共享白板与 +文件浏览：**HTTP（整文件 + 分片/断点续传）**，并内置 **SFTP 服务器** 用于原始文件暂存。 +采用 Spring 风格分层架构（`controllers` -> `services` -> `dao`，外加 `models` 与 +`schemas`），自带自动生成的 API 文档，全部运行在自包含的 `.venv` 中。 -## Features +## 功能 -- `GET /api/system/status` — CPU, memory, and per-disk usage (via `psutil`). -- `POST /api/files/upload` — **streamed** multipart upload (flat memory, multi-GB friendly), with SHA-256. -- `GET /api/files`, `GET /api/files/{id}`, `GET /api/files/{id}/download`. -- **Embedded SFTP server** (asyncssh) supporting **password + public-key** auth, sharing the same storage as HTTP. -- `/docs` (Swagger UI) and `/redoc` — interactive, auto-lists all APIs. -- Metadata persisted in an **independent MySQL database** (`zikai_filesvc`). -- `start.sh` / `stop.sh` lifecycle; `setup.sh` for one-time provisioning. +- `GET /api/system/status` - CPU、内存、各磁盘使用率（via `psutil`）。 +- `POST /api/files/upload` - **流式** multipart 上传（内存恒定，支持多 GB），落盘时算 SHA-256。 +- `GET /upload` - 拖拽上传页面：多文件、**分片（4 MiB）**、**断点续传**、sha256 去重。 +- `POST /api/files/chunk-uploads/*` - 支撑 `/upload` 的分片上传 API（建会话 / 查状态 / 传分片 / 完成）。 +- `GET /api/files`、`GET /api/files/{id}`、`GET /api/files/{id}/download`。 +- **文件浏览页** `GET /files`（Basic Auth，同 docs）：列出/下载/**硬删除**已上传文件；删除后不再显示。 + 管理 API：`GET /api/admin/files`、`GET /api/admin/files/{id}`、`GET /api/admin/files/{id}/download`、 + `DELETE /api/admin/files/{id}`（均 Basic Auth）。 +- **共享白板** `GET /whiteboard/{id}`（公开，不存在则新建）：Canvas 实时协作 + **清空 / 复制链接**，兼容移动端。 + 实时同步走 `WS /ws/whiteboard/{id}`（**心跳 3s，连续 5 次丢失判失活并移除**）。 +- **白板管理页** `GET /whiteboard-admin`（Basic Auth，同 docs）：查看创建时间/修改次数/上次修改时间/删除。 + 管理 API：`GET /api/admin/whiteboards`、`DELETE /api/admin/whiteboards/{id}`（均 Basic Auth）。 +- **反向隧道反代**：`ALL /api/userPort/{userName}` -- 把请求经 SSH 反向隧道转发到该 user 的本机服务。 +- **内置 SFTP/SSH 服务器**（asyncssh），支持 **密码 + 公钥** 鉴权，同时承载 SFTP 文件暂存与反向隧道。 +- `/docs`（Swagger UI）与 `/redoc` - 交互式文档，自动列出所有 API。 +- 元数据持久化在 **独立的 MySQL 数据库**（`zikai_filesvc`）。 +- `start.sh` / `stop.sh` 生命周期管理；`setup.sh` 一次性初始化。 -## Architecture (Spring-style layers) +## 架构（Spring 风格分层） ``` app/ -├── controllers/ # FastAPI routers — HTTP boundary (like @RestController) -├── services/ # business logic (SystemService, UploadService, SFTP server) -├── dao/ # data access objects — the only layer that issues SQL/ORM -├── models/ # SQLAlchemy ORM entities -├── schemas/ # pydantic DTOs (request/response validation) -├── database.py # engine, session, Base, get_db() dependency -├── config.py # typed Settings loaded from config.yaml -└── scripts/ # init_db.py — DB provisioning +├── controllers/ # FastAPI 路由 -- HTTP 边界（类似 @RestController） +├── services/ # 业务逻辑（SystemService, UploadService, ChunkUploadService, +│ # WhiteboardService, WhiteboardHub, SFTP 服务） +├── dao/ # 数据访问对象 -- 唯一发出 SQL/ORM 的层 +├── models/ # SQLAlchemy ORM 实体（UploadedFile, UploadSession, Whiteboard, ...） +├── schemas/ # pydantic DTO（请求/响应校验） +├── views/ # 服务端渲染的 HTML 页面（系统状态、上传页） +├── static/ # 前端静态资源（文件浏览/白板/白板管理的 HTML+JS+CSS，经 StaticFiles 挂载） +├── database.py # 引擎、Session、Base、get_db() 依赖 +├── config.py # 从 config.yaml 加载的类型化 Settings +└── scripts/ # init_db.py -- 数据库初始化 ``` -Request flow: **controller** → **service** → **dao** → **ORM model** → MySQL. -The DB session is injected by FastAPI's `get_db` dependency and passed down. +请求流程：**controller** -> **service** -> **dao** -> **ORM model** -> MySQL。 +DB Session 由 FastAPI 的 `get_db` 依赖注入并向下传递。前端三套页面走「独立静态文件 + +StaticFiles 挂载」的前后端分离模式，HTML 壳由具名路由返回（便于各自挂 Basic Auth）， +JS 调用同源 `/api/...`。 -## Quick start +## 快速开始 ```bash cd /root/zikai -./setup.sh # one-time: venv, deps, provision DB + user, SFTP host key -./start.sh # start HTTP (127.0.0.1:6867) + SFTP (0.0.0.0:2022) -./stop.sh # stop both +./setup.sh # 一次性：venv、依赖、建库建账、SFTP 主机密钥 +./start.sh # 启动 HTTP（127.0.0.1:6867）+ SFTP（0.0.0.0:2022） +./stop.sh # 停止两者 ``` -`setup.sh` is re-runnable. It creates the `.venv`, installs `requirements.txt`, -copies `config.example.yaml` → `config.yaml` (if absent), provisions a **new -independent MySQL database and app user** via the local root socket, and -generates the SFTP host key. +`setup.sh` 可重复执行。它会创建 `.venv`、安装 `requirements.txt`、复制 +`config.example.yaml` → `config.yaml`（若不存在）、通过本机 root socket 建一个 +**全新的独立 MySQL 数据库与应用账户**，并生成 SFTP 主机密钥。 -## Access +## 访问方式 -| Where | URL | +| 入口 | URL | |------|-----| -| Status page (HTML) | https://f.zikai.wang/api/system/status | -| Status page (JSON) | https://f.zikai.wang/api/system/status?format=json (or `Accept: application/json`) | -| API docs (Swagger) | https://f.zikai.wang/docs **(HTTP Basic Auth — see `docs:` in config.yaml)** | -| API docs (ReDoc) | https://f.zikai.wang/redoc (same auth) | -| Upload | `curl -F file=@big.iso https://f.zikai.wang/api/files/upload` | -| SFTP | `sftp -P 2022 uploader@f.zikai.wang` | +| 状态页（HTML） | https://f.zikai.wang/api/system/status | +| 状态页（JSON） | https://f.zikai.wang/api/system/status?format=json（或 `Accept: application/json`） | +| API 文档（Swagger） | https://f.zikai.wang/docs **（HTTP Basic Auth -- 见 config.yaml 的 `docs:` 段）** | +| API 文档（ReDoc） | https://f.zikai.wang/redoc（同样鉴权） | +| 上传页（拖拽、分片、断点续传） | https://f.zikai.wang/upload | +| 文件浏览页（列出/下载/删除） | https://f.zikai.wang/files **（Basic Auth，同 docs）** | +| 共享白板（实时协作） | https://f.zikai.wang/whiteboard/{id}（公开，`{id}` 为 `[a-zA-Z0-9_-]{1,64}`，不存在则新建） | +| 白板管理页 | https://f.zikai.wang/whiteboard-admin **（Basic Auth，同 docs）** | +| 上传（curl） | `curl -F file=@big.iso https://f.zikai.wang/api/files/upload` | +| SFTP | `sftp -P 2022 uploader@f.zikai.wang` | -`/api/system/status` content-negotiates: browsers (`Accept: text/html`) get a -human-readable page with progress bars; API clients get JSON. Force one with -`?format=html` or `?format=json`. +`/api/system/status` 做内容协商：浏览器（`Accept: text/html`）拿到带进度条的可读页面； +API 客户端拿到 JSON。可用 `?format=html` 或 `?format=json` 强制指定。 -`/docs`, `/redoc`, and `/openapi.json` require HTTP Basic Auth — the browser -will prompt you. Username + plaintext password live in `config.yaml` under -`docs:`. `/health` and `/` remain public. +`/docs`、`/redoc`、`/openapi.json` 需要 HTTP Basic Auth —— 浏览器会弹出登录框。用户名与 +明文密码写在 `config.yaml` 的 `docs:` 段。`/health` 与 `/` 保持公开。 -Apache (`/etc/apache2/sites-available/f.zikai.wang-le-ssl.conf`) proxies -`f.zikai.wang` → `127.0.0.1:6867` with `ProxyPreserveHost On`, so the service -only binds the loopback. +Apache（`/etc/apache2/sites-available/f.zikai.wang-le-ssl.conf`）把 `f.zikai.wang` 反代到 +`127.0.0.1:6867`（`ProxyPreserveHost On`），因此服务只绑 loopback。 -> **Large/slow HTTP uploads:** Apache's proxy leg inherits the global `Timeout 300`. -> For multi-GB transfers over a slow link, prefer **SFTP** (it bypasses the HTTP -> proxy entirely). To raise the HTTP ceiling you can add `ProxyTimeout`/`Timeout` -> in the Apache vhost. +> **大文件/慢速 HTTP 上传：** Apache 代理段继承全局 `Timeout 300`。多 GB 慢链路传输建议走 +> **分片上传**（`/upload` 页面或 `/api/files/chunk-uploads`，单片 4 MiB 在超时内可传完）或 +> **SFTP**（完全绕过 HTTP 代理）。要提高 HTTP 上限可在 Apache vhost 加 `ProxyTimeout`/`Timeout`。 -## Configuration +## 配置 -All runtime config lives in **`config.yaml`** (git-ignored). See -`config.example.yaml` for the full schema. Notable keys: +所有运行时配置都在 **`config.yaml`**（git-ignored）。完整 schema 见 `config.example.yaml`。 +关键配置项： -- `server` — bind host/port (keep `127.0.0.1:6867` to match Apache). -- `database` — host/port/user/password/database. The password is auto-generated - and written here by `setup.sh`/`init_db.py`. -- `storage.upload_dir`, `storage.chunk_bytes` (default 1 MiB streaming chunk). -- `sftp` — enabled, host/port, host key + authorized_keys paths, and `users`. +- `server` — 绑定 host/port（保持 `127.0.0.1:6867` 以对齐 Apache）。 +- `database` — host/port/user/password/database。密码由 `setup.sh`/`init_db.py` 自动生成并写回。 +- `storage.upload_dir`、`storage.chunk_bytes`（默认 1 MiB 流式分片）、`storage.chunk_session_dir` + （分片会话暂存目录）、`storage.chunk_session_ttl_seconds`（被放弃会话的存活秒数，默认 300）。 +- `sftp` — enabled、host/port、host key + authorized_keys 路径、`users`。 +- `whiteboard` - `heartbeat_interval_seconds`（默认 3）、`heartbeat_miss_threshold`（默认 5）、`max_board_id_length`（默认 64）、`list_limit`（默认 100）。 -### Setting the /docs admin password +### 设置 /docs 管理密码 -Edit `config.yaml` directly — no hashing required: +直接编辑 `config.yaml`，无需哈希： ```yaml docs: @@ -98,38 +112,102 @@ docs: realm: "zikai docs" ``` -Then `./stop.sh && ./start.sh`. The file is root-owned and stays on this -host; comparison is constant-time (`secrets.compare_digest`). +然后 `./stop.sh && ./start.sh`。该文件 root 持有且仅在本机；比较使用常量时间 +（`secrets.compare_digest`）。 -### Setting SFTP credentials +### 设置 SFTP 凭据 -**Password auth** — generate a bcrypt hash and put it in `config.yaml`: +**密码鉴权** —— 生成 bcrypt hash 写入 `config.yaml`： ```bash .venv/bin/python -c "import bcrypt;print(bcrypt.hashpw(b'yourpass',bcrypt.gensalt()).decode())" -# paste the output into sftp.users[].password_hash, then ./stop.sh && ./start.sh +# 输出粘贴到 sftp.users[].password_hash，然后 ./stop.sh && ./start.sh ``` -**Public-key auth** — append each client's public key (OpenSSH format) to -`keys/authorized_keys` (one per line). Clients in `sftp.users[]` may then log -in with either method. +**公钥鉴权** —— 把每个客户端的公钥（OpenSSH 格式）追加到 `keys/authorized_keys`（每行一个）。 +`sftp.users[]` 中的用户随后可用任一方式登录。 -### Regenerating the DB password +### 重新生成数据库密码 ```bash -.venv/bin/python -m app.scripts.init_db # new random password -KEEP_DB_PASSWORD=1 .venv/bin/python -m app.scripts.init_db # keep current +.venv/bin/python -m app.scripts.init_db # 生成新随机密码 +KEEP_DB_PASSWORD=1 .venv/bin/python -m app.scripts.init_db # 保留现有密码 ``` -## SFTP notes +## SFTP 说明 -- The SFTP server can't traverse Apache's HTTP proxy, so it binds `0.0.0.0:2022` - directly. **Open port 2022** in your firewall for external clients - (FileZilla/WinSCP/scp). -- Sessions are chrooted to the upload root (`uploads/`), shared with HTTP. -- Only users listed in `sftp.users` may connect; only SFTP (no shell/exec) is allowed. +- SFTP 服务无法穿透 Apache 的 HTTP 代理，因此直接绑 `0.0.0.0:2022`。**请在防火墙放开 + 2022 端口** 供外部客户端（FileZilla/WinSCP/scp）连接。 +- 会话 chroot 到上传根目录（`uploads/`），与 HTTP 共用存储。 +- 仅 `sftp.users` 中列出的用户可连接；只允许 SFTP（无 shell/exec）。 +- SFTP 服务器作为文件暂存通道；不再提供 HTTP 登记接口。 -## Logs & pidfiles +## 反向隧道 -- HTTP logs → `logs/app.log`; SFTP logs → `logs/sftp.log`. -- Pidfiles: `app.pid`, `sftp.pid` (used by `stop.sh`). +SSH 服务器（2022）同时承载 SFTP 文件暂存与反向隧道。隧道 user 在 `config.yaml` 的 +`tunnel.users[]` 独立配置（与 `sftp.users[]` 分开）： + +- user 端跑 `user/tunnel.py`，连 2022 请求 remote port forward 绑 `tunnel_port`。 +- `ZikaiSSHServer.server_requested` 校验该 user 是否允许绑该端口，记一条 `tunnel_session` + 到 DB（user IP、local_port、tunnel_port、起止时间）。 +- `GET /api/userPort/{userName}` 查 DB 该 user 活跃隧道的端口，反代到 `127.0.0.1:tunnel_port` + （经隧道回指 user 本地服务）。无活跃隧道返回 502。 +- user 断开时 `connection_lost` 回调关闭 DB 会话（记 `ended_at`）；另有启动 reaper 兜底 + 清理进程异常重启后的孤儿记录。 + +配置示例见 `config.example.yaml` 的 `tunnel:` 段。生成 bcrypt hash 的方式同 SFTP。 + +## 文件浏览页 + +- `GET /files`（Basic Auth，同 docs）渲染 `static/file_browser.html`，JS 调同源管理 API。 +- 管理 API（均 Basic Auth）： + - `GET /api/admin/files?limit=&offset=` -> `{total, items:[UploadedFileOut]}` + - `GET /api/admin/files/{id}` -> `UploadedFileOut` + - `GET /api/admin/files/{id}/download` -> 文件流（磁盘缺失返回 410） + - `DELETE /api/admin/files/{id}` -> 硬删除：删 DB 行 + 删磁盘文件（`unlink missing_ok`）。 +- **删除后不再显示**：列表每次进入或删除后重新 fetch，前端不缓存；DB 行已删，列表自然不含。 +- 公开 `/api/files` 系列（user.py 依赖的查重/查询/下载）保留不变。 + +## 共享白板 + +白板无鉴权，任何人凭 `/whiteboard/{id}` 即可访问并实时协作；`{id}` 须匹配 +`[a-zA-Z0-9_-]{1,64}`，非法返回 400。访问不存在的 id 自动新建空板。白板长期留存 +（存 MySQL `whiteboard` 表），进程重启后内容仍在。 + +### 实时同步与心跳 + +- 连接：`WS /ws/whiteboard/{id}`（公开）。JSON 文本帧协议： + - client -> server：`{"type":"hello","client_id":"..."}`（首帧，可选）、 + `{"type":"ping"}`（心跳）、`{"type":"stroke","stroke":{points,color,width}}`、`{"type":"clear"}` + - server -> client：`{"type":"init","strokes":[...],"stroke_count":n}`、`{"type":"pong"}`、 + `{"type":"stroke","stroke":{...},"client_id":"..."}`（广播给他人，不含发送者）、 + `{"type":"cleared","client_id":"..."}`（广播给所有人）、`{"type":"error","msg":"..."}` +- **心跳**：客户端每 `whiteboard.heartbeat_interval_seconds`（默认 3s）发一次 `ping`，服务端回 `pong` + 并刷新计时。后台 reaper 每秒扫描，连续 `heartbeat_miss_threshold`（默认 5）次未收到心跳 + （即 15s）判失活，**关闭该连接并从 hub 移除**。 +- **内存安全**：`WhiteboardHub` 维护 `{board_id: set[Connection]}`： + - `disconnect` 幂等，空 set 从 dict 删除（防 board 键无限增长）； + - WS 主循环 `try/finally` 必调 `disconnect`，异常/断连均清理； + - `broadcast` 对单连接发送失败立即 `disconnect`，不影响其他连接； + - 删除白板时 `close_board` 关闭并清理该 board 的全部连接。 +- **多 worker 限制**：hub 是进程内存，多 uvicorn worker 下不同进程的连接不互通。生产部署需 + 保持 `server.workers: 1`，或后续接 Redis pub/sub 跨进程广播。 + +### 白板管理 + +- `GET /whiteboard-admin`（Basic Auth，同 docs）渲染 `static/whiteboard_admin.html`。 +- 管理 API（均 Basic Auth）： + - `GET /api/admin/whiteboards?limit=&offset=` -> `{total, items:[{board_id, stroke_count, created_at, updated_at}]}` + - `DELETE /api/admin/whiteboards/{id}` -> 删 DB 行 + 关闭该 board 所有在线 WS 连接。 + +## 临时文件清理 + +- `complete` 成功（含去重命中）后，会话目录 `uploads/.work//` 立即删除。 +- 被放弃的上传（`pending` 状态且超过 `chunk_session_ttl_seconds` 无活动，默认 5 分钟）由 + **后台 reaper** 清理：每 60 秒扫一次，删 `.work//` 目录 + DB 会话行。 +- `start.sh` 启动时仍会兜底清掉残留的 `.work/` 与 `*.part`（进程异常退出时的半成品）。 + +## 日志与 pidfile + +- HTTP 日志 → `logs/app.log`；SFTP 日志 → `logs/sftp.log`。 +- pidfile：`app.pid`、`sftp.pid`（`stop.sh` 使用）。 diff --git a/app/config.py b/app/config.py index 8fc02ed..a80d508 100644 --- a/app/config.py +++ b/app/config.py @@ -33,6 +33,10 @@ class StorageConfig(BaseModel): upload_dir: str = "./uploads" chunk_bytes: int = 1024 * 1024 sha256_on_upload: bool = True + # 分片上传会话的暂存目录（相对 upload_dir），完整文件拼接在此完成 + chunk_session_dir: str = "./.work" + # 分片上传会话的存活秒数：超过该时长无活动的 pending 会话由后台 reaper 清理 + chunk_session_ttl_seconds: int = 300 class SftpUser(BaseModel): @@ -65,12 +69,51 @@ class DocsConfig(BaseModel): return "" if v is None else str(v) +class TunnelUser(BaseModel): + """一个反向隧道用户：凭据 + 固定的隧道端口与本地端口。""" + + username: str + password_hash: str = "" # bcrypt，与 SFTP 用户同款 + # 该 user 在 server 侧绑定的隧道端口（SSH remote forward 的 listen port） + tunnel_port: int = 0 + # 该 user 要暴露的本地服务端口（仅用于记录，实际转发由 user 端完成） + local_port: int = 0 + + +class TunnelConfig(BaseModel): + """反向隧道总开关与用户列表。""" + + enabled: bool = False + users: list[TunnelUser] = Field(default_factory=list) + + def find_user(self, username: str) -> TunnelUser | None: + return next((u for u in self.users if u.username == username), None) + + +class WhiteboardConfig(BaseModel): + """共享白板配置。 + + 白板本身无鉴权（任何人凭 /whiteboard/{id} 即可访问并实时协作）； + 管理页（/whiteboard-admin、/api/admin/whiteboards）走 docs 同款 Basic Auth。 + 心跳按 heartbeat_interval_seconds 发送，连续丢失 heartbeat_miss_threshold 次即判失活。 + """ + + heartbeat_interval_seconds: int = 3 + heartbeat_miss_threshold: int = 5 + # board_id 合法字符集与长度上限，防路径/注入 + max_board_id_length: int = 64 + # 列表/管理页分页默认值 + list_limit: int = 100 + + class Settings(BaseModel): server: ServerConfig = ServerConfig() database: DatabaseConfig = DatabaseConfig() storage: StorageConfig = StorageConfig() sftp: SftpConfig = SftpConfig() docs: DocsConfig = DocsConfig() + tunnel: TunnelConfig = TunnelConfig() + whiteboard: WhiteboardConfig = WhiteboardConfig() def db_url(self) -> str: c = self.database diff --git a/app/controllers/__init__.py b/app/controllers/__init__.py index a21b3e2..5f6dbc4 100644 --- a/app/controllers/__init__.py +++ b/app/controllers/__init__.py @@ -1,6 +1,17 @@ """Controller 层：API 路由。""" +from .chunk_upload_controller import router as chunk_upload_router +from .file_admin_controller import router as file_admin_router from .file_controller import router as file_router from .system_controller import router as system_router +from .tunnel_controller import router as tunnel_router +from .whiteboard_controller import router as whiteboard_router -__all__ = ["file_router", "system_router"] +__all__ = [ + "chunk_upload_router", + "file_admin_router", + "file_router", + "system_router", + "tunnel_router", + "whiteboard_router", +] diff --git a/app/controllers/chunk_upload_controller.py b/app/controllers/chunk_upload_controller.py new file mode 100644 index 0000000..a6b9c20 --- /dev/null +++ b/app/controllers/chunk_upload_controller.py @@ -0,0 +1,93 @@ +"""分片上传接口：创建会话 / 查状态 / 传分片 / 完成拼接。 + +支持大文件分片上传与断点续传。前端先创建会话拿 upload_id，逐片上传， +可随时查 status 获取已传分片以补传缺失部分，最后 complete 触发拼接入库。 +""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, Request +from sqlalchemy.orm import Session + +from ..database import get_db +from ..dao.upload_session_dao import UploadSessionDAO +from ..dao.uploaded_file_dao import UploadedFileDAO +from ..schemas.chunk import ( + ChunkUploadResponse, + CreateSessionRequest, + CreateSessionResponse, + SessionStatusResponse, +) +from ..schemas.file import FileUploadResponse +from ..services.chunk_upload_service import ChunkUploadService + +router = APIRouter(prefix="/api/files/chunk-uploads", tags=["chunk-upload"]) + + +def _service(db: Session = Depends(get_db)) -> ChunkUploadService: + return ChunkUploadService(UploadSessionDAO(db), UploadedFileDAO(db)) + + +@router.post( + "", + response_model=CreateSessionResponse, + summary="创建分片上传会话", + description=( + "客户端把文件切成固定大小的分片后，先调本接口创建会话。" + "服务端生成 upload_id 返回，后续上传分片、查状态、完成拼接都需要它。" + ), +) +def create_session( + body: CreateSessionRequest, + service: ChunkUploadService = Depends(_service), +) -> CreateSessionResponse: + return service.create_session(body) + + +@router.get( + "/{upload_id}/status", + response_model=SessionStatusResponse, + summary="查询会话状态（断点续传）", + description="返回已上传分片下标集合，前端据此只补传缺失分片。", +) +def session_status( + upload_id: str, + service: ChunkUploadService = Depends(_service), +) -> SessionStatusResponse: + return service.get_status(upload_id) + + +@router.post( + "/{upload_id}/chunks/{index}", + response_model=ChunkUploadResponse, + summary="上传单个分片", + description=( + "请求体为单个分片的原始二进制。分片可乱序上传，重传同一分片会覆盖。" + "返回当前已上传的分片下标集合。" + ), +) +async def upload_chunk( + upload_id: str, + index: int, + request: Request, + service: ChunkUploadService = Depends(_service), +) -> ChunkUploadResponse: + uploaded = service.write_chunk(upload_id, index, await request.body()) + return ChunkUploadResponse(upload_id=upload_id, index=index, uploaded_chunks=uploaded) + + +@router.post( + "/{upload_id}/complete", + response_model=FileUploadResponse, + summary="完成拼接并入库", + description=( + "服务端校验分片齐全后，按顺序拼接为完整文件、流式计算 sha256，" + "按 sha256 去重（命中则返回旧行不重复落盘），最后原子改名入库。" + "重复调用幂等，返回同一 file_id。" + ), +) +def complete_session( + upload_id: str, + service: ChunkUploadService = Depends(_service), +) -> FileUploadResponse: + return service.complete(upload_id) diff --git a/app/controllers/file_admin_controller.py b/app/controllers/file_admin_controller.py new file mode 100644 index 0000000..113c32c --- /dev/null +++ b/app/controllers/file_admin_controller.py @@ -0,0 +1,111 @@ +"""文件管理接口（Basic Auth，鉴权同 docs）。 + +与公开的 /api/files 区分：本路由面向「文件浏览页」，提供列表 / 查询 / 下载 / 删除， +均需 docs 凭据。公开路由（user.py 依赖的查重、查询、下载）保留不变。 + +硬删除策略：删 DB 行 + 删磁盘文件（unlink missing_ok），列表只展示仍存在的行。 +""" + +from __future__ import annotations + +from pathlib import Path + +from fastapi import APIRouter, Depends, HTTPException +from fastapi.responses import FileResponse +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from ..database import get_db +from ..dao.uploaded_file_dao import UploadedFileDAO +from ..schemas.file import FileListResponse, UploadedFileOut +from ..security import require_docs_auth +from ..services.upload_service import UploadService + +router = APIRouter(prefix="/api/admin/files", tags=["files-admin"]) + + +def _service(db: Session = Depends(get_db)) -> UploadService: + return UploadService(UploadedFileDAO(db)) + + +class DeleteResult(BaseModel): + deleted: bool + + +@router.get( + "", + response_model=FileListResponse, + summary="列出已上传文件（需鉴权）", + description="供文件浏览页使用；字段同公开 /api/files，但要求 docs Basic Auth。", +) +def list_files( + limit: int = 100, + offset: int = 0, + service: UploadService = Depends(_service), + _: str = Depends(require_docs_auth), +) -> FileListResponse: + total, items = service.list_files(limit=limit, offset=offset) + return FileListResponse(total=total, items=items) + + +@router.get( + "/{file_id}", + response_model=UploadedFileOut, + summary="查询单个文件元数据（需鉴权）", +) +def get_file( + file_id: int, + service: UploadService = Depends(_service), + _: str = Depends(require_docs_auth), +) -> UploadedFileOut: + out = service.get_out(file_id) + if out is None: + raise HTTPException(404, "文件不存在") + return out + + +@router.get( + "/{file_id}/download", + summary="下载文件（需鉴权，校验磁盘存在）", + description="文件实体不在磁盘上时返回 410 Gone。", +) +def download_file( + file_id: int, + service: UploadService = Depends(_service), + _: str = Depends(require_docs_auth), +) -> FileResponse: + out, path = service.get_out_with_disk_path(file_id) + if out is None: + raise HTTPException(404, "文件不存在") + if path is None or not path.exists(): + raise HTTPException(410, "文件实体已不在磁盘上") + return FileResponse( + path=str(path), + media_type=out.content_type or "application/octet-stream", + filename=out.original_filename, + ) + + +@router.delete( + "/{file_id}", + response_model=DeleteResult, + summary="硬删除文件（需鉴权）", + description="删除 DB 行与磁盘文件实体；不可恢复。列表随后不再显示。", +) +def delete_file( + file_id: int, + service: UploadService = Depends(_service), + _: str = Depends(require_docs_auth), +) -> DeleteResult: + out, path = service.get_out_with_disk_path(file_id) + if out is None: + return DeleteResult(deleted=False) + # 先删磁盘文件，再删 DB 行；磁盘文件缺失不阻断 DB 清理 + if path is not None: + try: + Path(path).unlink(missing_ok=True) + except Exception: + # 即使磁盘删除失败也继续清 DB 行，保证列表不再显示 + pass + service.dao.delete(file_id) + return DeleteResult(deleted=True) diff --git a/app/controllers/file_controller.py b/app/controllers/file_controller.py index a326ef5..4b4ed70 100644 --- a/app/controllers/file_controller.py +++ b/app/controllers/file_controller.py @@ -2,8 +2,6 @@ from __future__ import annotations -from pathlib import Path - from fastapi import APIRouter, Depends, File, HTTPException, UploadFile from fastapi.responses import FileResponse from sqlalchemy.orm import Session @@ -13,7 +11,6 @@ from ..dao.uploaded_file_dao import UploadedFileDAO from ..schemas.file import ( FileListResponse, FileUploadResponse, - SftpRegisterRequest, UploadedFileOut, ) from ..services.upload_service import UploadService @@ -32,7 +29,7 @@ def _service(db: Session = Depends(get_db)) -> UploadService: description=( "multipart/form-data 上传，按 1 MiB 分片流式落盘，内存占用恒定；" "落盘过程中计算 SHA-256 并入库。\n\n" - "极大或极慢的传输建议改用 SFTP（详见 README），HTTP 链路受 Apache 代理 300s 超时限制。" + "极大或极慢的传输建议改用分片上传接口（/api/files/chunk-uploads）或 SFTP。" ), ) async def upload_file( @@ -70,26 +67,6 @@ def file_exists( return out -@router.post( - "/register-sftp", - response_model=FileUploadResponse, - summary="登记一个已通过 SFTP 落盘的文件", - description=( - "客户端先把文件 SFTP 到 ``incoming/``，再用本接口登记入库。" - "服务端会计算 sha256（已存在则去重）、把文件原子改名到 ``YYYY/MM/.``、写 DB 行。" - ), -) -def register_sftp( - body: SftpRegisterRequest, - service: UploadService = Depends(_service), -) -> FileUploadResponse: - return service.register_sftp( - filename=body.filename, - original_filename=body.original_filename, - uploaded_by=body.uploaded_by, - ) - - @router.get("/{file_id}", response_model=UploadedFileOut, summary="查询单个文件元数据") def get_file(file_id: int, service: UploadService = Depends(_service)) -> UploadedFileOut: out = service.get_out(file_id) @@ -100,10 +77,9 @@ def get_file(file_id: int, service: UploadService = Depends(_service)) -> Upload @router.get("/{file_id}/download", summary="下载文件") def download_file(file_id: int, service: UploadService = Depends(_service)) -> FileResponse: - out = service.get_out(file_id) + out, path = service.get_out_with_disk_path(file_id) if out is None: raise HTTPException(404, "文件不存在") - path: Path | None = service.resolve_disk_path(file_id) if path is None or not path.exists(): raise HTTPException(410, "文件实体已不在磁盘上") return FileResponse( diff --git a/app/controllers/tunnel_controller.py b/app/controllers/tunnel_controller.py new file mode 100644 index 0000000..cf1623e --- /dev/null +++ b/app/controllers/tunnel_controller.py @@ -0,0 +1,97 @@ +"""反向隧道 HTTP 路由：/api/userPort/{userName}。 + +把进来的 HTTP 请求反代到该 user 当前活跃隧道对应的本地端口 +（SSH remote forward 绑定的 127.0.0.1:tunnel_port），经隧道回指 user 的本地服务。 +""" + +from __future__ import annotations + +import httpx +from fastapi import APIRouter, Depends, HTTPException, Request, Response +from sqlalchemy.orm import Session + +from ..database import get_db +from ..dao.tunnel_session_dao import TunnelSessionDAO +from ..services.tunnel_service import TunnelService + +router = APIRouter(prefix="/api/userPort", tags=["tunnel"]) + +# 不应透传给上游的 hop-by-hop / 控制头 +_HOP_BY_HOP = { + "connection", "keep-alive", "proxy-authenticate", "proxy-authorization", + "te", "trailers", "transfer-encoding", "upgrade", "host", "content-length", +} + + +def _service(db: Session = Depends(get_db)) -> TunnelService: + return TunnelService(TunnelSessionDAO(db)) + + +async def _proxy(request: Request, session, prefix: str) -> Response: + """把请求透传到该 user 当前活跃隧道对应的本地端口。 + + 去掉 /api/userPort/{userName} 前缀后才是上游路径，根路径补 /。 + """ + upstream_path = request.url.path.replace(prefix, "", 1) or "/" + url = f"http://127.0.0.1:{session.tunnel_port}{upstream_path}" + if request.url.query: + url += f"?{request.url.query}" + + body = await request.body() + headers = {k: v for k, v in request.headers.items() if k.lower() not in _HOP_BY_HOP} + + try: + async with httpx.AsyncClient(timeout=30.0) as client: + upstream = await client.request( + request.method, url, content=body, headers=headers, + ) + except httpx.RequestError as exc: + raise HTTPException(502, f"隧道端口不可达：{exc}") from exc + + resp_headers = {k: v for k, v in upstream.headers.items() if k.lower() not in _HOP_BY_HOP} + return Response(content=upstream.content, status_code=upstream.status_code, + headers=resp_headers) + + +# 根路径：/api/userPort/{userName} +@router.api_route( + "/{userName}", + methods=["GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"], + summary="反代到指定 user 的隧道端口（根路径）", + description=( + "查 DB 该 user 当前活跃的隧道端口，把请求透传到 127.0.0.1:tunnel_port" + "（经 SSH 反向隧道回指 user 的本地服务）。无活跃隧道返回 502。" + ), +) +async def proxy_to_tunnel( + userName: str, + request: Request, + service: TunnelService = Depends(_service), +) -> Response: + session = service.get_active(userName) + if session is None: + raise HTTPException(502, f"无活跃隧道：user={userName}") + return await _proxy(request, session, f"/api/userPort/{userName}") + + +# 子路径：/api/userPort/{userName}/... —— 反向代理必须能透传任意路径与查询串， +# 否则上游服务里所有非根路由都会 404。 +@router.api_route( + "/{userName}/{upstream_path:path}", + methods=["GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"], + summary="反代到指定 user 的隧道端口（子路径透传）", + description=( + "把 /api/userPort/{userName}/ 透传到 127.0.0.1:tunnel_port/。" + "无活跃隧道返回 502。" + ), +) +async def proxy_to_tunnel_path( + userName: str, + upstream_path: str, + request: Request, + service: TunnelService = Depends(_service), +) -> Response: + session = service.get_active(userName) + if session is None: + raise HTTPException(502, f"无活跃隧道：user={userName}") + return await _proxy(request, session, f"/api/userPort/{userName}") diff --git a/app/controllers/whiteboard_controller.py b/app/controllers/whiteboard_controller.py new file mode 100644 index 0000000..be12315 --- /dev/null +++ b/app/controllers/whiteboard_controller.py @@ -0,0 +1,220 @@ +"""白板接口：REST（访问/管理）+ WebSocket（实时同步）。 + +路由： + GET /whiteboard/{board_id} 公开：访问白板，不存在则新建 + WS /ws/whiteboard/{board_id} 公开：实时协作 + 心跳 + GET /api/admin/whiteboards Basic Auth：管理页列表 + DELETE /api/admin/whiteboards/{id} Basic Auth：删除白板 +""" + +from __future__ import annotations + +import json +import logging +import uuid + +from fastapi import APIRouter, Depends, HTTPException, Query, WebSocket, WebSocketDisconnect +from sqlalchemy.orm import Session + +from ..database import get_db +from ..dao.whiteboard_dao import WhiteboardDAO +from ..schemas.whiteboard import WhiteboardListResponse, WhiteboardOut +from ..security import require_docs_auth +from ..services.whiteboard_hub import Connection, get_hub +from ..services.whiteboard_service import WhiteboardService + +logger = logging.getLogger("zikai.whiteboard") + +router = APIRouter(tags=["whiteboard"]) + + +def _service(db: Session = Depends(get_db)) -> WhiteboardService: + """REST 路径的 service：注入 hub 以便删除时踢出连接。""" + return WhiteboardService(WhiteboardDAO(db), hub=get_hub()) + + +# ---------------- 公开 REST ---------------- + +@router.get( + "/whiteboard/{board_id}", + response_model=WhiteboardOut, + summary="访问白板（不存在则新建）", + description="任何人凭 board_id 即可访问；不存在时自动创建空板并返回。", +) +def get_whiteboard(board_id: str, service: WhiteboardService = Depends(_service)) -> WhiteboardOut: + return service.get_or_create(board_id) + + +# ---------------- 管理 REST（Basic Auth） ---------------- + +@router.get( + "/api/admin/whiteboards", + response_model=WhiteboardListResponse, + summary="列出所有白板（需鉴权）", + description="供白板管理页使用：board_id / 创建时间 / 修改次数 / 上次修改时间。", +) +def list_whiteboards( + limit: int = Query(100, ge=1, le=500), + offset: int = Query(0, ge=0), + service: WhiteboardService = Depends(_service), + _: str = Depends(require_docs_auth), +) -> WhiteboardListResponse: + total, items = service.list_all(limit=limit, offset=offset) + return WhiteboardListResponse(total=total, items=items) + + +@router.delete( + "/api/admin/whiteboards/{board_id}", + summary="删除白板（需鉴权）", + description="删 DB 行，并关闭该 board 的所有在线 WebSocket 连接。", +) +def delete_whiteboard( + board_id: str, + service: WhiteboardService = Depends(_service), + _: str = Depends(require_docs_auth), +) -> dict: + ok = service.delete(board_id) + if not ok: + raise HTTPException(404, "白板不存在") + return {"deleted": True} + + +# ---------------- WebSocket（公开，实时同步 + 心跳） ---------------- + +@router.websocket("/ws/whiteboard/{board_id}") +async def whiteboard_ws(websocket: WebSocket, board_id: str) -> None: + """白板实时协作端点。 + + 协议（JSON 文本帧）： + client -> server: + {"type":"hello","client_id":"..."} 首帧（可选，未带则服务端生成） + {"type":"ping"} 心跳，服务端回 pong 并刷新计时 + {"type":"stroke","stroke":{...}} 新增笔画，持久化并广播给他人 + {"type":"clear"} 清空，持久化并广播给所有人 + server -> client: + {"type":"init","strokes":[...],"stroke_count":n} + {"type":"pong"} + {"type":"stroke","stroke":{...},"client_id":"..."} + {"type":"cleared","client_id":"..."} + {"type":"error","msg":"..."} + """ + # 路径层只做最基本校验，详细校验交给 service（service 会查表） + hub = get_hub() + # 先 accept，便于对非法 board_id 也回一条 error 再关闭 + await websocket.accept() + + # 读取首帧 hello（或任意帧）拿 client_id + try: + first = await websocket.receive_text() + except WebSocketDisconnect: + return + + client_id = _extract_client_id(first) or uuid.uuid4().hex[:12] + + # 校验 board_id 并加载白板（不存在则新建） + from ..database import get_session_local + try: + board = _with_db(lambda db: WhiteboardService(WhiteboardDAO(db), hub=hub).get_or_create(board_id)) + except HTTPException as exc: + await _safe_send(websocket, {"type": "error", "msg": exc.detail}) + await _safe_close(websocket) + return + + # 注册连接并下发 init + conn = Connection(websocket=websocket, board_id=board_id, client_id=client_id) + await hub.register(conn) + await _safe_send(websocket, { + "type": "init", + "strokes": board.strokes, + "stroke_count": board.stroke_count, + }) + + # 主循环：收消息 -> 处理 -> 广播 + try: + while True: + raw = await websocket.receive_text() + msg = _parse(raw) + if msg is None: + continue + mtype = msg.get("type") + if mtype == "ping": + conn.touch() + await _safe_send(websocket, {"type": "pong"}) + continue + # 任何有效业务帧都视为活性证据 + conn.touch() + if mtype == "stroke": + stroke = msg.get("stroke") or {} + try: + _with_db(lambda db: WhiteboardService(WhiteboardDAO(db), hub=hub).append_stroke(board_id, stroke)) + except HTTPException as exc: + await _safe_send(websocket, {"type": "error", "msg": exc.detail}) + continue + # 广播给他人（发送者本地已画，不回推） + await hub.broadcast( + board_id, + {"type": "stroke", "stroke": stroke, "client_id": client_id}, + exclude=conn, + ) + elif mtype == "clear": + try: + _with_db(lambda db: WhiteboardService(WhiteboardDAO(db), hub=hub).clear(board_id)) + except HTTPException as exc: + await _safe_send(websocket, {"type": "error", "msg": exc.detail}) + continue + # clear 广播给所有人（含发送者，用于确认） + await hub.broadcast( + board_id, {"type": "cleared", "client_id": client_id} + ) + else: + await _safe_send(websocket, {"type": "error", "msg": f"未知消息类型 {mtype}"}) + except WebSocketDisconnect: + pass + except Exception as exc: # pragma: no cover + logger.warning("白板 WS 异常 board=%s client=%s: %s", board_id, client_id, exc) + finally: + await hub.disconnect(conn) + + +# ---------------- helpers ---------------- + +def _with_db(fn): + """在独立 Session 中执行 fn 并返回结果；用完即关。供 WS 路径每帧独立事务使用。""" + from ..database import get_session_local + db = get_session_local()() + try: + return fn(db) + finally: + db.close() + +def _extract_client_id(raw: str) -> str | None: + try: + data = json.loads(raw) + except (json.JSONDecodeError, TypeError): + return None + cid = data.get("client_id") if isinstance(data, dict) else None + if isinstance(cid, str) and cid: + return cid + return None + + +def _parse(raw: str) -> dict | None: + try: + data = json.loads(raw) + except (json.JSONDecodeError, TypeError): + return None + return data if isinstance(data, dict) else None + + +async def _safe_send(ws: WebSocket, msg: dict) -> None: + try: + await ws.send_json(msg) + except Exception: # pragma: no cover + pass + + +async def _safe_close(ws: WebSocket) -> None: + try: + await ws.close() + except Exception: # pragma: no cover + pass diff --git a/app/dao/__init__.py b/app/dao/__init__.py index 211aac4..345df0b 100644 --- a/app/dao/__init__.py +++ b/app/dao/__init__.py @@ -1,5 +1,7 @@ """DAO 层：数据库访问的唯一入口。""" +from .tunnel_session_dao import TunnelSessionDAO from .uploaded_file_dao import UploadedFileDAO +from .upload_session_dao import UploadSessionDAO -__all__ = ["UploadedFileDAO"] +__all__ = ["TunnelSessionDAO", "UploadedFileDAO", "UploadSessionDAO"] diff --git a/app/dao/tunnel_session_dao.py b/app/dao/tunnel_session_dao.py new file mode 100644 index 0000000..040c14e --- /dev/null +++ b/app/dao/tunnel_session_dao.py @@ -0,0 +1,63 @@ +"""TunnelSession 的 DAO。""" + +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import select, update +from sqlalchemy.orm import Session + +from ..models.tunnel_session import TunnelSession + + +class TunnelSessionDAO: + def __init__(self, db: Session) -> None: + self.db = db + + def create(self, session: TunnelSession) -> TunnelSession: + self.db.add(session) + self.db.commit() + self.db.refresh(session) + return session + + def get_active_by_user(self, user_name: str) -> TunnelSession | None: + """返回该 user 当前活跃的隧道会话(至多一条)。""" + stmt = ( + select(TunnelSession) + .where(TunnelSession.user_name == user_name) + .where(TunnelSession.status == "active") + .order_by(TunnelSession.started_at.desc()) + .limit(1) + ) + return self.db.scalars(stmt).first() + + def get_active_by_port(self, tunnel_port: int) -> TunnelSession | None: + stmt = ( + select(TunnelSession) + .where(TunnelSession.tunnel_port == tunnel_port) + .where(TunnelSession.status == "active") + .limit(1) + ) + return self.db.scalars(stmt).first() + + def list_active(self) -> list[TunnelSession]: + stmt = select(TunnelSession).where(TunnelSession.status == "active") + return list(self.db.scalars(stmt).all()) + + def close(self, session: TunnelSession) -> None: + """标记会话结束。""" + session.ended_at = datetime.now() + session.status = "closed" + self.db.commit() + + def close_active_by_user(self, user_name: str) -> int: + """关闭该 user 所有 active 会话(断开清理用),返回关闭条数。""" + stmt = ( + update(TunnelSession) + .where(TunnelSession.user_name == user_name) + .where(TunnelSession.status == "active") + .values(status="closed", ended_at=datetime.now()) + ) + result = self.db.execute(stmt) + self.db.commit() + return result.rowcount or 0 diff --git a/app/dao/upload_session_dao.py b/app/dao/upload_session_dao.py new file mode 100644 index 0000000..9a705ad --- /dev/null +++ b/app/dao/upload_session_dao.py @@ -0,0 +1,57 @@ +"""UploadSession 的 DAO。""" + +from __future__ import annotations + +from datetime import datetime, timedelta + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from ..models.upload_session import UploadSession + + +class UploadSessionDAO: + def __init__(self, db: Session) -> None: + self.db = db + + def create(self, session: UploadSession) -> UploadSession: + self.db.add(session) + self.db.commit() + self.db.refresh(session) + return session + + def get_by_upload_id(self, upload_id: str) -> UploadSession | None: + stmt = ( + select(UploadSession) + .where(UploadSession.upload_id == upload_id) + .limit(1) + ) + return self.db.scalars(stmt).first() + + def mark_uploaded(self, session: UploadSession, chunks: list[int]) -> UploadSession: + """更新已上传分片集合（整体替换，避免并发追加丢失）。""" + session.uploaded_chunks = sorted(set(chunks)) + self.db.commit() + self.db.refresh(session) + return session + + def mark_completed(self, session: UploadSession, file_id: int) -> UploadSession: + session.file_id = file_id + session.status = "completed" + self.db.commit() + self.db.refresh(session) + return session + + def list_stale(self, ttl_seconds: int) -> list[UploadSession]: + """返回 pending 且 updated_at 早于 cutoff 的会话（被放弃的上传）。""" + cutoff = datetime.now() - timedelta(seconds=ttl_seconds) + stmt = ( + select(UploadSession) + .where(UploadSession.status == "pending") + .where(UploadSession.updated_at < cutoff) + ) + return list(self.db.scalars(stmt).all()) + + def delete(self, session: UploadSession) -> None: + self.db.delete(session) + self.db.commit() diff --git a/app/dao/whiteboard_dao.py b/app/dao/whiteboard_dao.py new file mode 100644 index 0000000..fb1e542 --- /dev/null +++ b/app/dao/whiteboard_dao.py @@ -0,0 +1,84 @@ +"""Whiteboard 的 DAO。 + +所有写操作均在该层 commit，service 不直接操作 session。 +get_or_create 用于「访问即新建」语义（路由 GET /whiteboard/{id} 不存在则建）。 +""" + +from __future__ import annotations + +from typing import Any + +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from ..models.whiteboard import Whiteboard + + +class WhiteboardDAO: + def __init__(self, db: Session) -> None: + self.db = db + + def create(self, board: Whiteboard) -> Whiteboard: + self.db.add(board) + self.db.commit() + self.db.refresh(board) + return board + + def get(self, board_id: str) -> Whiteboard | None: + stmt = select(Whiteboard).where(Whiteboard.board_id == board_id).limit(1) + return self.db.scalars(stmt).first() + + def get_or_create(self, board_id: str) -> Whiteboard: + """存在则返回，否则新建空板。利用 unique 约束兜底并发首访。""" + board = self.get(board_id) + if board is not None: + return board + board = Whiteboard(board_id=board_id, strokes=[], stroke_count=0) + try: + return self.create(board) + except Exception: + # 并发下另一事务已插入：回滚后重新读 + self.db.rollback() + return self.get(board_id) # type: ignore[return-value] + + def append_strokes(self, board_id: str, new_strokes: list[Any]) -> Whiteboard | None: + """把新笔画追加到 strokes 数组尾部，stroke_count 自增。""" + board = self.get(board_id) + if board is None: + return None + board.strokes = [*board.strokes, *new_strokes] + board.stroke_count = (board.stroke_count or 0) + len(new_strokes) + self.db.commit() + self.db.refresh(board) + return board + + def replace_strokes(self, board_id: str, strokes: list[Any]) -> Whiteboard | None: + """整体替换 strokes（清空时传 []），stroke_count 自增 1。""" + board = self.get(board_id) + if board is None: + return None + board.strokes = list(strokes) + board.stroke_count = (board.stroke_count or 0) + 1 + self.db.commit() + self.db.refresh(board) + return board + + def list_all(self, limit: int = 100, offset: int = 0) -> list[Whiteboard]: + stmt = ( + select(Whiteboard) + .order_by(Whiteboard.updated_at.desc()) + .limit(limit) + .offset(offset) + ) + return list(self.db.scalars(stmt).all()) + + def count(self) -> int: + return self.db.scalar(select(func.count()).select_from(Whiteboard)) or 0 + + def delete(self, board_id: str) -> bool: + board = self.get(board_id) + if board is None: + return False + self.db.delete(board) + self.db.commit() + return True diff --git a/app/main.py b/app/main.py index 81240fa..fecf66b 100644 --- a/app/main.py +++ b/app/main.py @@ -1,29 +1,103 @@ """FastAPI 应用工厂。 路由概览： - GET / -> 仅返回版本号 - GET /docs -> Swagger UI（Basic Auth） - GET /redoc -> ReDoc （Basic Auth） - GET /openapi.json -> OpenAPI 文档（Basic Auth） - GET /health -> 存活探针（公开） - GET /api/... -> 业务接口 + GET / -> 仅返回版本号 + GET /docs -> Swagger UI（Basic Auth） + GET /redoc -> ReDoc （Basic Auth） + GET /openapi.json -> OpenAPI 文档（Basic Auth） + GET /health -> 存活探针（公开） + GET /upload -> 上传页面（公开 HTML） + GET /files -> 文件浏览页（Basic Auth，同 docs） + GET /whiteboard/{id} -> 白板页面（公开，不存在则新建） + GET /whiteboard-admin -> 白板管理页（Basic Auth，同 docs） + GET /api/... -> 业务接口 + WS /ws/whiteboard/{id} -> 白板实时同步（公开） + /static/... -> 前端静态资源（JS/CSS） """ from __future__ import annotations +import asyncio import logging from contextlib import asynccontextmanager +from pathlib import Path from fastapi import Depends, FastAPI from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html -from fastapi.responses import JSONResponse, PlainTextResponse +from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse +from fastapi.staticfiles import StaticFiles -from .controllers import file_router, system_router +from .controllers import ( + chunk_upload_router, + file_admin_router, + file_router, + system_router, + tunnel_router, + whiteboard_router, +) from .database import init_db_schema from .security import require_docs_auth +from .services.chunk_upload_service import ChunkUploadService +from .services.whiteboard_hub import get_hub +from .views.upload_html import render as render_upload_html logger = logging.getLogger("zikai") +# 后台 reaper 的扫描间隔（秒）。不依赖 start.sh，进程存活期间持续清理过期会话。 +_REAPER_INTERVAL_SECONDS = 60 + +_STATIC_DIR = Path(__file__).resolve().parent.parent / "static" + + +def _reap_once() -> None: + """同步执行一次过期会话清理（在 worker 线程里跑，避免阻塞事件循环）。""" + try: + from .database import get_session_local + from .dao.upload_session_dao import UploadSessionDAO + from .dao.uploaded_file_dao import UploadedFileDAO + + db = get_session_local()() + try: + n = ChunkUploadService(UploadSessionDAO(db), UploadedFileDAO(db)).reap_stale_sessions() + if n: + logger.info("reaper 清理了 %d 个过期分片会话", n) + finally: + db.close() + except Exception as exc: # pragma: no cover + logger.warning("reaper 执行失败：%s", exc) + + +async def _reaper_loop(stop: asyncio.Event) -> None: + """周期性清理被放弃的分片上传会话。""" + while not stop.is_set(): + try: + await asyncio.to_thread(_reap_once) + except Exception as exc: # pragma: no cover + logger.warning("reaper 循环异常：%s", exc) + # 用 wait_for 实现「可被 stop 提前唤醒的 sleep」 + try: + await asyncio.wait_for(stop.wait(), timeout=_REAPER_INTERVAL_SECONDS) + except asyncio.TimeoutError: + pass + + +def _tunnel_reap_once() -> None: + """启动时清理 DB 里残留的 active 隧道会话（进程异常重启后的孤儿记录）。""" + try: + from .database import get_session_local + from .dao.tunnel_session_dao import TunnelSessionDAO + from .services.tunnel_service import TunnelService + + db = get_session_local()() + try: + n = TunnelService(TunnelSessionDAO(db)).reap_orphans() + if n: + logger.info("tunnel reaper 清理了 %d 个孤儿隧道会话", n) + finally: + db.close() + except Exception as exc: # pragma: no cover + logger.warning("tunnel reaper 执行失败：%s", exc) + @asynccontextmanager async def lifespan(app: FastAPI): @@ -32,7 +106,24 @@ async def lifespan(app: FastAPI): logger.info("数据库表已就绪。") except Exception as exc: # pragma: no cover logger.error("初始化数据库失败：%s", exc) - yield + # 启动时清理残留的 active 隧道会话（进程重启后的孤儿记录） + await asyncio.to_thread(_tunnel_reap_once) + # 启动后台 reaper，清理被放弃的分片上传会话与临时文件 + stop = asyncio.Event() + reaper = asyncio.create_task(_reaper_loop(stop)) + logger.info("分片会话 reaper 已启动（间隔 %ds）。", _REAPER_INTERVAL_SECONDS) + # 启动白板心跳 reaper，清理失活的 WebSocket 连接 + wb_reaper = asyncio.create_task(get_hub().reap_loop(stop)) + logger.info("白板心跳 reaper 已启动。") + try: + yield + finally: + stop.set() + for task in (reaper, wb_reaper): + try: + await asyncio.wait_for(task, timeout=5) + except (asyncio.TimeoutError, asyncio.CancelledError): # pragma: no cover + task.cancel() def create_app() -> FastAPI: @@ -54,34 +145,69 @@ def create_app() -> FastAPI: app.include_router(system_router) app.include_router(file_router) + app.include_router(file_admin_router) + app.include_router(chunk_upload_router) + app.include_router(tunnel_router) + app.include_router(whiteboard_router) + + # 前端静态资源（JS/CSS）；HTML 壳由下面的具名路由返回，便于各自挂 Basic Auth + if _STATIC_DIR.is_dir(): + app.mount("/static", StaticFiles(directory=str(_STATIC_DIR)), name="static") # 受 Basic Auth 保护的文档接口 - @app.get("/openapi.json", include_in_schema=False) + @app.get("/openapi.json") def protected_openapi(_: str = Depends(require_docs_auth)) -> JSONResponse: return JSONResponse(app.openapi()) - @app.get("/docs", include_in_schema=False) + @app.get("/docs") def protected_docs(_: str = Depends(require_docs_auth)): return get_swagger_ui_html( openapi_url="/openapi.json", title="zikai docs", swagger_favicon_url="" ) - @app.get("/redoc", include_in_schema=False) + @app.get("/redoc") def protected_redoc(_: str = Depends(require_docs_auth)): return get_redoc_html( openapi_url="/openapi.json", title="zikai docs", redoc_favicon_url="" ) - # 公开元信息（不暴露接口列表） - @app.get("/", include_in_schema=False, response_class=PlainTextResponse) + @app.get("/", response_class=PlainTextResponse) def root() -> PlainTextResponse: return PlainTextResponse(f"zikai {app.version}\n") - @app.get("/health", include_in_schema=False) + @app.get("/health") def health() -> dict: return {"status": "ok"} + @app.get("/upload", response_class=HTMLResponse) + def upload_page() -> HTMLResponse: + """拖拽 / 多文件 / 分片上传页面（公开，对齐 /api/files/upload）。""" + return HTMLResponse(render_upload_html()) + + @app.get("/files", response_class=HTMLResponse) + def files_page(_: str = Depends(require_docs_auth)) -> HTMLResponse: + """文件浏览页（Basic Auth，同 docs）：列出/下载/删除已上传文件。""" + return _serve_static_html("file_browser.html") + + @app.get("/whiteboard-admin", response_class=HTMLResponse) + def whiteboard_admin_page(_: str = Depends(require_docs_auth)) -> HTMLResponse: + """白板管理页（Basic Auth，同 docs）：查看/删除白板。""" + return _serve_static_html("whiteboard_admin.html") + + @app.get("/whiteboard/{board_id}", response_class=HTMLResponse) + def whiteboard_page(board_id: str) -> HTMLResponse: + """白板页面（公开）：访问即协作，不存在则前端拉取时自动新建。""" + return _serve_static_html("whiteboard.html") + return app +def _serve_static_html(filename: str) -> HTMLResponse: + """读取 static/ 下的 HTML 文件并返回；缺失时返回 404 文本。""" + path = _STATIC_DIR / filename + if not path.is_file(): + return HTMLResponse(f"

未找到 {filename}

", status_code=404) + return HTMLResponse(path.read_text(encoding="utf-8")) + + app = create_app() diff --git a/app/models/__init__.py b/app/models/__init__.py index fa254e9..f4aa5ed 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -1,5 +1,8 @@ """ORM 模型包；import 本包即把所有实体注册到 Base.metadata。""" +from .tunnel_session import TunnelSession from .uploaded_file import UploadedFile +from .upload_session import UploadSession +from .whiteboard import Whiteboard -__all__ = ["UploadedFile"] +__all__ = ["TunnelSession", "UploadedFile", "UploadSession", "Whiteboard"] diff --git a/app/models/tunnel_session.py b/app/models/tunnel_session.py new file mode 100644 index 0000000..3acbd03 --- /dev/null +++ b/app/models/tunnel_session.py @@ -0,0 +1,41 @@ +"""反向隧道会话实体。 + +一条记录对应一次 user → server 的 SSH 反向隧道:user 连上 server 的 SSH(2022), +请求 remote port forwarding,server 在本地绑一个隧道端口,该端口经隧道回指 user +的本地服务。HTTP 路由 /api/userPort/{userName} 反代到该隧道端口。 +""" + +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import BigInteger, DateTime, Integer, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from ..database import Base + + +class TunnelSession(Base): + __tablename__ = "tunnel_session" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + # 发起隧道的 user 名(对应 config.tunnel.users[].username) + user_name: Mapped[str] = mapped_column(String(128), nullable=False, index=True) + # user 端的公网/内网 IP(SSH 连接的 peer 地址) + user_ip: Mapped[str] = mapped_column(String(64), nullable=False, default="") + # user 要暴露的本地服务端口 + local_port: Mapped[int] = mapped_column(Integer, nullable=False) + # server 侧的隧道端口(SSH 反向转发绑定的本地端口) + tunnel_port: Mapped[int] = mapped_column(Integer, nullable=False, index=True) + started_at: Mapped[datetime] = mapped_column( + DateTime, server_default=func.now(), nullable=False + ) + ended_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None) + # active / closed + status: Mapped[str] = mapped_column(String(32), nullable=False, default="active") + + def __repr__(self) -> str: # pragma: no cover + return ( + f"" + ) diff --git a/app/models/upload_session.py b/app/models/upload_session.py new file mode 100644 index 0000000..96d8c0a --- /dev/null +++ b/app/models/upload_session.py @@ -0,0 +1,44 @@ +"""分片上传会话实体。 + +一个会话对应一次「分片上传 + 拼接入库」流程：客户端创建会话拿到 upload_id， +逐片上传，最后 complete 触发服务端拼接、算 sha256、去重并落 DB。 +""" + +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import BigInteger, DateTime, JSON, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from ..database import Base + + +class UploadSession(Base): + __tablename__ = "upload_session" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + # 客户端持有的会话标识（uuid4().hex），服务端生成，防伪造路径 + upload_id: Mapped[str] = mapped_column(String(64), nullable=False, unique=True, index=True) + filename: Mapped[str] = mapped_column(String(512), nullable=False) + size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + chunk_size: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + total_chunks: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) + # 已上传分片下标数组，例如 [0, 1, 3]；JSON 列 + uploaded_chunks: Mapped[list] = mapped_column(JSON, nullable=False, default=list) + # complete 成功后指向 UploadedFile.id；失败/未完成时为 None + file_id: Mapped[int | None] = mapped_column(BigInteger, nullable=True, default=None) + # pending / completed + status: Mapped[str] = mapped_column(String(32), nullable=False, default="pending") + created_at: Mapped[datetime] = mapped_column( + DateTime, server_default=func.now(), nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime, server_default=func.now(), onupdate=func.now(), nullable=False + ) + + def __repr__(self) -> str: # pragma: no cover + return ( + f"" + ) diff --git a/app/models/whiteboard.py b/app/models/whiteboard.py new file mode 100644 index 0000000..9ac6f38 --- /dev/null +++ b/app/models/whiteboard.py @@ -0,0 +1,39 @@ +"""共享白板实体。 + +一个白板由 board_id 唯一标识（用户可读的 url id），strokes 以 JSON 列保存全部笔画。 +白板长期留存，进程重启后仍可恢复；实时协作由 WebSocket hub 在内存中维护在线连接， +笔画变更经 service 落库后由 hub 广播给同 board 的其它在线连接。 +""" + +from __future__ import annotations + +from datetime import datetime + +from sqlalchemy import BigInteger, DateTime, Integer, JSON, String, func +from sqlalchemy.orm import Mapped, mapped_column + +from ..database import Base + + +class Whiteboard(Base): + __tablename__ = "whiteboard" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) + # 用户可读的 url id（[a-zA-Z0-9_-]{1,64}），全局唯一 + board_id: Mapped[str] = mapped_column(String(64), nullable=False, unique=True, index=True) + # 笔画数组 [{points:[[x,y],...], color, width}, ...] + strokes: Mapped[list] = mapped_column(JSON, nullable=False, default=list) + # 修改次数：每次新增笔画或清空 +1，供管理页统计 + stroke_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + created_at: Mapped[datetime] = mapped_column( + DateTime, server_default=func.now(), nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime, server_default=func.now(), onupdate=func.now(), nullable=False + ) + + def __repr__(self) -> str: # pragma: no cover + return ( + f"" + ) diff --git a/app/schemas/__init__.py b/app/schemas/__init__.py index b02c0ab..980df3a 100644 --- a/app/schemas/__init__.py +++ b/app/schemas/__init__.py @@ -1,13 +1,37 @@ """请求 / 响应的 Pydantic DTO。""" +from .chunk import ( + ChunkUploadResponse, + CreateSessionRequest, + CreateSessionResponse, + SessionStatusResponse, +) from .file import FileListResponse, FileUploadResponse, UploadedFileOut from .system import DiskUsage, MemoryUsage, SystemStatus +from .tunnel import TunnelStatusResponse +from .whiteboard import ( + Stroke, + StrokeOp, + WhiteboardListItem, + WhiteboardListResponse, + WhiteboardOut, +) __all__ = [ + "ChunkUploadResponse", + "CreateSessionRequest", + "CreateSessionResponse", "DiskUsage", "FileListResponse", "FileUploadResponse", "MemoryUsage", + "SessionStatusResponse", + "Stroke", + "StrokeOp", "SystemStatus", + "TunnelStatusResponse", "UploadedFileOut", + "WhiteboardListItem", + "WhiteboardListResponse", + "WhiteboardOut", ] diff --git a/app/schemas/chunk.py b/app/schemas/chunk.py new file mode 100644 index 0000000..9791ed4 --- /dev/null +++ b/app/schemas/chunk.py @@ -0,0 +1,43 @@ +"""分片上传接口 DTO。""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class CreateSessionRequest(BaseModel): + """创建一个分片上传会话。""" + + filename: str = Field(..., description="客户端原始文件名") + size_bytes: int = Field(..., ge=0, description="文件总字节数") + chunk_size: int = Field(..., gt=0, description="分片大小（字节）") + total_chunks: int = Field(..., gt=0, description="分片总数") + + +class CreateSessionResponse(BaseModel): + upload_id: str = Field(..., description="服务端生成的会话标识，后续接口都需要它") + filename: str + size_bytes: int + chunk_size: int + total_chunks: int + + +class SessionStatusResponse(BaseModel): + """会话状态：前端据此决定还需补传哪些分片（断点续传）。""" + + upload_id: str + filename: str + size_bytes: int + chunk_size: int + total_chunks: int + uploaded_chunks: list[int] = Field(..., description="已上传的分片下标集合") + completed: bool = Field(..., description="是否已完成拼接入库") + file_id: int | None = Field(None, description="completed=true 时指向 UploadedFile.id") + + +class ChunkUploadResponse(BaseModel): + """单个分片上传成功的回执。""" + + upload_id: str + index: int + uploaded_chunks: list[int] diff --git a/app/schemas/file.py b/app/schemas/file.py index ef2c3a7..59ff262 100644 --- a/app/schemas/file.py +++ b/app/schemas/file.py @@ -28,19 +28,11 @@ class FileUploadResponse(BaseModel): sha256: str storage_path: str uploaded_at: datetime + deduplicated: bool = Field( + False, description="true=服务端已有同 sha256 文件，直接返回旧行，未重复落盘" + ) class FileListResponse(BaseModel): total: int items: list[UploadedFileOut] - - -class SftpRegisterRequest(BaseModel): - """登记一个通过 SFTP 上传到 ``incoming/`` 下的文件。""" - - filename: str = Field( - ..., - description="文件在 SFTP chroot 下的相对路径，必须落在 incoming/ 之下，例如 incoming/abc.bin", - ) - original_filename: str = Field(..., description="客户端原始文件名") - uploaded_by: str = Field(default="sftp", description="登记者标识，写入 uploaded_by 字段") diff --git a/app/schemas/tunnel.py b/app/schemas/tunnel.py new file mode 100644 index 0000000..bea730a --- /dev/null +++ b/app/schemas/tunnel.py @@ -0,0 +1,19 @@ +"""反向隧道接口 DTO。""" + +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel + + +class TunnelStatusResponse(BaseModel): + """隧道的当前状态(查 /api/userPort/{userName} 前可用于探测)。""" + + user_name: str + local_port: int + tunnel_port: int + started_at: datetime + status: str + + model_config = {"from_attributes": True} diff --git a/app/schemas/whiteboard.py b/app/schemas/whiteboard.py new file mode 100644 index 0000000..1c41dd9 --- /dev/null +++ b/app/schemas/whiteboard.py @@ -0,0 +1,53 @@ +"""白板接口 DTO。""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, Field + + +class Stroke(BaseModel): + """一条笔画：点序列 + 样式。结构宽松（Any）以兼容前端扩展字段。""" + + points: list[list[float]] = Field(default_factory=list, description="[[x,y],...]") + color: str = Field(default="#1565c0", description="笔画颜色") + width: float = Field(default=3, description="笔画宽度") + + +class WhiteboardOut(BaseModel): + """白板完整内容（GET /whiteboard/{id} 与 WS init 帧）。""" + + board_id: str + strokes: list[Any] = Field(default_factory=list, description="笔画数组") + stroke_count: int = Field(0, description="累计修改次数") + created_at: datetime + updated_at: datetime + + model_config = {"from_attributes": True} + + +class WhiteboardListItem(BaseModel): + """管理页列表项。""" + + board_id: str + stroke_count: int + created_at: datetime + updated_at: datetime + + model_config = {"from_attributes": True} + + +class WhiteboardListResponse(BaseModel): + """管理页列表响应。""" + + total: int + items: list[WhiteboardListItem] + + +class StrokeOp(BaseModel): + """WS 笔画操作（type=stroke 时携带）。""" + + type: str = Field(..., description="add / clear") + stroke: dict[str, Any] | None = Field(None, description="type=add 时携带的笔画对象") diff --git a/app/services/__init__.py b/app/services/__init__.py index dd1b519..250253c 100644 --- a/app/services/__init__.py +++ b/app/services/__init__.py @@ -1,6 +1,7 @@ """业务逻辑层（Service）。""" +from .chunk_upload_service import ChunkUploadService from .system_service import SystemService from .upload_service import UploadService -__all__ = ["SystemService", "UploadService"] +__all__ = ["ChunkUploadService", "SystemService", "UploadService"] diff --git a/app/services/chunk_upload_service.py b/app/services/chunk_upload_service.py new file mode 100644 index 0000000..17f55df --- /dev/null +++ b/app/services/chunk_upload_service.py @@ -0,0 +1,247 @@ +"""分片上传服务：会话管理 + 分片落盘 + 拼接 + sha256 去重 + 原子入库。 + +存储布局:: + + uploads/.work//0.part 分片暂存 + uploads/.work//1.part + ... + uploads/2026/07/. complete 后的正式文件 + +complete 流程： + 1. 校验分片齐全； + 2. 顺序拼接为 .part，流式算 sha256； + 3. 复用 UploadService.dedup_or_commit：按 sha256 去重命中则删会话返回旧行， + 否则写 DB 行 + os.replace 原子改名到正式路径； + 4. 清理 .work//。 +""" + +from __future__ import annotations + +import io +import logging +import os +import shutil +import uuid +from collections.abc import Iterator +from pathlib import Path + +from fastapi import HTTPException + +from ..config import get_settings +from ..dao.upload_session_dao import UploadSessionDAO +from ..dao.uploaded_file_dao import UploadedFileDAO +from ..models.upload_session import UploadSession +from ..models.uploaded_file import UploadedFile +from ..schemas.chunk import ( + CreateSessionRequest, + CreateSessionResponse, + SessionStatusResponse, +) +from ..schemas.file import FileUploadResponse +from .upload_service import UploadService, hash_stream + +logger = logging.getLogger("zikai.chunk") + +# 会话暂存目录名（位于 upload_root 下） +SESSION_WORK_DIR = ".work" + + +class ChunkUploadService: + def __init__( + self, + session_dao: UploadSessionDAO, + file_dao: UploadedFileDAO, + ) -> None: + s = get_settings() + self.session_dao = session_dao + self.file_dao = file_dao + self.upload_root = s.resolved_upload_dir() + self.chunk_bytes = s.storage.chunk_bytes + # 会话暂存目录：配置给出的是相对 upload_root 的子目录名 + session_sub = os.path.basename(s.storage.chunk_session_dir) or SESSION_WORK_DIR + self.work_root = (self.upload_root / session_sub).resolve() + self.work_root.mkdir(parents=True, exist_ok=True) + self.session_ttl = s.storage.chunk_session_ttl_seconds + # 复用 UploadService 的存储路径生成 / 落库提交 / sha256 去重逻辑 + self._upload = UploadService(file_dao) + + # ---------------- 会话生命周期 ---------------- + + def create_session(self, body: CreateSessionRequest) -> CreateSessionResponse: + upload_id = uuid.uuid4().hex + session = UploadSession( + upload_id=upload_id, + filename=body.filename, + size_bytes=body.size_bytes, + chunk_size=body.chunk_size, + total_chunks=body.total_chunks, + uploaded_chunks=[], + status="pending", + ) + self.session_dao.create(session) + self._session_dir(upload_id).mkdir(parents=True, exist_ok=True) + return CreateSessionResponse( + upload_id=upload_id, + filename=body.filename, + size_bytes=body.size_bytes, + chunk_size=body.chunk_size, + total_chunks=body.total_chunks, + ) + + def get_status(self, upload_id: str) -> SessionStatusResponse: + session = self._require_session(upload_id) + return SessionStatusResponse( + upload_id=session.upload_id, + filename=session.filename, + size_bytes=session.size_bytes, + chunk_size=session.chunk_size, + total_chunks=session.total_chunks, + uploaded_chunks=list(session.uploaded_chunks or []), + completed=(session.status == "completed"), + file_id=session.file_id, + ) + + # ---------------- 分片写入 ---------------- + + def write_chunk( + self, upload_id: str, index: int, data: bytes, + ) -> list[int]: + session = self._require_session(upload_id) + self._validate_index(session, index) + + session_dir = self._session_dir(upload_id) + session_dir.mkdir(parents=True, exist_ok=True) + chunk_path = session_dir / f"{index}.part" + + # 幂等覆盖：同一分片重传时直接覆盖旧文件 + try: + with chunk_path.open("wb") as out: + out.write(data) + out.flush() + os.fsync(out.fileno()) + except Exception: + chunk_path.unlink(missing_ok=True) + raise + + uploaded = list(session.uploaded_chunks or []) + if index not in uploaded: + uploaded.append(index) + self.session_dao.mark_uploaded(session, uploaded) + return sorted(uploaded) + + # ---------------- 拼接入库 ---------------- + + def complete(self, upload_id: str) -> FileUploadResponse: + session = self._require_session(upload_id) + + # 幂等：已 complete 直接复用结果 + if session.status == "completed" and session.file_id is not None: + row = self.file_dao.get_by_id(session.file_id) + if row is not None: + return UploadService.to_response(row, deduplicated=True) + + uploaded = set(session.uploaded_chunks or []) + missing = [i for i in range(session.total_chunks) if i not in uploaded] + if missing: + raise HTTPException( + 409, + f"分片不齐全：缺失 {len(missing)} 个，例如 {sorted(missing)[:10]}", + ) + + session_dir = self._session_dir(upload_id) + part_path = session_dir / "_assembled.part" + + size, digest = self._assemble_and_hash(session, session_dir, part_path) + + # 复用 UploadService 的「sha256 去重 + 落库 + 原子改名」公共尾部 + entity = UploadedFile( + storage_path="", # 由 dedup_or_commit 内部生成 + original_filename=os.path.basename(session.filename), + content_type="", + size_bytes=size, + sha256=digest, + source="chunk", + uploaded_by="web", + ) + resp = self._upload.dedup_or_commit(entity, part_path) + self.session_dao.mark_completed(session, resp.id) + # 会话目录里的分片已拼接走，清理残留 + self._cleanup_session_dir(upload_id) + return resp + + # ---------------- 过期会话清理 ---------------- + + def reap_stale_sessions(self) -> int: + """清理被放弃的会话：删 .work// 目录 + DB 记录。 + + 判定标准：status=pending 且 updated_at 距今超过 session_ttl 秒。 + 返回清理的会话数。不依赖 start.sh，由后台任务周期调用。 + """ + stale = self.session_dao.list_stale(self.session_ttl) + for session in stale: + self._cleanup_session_dir(session.upload_id) + self.session_dao.delete(session) + logger.info("清理过期分片会话 upload_id=%s file=%s", session.upload_id, session.filename) + return len(stale) + + def _cleanup_session_dir(self, upload_id: str) -> None: + """删除 .work// 目录；失败只记日志，不抛异常。""" + session_dir = self._session_dir(upload_id) + try: + if session_dir.exists(): + shutil.rmtree(session_dir, ignore_errors=True) + except Exception as exc: # pragma: no cover + logger.warning("清理会话目录失败 upload_id=%s: %s", upload_id, exc) + + # ---------------- 内部 ---------------- + + def _require_session(self, upload_id: str) -> UploadSession: + if not upload_id: + raise HTTPException(400, "upload_id 不能为空") + session = self.session_dao.get_by_upload_id(upload_id) + if session is None: + raise HTTPException(404, f"会话不存在或已过期：{upload_id}") + return session + + @staticmethod + def _validate_index(session: UploadSession, index: int) -> None: + if index < 0 or index >= session.total_chunks: + raise HTTPException( + 400, + f"分片下标越界：{index} 不在 [0, {session.total_chunks})", + ) + + def _session_dir(self, upload_id: str) -> Path: + return self.work_root / upload_id + + def _assemble_and_hash( + self, + session: UploadSession, + session_dir: Path, + out_path: Path, + ) -> tuple[int, str]: + """按 index 顺序拼接全部分片为 out_path，流式计算 (size, sha256)。""" + try: + with out_path.open("wb") as out: + size, digest = hash_stream( + self._iter_assembled_bytes(session, session_dir, out) + ) + out.flush() + os.fsync(out.fileno()) + except Exception: + out_path.unlink(missing_ok=True) + raise + return size, digest + + def _iter_assembled_bytes( + self, session: UploadSession, session_dir: Path, out: io.BufferedWriter, + ) -> Iterator[bytes]: + """按 index 顺序读各分片，写入 out 同时 yield 每个字节块（供 hash_stream）。""" + for index in range(session.total_chunks): + chunk_path = session_dir / f"{index}.part" + if not chunk_path.is_file(): + raise HTTPException(409, f"拼接时发现分片缺失：{index}.part") + with chunk_path.open("rb") as src: + while buf := src.read(self.chunk_bytes): + out.write(buf) + yield buf diff --git a/app/services/sftp_server.py b/app/services/sftp_server.py index dbb74fc..7cb30a3 100644 --- a/app/services/sftp_server.py +++ b/app/services/sftp_server.py @@ -1,4 +1,4 @@ -"""嵌入式 SFTP 服务，与 HTTP API 共用 uploads/ 目录。 +"""嵌入式 SFTP 服务，与 HTTP API 共用 uploads/ 目录；同时承载反向隧道的 SSH 转发。 启动方式： python -m app.services.sftp_server @@ -35,16 +35,34 @@ class ZikaiSFTPServer(asyncssh.SFTPServer): logger.info("SFTP 会话结束 user=%s", self._username) +def _tunnel_dao(): + """惰性构造一个 TunnelSessionDAO（避免 import 时的副作用）。""" + from ..database import get_session_local + from ..dao.tunnel_session_dao import TunnelSessionDAO + return TunnelSessionDAO(get_session_local()()) + + +def _close_tunnel_dao(dao) -> None: + try: + dao.db.close() + except Exception: # pragma: no cover + pass + + class ZikaiSSHServer(asyncssh.SSHServer): - """支持密码（bcrypt）与公钥两种鉴权。""" + """支持密码（bcrypt）与公钥两种鉴权；允许反向隧道 user 的 remote forwarding。""" def __init__(self, settings, authorized_keys: asyncssh.SSHAuthorizedKeys | None) -> None: self._settings = settings self._authorized_keys = authorized_keys self._conn: asyncssh.SSHServerConnection | None = None + self._username: str | None = None # 认证成功后填入 + self._peer_ip: str = "" def connection_made(self, conn: asyncssh.SSHServerConnection) -> None: # type: ignore[override] self._conn = conn + peer = conn.get_extra_info("peername") + self._peer_ip = peer[0] if isinstance(peer, tuple) and peer else "" def begin_auth(self, username: str) -> bool: # 始终要求鉴权；用户不存在时所有方法会失败 → 干净的 permission denied @@ -56,14 +74,17 @@ class ZikaiSSHServer(asyncssh.SSHServer): return True def validate_password(self, username: str, password: str) -> bool: - user = next((u for u in self._settings.sftp.users if u.username == username), None) - if user is None or not user.password_hash or user.password_hash == "CHANGE_ME_BCRYPT_HASH": + user = self._find_sftp_user(username) or self._find_tunnel_user(username) + if user is None or not getattr(user, "password_hash", "") \ + or user.password_hash == "CHANGE_ME_BCRYPT_HASH": return False try: ok = bcrypt.checkpw(password.encode(), user.password_hash.encode()) except (ValueError, TypeError): ok = False - logger.info("SFTP 密码鉴权 %s user=%s", "OK" if ok else "FAIL", username) + if ok: + self._username = username + logger.info("SSH 密码鉴权 %s user=%s", "OK" if ok else "FAIL", username) return ok # 公钥 @@ -74,21 +95,88 @@ class ZikaiSSHServer(asyncssh.SSHServer): def validate_public_key(self, username: str, key: asyncssh.SSHKey) -> bool: if self._authorized_keys is None: return False - if not any(u.username == username for u in self._settings.sftp.users): + if self._find_sftp_user(username) is None and self._find_tunnel_user(username) is None: return False - addr = "" - if self._conn: - peer = self._conn.get_extra_info("peername") - addr = peer[0] if isinstance(peer, tuple) and peer else "" + addr = self._peer_ip try: # asyncssh 命中返回 dict（可能为空），未命中返回 None result = self._authorized_keys.validate(key, client_host=addr, client_addr=addr) except Exception: result = None ok = result is not None - logger.info("SFTP 公钥鉴权 %s user=%s", "OK" if ok else "FAIL", username) + if ok: + self._username = username + logger.info("SSH 公钥鉴权 %s user=%s", "OK" if ok else "FAIL", username) return ok + # 反向隧道：remote port-forward 请求 + + def server_requested(self, listen_host: str, listen_port: int) -> bool: + """客户端请求在本地（server 侧）绑端口做反向转发时回调。 + + 仅允许已认证的 tunnel user 绑定其配置中预定的 tunnel_port；记一条 active + 隧道会话到 DB。其他情况拒绝。 + """ + if not self._settings.tunnel.enabled: + logger.warning("拒绝 remote forward：tunnel 未启用 user=%s", self._username) + return False + if not self._username: + logger.warning("拒绝 remote forward：未认证") + return False + tunnel_user = self._find_tunnel_user(self._username) + if tunnel_user is None: + logger.warning("拒绝 remote forward：%s 不是 tunnel user", self._username) + return False + if listen_port != tunnel_user.tunnel_port: + logger.warning( + "拒绝 remote forward：user=%s 端口 %d 不等于配置 %d", + self._username, listen_port, tunnel_user.tunnel_port, + ) + return False + + dao = _tunnel_dao() + try: + from ..services.tunnel_service import TunnelService + TunnelService(dao).register( + user_name=self._username, + user_ip=self._peer_ip, + tunnel_port=listen_port, + local_port=tunnel_user.local_port, + ) + except Exception as exc: # pragma: no cover + _close_tunnel_dao(dao) + logger.error("登记隧道会话失败：%s", exc) + return False + _close_tunnel_dao(dao) + logger.info( + "允许 remote forward user=%s listen=%s:%d -> local_port=%d", + self._username, listen_host, listen_port, tunnel_user.local_port, + ) + return True + + def connection_lost(self, exc: Exception | None) -> None: # type: ignore[override] + """SSH 连接断开时清理该 user 的活跃隧道会话。""" + if self._username and self._settings.tunnel.enabled: + dao = _tunnel_dao() + try: + from ..services.tunnel_service import TunnelService + TunnelService(dao).close(self._username) + except Exception as e: # pragma: no cover + logger.warning("清理隧道会话失败 user=%s: %s", self._username, e) + _close_tunnel_dao(dao) + if exc: + logger.info("SSH 连接异常断开 user=%s: %s", self._username, exc) + else: + logger.info("SSH 连接关闭 user=%s", self._username) + + # 内部 + + def _find_sftp_user(self, username: str): + return next((u for u in self._settings.sftp.users if u.username == username), None) + + def _find_tunnel_user(self, username: str): + return self._settings.tunnel.find_user(username) + def _load_authorized_keys(path: Path) -> asyncssh.SSHAuthorizedKeys | None: if not path.exists() or path.stat().st_size == 0: @@ -119,7 +207,7 @@ async def _run() -> None: upload_root = settings.resolved_upload_dir() upload_root.mkdir(parents=True, exist_ok=True) - # SFTP 客户端登记前的暂存目录；register-sftp 只接受此目录下的路径。 + # SFTP 客户端的暂存目录（chroot 内）。 (upload_root / "incoming").mkdir(parents=True, exist_ok=True) host_key_path = (PROJECT_ROOT / settings.sftp.host_key_path).resolve() diff --git a/app/services/tunnel_service.py b/app/services/tunnel_service.py new file mode 100644 index 0000000..69830c3 --- /dev/null +++ b/app/services/tunnel_service.py @@ -0,0 +1,70 @@ +"""反向隧道业务逻辑：会话注册 / 注销 / 查询。 + +SSH 服务收到 remote port-forward 请求时调 register 记一条 active 会话； +user 断开时调 close 标记 ended_at；HTTP 路由调 get_active 查隧道端口做反代。 +""" + +from __future__ import annotations + +import logging + +from ..config import get_settings +from ..dao.tunnel_session_dao import TunnelSessionDAO +from ..models.tunnel_session import TunnelSession + +logger = logging.getLogger("zikai.tunnel") + + +class TunnelService: + def __init__(self, dao: TunnelSessionDAO) -> None: + self.dao = dao + self.settings = get_settings().tunnel + + def register( + self, user_name: str, user_ip: str, tunnel_port: int, local_port: int, + ) -> TunnelSession: + """登记一条活跃隧道会话。 + + 若该 user 已有活跃会话先关闭旧的（同一 user 同一时刻只保留一条）。 + """ + self.dao.close_active_by_user(user_name) + session = TunnelSession( + user_name=user_name, + user_ip=user_ip, + local_port=local_port, + tunnel_port=tunnel_port, + status="active", + ) + saved = self.dao.create(session) + logger.info( + "隧道建立 user=%s ip=%s tunnel_port=%d local_port=%d", + user_name, user_ip, tunnel_port, local_port, + ) + return saved + + def close(self, user_name: str) -> int: + """关闭该 user 的活跃会话（SSH 断开时调用），返回关闭条数。""" + n = self.dao.close_active_by_user(user_name) + if n: + logger.info("隧道关闭 user=%s 条数=%d", user_name, n) + return n + + def get_active(self, user_name: str) -> TunnelSession | None: + return self.dao.get_active_by_user(user_name) + + def is_port_allowed(self, user_name: str, tunnel_port: int) -> bool: + """校验该 user 是否被允许绑定该隧道端口（防 user 乱绑端口）。""" + user = self.settings.find_user(user_name) + return user is not None and user.tunnel_port == tunnel_port + + def reap_orphans(self) -> int: + """兜底清理：关闭所有 active 会话（进程重启时 DB 里残留的孤儿记录）。 + + 由后台 reaper 在启动后调用一次。SSH 实际断开时已有 close() 处理， + 这里只兜底进程异常退出后 DB 与实际状态不一致的情况。 + """ + active = self.dao.list_active() + for session in active: + self.dao.close(session) + logger.info("reaper 清理孤儿隧道 id=%d user=%s", session.id, session.user_name) + return len(active) diff --git a/app/services/upload_service.py b/app/services/upload_service.py index 153eb7d..05ffaf2 100644 --- a/app/services/upload_service.py +++ b/app/services/upload_service.py @@ -1,22 +1,35 @@ -"""上传服务：流式落盘 + 原子改名 + 元数据入库。""" +"""上传服务：流式落盘 + 原子改名 + 元数据入库 + sha256 去重。 + +两条上传路径（HTTP 整文件 / 分片拼接）共享本类的「存储路径生成 + 落库提交 + +sha256 去重」逻辑，避免重复实现。SFTP 服务器仅作为文件暂存通道（chroot 到 +upload_root），不再有 HTTP 登记接口。 +""" from __future__ import annotations import hashlib import os import uuid +from collections.abc import Iterator from datetime import datetime, timezone from pathlib import Path -from fastapi import HTTPException, UploadFile +from fastapi import UploadFile from ..config import get_settings from ..dao.uploaded_file_dao import UploadedFileDAO from ..models.uploaded_file import UploadedFile from ..schemas.file import FileUploadResponse, UploadedFileOut -# SFTP 客户端登记前必须把文件先落到此目录（chroot 内） -SFTP_INCOMING_DIR = "incoming" + +def hash_stream(chunks: Iterator[bytes]) -> tuple[int, str]: + """对一段字节块序列流式计算 (size, sha256)。供多条上传路径复用。""" + h = hashlib.sha256() + size = 0 + for chunk in chunks: + size += len(chunk) + h.update(chunk) + return size, h.hexdigest() class UploadService: @@ -36,7 +49,7 @@ class UploadService: 失败仅会留下可识别的 ``.part`` 文件，由 start.sh 启动时统一清理。 """ - rel_path, abs_path = self._make_storage_path(file.filename or "") + rel_path, abs_path = self.make_storage_path(file.filename or "") part_path = abs_path.with_name(abs_path.name + ".part") size, digest = self._write_part(file, part_path) @@ -49,34 +62,7 @@ class UploadService: source=source, uploaded_by=uploaded_by, ) - return self._commit(entity, part_path, abs_path) - - def register_sftp( - self, filename: str, original_filename: str, uploaded_by: str = "sftp", - ) -> FileUploadResponse: - """登记一个已通过 SFTP 落到 ``incoming/`` 下的文件。 - - 相同 sha256 已存在时直接返回旧行并删掉新副本（去重）。 - """ - src_abs = self._validate_incoming_path(filename) - size, digest = self._hash_disk_file(src_abs) - - existing = self.dao.get_by_sha256(digest) - if existing is not None: - src_abs.unlink(missing_ok=True) - return self._to_response(existing) - - rel_path, abs_path = self._make_storage_path(original_filename or filename) - entity = UploadedFile( - storage_path=str(rel_path), - original_filename=os.path.basename(original_filename or src_abs.name), - content_type="", - size_bytes=size, - sha256=digest, - source="sftp", - uploaded_by=uploaded_by, - ) - return self._commit(entity, src_abs, abs_path) + return self.commit_entity(entity, part_path, abs_path) # ---------------- 查询 ---------------- @@ -94,18 +80,26 @@ class UploadService: row = self.dao.get_by_sha256(sha256) return UploadedFileOut.model_validate(row) if row else None - def resolve_disk_path(self, file_id: int) -> Path | None: - row = self.dao.get_by_id(file_id) - return (self.upload_root / row.storage_path).resolve() if row else None + def get_out_with_disk_path(self, file_id: int) -> tuple[UploadedFileOut | None, Path | None]: + """合并查询：一次 DB 读取同时返回 (元数据, 磁盘绝对路径)。 - # ---------------- 内部 ---------------- + 供下载/删除路径复用，避免原先 get_out + resolve_disk_path 各查一次的重复读。 + """ + row = self.dao.get_by_id(file_id) + if row is None: + return None, None + out = UploadedFileOut.model_validate(row) + path = (self.upload_root / row.storage_path).resolve() + return out, path + + # ---------------- 共享 helper（供本类与 ChunkUploadService 复用） ---------------- @staticmethod def _safe_ext(filename: str) -> str: return os.path.splitext(os.path.basename(filename))[1] @staticmethod - def _to_response(row: UploadedFile) -> FileUploadResponse: + def to_response(row: UploadedFile, *, deduplicated: bool = False) -> FileUploadResponse: return FileUploadResponse( id=row.id, filename=row.original_filename, @@ -113,9 +107,10 @@ class UploadService: sha256=row.sha256, storage_path=row.storage_path, uploaded_at=row.uploaded_at, + deduplicated=deduplicated, ) - def _make_storage_path(self, original_filename: str) -> tuple[Path, Path]: + def make_storage_path(self, original_filename: str) -> tuple[Path, Path]: """生成 ``(rel_path, abs_path)``；abs_path 的父目录已创建。""" now = datetime.now(timezone.utc) rel_dir = Path(f"{now:%Y}/{now:%m}") @@ -123,21 +118,7 @@ class UploadService: rel_path = rel_dir / f"{uuid.uuid4().hex}{self._safe_ext(original_filename)}" return rel_path, self.upload_root / rel_path - def _validate_incoming_path(self, filename: str) -> Path: - """校验客户端给出的相对路径必须落在 upload_root/incoming/ 之下且是文件。""" - if not filename or filename.startswith(("/", "\\")): - raise HTTPException(400, "filename 必须是 incoming/ 下的相对路径") - incoming_root = (self.upload_root / SFTP_INCOMING_DIR).resolve() - try: - abs_path = (self.upload_root / filename).resolve() - abs_path.relative_to(incoming_root) - except ValueError: - raise HTTPException(400, f"filename 必须落在 {SFTP_INCOMING_DIR}/ 之下") - if not abs_path.is_file(): - raise HTTPException(404, f"文件不存在或不是普通文件：{filename}") - return abs_path - - def _commit( + def commit_entity( self, entity: UploadedFile, src_path: Path, dest_path: Path, ) -> FileUploadResponse: """落 DB 行后把 src_path 原子改名到 dest_path；任一失败回滚已生成的副作用。""" @@ -154,17 +135,25 @@ class UploadService: finally: src_path.unlink(missing_ok=True) raise - return self._to_response(saved) + return self.to_response(saved) - def _hash_disk_file(self, path: Path) -> tuple[int, str]: - """流式读取磁盘文件，返回 (size, sha256)。""" - h = hashlib.sha256() - size = 0 - with path.open("rb") as fh: - while chunk := fh.read(self.chunk_bytes): - size += len(chunk) - h.update(chunk) - return size, h.hexdigest() + def dedup_or_commit( + self, entity: UploadedFile, src_path: Path, + ) -> FileUploadResponse: + """公共尾部：按 entity.sha256 去重，命中则删 src 返回旧行；否则 commit。 + + 供分片拼接等「先算出 sha256 再决定落盘」的路径复用，与 stream_to_disk + （边写边算、无独立 src）的区别在于这里 sha256 已在 entity 上。 + """ + existing = self.dao.get_by_sha256(entity.sha256) + if existing is not None: + src_path.unlink(missing_ok=True) + return self.to_response(existing, deduplicated=True) + rel_path, abs_path = self.make_storage_path(entity.original_filename) + entity.storage_path = str(rel_path) + return self.commit_entity(entity, src_path, abs_path) + + # ---------------- 内部 ---------------- def _write_part(self, file: UploadFile, part_path: Path) -> tuple[int, str]: """把上传流写到 part_path 并 fsync；返回 (size, sha256)。失败时清理残品。""" diff --git a/app/services/whiteboard_hub.py b/app/services/whiteboard_hub.py new file mode 100644 index 0000000..5925089 --- /dev/null +++ b/app/services/whiteboard_hub.py @@ -0,0 +1,175 @@ +"""白板 WebSocket 连接管理器（实时同步 + 心跳 + 失活清理）。 + +设计要点： +- 进程内单例 ``WhiteboardHub``，维护 ``{board_id: set[Connection]}``。 +- 每个 Connection 封装 websocket / board_id / client_id / last_heartbeat。 +- 心跳：客户端每 ``heartbeat_interval_seconds``（默认 3s）发一次 ping，服务端回 pong 并 + 刷新 last_heartbeat。reaper 每秒扫描，超过 ``interval * threshold``（默认 15s）未心跳 + 的连接判为失活，关闭并从 hub 移除。 +- 内存安全：disconnect 幂等；空 set 从 dict 删除；broadcast 对单连接异常立即 disconnect； + close_board 关闭并清理整个 board 的连接集合。 +- 并发：用一个 asyncio.Lock 保护 ``_boards`` 的结构变更（add/remove board key）， + 集合内的连接增删用 set 原子操作（Python 单线程事件循环下安全）。 + +注意：hub 是进程内存，多 worker 下不互通。生产部署需单 worker 或后续接 Redis pub/sub。 +""" + +from __future__ import annotations + +import asyncio +import logging +import time +from dataclasses import dataclass, field + +from fastapi import WebSocket + +from ..config import get_settings + +logger = logging.getLogger("zikai.whiteboard") + + +@dataclass(eq=False) +class Connection: + """一个白板在线连接。eq=False 使其按对象 identity 哈希/比较，可放入 set。""" + + websocket: WebSocket + board_id: str + client_id: str + last_heartbeat: float = field(default_factory=time.monotonic) + + async def send_json(self, msg: dict) -> bool: + """发送一条消息；失败返回 False（调用方据此 disconnect）。""" + try: + await self.websocket.send_json(msg) + return True + except Exception as exc: # WebSocketDisconnect / 已关闭 / 编码失败 + logger.debug("发送失败 board=%s client=%s: %s", self.board_id, self.client_id, exc) + return False + + def touch(self) -> None: + self.last_heartbeat = time.monotonic() + + +class WhiteboardHub: + """白板连接管理器单例。""" + + def __init__(self) -> None: + cfg = get_settings().whiteboard + self.heartbeat_interval = cfg.heartbeat_interval_seconds + self.heartbeat_miss_threshold = cfg.heartbeat_miss_threshold + self.timeout_seconds = self.heartbeat_interval * self.heartbeat_miss_threshold + # {board_id: set[Connection]} + self._boards: dict[str, set[Connection]] = {} + self._lock = asyncio.Lock() + + # ---------------- 连接生命周期 ---------------- + + async def register(self, conn: Connection) -> None: + """把已 accept 的连接加入 board 集合（WebSocket accept 由 controller 负责）。""" + async with self._lock: + conns = self._boards.setdefault(conn.board_id, set()) + conns.add(conn) + logger.info("连接接入 board=%s client=%s（当前 %d 人）", + conn.board_id, conn.client_id, self.connection_count(conn.board_id)) + + async def disconnect(self, conn: Connection) -> None: + """幂等移除连接；空 set 从 dict 删除以防内存泄漏。""" + async with self._lock: + conns = self._boards.get(conn.board_id) + if conns is None: + return + conns.discard(conn) + if not conns: + self._boards.pop(conn.board_id, None) + # 尽力关闭 websocket（可能已关闭） + try: + await conn.websocket.close() + except Exception: # pragma: no cover + pass + logger.info("连接移除 board=%s client=%s（剩余 %d 人）", + conn.board_id, conn.client_id, self.connection_count(conn.board_id)) + + def connection_count(self, board_id: str) -> int: + """调试/监控用：某 board 当前在线人数。""" + return len(self._boards.get(board_id, ())) + + # ---------------- 广播 ---------------- + + async def broadcast(self, board_id: str, msg: dict, exclude: Connection | None = None) -> None: + """把 msg 发给 board 内所有在线连接（可排除发送者）。单连接失败不影响其他。""" + async with self._lock: + conns = list(self._boards.get(board_id, ())) + dead: list[Connection] = [] + for conn in conns: + if exclude is not None and conn is exclude: + continue + ok = await conn.send_json(msg) + if not ok: + dead.append(conn) + # 发送失败的连接统一清理 + for conn in dead: + await self.disconnect(conn) + + # ---------------- 心跳 reaper ---------------- + + async def reap_loop(self, stop: asyncio.Event) -> None: + """后台循环：扫描失活连接。每秒一次，粒度细于 timeout。""" + logger.info("白板心跳 reaper 已启动（间隔 1s，超时 %ds）", self.timeout_seconds) + while not stop.is_set(): + try: + await self._reap_once() + except Exception as exc: # pragma: no cover + logger.warning("reaper 循环异常：%s", exc) + try: + await asyncio.wait_for(stop.wait(), timeout=1.0) + except asyncio.TimeoutError: + pass + + async def _reap_once(self) -> None: + now = time.monotonic() + async with self._lock: + # 快照待检查连接，避免持锁时 await + stale: list[Connection] = [] + for board_id, conns in self._boards.items(): + for conn in conns: + if now - conn.last_heartbeat > self.timeout_seconds: + stale.append(conn) + for conn in stale: + logger.warning("心跳失活，移除 board=%s client=%s（静默 %ds）", + conn.board_id, conn.client_id, + int(now - conn.last_heartbeat)) + await self.disconnect(conn) + + async def close_board(self, board_id: str) -> None: + """关闭并清理某 board 的所有连接（删除白板时调用）。""" + async with self._lock: + conns = self._boards.pop(board_id, None) + if not conns: + return + await asyncio.gather( + *(c.send_json({"type": "error", "msg": "白板已被删除"}) for c in conns), + return_exceptions=True, + ) + for conn in conns: + try: + await conn.websocket.close() + except Exception: # pragma: no cover + pass + logger.info("关闭白板 board=%s，踢出 %d 个连接", board_id, len(conns)) + + +# 进程内单例（由 main.py lifespan / controller 共享） +_hub: WhiteboardHub | None = None + + +def get_hub() -> WhiteboardHub: + global _hub + if _hub is None: + _hub = WhiteboardHub() + return _hub + + +def reset_hub() -> None: + """测试用：重置单例。""" + global _hub + _hub = None diff --git a/app/services/whiteboard_service.py b/app/services/whiteboard_service.py new file mode 100644 index 0000000..ccf02e8 --- /dev/null +++ b/app/services/whiteboard_service.py @@ -0,0 +1,92 @@ +"""白板服务：CRUD + 笔画操作。 + +不持有 WebSocket 连接状态（那是 hub 的职责）；删除白板时通过可选的 hub 回调 +通知 hub 踢出该 board 的所有在线连接，由 controller 在装配时注入，避免 service -> hub +的硬依赖（保持低耦合）。 +""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Any + +from fastapi import HTTPException + +from ..config import get_settings +from ..dao.whiteboard_dao import WhiteboardDAO +from ..schemas.whiteboard import WhiteboardListItem, WhiteboardOut + +if TYPE_CHECKING: # 避免运行时循环导入 + from .whiteboard_hub import WhiteboardHub + +# board_id 合法字符集：字母数字下划线短横线 +_BOARD_ID_RE = re.compile(r"^[a-zA-Z0-9_-]+$") + + +class WhiteboardService: + def __init__(self, dao: WhiteboardDAO, hub: "WhiteboardHub | None" = None) -> None: + self.dao = dao + self.hub = hub + cfg = get_settings().whiteboard + self.max_board_id_length = cfg.max_board_id_length + self.list_limit = cfg.list_limit + + # ---------------- 校验 ---------------- + + def validate_board_id(self, board_id: str) -> None: + """非法 board_id 直接 400（防路径穿越/注入）。""" + if ( + not board_id + or len(board_id) > self.max_board_id_length + or not _BOARD_ID_RE.match(board_id) + ): + raise HTTPException(400, "board_id 非法（仅允许字母数字下划线短横线，1-64 字符）") + + # ---------------- 读 ---------------- + + def get_or_create(self, board_id: str) -> WhiteboardOut: + self.validate_board_id(board_id) + board = self.dao.get_or_create(board_id) + return WhiteboardOut.model_validate(board) + + def list_all(self, limit: int = 100, offset: int = 0) -> tuple[int, list[WhiteboardListItem]]: + limit = min(max(limit, 0), self.list_limit) or self.list_limit + offset = max(offset, 0) + total = self.dao.count() + rows = self.dao.list_all(limit=limit, offset=offset) + items = [WhiteboardListItem.model_validate(r) for r in rows] + return total, items + + # ---------------- 写 ---------------- + + def append_stroke(self, board_id: str, stroke: dict[str, Any]) -> WhiteboardOut: + """追加一条笔画并返回最新状态。""" + self.validate_board_id(board_id) + board = self.dao.append_strokes(board_id, [stroke]) + if board is None: + raise HTTPException(404, "白板不存在") + return WhiteboardOut.model_validate(board) + + def clear(self, board_id: str) -> WhiteboardOut: + """清空白板；stroke_count 仍自增以记录这次修改。""" + self.validate_board_id(board_id) + board = self.dao.replace_strokes(board_id, []) + if board is None: + raise HTTPException(404, "白板不存在") + return WhiteboardOut.model_validate(board) + + def delete(self, board_id: str) -> bool: + """删除白板；同时通知 hub 踢出该 board 的所有在线连接。""" + self.validate_board_id(board_id) + ok = self.dao.delete(board_id) + if ok and self.hub is not None: + # hub.close_board 是 async，但删除走 REST 同步路径；安排到事件循环里执行 + import asyncio + + try: + loop = asyncio.get_running_loop() + loop.create_task(self.hub.close_board(board_id)) + except RuntimeError: + # 无运行中事件循环（如脚本调用）：同步调用会报错，忽略即可 + pass + return ok diff --git a/app/views/upload_html.py b/app/views/upload_html.py new file mode 100644 index 0000000..659adb9 --- /dev/null +++ b/app/views/upload_html.py @@ -0,0 +1,288 @@ +"""上传页面 HTML 渲染（拖拽 + 多文件 + 分片 + 断点续传）。 + +单文件 server-rendered，内联 CSS+JS，风格对齐 system_status_html.py： +深色模式自适应、卡片、进度条。无构建步骤、无外部依赖。 +""" + +from __future__ import annotations + +from html import escape + +# 默认分片大小 4 MiB：大于 Apache 300s 限制下单片可数秒传完，小到内存恒定。 +DEFAULT_CHUNK_SIZE = 4 * 1024 * 1024 +# 同一文件分片并发数 +DEFAULT_CONCURRENCY = 3 +# 单分片失败重试次数 +MAX_RETRY = 2 + + +def render() -> str: + return f""" + + + + +上传文件 — zikai + + + +

上传文件

+

拖拽文件到下方，或点击选择。支持多文件、大文件分片上传与断点续传。

+ +
+

把文件拖到这里，或

+ +
+ +
+ +
+ +

系统状态 · zikai file service

+ + + + +""" + +_CSS = """ +:root { color-scheme: light dark; } +body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; + max-width: 880px; margin: 2em auto; padding: 0 1em; line-height: 1.5; } +h1 { margin-bottom: 0.1em; } +.sub { color: #777; margin-top: 0; font-size: 0.95em; } +.drop { border: 2px dashed #bbb; border-radius: 12px; padding: 2.5em 1em; + text-align: center; margin: 1.5em 0; transition: background 0.15s, border-color 0.15s; } +.drop.drag { background: rgba(21,101,192,0.08); border-color: #1565c0; } +.drop-hint { margin: 0 0 1em; color: #888; } +.btn { display: inline-block; padding: 0.5em 1.2em; border-radius: 6px; + background: #1565c0; color: #fff; cursor: pointer; font-size: 0.95em; } +.btn:hover { background: #0d47a1; } +.btn input { display: none; } +.tasks { margin: 1em 0; } +.card { border: 1px solid #ddd; border-radius: 8px; padding: 0.9em 1.1em; + margin: 0.7em 0; background: rgba(0,0,0,0.02); } +.task-head { display: flex; align-items: center; gap: 0.6em; margin-bottom: 0.5em; } +.fname { font-weight: 600; word-break: break-all; flex: 1; } +.fsize { color: #888; font-size: 0.85em; white-space: nowrap; } +.fstate { font-size: 0.85em; padding: 0.1em 0.6em; border-radius: 10px; + background: #eee; white-space: nowrap; } +.state-running { background: #e3f2fd; color: #1565c0; } +.state-hashing { background: #fff3e0; color: #e65100; } +.state-done { background: #e8f5e9; color: #2e7d32; } +.state-dedup { background: #f3e5f5; color: #7b1fa2; } +.state-fail { background: #ffebee; color: #c62828; } +.bar { position: relative; background: #e6e6e6; border-radius: 4px; + height: 18px; width: 100%; overflow: hidden; } +.bar .fill { height: 100%; width: 0; background: #43a047; transition: width 0.2s; } +.bar .pct { position: absolute; top: 0; left: 0; width: 100%; height: 100%; + text-align: center; font-size: 12px; line-height: 18px; + color: #fff; mix-blend-mode: difference; } +.task-meta { margin-top: 0.5em; font-size: 0.82em; color: #666; word-break: break-all; } +.task-meta code { background: rgba(0,0,0,0.06); padding: 0 0.3em; border-radius: 3px; } +.foot { color: #888; font-size: 0.85em; margin-top: 1.5em; text-align: center; } +a.json { color: #1565c0; text-decoration: none; } +@media (prefers-color-scheme: dark) {{ + body {{ background: #1a1a1a; color: #e0e0e0; }} + .card {{ background: rgba(255,255,255,0.04); border-color: #333; }} + .drop {{ border-color: #555; }} + .drop.drag {{ background: rgba(21,101,192,0.18); }} + .fstate {{ background: #333; }} + .bar {{ background: #333; }} + .task-meta code {{ background: rgba(255,255,255,0.08); }} +}} +""" diff --git a/config.example.yaml b/config.example.yaml index dc6e2e2..58a162a 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -19,6 +19,8 @@ storage: upload_dir: ./uploads # where uploaded files are written (shared with SFTP) chunk_bytes: 1048576 # 1 MiB streaming chunk for HTTP upload (keeps RAM flat) sha256_on_upload: true # compute sha256 while streaming to disk + chunk_session_dir: ./.work # 分片上传会话暂存目录（相对 upload_dir），拼接在此完成 + chunk_session_ttl_seconds: 300 # 被放弃会话的存活秒数；后台 reaper 据此清理临时文件 sftp: enabled: true @@ -42,3 +44,23 @@ docs: username: admin password: "CHANGE_ME" realm: "zikai docs" + +tunnel: + # 反向隧道：user 端通过 SSH 反向转发把本地服务暴露到 server，server 再经 + # HTTP 路由 /api/userPort/{userName} 对公网提供访问。disabled 时 SSH 拒绝转发请求。 + enabled: false + users: + - username: tunneluser + password_hash: "CHANGE_ME_BCRYPT_HASH" # bcrypt，生成方式同 sftp.users + tunnel_port: 9001 # server 侧 SSH remote forward 绑定的本地端口 + local_port: 8080 # user 端要暴露的本地服务端口（仅记录用） + +whiteboard: + # 共享白板：/whiteboard/{id} 公开访问并实时协作（WebSocket）；管理页 /whiteboard-admin + # 与 /api/admin/whiteboards 走 docs 同款 Basic Auth。 + # 心跳：客户端每 heartbeat_interval_seconds 发一次 ping；连续丢失 heartbeat_miss_threshold + # 次判失活，服务端关闭并移除该连接。 + heartbeat_interval_seconds: 3 + heartbeat_miss_threshold: 5 + max_board_id_length: 64 # board_id 合法字符 [a-zA-Z0-9_-]，长度上限 + list_limit: 100 # 管理页单次列表上限 diff --git a/requirements.txt b/requirements.txt index 0a853fa..173e6d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ pydantic-settings==2.7.0 PyYAML==6.0.2 asyncssh==2.18.0 bcrypt==4.2.1 +httpx==0.28.1 diff --git a/sql/schema.sql b/sql/schema.sql index 0fc5568..4139d09 100644 --- a/sql/schema.sql +++ b/sql/schema.sql @@ -18,3 +18,51 @@ CREATE TABLE IF NOT EXISTS `uploaded_file` ( KEY `idx_source` (`source`), KEY `idx_sha256` (`sha256`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- 分片上传会话表。由 app/services/chunk_upload_service.py 使用。 +-- 运行时也由 Base.metadata.create_all 幂等创建，此处供 init_db.py 全新建库时一并导入。 +CREATE TABLE IF NOT EXISTS `upload_session` ( + `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `upload_id` CHAR(64) NOT NULL COMMENT 'uuid4().hex，客户端持有的会话标识', + `filename` VARCHAR(512) NOT NULL, + `size_bytes` BIGINT UNSIGNED NOT NULL DEFAULT 0, + `chunk_size` BIGINT UNSIGNED NOT NULL DEFAULT 0, + `total_chunks` BIGINT UNSIGNED NOT NULL DEFAULT 0, + `uploaded_chunks` JSON NOT NULL COMMENT '已上传分片下标数组，如 [0,1,3]', + `file_id` BIGINT UNSIGNED NULL DEFAULT NULL COMMENT 'complete 后指向 uploaded_file.id', + `status` VARCHAR(32) NOT NULL DEFAULT 'pending' COMMENT 'pending/completed/failed', + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `uq_upload_id` (`upload_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- 反向隧道会话表。由 app/services/tunnel_service.py + sftp_server.py 使用。 +CREATE TABLE IF NOT EXISTS `tunnel_session` ( + `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `user_name` VARCHAR(128) NOT NULL COMMENT '发起隧道的 user 名', + `user_ip` VARCHAR(64) NOT NULL DEFAULT '' COMMENT 'user 端 SSH peer 地址', + `local_port` INT NOT NULL COMMENT 'user 要暴露的本地服务端口', + `tunnel_port` INT NOT NULL COMMENT 'server 侧 SSH remote forward 绑定的本地端口', + `started_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `ended_at` DATETIME NULL DEFAULT NULL, + `status` VARCHAR(32) NOT NULL DEFAULT 'active' COMMENT 'active/closed', + PRIMARY KEY (`id`), + KEY `idx_user_name` (`user_name`), + KEY `idx_tunnel_port` (`tunnel_port`), + KEY `idx_status` (`status`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- 共享白板表。由 app/services/whiteboard_service.py 使用。 +-- 白板长期留存，strokes 以 JSON 保存全部笔画；实时同步由 WebSocket hub 在内存维护。 +CREATE TABLE IF NOT EXISTS `whiteboard` ( + `id` BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, + `board_id` VARCHAR(64) NOT NULL COMMENT '用户可读的 url id，[a-zA-Z0-9_-]{1,64}', + `strokes` JSON NOT NULL COMMENT '笔画数组 [{points,color,width}, ...]', + `stroke_count` INT NOT NULL DEFAULT 0 COMMENT '累计修改次数（新增笔画/清空各 +1）', + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `uq_board_id` (`board_id`), + KEY `idx_updated_at` (`updated_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/start.sh b/start.sh index 8b6c97e..41649c9 100755 --- a/start.sh +++ b/start.sh @@ -12,12 +12,22 @@ PY="$VENV/bin/python" mkdir -p "$ROOT/logs" "$ROOT/uploads" -# 启动前清理：upload_service.py 用 .part 后缀做原子写，进程已停时残留必为半成品。 +# 启动前清理： +# - upload_service.py 用 .part 后缀做原子写，进程已停时残留必为半成品。 +# - chunk_upload_service.py 的分片会话目录 .work/ 同理，残留的未完成会话无法续传。 PART_COUNT=$(find "$ROOT/uploads" -type f -name '*.part' -printf '.' 2>/dev/null | wc -c) if [ "$PART_COUNT" -gt 0 ]; then find "$ROOT/uploads" -type f -name '*.part' -delete echo "清理：删除残留 .part 文件 $PART_COUNT 个" fi +WORK_COUNT=0 +if [ -d "$ROOT/uploads/.work" ]; then + WORK_COUNT=$(find "$ROOT/uploads/.work" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | wc -l) +fi +if [ "$WORK_COUNT" -gt 0 ]; then + rm -rf "$ROOT/uploads/.work"/* + echo "清理：删除残留分片会话目录 $WORK_COUNT 个" +fi is_running() { [ -f "$1" ] && kill -0 "$(cat "$1")" 2>/dev/null; } diff --git a/static/common.css b/static/common.css new file mode 100644 index 0000000..eb8c04c --- /dev/null +++ b/static/common.css @@ -0,0 +1,112 @@ +/* zikai 共享前端样式：与 upload_html / system_status_html 风格统一。 + 深色模式自适应、卡片、主色 #1565c0。各页面在此基础上叠加专属样式。 */ +:root { + color-scheme: light dark; + --bg: #f6f7f9; + --surface: #ffffff; + --surface-2: rgba(0, 0, 0, 0.02); + --border: #e0e3e7; + --text: #1a1a1a; + --text-dim: #6b7280; + --primary: #1565c0; + --primary-strong: #0d47a1; + --primary-soft: rgba(21, 101, 192, 0.1); + --success: #2e7d32; + --success-soft: #e8f5e9; + --danger: #c62828; + --danger-soft: #ffebee; + --warn: #e65100; + --warn-soft: #fff3e0; + --radius: 10px; + --shadow: 0 1px 3px rgba(0, 0, 0, 0.04), 0 1px 2px rgba(0, 0, 0, 0.06); +} +@media (prefers-color-scheme: dark) { + :root { + --bg: #15161a; + --surface: #1c1d22; + --surface-2: rgba(255, 255, 255, 0.04); + --border: #2c2d33; + --text: #e6e6e6; + --text-dim: #9aa0a6; + --primary: #5b9bf0; + --primary-strong: #82b4f5; + --primary-soft: rgba(91, 155, 240, 0.16); + --success: #66bb6a; + --success-soft: rgba(102, 187, 106, 0.16); + --danger: #ef5350; + --danger-soft: rgba(239, 83, 80, 0.16); + --warn: #fb8c00; + --warn-soft: rgba(251, 140, 0, 0.16); + --shadow: 0 1px 3px rgba(0, 0, 0, 0.3); + } +} +* { box-sizing: border-box; } +html, body { margin: 0; padding: 0; } +body { + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", + "PingFang SC", "Microsoft YaHei", sans-serif; + background: var(--bg); + color: var(--text); + line-height: 1.55; + -webkit-font-smoothing: antialiased; +} +.wrap { max-width: 960px; margin: 0 auto; padding: 2em 1.2em 4em; } +h1 { margin: 0 0 0.2em; font-size: 1.6em; letter-spacing: -0.01em; } +.sub { color: var(--text-dim); margin: 0 0 1.6em; font-size: 0.95em; } +.card { + border: 1px solid var(--border); + border-radius: var(--radius); + padding: 1em 1.2em; + margin: 0.8em 0; + background: var(--surface); + box-shadow: var(--shadow); +} +.toolbar { display: flex; align-items: center; gap: 0.6em; flex-wrap: wrap; margin: 0 0 1.2em; } +.btn { + display: inline-flex; align-items: center; gap: 0.4em; + padding: 0.5em 1.1em; border-radius: 8px; + border: 1px solid var(--border); background: var(--surface); + color: var(--text); cursor: pointer; font-size: 0.92em; + transition: background 0.15s, border-color 0.15s, transform 0.05s; + user-select: none; +} +.btn:hover { background: var(--surface-2); } +.btn:active { transform: translateY(1px); } +.btn.primary { background: var(--primary); border-color: var(--primary); color: #fff; } +.btn.primary:hover { background: var(--primary-strong); } +.btn.danger { color: var(--danger); border-color: var(--danger-soft); } +.btn.danger:hover { background: var(--danger-soft); } +.btn.ghost { background: transparent; } +.btn[disabled] { opacity: 0.5; cursor: not-allowed; } +table { width: 100%; border-collapse: collapse; font-size: 0.9em; } +th, td { text-align: left; padding: 0.6em 0.8em; border-bottom: 1px solid var(--border); } +th { color: var(--text-dim); font-weight: 600; font-size: 0.82em; + text-transform: uppercase; letter-spacing: 0.04em; } +tr:last-child td { border-bottom: none; } +tr:hover td { background: var(--surface-2); } +.mono { font-family: ui-monospace, "SF Mono", Menlo, Consolas, monospace; font-size: 0.85em; } +.dim { color: var(--text-dim); } +.muted { color: var(--text-dim); font-size: 0.82em; } +.tag { + display: inline-block; padding: 0.1em 0.6em; border-radius: 10px; + font-size: 0.78em; background: var(--surface-2); color: var(--text-dim); +} +.tag.ok { background: var(--success-soft); color: var(--success); } +.tag.warn { background: var(--warn-soft); color: var(--warn); } +.tag.err { background: var(--danger-soft); color: var(--danger); } +a.link { color: var(--primary); text-decoration: none; } +a.link:hover { text-decoration: underline; } +.foot { color: var(--text-dim); font-size: 0.82em; margin-top: 2em; text-align: center; } +.toast { + position: fixed; bottom: 1.5em; left: 50%; transform: translateX(-50%); + background: var(--text); color: var(--bg); + padding: 0.6em 1.2em; border-radius: 8px; font-size: 0.88em; + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.2); + opacity: 0; transition: opacity 0.2s, transform 0.2s; pointer-events: none; + z-index: 50; +} +.toast.show { opacity: 0.95; transform: translateX(-50%) translateY(-4px); } +.copyable { cursor: pointer; } +.copyable:hover { color: var(--primary); } +.empty { text-align: center; color: var(--text-dim); padding: 3em 1em; font-size: 0.95em; } +.skel { color: var(--text-dim); padding: 2em; text-align: center; } diff --git a/static/common.js b/static/common.js new file mode 100644 index 0000000..7c98781 --- /dev/null +++ b/static/common.js @@ -0,0 +1,87 @@ +/* zikai 共享前端工具：toast、复制、字节/时间格式化、fetch 封装。 + Basic Auth 同源时浏览器自动带缓存的凭据，无需额外处理；fetch 默认 same-origin。 */ +(function (global) { + "use strict"; + + function el(tag, attrs, ...children) { + const node = document.createElement(tag); + if (attrs) { + for (const [k, v] of Object.entries(attrs)) { + if (k === "class") node.className = v; + else if (k === "dataset") Object.assign(node.dataset, v); + else if (k.startsWith("on") && typeof v === "function") + node.addEventListener(k.slice(2).toLowerCase(), v); + else if (v !== null && v !== undefined) node.setAttribute(k, v); + } + } + for (const c of children) { + if (c == null || c === false) continue; + node.appendChild(typeof c === "string" ? document.createTextNode(c) : c); + } + return node; + } + + let toastTimer = null; + function toast(msg, kind) { + let node = document.querySelector(".toast"); + if (!node) { + node = el("div", { class: "toast" }); + document.body.appendChild(node); + } + node.textContent = msg; + node.className = "toast show" + (kind ? " " + kind : ""); + clearTimeout(toastTimer); + toastTimer = setTimeout(() => (node.className = "toast"), 2200); + } + + async function copyText(text) { + try { + await navigator.clipboard.writeText(text); + return true; + } catch { + // 降级：临时 textarea + const ta = document.createElement("textarea"); + ta.value = text; + ta.style.position = "fixed"; + ta.style.opacity = "0"; + document.body.appendChild(ta); + ta.select(); + let ok = false; + try { ok = document.execCommand("copy"); } catch {} + document.body.removeChild(ta); + return ok; + } + } + + function fmtBytes(n) { + if (n == null) return "-"; + const x = Number(n); + if (!isFinite(x)) return "-"; + for (const unit of ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]) { + if (Math.abs(x) < 1024 || unit === "PiB") + return unit === "B" ? `${x} B` : `${x.toFixed(1)} ${unit}`; + n = x / 1024; + } + return `${n.toFixed(1)} PiB`; + } + + function fmtTime(s) { + if (!s) return "-"; + const d = new Date(s); + if (isNaN(d.getTime())) return s; + const p = (n) => String(n).padStart(2, "0"); + return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())} ${p(d.getHours())}:${p(d.getMinutes())}`; + } + + async function api(path, opts) { + const res = await fetch(path, opts); + if (res.status === 401) { + // 触发浏览器 Basic Auth 弹窗（同源 reload 即可带上凭据） + toast("需要登录"); + throw new Error("UNAUTHORIZED"); + } + return res; + } + + global.ZK = { el, toast, copyText, fmtBytes, fmtTime, api }; +})(window); diff --git a/static/file_browser.css b/static/file_browser.css new file mode 100644 index 0000000..0323823 --- /dev/null +++ b/static/file_browser.css @@ -0,0 +1,18 @@ +/* 文件浏览页专属样式。表格、行内操作、sha 截断。 */ +.files-table th.col-name { min-width: 30%; } +.files-table th.col-size { width: 9em; } +.files-table th.col-src { width: 6em; } +.files-table th.col-time { width: 11em; } +.files-table th.col-act { width: 9em; text-align: right; } +.files-table td.col-act { text-align: right; white-space: nowrap; } +.files-table td.col-name .fname { font-weight: 600; word-break: break-all; } +.files-table td.col-name .sha { display: block; margin-top: 0.15em; color: var(--text-dim); } +.files-table td.col-act .btn { padding: 0.3em 0.8em; font-size: 0.85em; margin-left: 0.3em; } +.sha-short { cursor: pointer; } +.sha-short:hover { color: var(--primary); } +.row-removed { opacity: 0; transition: opacity 0.25s; } +@media (max-width: 640px) { + .files-table th, .files-table td { padding: 0.5em 0.4em; } + .files-table th.col-src, .files-table td.col-src { display: none; } + .files-table th.col-time, .files-table td.col-time { font-size: 0.78em; } +} diff --git a/static/file_browser.html b/static/file_browser.html new file mode 100644 index 0000000..1f19d0b --- /dev/null +++ b/static/file_browser.html @@ -0,0 +1,29 @@ + + + + + +文件浏览 - zikai + + + + +
+

文件浏览

+

查看已上传的文件、下载或删除。删除后不再显示。

+ +
+ + +
+ +
+
加载中…
+
+ +

上传文件 · zikai file service

+
+ + + + diff --git a/static/file_browser.js b/static/file_browser.js new file mode 100644 index 0000000..6916ec9 --- /dev/null +++ b/static/file_browser.js @@ -0,0 +1,94 @@ +/* 文件浏览页：拉取 /api/admin/files、渲染表格、下载/删除/复制 sha。 + 每次进入或删除后重新拉取，前端不缓存列表，保证已删除文件不显示。 */ +(function () { + "use strict"; + const { el, toast, copyText, fmtBytes, fmtTime, api } = window.ZK; + const listEl = document.getElementById("list"); + const countEl = document.getElementById("count"); + const refreshBtn = document.getElementById("refresh"); + + refreshBtn.addEventListener("click", load); + + async function load() { + listEl.innerHTML = '
加载中…
'; + countEl.textContent = ""; + try { + const res = await api("/api/admin/files?limit=500&offset=0"); + if (!res.ok) throw new Error("HTTP " + res.status); + const body = await res.json(); + render(body.items || []); + countEl.textContent = `共 ${body.total} 个文件`; + } catch (e) { + listEl.innerHTML = '
加载失败：' + (e.message || e) + "
"; + } + } + + function render(items) { + if (!items.length) { + listEl.innerHTML = '
还没有文件。去 上传 一个吧。
'; + return; + } + const table = el("table", { class: "files-table" }); + const thead = el("thead", null, + el("tr", null, + el("th", { class: "col-name" }, "文件名 / SHA-256"), + el("th", { class: "col-size" }, "大小"), + el("th", { class: "col-src" }, "来源"), + el("th", { class: "col-time" }, "上传时间"), + el("th", { class: "col-act" }, "操作") + ) + ); + const tbody = el("tbody", null); + for (const f of items) { + const row = el("tr", null, + el("td", { class: "col-name" }, + el("div", { class: "fname" }, f.original_filename), + el("span", { class: "sha mono sha-short", title: "点击复制完整 SHA-256" }, + shortSha(f.sha256)) + ), + el("td", { class: "col-size mono" }, fmtBytes(f.size_bytes)), + el("td", { class: "col-src" }, el("span", { class: "tag" }, f.source || "-")), + el("td", { class: "col-time muted" }, fmtTime(f.uploaded_at)), + el("td", { class: "col-act" }, + el("a", { class: "btn primary", href: `/api/admin/files/${f.id}/download`, download: "" }, "下载"), + el("button", { class: "btn danger", onclick: () => remove(f, row) }, "删除") + ) + ); + const shaNode = row.querySelector(".sha-short"); + shaNode.addEventListener("click", async () => { + const ok = await copyText(f.sha256); + toast(ok ? "已复制 SHA-256" : "复制失败"); + }); + tbody.appendChild(row); + } + table.appendChild(thead); + table.appendChild(tbody); + listEl.innerHTML = ""; + listEl.appendChild(table); + } + + function shortSha(sha) { + if (!sha) return "-"; + return sha.length > 16 ? sha.slice(0, 12) + "…" + sha.slice(-4) : sha; + } + + async function remove(f, row) { + if (!confirm(`确定删除「${f.original_filename}」？\n此操作不可恢复，将同时删除磁盘文件。`)) return; + try { + const res = await api(`/api/admin/files/${f.id}`, { method: "DELETE" }); + if (!res.ok) throw new Error("HTTP " + res.status); + const body = await res.json(); + if (!body.deleted) { toast("文件已不存在"); } + row.classList.add("row-removed"); + setTimeout(() => row.remove(), 250); + toast("已删除"); + // 更新计数 + const m = (countEl.textContent || "").match(/(\d+)/); + if (m) countEl.textContent = `共 ${Math.max(0, Number(m[1]) - 1)} 个文件`; + } catch (e) { + toast("删除失败：" + (e.message || e), "err"); + } + } + + load(); +})(); diff --git a/static/whiteboard.css b/static/whiteboard.css new file mode 100644 index 0000000..d81493a --- /dev/null +++ b/static/whiteboard.css @@ -0,0 +1,50 @@ +/* 白板页专属样式：全屏画布、悬浮工具栏、移动端适配。 */ +:root { --bar-h: 52px; } +body { overflow: hidden; background: var(--bg); } +.wb-app { display: flex; flex-direction: column; height: 100vh; height: 100dvh; } +.wb-bar { + display: flex; align-items: center; justify-content: space-between; + gap: 0.6em; padding: 0.5em 0.9em; + background: var(--surface); border-bottom: 1px solid var(--border); + box-shadow: var(--shadow); min-height: var(--bar-h); + flex-wrap: wrap; +} +.wb-bar-left, .wb-bar-right { display: flex; align-items: center; gap: 0.6em; } +.wb-title { font-weight: 700; font-size: 1.05em; } +.wb-id { background: var(--surface-2); padding: 0.15em 0.5em; border-radius: 6px; font-size: 0.82em; color: var(--text-dim); } +.wb-online { color: var(--success); font-size: 0.7em; } +.wb-online.off { color: var(--text-dim); } +.wb-tool { display: inline-flex; align-items: center; gap: 0.3em; font-size: 0.85em; color: var(--text-dim); } +.wb-tool input[type="color"] { width: 28px; height: 28px; padding: 0; border: 1px solid var(--border); border-radius: 6px; background: transparent; cursor: pointer; } +.wb-tool input[type="range"] { width: 80px; accent-color: var(--primary); } +.wb-width-val { width: 1.4em; text-align: center; } +.wb-bar .btn { padding: 0.4em 0.9em; font-size: 0.86em; } + +.wb-stage { position: relative; flex: 1; overflow: hidden; } +#canvas { + position: absolute; inset: 0; width: 100%; height: 100%; + display: block; touch-action: none; cursor: crosshair; + background: + linear-gradient(var(--border) 1px, transparent 1px) 0 0 / 24px 24px, + linear-gradient(90deg, var(--border) 1px, transparent 1px) 0 0 / 24px 24px, + var(--surface); + background-blend-mode: normal; +} +.wb-status { + position: absolute; bottom: 0.8em; left: 50%; transform: translateX(-50%); + background: var(--surface); border: 1px solid var(--border); + padding: 0.3em 0.9em; border-radius: 16px; font-size: 0.8em; + color: var(--text-dim); box-shadow: var(--shadow); + opacity: 0; transition: opacity 0.3s; pointer-events: none; +} +.wb-status.show { opacity: 1; } +.wb-status.err { color: var(--danger); border-color: var(--danger-soft); } + +/* 移动端：工具栏紧凑、按钮变大易触 */ +@media (max-width: 640px) { + .wb-bar { padding: 0.4em 0.5em; gap: 0.4em; } + .wb-id { max-width: 8em; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } + .wb-tool input[type="range"] { width: 56px; } + .wb-bar .btn { padding: 0.45em 0.7em; } + .wb-title { display: none; } +} diff --git a/static/whiteboard.html b/static/whiteboard.html new file mode 100644 index 0000000..0d96662 --- /dev/null +++ b/static/whiteboard.html @@ -0,0 +1,39 @@ + + + + + + +白板 - zikai + + + + +
+
+
+ 白板 + + ● +
+
+ + + + +
+
+
+ +
连接中…
+
+
+ + + + diff --git a/static/whiteboard.js b/static/whiteboard.js new file mode 100644 index 0000000..ced60d0 --- /dev/null +++ b/static/whiteboard.js @@ -0,0 +1,271 @@ +/* 白板：Canvas 绘画 + WebSocket 实时同步 + 心跳。 + - 笔画以 {points:[[x,y],...], color, width} 表示，增量广播。 + - 心跳 3s 一次 ping；服务端 15s 无心跳判失活会主动断连，前端据此重连。 + - 收到他人 stroke 增量重绘该笔；收到 cleared 清空本地画布。 + - 兼容鼠标 + 触摸：统一用 pointer events，touch-action:none 防滚动缩放。 */ +(function () { + "use strict"; + const { el, toast, copyText } = window.ZK; + + // ---------- 从 URL 解析 board_id ---------- + // 路径形如 /whiteboard/{id}；id 为 [a-zA-Z0-9_-]{1,64} + const m = location.pathname.match(/^\/whiteboard\/([^/]+)\/?$/); + let boardId = m ? decodeURIComponent(m[1]) : "default"; + // 合法性兜底：前端非法字符直接回退到 default，真正校验在服务端 + if (!/^[a-zA-Z0-9_-]{1,64}$/.test(boardId)) boardId = "default"; + document.getElementById("boardId").textContent = boardId; + + // ---------- DOM ---------- + const canvas = document.getElementById("canvas"); + const ctx = canvas.getContext("2d"); + const colorInput = document.getElementById("color"); + const widthInput = document.getElementById("width"); + const widthVal = document.getElementById("widthVal"); + const clearBtn = document.getElementById("clearBtn"); + const copyBtn = document.getElementById("copyBtn"); + const statusEl = document.getElementById("status"); + const onlineEl = document.getElementById("online"); + + // ---------- 状态 ---------- + let strokes = []; // 已确认的笔画 + let current = null; // 正在画的笔画（本地未提交） + let drawing = false; + let ws = null; + let clientId = localStorage.getItem("wb_cid") || ""; + if (!clientId) { + clientId = "c_" + Math.random().toString(36).slice(2, 10); + localStorage.setItem("wb_cid", clientId); + } + let heartbeatTimer = null; + let reconnectTimer = null; + let connected = false; + + // ---------- 画布尺寸 ---------- + function resize() { + const dpr = window.devicePixelRatio || 1; + const w = canvas.clientWidth; + const h = canvas.clientHeight; + canvas.width = Math.max(1, Math.floor(w * dpr)); + canvas.height = Math.max(1, Math.floor(h * dpr)); + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + redraw(); + } + window.addEventListener("resize", resize); + + // ---------- 绘制 ---------- + function drawStroke(s) { + if (!s || !s.points || s.points.length < 1) return; + ctx.strokeStyle = s.color || "#1565c0"; + ctx.lineWidth = Number(s.width) || 3; + ctx.lineCap = "round"; + ctx.lineJoin = "round"; + const pts = s.points; + ctx.beginPath(); + ctx.moveTo(pts[0][0], pts[0][1]); + if (pts.length === 1) { + // 单点：画一个小圆点 + ctx.arc(pts[0][0], pts[0][1], (ctx.lineWidth || 3) / 2, 0, Math.PI * 2); + ctx.fillStyle = ctx.strokeStyle; + ctx.fill(); + return; + } + for (let i = 1; i < pts.length; i++) ctx.lineTo(pts[i][0], pts[i][1]); + ctx.stroke(); + } + + function redraw() { + ctx.clearRect(0, 0, canvas.width, canvas.height); + for (const s of strokes) drawStroke(s); + if (current) drawStroke(current); + } + + // ---------- 指针事件 ---------- + function pos(e) { + const r = canvas.getBoundingClientRect(); + return [e.clientX - r.left, e.clientY - r.top]; + } + + canvas.addEventListener("pointerdown", (e) => { + if (!connected) { flashStatus("未连接，正在重连…"); return; } + e.preventDefault(); + canvas.setPointerCapture(e.pointerId); + drawing = true; + current = { points: [pos(e)], color: colorInput.value, width: Number(widthInput.value) }; + drawStroke(current); + }); + canvas.addEventListener("pointermove", (e) => { + if (!drawing) return; + e.preventDefault(); + const p = pos(e); + const last = current.points[current.points.length - 1]; + // 跳过过近的点，减少数据量 + if (Math.hypot(p[0] - last[0], p[1] - last[1]) < 1.5) return; + current.points.push(p); + // 增量画最后一段 + ctx.strokeStyle = current.color; + ctx.lineWidth = current.width; + ctx.lineCap = "round"; ctx.lineJoin = "round"; + ctx.beginPath(); + ctx.moveTo(last[0], last[1]); + ctx.lineTo(p[0], p[1]); + ctx.stroke(); + }); + function endStroke(e) { + if (!drawing) return; + drawing = false; + if (e && e.pointerId !== undefined) { + try { canvas.releasePointerCapture(e.pointerId); } catch {} + } + if (current && current.points.length) { + strokes.push(current); + send({ type: "stroke", stroke: current }); + } + current = null; + } + canvas.addEventListener("pointerup", endStroke); + canvas.addEventListener("pointercancel", endStroke); + canvas.addEventListener("pointerleave", (e) => { + // 仅在抬起时结束；离开但按住不放不结束（pointer capture 已处理） + if (!drawing) return; + if (e.buttons === 0) endStroke(e); + }); + + widthInput.addEventListener("input", () => (widthVal.textContent = widthInput.value)); + + clearBtn.addEventListener("click", () => { + if (!connected) { flashStatus("未连接"); return; } + if (!confirm("确定清空白板？所有人的内容都会被清除。")) return; + send({ type: "clear" }); + }); + + copyBtn.addEventListener("click", async () => { + const url = `${location.origin}/whiteboard/${boardId}`; + const ok = await copyText(url); + toast(ok ? "链接已复制" : "复制失败"); + }); + + // ---------- WebSocket ---------- + function wsUrl() { + const proto = location.protocol === "https:" ? "wss:" : "ws:"; + return `${proto}//${location.host}/ws/whiteboard/${encodeURIComponent(boardId)}`; + } + + function connect() { + setStatus("连接中…"); + try { + ws = new WebSocket(wsUrl()); + } catch (e) { + scheduleReconnect(); + return; + } + ws.onopen = () => { + connected = true; + setStatus("已连接", true); + onlineEl.classList.remove("off"); + send({ type: "hello", client_id: clientId }); + startHeartbeat(); + }; + ws.onmessage = (ev) => onMessage(ev.data); + ws.onclose = () => onLost("连接已关闭"); + ws.onerror = () => { /* close 会跟进 */ }; + } + + function onMessage(raw) { + let msg; + try { msg = JSON.parse(raw); } catch { return; } + switch (msg.type) { + case "init": + strokes = Array.isArray(msg.strokes) ? msg.strokes : []; + redraw(); + setStatus(`已同步 ${strokes.length} 笔`, true); + break; + case "pong": + // 心跳回声，保持连接 + break; + case "stroke": + if (msg.client_id === clientId) break; // 自己的，已本地画 + strokes.push(msg.stroke); + drawStroke(msg.stroke); + break; + case "cleared": + strokes = []; + current = null; + redraw(); + flashStatus(msg.client_id === clientId ? "已清空" : "对方清空了白板"); + break; + case "error": + flashStatus(msg.msg || "错误", true); + break; + } + } + + function send(obj) { + if (ws && ws.readyState === WebSocket.OPEN) { + try { ws.send(JSON.stringify(obj)); } catch {} + } + } + + function startHeartbeat() { + stopHeartbeat(); + heartbeatTimer = setInterval(() => send({ type: "ping" }), 3000); + } + function stopHeartbeat() { + if (heartbeatTimer) { clearInterval(heartbeatTimer); heartbeatTimer = null; } + } + + function onLost(reason) { + connected = false; + stopHeartbeat(); + onlineEl.classList.add("off"); + setStatus(reason + "，重连中…", true); + scheduleReconnect(); + } + + function scheduleReconnect() { + if (reconnectTimer) return; + reconnectTimer = setTimeout(() => { + reconnectTimer = null; + connect(); + }, 2000); + } + + function setStatus(text, isErr) { + statusEl.textContent = text; + statusEl.classList.add("show"); + statusEl.classList.toggle("err", !!isErr); + } + let statusTimer = null; + function flashStatus(text, isErr) { + setStatus(text, isErr); + clearTimeout(statusTimer); + statusTimer = setTimeout(() => statusEl.classList.remove("show"), 1600); + } + + // ---------- 启动 ---------- + // 先 GET /whiteboard/{id} 确保白板存在（不存在则服务端新建），再连 WS + fetch(`/whiteboard/${encodeURIComponent(boardId)}`, { headers: { accept: "application/json" } }) + .then((r) => r.ok ? r.json() : null) + .then((body) => { + if (body && Array.isArray(body.strokes)) { + strokes = body.strokes; + redraw(); + } + resize(); + connect(); + }) + .catch(() => { resize(); connect(); }); + + // 页面隐藏时不发心跳可被服务端判失活；重新可见时主动重连 + document.addEventListener("visibilitychange", () => { + if (document.visibilityState === "visible" && (!ws || ws.readyState !== WebSocket.OPEN)) { + if (reconnectTimer) { clearTimeout(reconnectTimer); reconnectTimer = null; } + connect(); + } + }); + + window.addEventListener("beforeunload", () => { + stopHeartbeat(); + if (reconnectTimer) clearTimeout(reconnectTimer); + try { ws && ws.close(); } catch {} + }); +})(); diff --git a/static/whiteboard_admin.css b/static/whiteboard_admin.css new file mode 100644 index 0000000..11b8a27 --- /dev/null +++ b/static/whiteboard_admin.css @@ -0,0 +1,15 @@ +/* 白板管理页专属样式。 */ +.wb-admin-table th.col-id { min-width: 28%; } +.wb-admin-table th.col-mods { width: 7em; } +.wb-admin-table th.col-created { width: 11em; } +.wb-admin-table th.col-updated { width: 11em; } +.wb-admin-table th.col-act { width: 7em; text-align: right; } +.wb-admin-table td.col-act { text-align: right; } +.wb-admin-table td.col-act .btn { padding: 0.3em 0.8em; font-size: 0.85em; } +.wb-admin-table td.col-id .bid { font-weight: 600; word-break: break-all; } +.wb-admin-table td.col-mods { text-align: right; } +.row-removed { opacity: 0; transition: opacity 0.25s; } +@media (max-width: 640px) { + .wb-admin-table th, .wb-admin-table td { padding: 0.5em 0.4em; } + .wb-admin-table th.col-created, .wb-admin-table td.col-created { display: none; } +} diff --git a/static/whiteboard_admin.html b/static/whiteboard_admin.html new file mode 100644 index 0000000..da3b7c6 --- /dev/null +++ b/static/whiteboard_admin.html @@ -0,0 +1,30 @@ + + + + + +白板管理 - zikai + + + + +
+

白板管理

+

查看所有白板的创建时间、修改次数、上次修改时间；可删除。

+ +
+ + + +
+ +
+
加载中…
+
+ +

上传文件 · 文件浏览 · zikai

+
+ + + + diff --git a/static/whiteboard_admin.js b/static/whiteboard_admin.js new file mode 100644 index 0000000..a28f9e4 --- /dev/null +++ b/static/whiteboard_admin.js @@ -0,0 +1,84 @@ +/* 白板管理页：拉取 /api/admin/whiteboards、渲染表格、删除、新建并跳转。 */ +(function () { + "use strict"; + const { el, toast, fmtTime, api } = window.ZK; + const listEl = document.getElementById("list"); + const countEl = document.getElementById("count"); + const refreshBtn = document.getElementById("refresh"); + const newBtn = document.getElementById("newBtn"); + + refreshBtn.addEventListener("click", load); + newBtn.addEventListener("click", () => { + // 生成一个随机 board_id 并打开（访问即创建） + const id = "b_" + Math.random().toString(36).slice(2, 10); + window.open(`/whiteboard/${id}`, "_blank"); + }); + + async function load() { + listEl.innerHTML = '
加载中…
'; + countEl.textContent = ""; + try { + const res = await api("/api/admin/whiteboards?limit=500&offset=0"); + if (!res.ok) throw new Error("HTTP " + res.status); + const body = await res.json(); + render(body.items || []); + countEl.textContent = `共 ${body.total} 个白板`; + } catch (e) { + listEl.innerHTML = '
加载失败：' + (e.message || e) + "
"; + } + } + + function render(items) { + if (!items.length) { + listEl.innerHTML = '
还没有白板。点「新建并打开」创建一个。
'; + return; + } + const table = el("table", { class: "wb-admin-table" }); + const thead = el("thead", null, + el("tr", null, + el("th", { class: "col-id" }, "白板 ID"), + el("th", { class: "col-mods" }, "修改次数"), + el("th", { class: "col-created" }, "创建时间"), + el("th", { class: "col-updated" }, "上次修改"), + el("th", { class: "col-act" }, "操作") + ) + ); + const tbody = el("tbody", null); + for (const b of items) { + const row = el("tr", null, + el("td", { class: "col-id" }, + el("a", { class: "bid link", href: `/whiteboard/${b.board_id}`, target: "_blank" }, b.board_id) + ), + el("td", { class: "col-mods mono" }, String(b.stroke_count ?? 0)), + el("td", { class: "col-created muted" }, fmtTime(b.created_at)), + el("td", { class: "col-updated muted" }, fmtTime(b.updated_at)), + el("td", { class: "col-act" }, + el("button", { class: "btn danger", onclick: () => remove(b, row) }, "删除") + ) + ); + tbody.appendChild(row); + } + table.appendChild(thead); + table.appendChild(tbody); + listEl.innerHTML = ""; + listEl.appendChild(table); + } + + async function remove(b, row) { + if (!confirm(`确定删除白板「${b.board_id}」？\n所有在线协作者会被断开，内容不可恢复。`)) return; + try { + const res = await api(`/api/admin/whiteboards/${encodeURIComponent(b.board_id)}`, { method: "DELETE" }); + if (res.status === 404) { toast("白板已不存在"); } + else if (!res.ok) throw new Error("HTTP " + res.status); + row.classList.add("row-removed"); + setTimeout(() => row.remove(), 250); + toast("已删除"); + const m = (countEl.textContent || "").match(/(\d+)/); + if (m) countEl.textContent = `共 ${Math.max(0, Number(m[1]) - 1)} 个白板`; + } catch (e) { + toast("删除失败：" + (e.message || e), "err"); + } + } + + load(); +})(); diff --git a/tests/manual_whiteboard_hub.py b/tests/manual_whiteboard_hub.py new file mode 100644 index 0000000..dac0106 --- /dev/null +++ b/tests/manual_whiteboard_hub.py @@ -0,0 +1,130 @@ +"""白板 hub 内存与心跳验证脚本（手动烟测，非 pytest）。 + +验证点： +1. 连接接入 -> hub._boards 出现该 board 的 set +2. disconnect -> 连接从 set 移除，空 set 从 dict 删除（无内存泄漏） +3. 心跳失活：reaper 扫描超过 timeout 的连接并移除 +4. 删除白板 -> close_board 踢出该 board 所有连接并清理 dict +5. broadcast 对失败连接自动 disconnect + +用伪造的 WebSocket 对象（不依赖真实网络）跑，直接断言 hub 内部状态。 +运行：.venv/bin/python tests/manual_whiteboard_hub.py +""" + +from __future__ import annotations + +import asyncio +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) + +from app.services.whiteboard_hub import Connection, WhiteboardHub # noqa: E402 + + +class FakeWS: + """最小化的 WebSocket 替身：记录发送/关闭，receive 永不返回。""" + + def __init__(self) -> None: + self.sent: list[dict] = [] + self.closed = False + self._fail_send = False + + async def accept(self) -> None: + pass + + async def send_json(self, msg: dict) -> None: + if self._fail_send: + raise RuntimeError("send failed (simulated)") + self.sent.append(msg) + + async def close(self) -> None: + self.closed = True + + +async def main() -> None: + # 用一个独立 hub 实例，避免污染全局单例；构造时读 config 默认 3s*5=15s + hub = WhiteboardHub() + print(f"timeout_seconds = {hub.timeout_seconds} (期望 15)") + assert hub.timeout_seconds == 15, "心跳超时应为 3*5=15s" + + # ---- 1. 接入 ---- + ws1 = FakeWS() + c1 = Connection(websocket=ws1, board_id="b1", client_id="c1") + await hub.register(c1) + assert "b1" in hub._boards, "接入后 b1 应存在" + assert hub.connection_count("b1") == 1 + print("1. 接入 OK") + + # ---- 2. disconnect 清理空 set ---- + ws2 = FakeWS() + c2 = Connection(websocket=ws2, board_id="b1", client_id="c2") + await hub.register(c2) + assert hub.connection_count("b1") == 2 + await hub.disconnect(c2) + assert hub.connection_count("b1") == 1, "c2 移除后应剩 1" + assert ws2.closed, "c2 的 ws 应被关闭" + await hub.disconnect(c1) + assert "b1" not in hub._boards, "空 set 应从 dict 删除（防泄漏）" + print("2. disconnect 清理空 set OK") + + # ---- 3. 心跳失活 reaper ---- + # 手动把 last_heartbeat 调到很久以前，触发 reaper 移除 + ws3 = FakeWS() + c3 = Connection(websocket=ws3, board_id="b2", client_id="c3") + # 模拟 30s 前心跳 + c3.last_heartbeat = asyncio.get_event_loop().time() - 30 + await hub.register(c3) + assert hub.connection_count("b2") == 1 + await hub._reap_once() + assert hub.connection_count("b2") == 0, "失活连接应被 reaper 移除" + assert ws3.closed, "失活连接的 ws 应被关闭" + assert "b2" not in hub._boards, "移除后空 set 应清理" + print("3. 心跳失活 reaper OK") + + # ---- 4. 心跳未超时的连接不被移除 ---- + ws4 = FakeWS() + c4 = Connection(websocket=ws4, board_id="b3", client_id="c4") + await hub.register(c4) + await hub._reap_once() + assert hub.connection_count("b3") == 1, "正常心跳连接不应被移除" + print("4. 正常连接保留 OK") + + # ---- 5. close_board 踢出所有连接 ---- + ws5 = FakeWS() + c5 = Connection(websocket=ws5, board_id="b3", client_id="c5") + await hub.register(c5) + assert hub.connection_count("b3") == 2 + await hub.close_board("b3") + assert hub.connection_count("b3") == 0, "close_board 后连接应清空" + assert ws4.closed and ws5.closed, "该 board 所有 ws 应被关闭" + assert "b3" not in hub._boards, "close_board 后 dict 应清理" + assert any(m.get("type") == "error" for m in ws4.sent), "应先发 error 通知" + print("5. close_board 踢出连接 OK") + + # ---- 6. broadcast 排除发送者 + 失败连接自动清理 ---- + wsA = FakeWS() + wsB = FakeWS() + wsB._fail_send = True # 模拟 B 发送失败 + cA = Connection(websocket=wsA, board_id="b4", client_id="A") + cB = Connection(websocket=wsB, board_id="b4", client_id="B") + await hub.register(cA) + await hub.register(cB) + await hub.broadcast("b4", {"type": "stroke"}, exclude=cA) + assert not wsA.sent, "排除发送者：A 不应收到" + assert wsB.closed, "发送失败的 B 应被自动 disconnect" + assert hub.connection_count("b4") == 1, "B 移除后应剩 A" + print("6. broadcast 排除 + 失败自动清理 OK") + + # ---- 7. 幂等 disconnect ---- + await hub.disconnect(cA) + await hub.disconnect(cA) # 重复调用不应报错 + assert "b4" not in hub._boards + print("7. 幂等 disconnect OK") + + print("\n全部通过 ✅") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/manual_whiteboard_kick.py b/tests/manual_whiteboard_kick.py new file mode 100644 index 0000000..56aae3a --- /dev/null +++ b/tests/manual_whiteboard_kick.py @@ -0,0 +1,52 @@ +"""验证删除白板时，在线 WS 连接被服务端踢出（收到 error 帧并断连）。""" + +from __future__ import annotations + +import asyncio +import json +import urllib.request + +import websockets + +BASE_WS = "ws://127.0.0.1:6890/ws/whiteboard" +BOARD = "kicktest" +AUTH = "Basic YTo2NjUxMTMxNQ==" # a:66511315 + + +async def main() -> None: + async with websockets.connect(f"{BASE_WS}/{BOARD}") as a: + await a.send(json.dumps({"type": "hello", "client_id": "A"})) + init = json.loads(await asyncio.wait_for(a.recv(), timeout=2)) + assert init["type"] == "init" + print("A connected, init received") + + # 通过 REST 删除白板（带 Basic Auth） + req = urllib.request.Request( + f"http://127.0.0.1:6890/api/admin/whiteboards/{BOARD}", + method="DELETE", + headers={"Authorization": AUTH}, + ) + with urllib.request.urlopen(req) as r: + print("delete response:", r.read().decode()) + + # A 应收到 error 帧随后连接关闭 + try: + raw = await asyncio.wait_for(a.recv(), timeout=3) + msg = json.loads(raw) + print("A received:", msg) + assert msg["type"] == "error", "应收到 error 通知" + except (websockets.ConnectionClosed, asyncio.TimeoutError) as e: + print("连接关闭/超时:", e) + + # 确认连接已断 + try: + await asyncio.wait_for(a.recv(), timeout=2) + print("ERROR: 连接仍存活（应已断开）") + except websockets.ConnectionClosed: + print("连接已被服务端关闭 ✅") + except asyncio.TimeoutError: + print("连接未关闭但无消息（部分通过）") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/manual_whiteboard_ws.py b/tests/manual_whiteboard_ws.py new file mode 100644 index 0000000..c3423d6 --- /dev/null +++ b/tests/manual_whiteboard_ws.py @@ -0,0 +1,92 @@ +"""白板 WebSocket 端到端烟测：两客户端实时同步 + 心跳 + 清空。 + +验证： +1. 客户端 A 连入 -> 收到 init +2. 客户端 B 连入 -> 收到 init +3. A 画一笔 -> B 收到 stroke 广播（A 不收自己的） +4. 心跳 ping -> pong +5. A 清空 -> A、B 都收到 cleared +6. 停发心跳的连接会被服务端 reaper 移除（15s，这里只验证 ping/pong 即可，reaper 已在 hub 单测覆盖） +""" + +from __future__ import annotations + +import asyncio +import json + +import websockets + +BASE_WS = "ws://127.0.0.1:6890/ws/whiteboard" +BOARD = "e2etest" + + +async def recv_msg(ws, timeout=2.0) -> dict | None: + try: + raw = await asyncio.wait_for(ws.recv(), timeout=timeout) + return json.loads(raw) + except asyncio.TimeoutError: + return None + + +async def main() -> None: + async with websockets.connect(f"{BASE_WS}/{BOARD}") as a, \ + websockets.connect(f"{BASE_WS}/{BOARD}") as b: + # hello + await a.send(json.dumps({"type": "hello", "client_id": "A"})) + await b.send(json.dumps({"type": "hello", "client_id": "B"})) + + init_a = await recv_msg(a) + init_b = await recv_msg(b) + print("A init:", init_a.get("type") if init_a else None) + print("B init:", init_b.get("type") if init_b else None) + assert init_a and init_a["type"] == "init" + assert init_b and init_b["type"] == "init" + + # A 画一笔 + stroke = {"points": [[10, 10], [20, 20]], "color": "#1565c0", "width": 3} + await a.send(json.dumps({"type": "stroke", "stroke": stroke})) + # A 不应收到自己的（排除发送者） + echo = await recv_msg(a, timeout=1.0) + print("A self-echo (expect None):", echo) + assert echo is None, "发送者不应收到自己的 stroke" + # B 应收到 + got = await recv_msg(b) + print("B recv:", got.get("type") if got else None, "client_id=", got.get("client_id") if got else None) + assert got and got["type"] == "stroke" and got["client_id"] == "A" + assert got["stroke"] == stroke + + # 心跳 + await a.send(json.dumps({"type": "ping"})) + pong = await recv_msg(a) + print("A pong:", pong.get("type") if pong else None) + assert pong and pong["type"] == "pong" + + # 清空 -> 两端都收 cleared + await b.send(json.dumps({"type": "clear"})) + cleared_b = await recv_msg(b) + cleared_a = await recv_msg(a) + print("B cleared:", cleared_b.get("type") if cleared_b else None, + "A cleared:", cleared_a.get("type") if cleared_a else None) + assert cleared_b and cleared_b["type"] == "cleared" and cleared_b["client_id"] == "B" + assert cleared_a and cleared_a["type"] == "cleared" and cleared_a["client_id"] == "B" + + # 验证持久化：重连后 init 应 strokes 为空（已清空） + async with websockets.connect(f"{BASE_WS}/{BOARD}") as c: + await c.send(json.dumps({"type": "hello", "client_id": "C"})) + init_c = await recv_msg(c) + print("C init after clear, strokes=", init_c.get("strokes") if init_c else None) + assert init_c and init_c["type"] == "init" + assert init_c["strokes"] == [], "清空后重连应得到空 strokes" + + # 验证 stroke_count 累计（之前 1 笔 + 1 次清空 = 2） + import urllib.request + with urllib.request.urlopen(f"http://127.0.0.1:6890/whiteboard/{BOARD}") as r: + meta = json.load(r) + print("stroke_count after ops:", meta["stroke_count"]) + assert meta["stroke_count"] == 2, "1 笔 + 1 清空 = 2 次修改" + + print("\nWS 端到端全部通过 ✅") + + +if __name__ == "__main__": + asyncio.run(main())