feat: SFTP 上传的文件经磁盘扫描补录后可在文件浏览页展示

SFTP 上传的文件直接落盘到 upload_root(chroot 根),不经 HTTP 路径,无 DB 记录,
文件浏览页(读 uploaded_file 表)看不到。新增磁盘扫描机制:

- UploadService.scan_sftp_files(force=False):遍历 uploads/ 下所有文件(排除 .work/
  与 .part 残品),与 DB 已有 storage_path 比对,磁盘有 DB 无的补录(source="sftp",
  流式算 size + sha256)。
- 频率限制:模块级 _last_scan_time,两次扫描间隔 < 3s 则跳过返回 0(只读 DB 缓存),
  force=True 强制扫。
- file_admin_controller 的 list 接口返回前触发 scan_sftp_files(),文件浏览页打开/
  刷新即自动同步 SFTP 文件(受 3s 频率限制,不会每次翻页都扫磁盘)。
- UploadedFileDAO 加 list_storage_paths() 批量查已记录路径集合供比对。

SFTP 文件补录后可正常下载/删除(与 HTTP 上传文件一致)。
This commit is contained in:
zikai
2026-07-22 01:25:41 +00:00
parent 30a263ed50
commit 09705a8843
3 changed files with 79 additions and 1 deletions

View File

@@ -43,7 +43,8 @@ class BatchDeleteResult(BaseModel):
"", "",
response_model=FileListResponse, response_model=FileListResponse,
summary="列出已上传文件(需鉴权)", summary="列出已上传文件(需鉴权)",
description="供文件浏览页使用;字段同公开 /api/files但要求 docs Basic Auth。支持分页。", description="供文件浏览页使用;字段同公开 /api/files但要求 docs Basic Auth。支持分页。"
"返回前会触发磁盘扫描(频率限制 3s把 SFTP 上传但未入库的文件补录进来。",
) )
def list_files( def list_files(
limit: int = Query(100, ge=1, le=10000, description="每页条数1-10000"), limit: int = Query(100, ge=1, le=10000, description="每页条数1-10000"),
@@ -51,6 +52,7 @@ def list_files(
service: UploadService = Depends(_service), service: UploadService = Depends(_service),
_: str = Depends(require_docs_auth), _: str = Depends(require_docs_auth),
) -> FileListResponse: ) -> FileListResponse:
service.scan_sftp_files() # 频率限制内置3s 内重复只返回 DB 缓存
total, items = service.list_files(limit=limit, offset=offset) total, items = service.list_files(limit=limit, offset=offset)
return FileListResponse(total=total, items=items) return FileListResponse(total=total, items=items)

View File

@@ -29,6 +29,11 @@ class UploadedFileDAO:
"""返回数据库中文件总条数。""" """返回数据库中文件总条数。"""
return self.db.scalar(select(func.count()).select_from(UploadedFile)) or 0 return self.db.scalar(select(func.count()).select_from(UploadedFile)) or 0
def list_storage_paths(self) -> set[str]:
"""返回所有已记录的 storage_path 集合(供磁盘扫描比对,识别未入库文件)。"""
stmt = select(UploadedFile.storage_path)
return {row for row in self.db.scalars(stmt).all()}
def list(self, limit: int = 100, offset: int = 0) -> list[UploadedFile]: def list(self, limit: int = 100, offset: int = 0) -> list[UploadedFile]:
stmt = ( stmt = (
select(UploadedFile) select(UploadedFile)

View File

@@ -10,6 +10,7 @@ from __future__ import annotations
import hashlib import hashlib
import logging import logging
import os import os
import time
import uuid import uuid
from collections.abc import Iterator from collections.abc import Iterator
from datetime import datetime, timezone from datetime import datetime, timezone
@@ -24,6 +25,10 @@ from ..schemas.file import FileUploadResponse, UploadedFileOut
logger = logging.getLogger("zikai.upload") logger = logging.getLogger("zikai.upload")
# 磁盘扫描频率限制:两次扫描至少间隔此秒数,否则跳过(只返回 DB 缓存)。
_SCAN_MIN_INTERVAL = 3.0
_last_scan_time: float = 0.0
def hash_stream(chunks: Iterator[bytes]) -> tuple[int, str]: def hash_stream(chunks: Iterator[bytes]) -> tuple[int, str]:
"""对一段字节块序列流式计算 (size, sha256)。供多条上传路径复用。""" """对一段字节块序列流式计算 (size, sha256)。供多条上传路径复用。"""
@@ -69,6 +74,72 @@ class UploadService:
# ---------------- 查询 ---------------- # ---------------- 查询 ----------------
def scan_sftp_files(self, force: bool = False) -> int:
"""扫描 uploads/ 目录,把磁盘上有但 DB 未记录的文件补录入库。
SFTP 上传的文件直接落盘到 upload_rootchroot 根),不经过 HTTP 路径,
因此没有 DB 记录。本方法遍历磁盘文件,与 DB 已有 storage_path 比对,
为缺失项创建记录source="sftp",计算 size + sha256
频率限制:两次扫描间隔 < _SCAN_MIN_INTERVAL3s则跳过force=True 强制扫。
返回本次新补录的条数(跳过时返回 0
"""
global _last_scan_time
now = time.monotonic()
if not force and (now - _last_scan_time) < _SCAN_MIN_INTERVAL:
return 0
_last_scan_time = now
known = self.dao.list_storage_paths()
new_count = 0
# 排除分片会话暂存目录与 .part 残品
skip_dirs = {".work"}
for abs_path in self.upload_root.rglob("*"):
if not abs_path.is_file():
continue
if abs_path.suffix == ".part":
continue
# 跳过 .work 目录下的任何文件
rel = abs_path.relative_to(self.upload_root)
if rel.parts and rel.parts[0] in skip_dirs:
continue
storage_path = str(rel).replace("\\", "/")
if storage_path in known:
continue
# 磁盘有、DB 无:补录
try:
size = abs_path.stat().st_size
sha256 = self._hash_file(abs_path)
except Exception as exc:
logger.warning("扫描文件失败 path=%s: %s", abs_path, exc)
continue
entity = UploadedFile(
storage_path=storage_path,
original_filename=abs_path.name,
content_type="",
size_bytes=size,
sha256=sha256,
source="sftp",
uploaded_by="sftp",
)
try:
self.dao.create(entity)
new_count += 1
known.add(storage_path)
except Exception as exc:
logger.warning("补录 SFTP 文件失败 path=%s: %s", storage_path, exc)
if new_count:
logger.info("SFTP 文件扫描补录 %d", new_count)
return new_count
def _hash_file(self, path: Path) -> str:
"""流式计算文件 sha256避免大文件一次性读入内存"""
h = hashlib.sha256()
with path.open("rb") as f:
while chunk := f.read(self.chunk_bytes):
h.update(chunk)
return h.hexdigest()
def list_files(self, limit: int = 100, offset: int = 0) -> tuple[int, list[UploadedFileOut]]: def list_files(self, limit: int = 100, offset: int = 0) -> tuple[int, list[UploadedFileOut]]:
total = self.dao.count() total = self.dao.count()
rows = self.dao.list(limit=limit, offset=offset) rows = self.dao.list(limit=limit, offset=offset)