create project

This commit is contained in:
2026-01-16 17:30:40 +08:00
commit effac6b017
157 changed files with 45997 additions and 0 deletions

View File

@@ -0,0 +1,103 @@
package database
import (
"context"
"strings"
"time"
"gorm.io/gorm/logger"
)
// CustomLogger 自定义 GORM logger截断过长的 SQL 参数(如 base64 数据)
type CustomLogger struct {
logger.Interface
}
// NewCustomLogger 创建自定义 logger
func NewCustomLogger() logger.Interface {
return &CustomLogger{
Interface: logger.Default.LogMode(logger.Silent),
}
}
// Trace 重写 Trace 方法,禁用 SQL 日志输出
func (l *CustomLogger) Trace(ctx context.Context, begin time.Time, fc func() (sql string, rowsAffected int64), err error) {
// 不输出任何 SQL 日志
// 如果需要调试,可以临时取消注释下面的代码
/*
sql, rows := fc()
sql = truncateLongValues(sql)
elapsed := time.Since(begin)
if err != nil {
l.Interface.Error(ctx, "SQL error: %v [%v] %s", err, elapsed, sql)
} else {
l.Interface.Info(ctx, "[%.3fms] [rows:%d] %s", float64(elapsed.Nanoseconds())/1e6, rows, sql)
}
*/
}
// truncateLongValues 截断 SQL 中的长字符串值
func truncateLongValues(sql string) string {
// 查找 base64 格式的数据 (data:image/...;base64,...)
if strings.Contains(sql, "data:image/") && strings.Contains(sql, ";base64,") {
parts := strings.Split(sql, "\"")
for i, part := range parts {
if strings.HasPrefix(part, "data:image/") && strings.Contains(part, ";base64,") {
if len(part) > 100 {
// 保留前50字符添加截断标记
parts[i] = part[:50] + "...[base64 data truncated]"
}
}
}
sql = strings.Join(parts, "\"")
}
// 截断其他过长的值
if len(sql) > 5000 {
// 查找 VALUES 或 SET 后的内容
if idx := strings.Index(sql, " VALUES "); idx > 0 && len(sql) > idx+5000 {
sql = sql[:idx+5000] + "...[truncated]"
} else if idx := strings.Index(sql, " SET "); idx > 0 && len(sql) > idx+3000 {
sql = sql[:idx+3000] + "...[truncated]"
} else if len(sql) > 5000 {
sql = sql[:5000] + "...[truncated]"
}
}
return sql
}
// Info 实现 Info 方法
func (l *CustomLogger) Info(ctx context.Context, msg string, data ...interface{}) {
l.Interface.Info(ctx, msg, data...)
}
// Warn 实现 Warn 方法
func (l *CustomLogger) Warn(ctx context.Context, msg string, data ...interface{}) {
l.Interface.Warn(ctx, msg, data...)
}
// Error 实现 Error 方法
func (l *CustomLogger) Error(ctx context.Context, msg string, data ...interface{}) {
// 检查并截断 data 中的长字符串
truncatedData := make([]interface{}, len(data))
for i, d := range data {
if str, ok := d.(string); ok && len(str) > 200 {
if strings.HasPrefix(str, "data:image/") {
truncatedData[i] = str[:50] + "...[base64 data]"
} else {
truncatedData[i] = str[:200] + "..."
}
} else {
truncatedData[i] = d
}
}
l.Interface.Error(ctx, msg, truncatedData...)
}
// LogMode 实现 LogMode 方法
func (l *CustomLogger) LogMode(level logger.LogLevel) logger.Interface {
newLogger := *l
newLogger.Interface = l.Interface.LogMode(level)
return &newLogger
}

View File

@@ -0,0 +1,97 @@
package database
import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/drama-generator/backend/domain/models"
"github.com/drama-generator/backend/pkg/config"
"gorm.io/driver/mysql"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
_ "modernc.org/sqlite"
)
func NewDatabase(cfg config.DatabaseConfig) (*gorm.DB, error) {
dsn := cfg.DSN()
if cfg.Type == "sqlite" {
dbDir := filepath.Dir(dsn)
if err := os.MkdirAll(dbDir, 0755); err != nil {
return nil, fmt.Errorf("failed to create database directory: %w", err)
}
}
gormConfig := &gorm.Config{
Logger: NewCustomLogger(),
}
var db *gorm.DB
var err error
if cfg.Type == "sqlite" {
// 使用 modernc.org/sqlite 纯 Go 驱动(无需 CGO
// 添加并发优化参数WAL 模式、busy_timeout、cache
dsnWithParams := dsn + "?_journal_mode=WAL&_busy_timeout=5000&_synchronous=NORMAL&cache=shared"
db, err = gorm.Open(sqlite.Dialector{
DriverName: "sqlite",
DSN: dsnWithParams,
}, gormConfig)
} else {
db, err = gorm.Open(mysql.Open(dsn), gormConfig)
}
if err != nil {
return nil, fmt.Errorf("failed to connect to database: %w", err)
}
sqlDB, err := db.DB()
if err != nil {
return nil, fmt.Errorf("failed to get database instance: %w", err)
}
// SQLite 连接池配置(限制并发连接数)
if cfg.Type == "sqlite" {
sqlDB.SetMaxIdleConns(1)
sqlDB.SetMaxOpenConns(1) // SQLite 单写入,限制为 1
} else {
sqlDB.SetMaxIdleConns(cfg.MaxIdle)
sqlDB.SetMaxOpenConns(cfg.MaxOpen)
}
sqlDB.SetConnMaxLifetime(time.Hour)
if err := sqlDB.Ping(); err != nil {
return nil, fmt.Errorf("failed to ping database: %w", err)
}
return db, nil
}
func AutoMigrate(db *gorm.DB) error {
return db.AutoMigrate(
// 核心模型
&models.Drama{},
&models.Episode{},
&models.Character{},
&models.Scene{},
&models.Storyboard{},
// 生成相关
&models.ImageGeneration{},
&models.VideoGeneration{},
&models.VideoMerge{},
// AI配置
&models.AIServiceConfig{},
&models.AIServiceProvider{},
// 资源管理
&models.Asset{},
&models.CharacterLibrary{},
// 任务管理
&models.AsyncTask{},
)
}

612
infrastructure/external/ffmpeg/ffmpeg.go vendored Normal file
View File

@@ -0,0 +1,612 @@
package ffmpeg
import (
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
"github.com/drama-generator/backend/pkg/logger"
)
type FFmpeg struct {
log *logger.Logger
tempDir string
}
func NewFFmpeg(log *logger.Logger) *FFmpeg {
tempDir := filepath.Join(os.TempDir(), "drama-video-merge")
os.MkdirAll(tempDir, 0755)
return &FFmpeg{
log: log,
tempDir: tempDir,
}
}
type VideoClip struct {
URL string
Duration float64
StartTime float64
EndTime float64
Transition map[string]interface{}
}
type MergeOptions struct {
OutputPath string
Clips []VideoClip
}
func (f *FFmpeg) MergeVideos(opts *MergeOptions) (string, error) {
if len(opts.Clips) == 0 {
return "", fmt.Errorf("no video clips to merge")
}
f.log.Infow("Starting video merge with trimming", "clips_count", len(opts.Clips))
// 下载并裁剪所有视频片段
trimmedPaths := make([]string, 0, len(opts.Clips))
downloadedPaths := make([]string, 0, len(opts.Clips))
for i, clip := range opts.Clips {
// 下载原始视频
downloadPath := filepath.Join(f.tempDir, fmt.Sprintf("download_%d_%d.mp4", time.Now().Unix(), i))
localPath, err := f.downloadVideo(clip.URL, downloadPath)
if err != nil {
f.cleanup(downloadedPaths)
f.cleanup(trimmedPaths)
return "", fmt.Errorf("failed to download clip %d: %w", i, err)
}
downloadedPaths = append(downloadedPaths, localPath)
// 裁剪视频片段根据StartTime和EndTime
trimmedPath := filepath.Join(f.tempDir, fmt.Sprintf("trimmed_%d_%d.mp4", time.Now().Unix(), i))
err = f.trimVideo(localPath, trimmedPath, clip.StartTime, clip.EndTime)
if err != nil {
f.cleanup(downloadedPaths)
f.cleanup(trimmedPaths)
return "", fmt.Errorf("failed to trim clip %d: %w", i, err)
}
trimmedPaths = append(trimmedPaths, trimmedPath)
f.log.Infow("Clip trimmed",
"index", i,
"start", clip.StartTime,
"end", clip.EndTime,
"duration", clip.EndTime-clip.StartTime)
}
// 清理下载的原始文件
f.cleanup(downloadedPaths)
// 确保输出目录存在
outputDir := filepath.Dir(opts.OutputPath)
if err := os.MkdirAll(outputDir, 0755); err != nil {
f.cleanup(trimmedPaths)
return "", fmt.Errorf("failed to create output directory: %w", err)
}
// 合并裁剪后的视频片段(支持转场效果)
err := f.concatenateVideosWithTransitions(trimmedPaths, opts.Clips, opts.OutputPath)
// 清理裁剪后的临时文件
f.cleanup(trimmedPaths)
if err != nil {
return "", fmt.Errorf("failed to concatenate videos: %w", err)
}
f.log.Infow("Video merge completed", "output", opts.OutputPath)
return opts.OutputPath, nil
}
func (f *FFmpeg) downloadVideo(url, destPath string) (string, error) {
f.log.Infow("Downloading video", "url", url, "dest", destPath)
resp, err := http.Get(url)
if err != nil {
return "", fmt.Errorf("failed to download: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("bad status: %s", resp.Status)
}
out, err := os.Create(destPath)
if err != nil {
return "", fmt.Errorf("failed to create file: %w", err)
}
defer out.Close()
_, err = io.Copy(out, resp.Body)
if err != nil {
return "", fmt.Errorf("failed to save file: %w", err)
}
return destPath, nil
}
func (f *FFmpeg) trimVideo(inputPath, outputPath string, startTime, endTime float64) error {
f.log.Infow("Trimming video",
"input", inputPath,
"output", outputPath,
"start", startTime,
"end", endTime)
// 如果startTime和endTime都为0或者endTime <= startTime复制整个视频
// 使用重新编码而非-c copy以确保输出文件完整性
if (startTime == 0 && endTime == 0) || endTime <= startTime {
f.log.Infow("No valid trim range, re-encoding entire video")
cmd := exec.Command("ffmpeg",
"-i", inputPath,
"-c:v", "libx264",
"-preset", "fast",
"-crf", "23",
"-c:a", "aac",
"-b:a", "128k",
"-movflags", "+faststart",
"-y",
outputPath,
)
output, err := cmd.CombinedOutput()
if err != nil {
f.log.Errorw("FFmpeg re-encode failed", "error", err, "output", string(output))
return fmt.Errorf("ffmpeg re-encode failed: %w, output: %s", err, string(output))
}
f.log.Infow("Video re-encoded successfully", "output", outputPath)
return nil
}
// 使用FFmpeg裁剪视频
// -ss: 开始时间(秒)
// -to/-t: 结束时间或持续时间
// 使用重新编码而非-c copy以确保输出文件完整性避免Windows环境下流信息丢失
var cmd *exec.Cmd
if endTime > 0 {
// 有明确的结束时间
cmd = exec.Command("ffmpeg",
"-i", inputPath,
"-ss", fmt.Sprintf("%.2f", startTime),
"-to", fmt.Sprintf("%.2f", endTime),
"-c:v", "libx264",
"-preset", "fast",
"-crf", "23",
"-c:a", "aac",
"-b:a", "128k",
"-movflags", "+faststart",
"-y",
outputPath,
)
} else {
// 只有开始时间,裁剪到视频末尾
cmd = exec.Command("ffmpeg",
"-i", inputPath,
"-ss", fmt.Sprintf("%.2f", startTime),
"-c:v", "libx264",
"-preset", "fast",
"-crf", "23",
"-c:a", "aac",
"-b:a", "128k",
"-movflags", "+faststart",
"-y",
outputPath,
)
}
output, err := cmd.CombinedOutput()
if err != nil {
f.log.Errorw("FFmpeg trim failed", "error", err, "output", string(output))
return fmt.Errorf("ffmpeg trim failed: %w, output: %s", err, string(output))
}
f.log.Infow("Video trimmed successfully", "output", outputPath)
return nil
}
func (f *FFmpeg) concatenateVideosWithTransitions(inputPaths []string, clips []VideoClip, outputPath string) error {
if len(inputPaths) == 0 {
return fmt.Errorf("no input paths")
}
// 如果只有一个视频,直接复制
if len(inputPaths) == 1 {
f.log.Infow("Only one clip, copying directly")
return f.copyFile(inputPaths[0], outputPath)
}
// 检查是否有转场效果
hasTransitions := false
for _, clip := range clips {
if clip.Transition != nil && len(clip.Transition) > 0 {
hasTransitions = true
break
}
}
// 如果没有转场效果,使用简单拼接
if !hasTransitions {
f.log.Infow("No transitions, using simple concatenation")
return f.concatenateVideos(inputPaths, outputPath)
}
// 使用xfade滤镜添加转场效果
f.log.Infow("Merging with transitions", "clips_count", len(inputPaths))
return f.mergeWithXfade(inputPaths, clips, outputPath)
}
func (f *FFmpeg) concatenateVideos(inputPaths []string, outputPath string) error {
// 创建文件列表
listFile := filepath.Join(f.tempDir, fmt.Sprintf("filelist_%d.txt", time.Now().Unix()))
defer os.Remove(listFile)
var content strings.Builder
for _, path := range inputPaths {
content.WriteString(fmt.Sprintf("file '%s'\n", path))
}
if err := os.WriteFile(listFile, []byte(content.String()), 0644); err != nil {
return fmt.Errorf("failed to create file list: %w", err)
}
// 使用FFmpeg合并视频
// -f concat: 使用concat demuxer
// -safe 0: 允许不安全的文件路径
// -i: 输入文件列表
// -c copy: 直接复制流,不重新编码(速度快)
cmd := exec.Command("ffmpeg",
"-f", "concat",
"-safe", "0",
"-i", listFile,
"-c", "copy",
"-y", // 覆盖输出文件
outputPath,
)
output, err := cmd.CombinedOutput()
if err != nil {
f.log.Errorw("FFmpeg failed", "error", err, "output", string(output))
return fmt.Errorf("ffmpeg execution failed: %w, output: %s", err, string(output))
}
f.log.Infow("FFmpeg concatenation completed", "output", outputPath)
return nil
}
func (f *FFmpeg) mergeWithXfade(inputPaths []string, clips []VideoClip, outputPath string) error {
// 使用xfade滤镜进行转场
// 构建输入参数
args := []string{}
for _, path := range inputPaths {
args = append(args, "-i", path)
}
// 检测每个视频是否有音频流
audioStreams := make([]bool, len(inputPaths))
hasAnyAudio := false
for i, path := range inputPaths {
audioStreams[i] = f.hasAudioStream(path)
if audioStreams[i] {
hasAnyAudio = true
}
f.log.Infow("Audio stream detection", "index", i, "path", path, "has_audio", audioStreams[i])
}
f.log.Infow("Overall audio detection", "has_any_audio", hasAnyAudio, "audio_streams", audioStreams)
// 检测视频分辨率,找到最大分辨率作为目标分辨率
maxWidth := 0
maxHeight := 0
for i, path := range inputPaths {
width, height := f.getVideoResolution(path)
if width > maxWidth {
maxWidth = width
}
if height > maxHeight {
maxHeight = height
}
f.log.Infow("Video resolution detection", "index", i, "width", width, "height", height)
}
f.log.Infow("Target resolution", "width", maxWidth, "height", maxHeight)
// 为每个视频流添加缩放滤镜,统一分辨率
var scaleFilters []string
for i := 0; i < len(inputPaths); i++ {
// 使用scale滤镜缩放到目标分辨率pad添加黑边保持长宽比
scaleFilters = append(scaleFilters,
fmt.Sprintf("[%d:v]scale=%d:%d:force_original_aspect_ratio=decrease,pad=%d:%d:(ow-iw)/2:(oh-ih)/2[v%d]",
i, maxWidth, maxHeight, maxWidth, maxHeight, i))
}
// 构建filter_complex
// 例如: [0:v][1:v]xfade=transition=fade:duration=1:offset=5[v01];[v01][2:v]xfade=transition=fade:duration=1:offset=10[out]
// 构建转场滤镜,使用缩放后的视频流
var transitionFilters []string
var offset float64 = 0
for i := 0; i < len(inputPaths)-1; i++ {
// 获取当前片段的时长
clipDuration := clips[i].Duration
if clips[i].EndTime > 0 && clips[i].StartTime >= 0 {
clipDuration = clips[i].EndTime - clips[i].StartTime
}
// 获取转场类型和时长
transitionType := "fade" // 默认淡入淡出
transitionDuration := 1.0 // 默认转场时长为1秒
if clips[i].Transition != nil {
// 读取转场类型
if tType, ok := clips[i].Transition["type"].(string); ok && tType != "" {
transitionType = f.mapTransitionType(tType)
f.log.Infow("Using transition type", "type", tType, "mapped", transitionType)
}
// 读取转场时长
if tDuration, ok := clips[i].Transition["duration"].(float64); ok && tDuration > 0 {
transitionDuration = tDuration
}
}
// 计算转场开始的时间点
// 转场在两个片段的交界处,从前一个片段结束前 transitionDuration/2 开始
// 这样转场效果会平均分布在两个片段的交界处
offset += clipDuration - (transitionDuration / 2)
if offset < 0 {
offset = 0
}
f.log.Infow("Transition settings",
"clip_index", i,
"type", transitionType,
"duration", transitionDuration,
"offset", offset,
"clip_duration", clipDuration)
var inputLabel, outputLabel string
if i == 0 {
inputLabel = fmt.Sprintf("[v0][v1]")
} else {
inputLabel = fmt.Sprintf("[vx%02d][v%d]", i-1, i+1)
}
if i == len(inputPaths)-2 {
outputLabel = "[outv]"
} else {
outputLabel = fmt.Sprintf("[vx%02d]", i)
}
filterPart := fmt.Sprintf("%sxfade=transition=%s:duration=%.1f:offset=%.1f%s",
inputLabel, transitionType, transitionDuration, offset, outputLabel)
transitionFilters = append(transitionFilters, filterPart)
}
// 合并缩放和转场滤镜
var videoFilters []string
videoFilters = append(videoFilters, scaleFilters...)
videoFilters = append(videoFilters, transitionFilters...)
filterComplex := strings.Join(videoFilters, ";")
// 音频处理:如果有任何视频包含音频流,则处理音频
var fullFilter string
if hasAnyAudio {
// 为没有音频的视频生成静音轨道,确保所有输入音频流一致
var silenceFilters []string
for i := 0; i < len(inputPaths); i++ {
if !audioStreams[i] {
// 计算该视频的时长
clipDuration := clips[i].Duration
if clips[i].EndTime > 0 && clips[i].StartTime >= 0 {
clipDuration = clips[i].EndTime - clips[i].StartTime
}
// anullsrc是源滤镜不接受输入使用duration参数指定时长
silenceFilters = append(silenceFilters,
fmt.Sprintf("anullsrc=channel_layout=stereo:sample_rate=44100:duration=%.2f[a%d]", clipDuration, i))
}
}
// 拼接所有音频流(包括生成的静音流)
var audioConcat strings.Builder
for i := 0; i < len(inputPaths); i++ {
if audioStreams[i] {
audioConcat.WriteString(fmt.Sprintf("[%d:a]", i))
} else {
audioConcat.WriteString(fmt.Sprintf("[a%d]", i))
}
}
audioConcat.WriteString(fmt.Sprintf("concat=n=%d:v=0:a=1[outa]", len(inputPaths)))
// 构建完整滤镜:先生成静音流,再拼接音频
if len(silenceFilters) > 0 {
fullFilter = filterComplex + ";" + strings.Join(silenceFilters, ";") + ";" + audioConcat.String()
} else {
fullFilter = filterComplex + ";" + audioConcat.String()
}
} else {
// 所有视频都无音频流,只处理视频
fullFilter = filterComplex
}
// 构建完整命令
args = append(args,
"-filter_complex", fullFilter,
"-map", "[outv]",
)
// 仅在有任何音频时映射音频输出
if hasAnyAudio {
args = append(args, "-map", "[outa]")
}
args = append(args,
"-c:v", "libx264",
"-preset", "medium",
"-crf", "23",
)
// 仅在有任何音频时设置音频编码参数
if hasAnyAudio {
args = append(args,
"-c:a", "aac",
"-b:a", "128k",
)
}
args = append(args,
"-y",
outputPath,
)
f.log.Infow("Running FFmpeg with transitions", "filter", fullFilter, "has_any_audio", hasAnyAudio)
cmd := exec.Command("ffmpeg", args...)
output, err := cmd.CombinedOutput()
if err != nil {
f.log.Errorw("FFmpeg xfade failed", "error", err, "output", string(output))
return fmt.Errorf("ffmpeg xfade failed: %w, output: %s", err, string(output))
}
f.log.Infow("Video merged with transitions successfully")
return nil
}
func (f *FFmpeg) mapTransitionType(transType string) string {
// 将前端传入的转场类型映射为FFmpeg xfade支持的类型
// FFmpeg xfade支持的完整转场列表: https://ffmpeg.org/ffmpeg-filters.html#xfade
switch strings.ToLower(transType) {
// 淡入淡出类
case "fade", "fadein", "fadeout":
return "fade"
case "fadeblack":
return "fadeblack"
case "fadewhite":
return "fadewhite"
case "fadegrays":
return "fadegrays"
// 滑动类
case "slideleft":
return "slideleft"
case "slideright":
return "slideright"
case "slideup":
return "slideup"
case "slidedown":
return "slidedown"
// 擦除类
case "wipeleft":
return "wipeleft"
case "wiperight":
return "wiperight"
case "wipeup":
return "wipeup"
case "wipedown":
return "wipedown"
// 圆形类
case "circleopen":
return "circleopen"
case "circleclose":
return "circleclose"
// 矩形打开/关闭类
case "horzopen":
return "horzopen"
case "horzclose":
return "horzclose"
case "vertopen":
return "vertopen"
case "vertclose":
return "vertclose"
// 其他特效
case "dissolve":
return "dissolve"
case "distance":
return "distance"
case "pixelize":
return "pixelize"
default:
return "fade" // 默认淡入淡出
}
}
func (f *FFmpeg) hasAudioStream(videoPath string) bool {
cmd := exec.Command("ffprobe",
"-v", "error",
"-select_streams", "a:0",
"-show_entries", "stream=codec_type",
"-of", "default=noprint_wrappers=1:nokey=1",
videoPath,
)
output, err := cmd.CombinedOutput()
if err != nil {
return false
}
result := strings.TrimSpace(string(output))
return result == "audio"
}
func (f *FFmpeg) getVideoResolution(videoPath string) (int, int) {
cmd := exec.Command("ffprobe",
"-v", "error",
"-select_streams", "v:0",
"-show_entries", "stream=width,height",
"-of", "csv=p=0",
videoPath,
)
output, err := cmd.CombinedOutput()
if err != nil {
f.log.Warnw("Failed to get video resolution", "path", videoPath, "error", err)
return 1920, 1080 // 默认分辨率
}
result := strings.TrimSpace(string(output))
parts := strings.Split(result, ",")
if len(parts) != 2 {
f.log.Warnw("Invalid resolution format", "output", result)
return 1920, 1080
}
var width, height int
fmt.Sscanf(parts[0], "%d", &width)
fmt.Sscanf(parts[1], "%d", &height)
if width <= 0 || height <= 0 {
return 1920, 1080
}
return width, height
}
func (f *FFmpeg) copyFile(src, dst string) error {
cmd := exec.Command("cp", src, dst)
output, err := cmd.CombinedOutput()
if err != nil {
f.log.Errorw("File copy failed", "error", err, "output", string(output))
return fmt.Errorf("copy failed: %w", err)
}
return nil
}
func (f *FFmpeg) cleanup(paths []string) {
for _, path := range paths {
if err := os.Remove(path); err != nil {
f.log.Warnw("Failed to cleanup file", "path", path, "error", err)
}
}
}
func (f *FFmpeg) CleanupTempDir() error {
return os.RemoveAll(f.tempDir)
}

View File

@@ -0,0 +1,240 @@
package scheduler
import (
"time"
"github.com/drama-generator/backend/application/services"
"github.com/drama-generator/backend/pkg/logger"
"github.com/robfig/cron/v3"
"gorm.io/gorm"
)
type ResourceTransferScheduler struct {
cron *cron.Cron
transferService *services.ResourceTransferService
db *gorm.DB
log *logger.Logger
running bool
}
func NewResourceTransferScheduler(
transferService *services.ResourceTransferService,
db *gorm.DB,
log *logger.Logger,
) *ResourceTransferScheduler {
return &ResourceTransferScheduler{
cron: cron.New(cron.WithSeconds()),
transferService: transferService,
db: db,
log: log,
running: false,
}
}
// Start 启动定时任务
func (s *ResourceTransferScheduler) Start() error {
if s.running {
s.log.Warn("Resource transfer scheduler already running")
return nil
}
s.log.Info("Starting resource transfer scheduler...")
// 每小时执行一次资源转存任务
_, err := s.cron.AddFunc("0 0 * * * *", func() {
s.log.Info("Starting scheduled resource transfer task")
s.transferPendingResources()
})
if err != nil {
return err
}
// 每天凌晨2点执行完整扫描
_, err = s.cron.AddFunc("0 0 2 * * *", func() {
s.log.Info("Starting daily full resource scan and transfer")
s.transferAllPendingResources()
})
if err != nil {
return err
}
s.cron.Start()
s.running = true
s.log.Info("Resource transfer scheduler started successfully")
return nil
}
// Stop 停止定时任务
func (s *ResourceTransferScheduler) Stop() {
if !s.running {
return
}
s.log.Info("Stopping resource transfer scheduler...")
ctx := s.cron.Stop()
<-ctx.Done()
s.running = false
s.log.Info("Resource transfer scheduler stopped")
}
// transferPendingResources 转存最近生成的待转存资源最近24小时
func (s *ResourceTransferScheduler) transferPendingResources() {
s.log.Info("Scanning for pending resources to transfer (last 24 hours)...")
// 查找最近24小时内完成的、还未转存的图片和视频
type DramaCount struct {
DramaID string
Count int64
}
// 统计每个剧本的待转存图片数量
var imageDramas []DramaCount
s.db.Raw(`
SELECT drama_id, COUNT(*) as count
FROM image_generations
WHERE status = 'completed'
AND image_url IS NOT NULL
AND image_url != ''
AND (minio_url IS NULL OR minio_url = '')
AND completed_at >= ?
GROUP BY drama_id
`, time.Now().Add(-24*time.Hour)).Scan(&imageDramas)
// 转存图片
imageCount := 0
for _, drama := range imageDramas {
count, err := s.transferService.BatchTransferImagesToMinio(drama.DramaID, 50) // 每个剧本最多转50个
if err != nil {
s.log.Errorw("Failed to transfer images for drama",
"drama_id", drama.DramaID,
"error", err)
continue
}
imageCount += count
s.log.Infow("Transferred images for drama",
"drama_id", drama.DramaID,
"count", count)
}
// 统计每个剧本的待转存视频数量
var videoDramas []DramaCount
s.db.Raw(`
SELECT drama_id, COUNT(*) as count
FROM video_generations
WHERE status = 'completed'
AND video_url IS NOT NULL
AND video_url != ''
AND (minio_url IS NULL OR minio_url = '')
AND completed_at >= ?
GROUP BY drama_id
`, time.Now().Add(-24*time.Hour)).Scan(&videoDramas)
// 转存视频
videoCount := 0
for _, drama := range videoDramas {
count, err := s.transferService.BatchTransferVideosToMinio(drama.DramaID, 50) // 每个剧本最多转50个
if err != nil {
s.log.Errorw("Failed to transfer videos for drama",
"drama_id", drama.DramaID,
"error", err)
continue
}
videoCount += count
s.log.Infow("Transferred videos for drama",
"drama_id", drama.DramaID,
"count", count)
}
s.log.Infow("Scheduled resource transfer task completed",
"images", imageCount,
"videos", videoCount)
}
// transferAllPendingResources 转存所有待转存的资源(全量扫描)
func (s *ResourceTransferScheduler) transferAllPendingResources() {
s.log.Info("Starting full scan for all pending resources...")
// 查找所有待转存的资源
type DramaCount struct {
DramaID string
Count int64
}
// 统计所有剧本的待转存图片
var imageDramas []DramaCount
s.db.Raw(`
SELECT drama_id, COUNT(*) as count
FROM image_generations
WHERE status = 'completed'
AND image_url IS NOT NULL
AND image_url != ''
AND (minio_url IS NULL OR minio_url = '')
GROUP BY drama_id
`).Scan(&imageDramas)
s.log.Infow("Found dramas with pending images", "count", len(imageDramas))
// 转存所有待转存图片
totalImageCount := 0
for _, drama := range imageDramas {
count, err := s.transferService.BatchTransferImagesToMinio(drama.DramaID, 0) // 0表示全部转存
if err != nil {
s.log.Errorw("Failed to transfer images for drama",
"drama_id", drama.DramaID,
"error", err)
continue
}
totalImageCount += count
s.log.Infow("Transferred all images for drama",
"drama_id", drama.DramaID,
"count", count)
}
// 统计所有剧本的待转存视频
var videoDramas []DramaCount
s.db.Raw(`
SELECT drama_id, COUNT(*) as count
FROM video_generations
WHERE status = 'completed'
AND video_url IS NOT NULL
AND video_url != ''
AND (minio_url IS NULL OR minio_url = '')
GROUP BY drama_id
`).Scan(&videoDramas)
s.log.Infow("Found dramas with pending videos", "count", len(videoDramas))
// 转存所有待转存视频
totalVideoCount := 0
for _, drama := range videoDramas {
count, err := s.transferService.BatchTransferVideosToMinio(drama.DramaID, 0) // 0表示全部转存
if err != nil {
s.log.Errorw("Failed to transfer videos for drama",
"drama_id", drama.DramaID,
"error", err)
continue
}
totalVideoCount += count
s.log.Infow("Transferred all videos for drama",
"drama_id", drama.DramaID,
"count", count)
}
s.log.Infow("Full resource scan and transfer completed",
"total_images", totalImageCount,
"total_videos", totalVideoCount,
"drama_count", len(imageDramas)+len(videoDramas))
}
// RunNow 立即执行一次转存任务(用于手动触发)
func (s *ResourceTransferScheduler) RunNow() {
s.log.Info("Manually triggering resource transfer task...")
go s.transferPendingResources()
}
// RunFullScan 立即执行一次全量扫描(用于手动触发)
func (s *ResourceTransferScheduler) RunFullScan() {
s.log.Info("Manually triggering full resource scan...")
go s.transferAllPendingResources()
}

View File

@@ -0,0 +1,137 @@
package storage
import (
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strings"
"time"
)
type LocalStorage struct {
basePath string
baseURL string
}
func NewLocalStorage(basePath, baseURL string) (*LocalStorage, error) {
if err := os.MkdirAll(basePath, 0755); err != nil {
return nil, fmt.Errorf("failed to create storage directory: %w", err)
}
return &LocalStorage{
basePath: basePath,
baseURL: baseURL,
}, nil
}
func (s *LocalStorage) Upload(file io.Reader, filename string, category string) (string, error) {
dir := filepath.Join(s.basePath, category)
if err := os.MkdirAll(dir, 0755); err != nil {
return "", fmt.Errorf("failed to create category directory: %w", err)
}
timestamp := time.Now().Format("20060102_150405")
newFilename := fmt.Sprintf("%s_%s", timestamp, filename)
filePath := filepath.Join(dir, newFilename)
dst, err := os.Create(filePath)
if err != nil {
return "", fmt.Errorf("failed to create file: %w", err)
}
defer dst.Close()
if _, err := io.Copy(dst, file); err != nil {
return "", fmt.Errorf("failed to save file: %w", err)
}
url := fmt.Sprintf("%s/%s/%s", s.baseURL, category, newFilename)
return url, nil
}
func (s *LocalStorage) Delete(url string) error {
return nil
}
func (s *LocalStorage) GetURL(path string) string {
return fmt.Sprintf("%s/%s", s.baseURL, path)
}
// DownloadFromURL 从远程URL下载文件到本地存储
func (s *LocalStorage) DownloadFromURL(url, category string) (string, error) {
// 发送HTTP请求下载文件
resp, err := http.Get(url)
if err != nil {
return "", fmt.Errorf("failed to download file: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("failed to download file: HTTP %d", resp.StatusCode)
}
// 从URL或Content-Type推断文件扩展名
ext := getFileExtension(url, resp.Header.Get("Content-Type"))
// 创建目录
dir := filepath.Join(s.basePath, category)
if err := os.MkdirAll(dir, 0755); err != nil {
return "", fmt.Errorf("failed to create category directory: %w", err)
}
// 生成唯一文件名
timestamp := time.Now().Format("20060102_150405_000")
filename := fmt.Sprintf("%s%s", timestamp, ext)
filePath := filepath.Join(dir, filename)
// 保存文件
dst, err := os.Create(filePath)
if err != nil {
return "", fmt.Errorf("failed to create file: %w", err)
}
defer dst.Close()
if _, err := io.Copy(dst, resp.Body); err != nil {
return "", fmt.Errorf("failed to save file: %w", err)
}
// 返回本地URL
localURL := fmt.Sprintf("%s/%s/%s", s.baseURL, category, filename)
return localURL, nil
}
// getFileExtension 从URL或Content-Type推断文件扩展名
func getFileExtension(url, contentType string) string {
// 首先尝试从URL获取扩展名
if idx := strings.LastIndex(url, "."); idx != -1 {
ext := url[idx:]
// 只取扩展名部分,忽略查询参数
if qIdx := strings.Index(ext, "?"); qIdx != -1 {
ext = ext[:qIdx]
}
if len(ext) <= 5 { // 合理的扩展名长度
return ext
}
}
// 根据Content-Type推断扩展名
switch {
case strings.Contains(contentType, "image/jpeg"):
return ".jpg"
case strings.Contains(contentType, "image/png"):
return ".png"
case strings.Contains(contentType, "image/gif"):
return ".gif"
case strings.Contains(contentType, "image/webp"):
return ".webp"
case strings.Contains(contentType, "video/mp4"):
return ".mp4"
case strings.Contains(contentType, "video/webm"):
return ".webm"
case strings.Contains(contentType, "video/quicktime"):
return ".mov"
default:
return ".bin"
}
}