2026_OA_AI_Integration/server
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
Joe
2026-01-16 15:49:34 +08:00
commit 550d3e1f42
380 changed files with 62024 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/console/commands/analyze_stats.go
+162
View File
@@ -0,0 +1,162 @@
package commands
import (
"fmt"
"strings"
"time"
"github.com/goravel/framework/contracts/console"
"github.com/goravel/framework/contracts/console/command"
"github.com/goravel/framework/facades"
"goravel/app/utils"
)
type AnalyzeStats struct {
}
func (r *AnalyzeStats) Signature() string {
return "db:analyze-stats"
}
func (r *AnalyzeStats) Description() string {
// # 默认：分析当前月+上个月的 orders/order_details 分表，并分析 payments 表
// go run . artisan db:analyze-stats
//
// # 指定向前分析几个月（含当前月）
// go run . artisan db:analyze-stats --months=2
// go run . artisan db:analyze-stats --months=6
//
// # 指定某一个月（只分析该月的分表）
// go run . artisan db:analyze-stats --month=202601
//
// # 只分析订单分表，不分析支付表
// go run . artisan db:analyze-stats --payments=false
//
// # 只分析 payments 表
// go run . artisan db:analyze-stats --orders=false --order-details=false
//
// # 帮助
// go run . artisan db:analyze-stats --help
return "更新订单分表与支付表统计信息（ANALYZE）"
}
func (r *AnalyzeStats) Extend() command.Extend {
return command.Extend{
Category: "db",
Flags: []command.Flag{
&command.StringFlag{
Name: "month",
Aliases: []string{"m"},
Usage: "指定月份(格式: YYYYMM,如:202512)，不指定则按 months 向前分析",
},
&command.IntFlag{
Name: "months",
Aliases: []string{"n"},
Value: 2,
Usage: "向前分析几个月（默认2：当前月+上个月）",
},
&command.BoolFlag{
Name: "orders",
Value: true,
Usage: "是否分析订单分表（orders_YYYYMM）",
},
&command.BoolFlag{
Name: "order-details",
Value: true,
Usage: "是否分析订单详情分表（order_details_YYYYMM）",
},
&command.BoolFlag{
Name: "payments",
Value: true,
Usage: "是否分析支付记录表（payments）",
},
},
}
}
func (r *AnalyzeStats) Handle(ctx console.Context) error {
dbConnection := strings.ToLower(facades.Config().GetString("database.default", "sqlite"))
monthsFlag := ctx.OptionInt("months")
if monthsFlag <= 0 {
monthsFlag = 2
}
monthFlag := strings.TrimSpace(ctx.Option("month"))
analyzeOrders := ctx.OptionBool("orders")
analyzeOrderDetails := ctx.OptionBool("order-details")
analyzePayments := ctx.OptionBool("payments")
var months []time.Time
if monthFlag != "" {
parsedTime, err := time.ParseInLocation("200601", monthFlag, time.UTC)
if err != nil {
return fmt.Errorf("月份格式错误，应为 YYYYMM 格式（如:202512): %v", err)
}
months = []time.Time{parsedTime}
} else {
now := time.Now().UTC()
currentMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)
for i := 0; i < monthsFlag; i++ {
months = append(months, currentMonth.AddDate(0, -i, 0))
}
}
ctx.Info("开始执行 ANALYZE...")
execAnalyze := func(table string) error {
var sql string
switch dbConnection {
case "mysql":
sql = fmt.Sprintf("ANALYZE TABLE `%s`", table)
case "postgres":
sql = fmt.Sprintf("ANALYZE %s", table)
default:
return fmt.Errorf("unsupported database: %s", dbConnection)
}
_, err := facades.Orm().Query().Exec(sql)
if err != nil {
return err
}
ctx.Info("✓ " + sql)
return nil
}
if analyzeOrders {
for _, m := range months {
table := utils.GetShardingTableName("orders", m)
if facades.Schema().HasTable(table) {
if err := execAnalyze(table); err != nil {
return fmt.Errorf("analyze %s failed: %v", table, err)
}
}
}
}
if analyzeOrderDetails {
for _, m := range months {
table := utils.GetShardingTableName("order_details", m)
if facades.Schema().HasTable(table) {
if err := execAnalyze(table); err != nil {
return fmt.Errorf("analyze %s failed: %v", table, err)
}
}
}
}
if analyzePayments {
if facades.Schema().HasTable("payments") {
if err := execAnalyze("payments"); err != nil {
return fmt.Errorf("analyze payments failed: %v", err)
}
}
}
ctx.Info("完成")
return nil
}
app/console/commands/clear_chunks.go
+183
View File
@@ -0,0 +1,183 @@
package commands
import (
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/goravel/framework/contracts/console"
"github.com/goravel/framework/contracts/console/command"
"github.com/goravel/framework/facades"
"github.com/goravel/framework/support/path"
"goravel/app/utils"
)
type ClearChunks struct {
}
// Signature The name and signature of the console command.
func (r *ClearChunks) Signature() string {
return "app:clear-chunks"
}
// Description The console command description.
func (r *ClearChunks) Description() string {
return "清理3天前的分片文件"
}
// Extend The console command extend.
func (r *ClearChunks) Extend() command.Extend {
return command.Extend{Category: "app"}
}
// Handle Execute the console command.
func (r *ClearChunks) Handle(ctx console.Context) error {
// 从数据库读取文件存储配置
disk := utils.GetConfigValue("storage", "file_disk", "")
if disk == "" {
disk = "local"
}
// 检查存储驱动是否为本地存储
if disk != "local" && disk != "public" {
ctx.Info(fmt.Sprintf("当前存储驱动为 %s，清理分片文件功能仅支持本地存储，跳过清理", disk))
return nil
}
storage := facades.Storage().Disk(disk)
// 计算3天前的时间
threeDaysAgo := time.Now().AddDate(0, 0, -3)
ctx.Info(fmt.Sprintf("开始清理3天前的分片文件（%s之前创建的文件）...", threeDaysAgo.Format(utils.DateTimeFormat)))
// 获取存储根目录
var storageRoot string
if disk == "public" {
storageRoot = path.Storage("app/public")
} else {
storageRoot = path.Storage("app")
}
chunksDir := filepath.Join(storageRoot, "chunks")
// 检查目录是否存在
if _, err := os.Stat(chunksDir); os.IsNotExist(err) {
ctx.Info("分片目录不存在，无需清理")
return nil
}
cleanedCount := 0
cleanedSize := int64(0)
errorCount := 0
// 遍历 chunks 目录下的所有文件
err := filepath.Walk(chunksDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
// 如果无法访问某个文件/目录，记录错误但继续
ctx.Info(fmt.Sprintf("无法访问路径 %s: %v", path, err))
errorCount++
return nil
}
// 跳过根目录本身
if path == chunksDir {
return nil
}
// 只处理文件（分片文件）
if !info.IsDir() {
// 跳过 .gitignore 文件，避免误删
if info.Name() == ".gitignore" {
return nil
}
// 检查文件修改时间
if info.ModTime().Before(threeDaysAgo) {
// 使用 Storage 接口删除文件（保持一致性）
relativePath := strings.TrimPrefix(path, storageRoot+string(filepath.Separator))
relativePath = strings.ReplaceAll(relativePath, string(filepath.Separator), "/")
if err := storage.Delete(relativePath); err != nil {
ctx.Info(fmt.Sprintf("删除分片文件失败 %s: %v", relativePath, err))
errorCount++
} else {
cleanedCount++
cleanedSize += info.Size()
}
}
}
return nil
})
if err != nil {
ctx.Error(fmt.Sprintf("遍历分片目录失败: %v", err))
return err
}
// 清理空目录（避免留下大量空目录）
emptyDirCount := r.cleanupEmptyDirs(chunksDir, ctx)
ctx.Info(fmt.Sprintf("分片文件清理完成！已清理 %d 个文件，释放空间 %s，删除 %d 个空目录，错误数: %d",
cleanedCount,
r.formatFileSize(cleanedSize),
emptyDirCount,
errorCount))
return nil
}
// cleanupEmptyDirs 清理空目录
func (r *ClearChunks) cleanupEmptyDirs(rootDir string, _ console.Context) int {
var dirs []string
err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return nil
}
if info.IsDir() && path != rootDir {
dirs = append(dirs, path)
}
return nil
})
if err != nil {
return 0
}
// 反向遍历目录（从最深层的开始）
removedCount := 0
for i := len(dirs) - 1; i >= 0; i-- {
dir := dirs[i]
// 检查目录是否为空
entries, err := os.ReadDir(dir)
if err != nil {
continue
}
if len(entries) == 0 {
// 目录为空，删除它
if err := os.Remove(dir); err == nil {
removedCount++
}
}
}
return removedCount
}
// formatFileSize 格式化文件大小
func (r *ClearChunks) formatFileSize(size int64) string {
const unit = 1024
if size < unit {
return fmt.Sprintf("%d B", size)
}
div, exp := int64(unit), 0
for n := size / unit; n >= unit; n /= unit {
div *= unit
exp++
}
return fmt.Sprintf("%.1f %cB", float64(size)/float64(div), "KMGTPE"[exp])
}
app/console/commands/clear_logs.go
+71
View File
@@ -0,0 +1,71 @@
package commands
import (
"fmt"
"time"
"github.com/goravel/framework/contracts/console"
"github.com/goravel/framework/contracts/console/command"
"github.com/goravel/framework/facades"
"goravel/app/models"
)
type ClearLogs struct {
}
// Signature The name and signature of the console command.
func (r *ClearLogs) Signature() string {
return "app:clear-logs"
}
// Description The console command description.
func (r *ClearLogs) Description() string {
return "清理6个月前的日志记录（操作日志、登录日志、系统日志）"
}
// Extend The console command extend.
func (r *ClearLogs) Extend() command.Extend {
return command.Extend{Category: "app"}
}
// Handle Execute the console command.
func (r *ClearLogs) Handle(ctx console.Context) error {
// 计算6个月前的日期
monthsAgo := time.Now().AddDate(0, -6, 0)
ctx.Info("开始清理6个月前的日志...")
// 清理操作日志
operationLogResult, err := facades.Orm().Query().Model(&models.OperationLog{}).
Where("created_at < ?", monthsAgo).
Delete(&models.OperationLog{})
if err != nil {
ctx.Error("清理操作日志失败: " + err.Error())
return err
}
ctx.Info(fmt.Sprintf("已清理操作日志: %d 条", operationLogResult.RowsAffected))
// 清理登录日志
loginLogResult, err := facades.Orm().Query().Model(&models.LoginLog{}).
Where("created_at < ?", monthsAgo).
Delete(&models.LoginLog{})
if err != nil {
ctx.Error("清理登录日志失败: " + err.Error())
return err
}
ctx.Info(fmt.Sprintf("已清理登录日志: %d 条", loginLogResult.RowsAffected))
// 清理系统日志
systemLogResult, err := facades.Orm().Query().Model(&models.SystemLog{}).
Where("created_at < ?", monthsAgo).
Delete(&models.SystemLog{})
if err != nil {
ctx.Error("清理系统日志失败: " + err.Error())
return err
}
ctx.Info(fmt.Sprintf("已清理系统日志: %d 条", systemLogResult.RowsAffected))
ctx.Info("日志清理完成！")
return nil
}
app/console/commands/create_order_sharding_tables.go
+141
View File
@@ -0,0 +1,141 @@
package commands
import (
"context"
"fmt"
"time"
"github.com/goravel/framework/contracts/console"
"github.com/goravel/framework/contracts/console/command"
"github.com/goravel/framework/facades"
"goravel/app/services"
"goravel/app/utils"
"goravel/app/utils/errorlog"
)
type CreateOrderShardingTables struct {
shardingService services.ShardingService
}
func NewCreateOrderShardingTables() *CreateOrderShardingTables {
return &CreateOrderShardingTables{
shardingService: services.NewShardingService(),
}
}
// Signature The name and signature of the console command.
func (r *CreateOrderShardingTables) Signature() string {
return "order:create-sharding-tables"
}
// Description The console command description.
func (r *CreateOrderShardingTables) Description() string {
// # 创建分表（默认创建上个月、当前月份及未来2个月，共4个月）
// go run . artisan order:create-sharding-tables
// # 创建指定月份的分表
// go run . artisan order:create-sharding-tables --month=202512
// # 创建上个月、当前月份及未来N个月的分表（--months 参数包括上个月和当前月）
// # 例如：--months=6 会创建上个月、当前月及未来4个月，共6个月
// go run . artisan order:create-sharding-tables --months=6
// # 帮助
// go run . artisan order:create-sharding-tables --help
return "创建订单分表（按月分表，可指定月份或创建上个月、当前月份及未来几个月）"
}
// Extend The console command extend.
func (r *CreateOrderShardingTables) Extend() command.Extend {
return command.Extend{
Category: "order",
Flags: []command.Flag{
&command.StringFlag{
Name: "month",
Aliases: []string{"m"},
Usage: "指定月份(格式: YYYYMM,如:202512),不指定则创建当前月份",
},
&command.IntFlag{
Name: "months",
Aliases: []string{"n"},
Value: 4,
Usage: "创建几个月(默认4个月,包括上个月、当前月份及未来2个月)",
},
},
}
}
// Handle Execute the console command.
func (r *CreateOrderShardingTables) Handle(ctx console.Context) error {
var months []time.Time
monthFlag := ctx.Option("month")
monthsFlag := ctx.OptionInt("months")
if monthFlag != "" {
// 指定月份（解析为 UTC 时区，与分表逻辑保持一致）
parsedTime, err := time.ParseInLocation("200601", monthFlag, time.UTC)
if err != nil {
return fmt.Errorf("月份格式错误，应为 YYYYMM 格式（如:202512): %v", err)
}
months = []time.Time{parsedTime}
} else {
// 创建上个月、当前月份及未来几个月（使用 UTC 时区，与分表逻辑保持一致）
// 默认：上个月(-1)、当前月(0)、未来2个月(1,2)，共4个月
now := time.Now().UTC()
currentMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)
// 从 -1 开始（上个月），到 monthsFlag-2（未来几个月）
// 例如：monthsFlag=4 时，创建：上个月(-1)、当前月(0)、未来1个月(1)、未来2个月(2)
startOffset := -1 // 从上个月开始
for i := startOffset; i < monthsFlag-1; i++ {
month := currentMonth.AddDate(0, i, 0)
months = append(months, month)
}
}
createdCount := 0
skippedCount := 0
for _, month := range months {
tableName := utils.GetShardingTableName("orders", month)
detailTableName := utils.GetShardingTableName("order_details", month)
// 创建订单主表
if facades.Schema().HasTable(tableName) {
ctx.Info(fmt.Sprintf("分表 %s 已存在，跳过", tableName))
skippedCount++
} else {
if err := r.shardingService.CreateShardingTable(tableName, "orders"); err != nil {
errorlog.Record(context.Background(), "sharding", "创建分表失败", map[string]any{
"table_name": tableName,
"base_table_name": "orders",
"error": err.Error(),
}, "创建分表 %s 失败: %v", tableName, err)
return fmt.Errorf("创建分表 %s 失败: %v", tableName, err)
}
ctx.Info(fmt.Sprintf("✓ 创建分表: %s", tableName))
createdCount++
}
// 创建订单详情表
if facades.Schema().HasTable(detailTableName) {
ctx.Info(fmt.Sprintf("分表 %s 已存在，跳过", detailTableName))
} else {
if err := r.shardingService.CreateShardingTable(detailTableName, "order_details"); err != nil {
errorlog.Record(context.Background(), "sharding", "创建分表失败", map[string]any{
"table_name": detailTableName,
"base_table_name": "order_details",
"error": err.Error(),
}, "创建分表 %s 失败: %v", detailTableName, err)
return fmt.Errorf("创建分表 %s 失败: %v", detailTableName, err)
}
ctx.Info(fmt.Sprintf("✓ 创建分表: %s", detailTableName))
}
}
ctx.Info(fmt.Sprintf("\n完成！创建了 %d 个分表，跳过了 %d 个已存在的分表", createdCount, skippedCount))
return nil
}
app/console/commands/create_payment_sharding_tables.go
+121
View File
@@ -0,0 +1,121 @@
package commands
import (
"context"
"fmt"
"time"
"github.com/goravel/framework/contracts/console"
"github.com/goravel/framework/contracts/console/command"
"github.com/goravel/framework/facades"
"goravel/app/utils"
"goravel/app/utils/errorlog"
"goravel/database/migrations"
)
type CreatePaymentShardingTables struct {
}
func NewCreatePaymentShardingTables() *CreatePaymentShardingTables {
return &CreatePaymentShardingTables{}
}
// Signature The name and signature of the console command.
func (r *CreatePaymentShardingTables) Signature() string {
return "payment:create-sharding-tables"
}
// Description The console command description.
func (r *CreatePaymentShardingTables) Description() string {
// # 创建分表（默认创建上个月、当前月份及未来2个月，共4个月）
// go run . artisan payment:create-sharding-tables
// # 创建指定月份的分表
// go run . artisan payment:create-sharding-tables --month=202512
// # 创建上个月、当前月份及未来N个月的分表（--months 参数包括上个月和当前月）
// # 例如：--months=6 会创建上个月、当前月及未来4个月，共6个月
// go run . artisan payment:create-sharding-tables --months=6
// # 帮助
// go run . artisan payment:create-sharding-tables --help
return "创建支付记录分表（按月分表，可指定月份或创建上个月、当前月份及未来几个月）"
}
// Extend The console command extend.
func (r *CreatePaymentShardingTables) Extend() command.Extend {
return command.Extend{
Category: "payment",
Flags: []command.Flag{
&command.StringFlag{
Name: "month",
Aliases: []string{"m"},
Usage: "指定月份(格式: YYYYMM,如:202512),不指定则创建当前月份",
},
&command.IntFlag{
Name: "months",
Aliases: []string{"n"},
Value: 4,
Usage: "创建几个月(默认4个月,包括上个月、当前月份及未来2个月)",
},
},
}
}
// Handle Execute the console command.
func (r *CreatePaymentShardingTables) Handle(ctx console.Context) error {
var months []time.Time
monthFlag := ctx.Option("month")
monthsFlag := ctx.OptionInt("months")
if monthFlag != "" {
// 指定月份（解析为 UTC 时区，与分表逻辑保持一致）
parsedTime, err := time.ParseInLocation("200601", monthFlag, time.UTC)
if err != nil {
return fmt.Errorf("月份格式错误，应为 YYYYMM 格式（如:202512): %v", err)
}
months = []time.Time{parsedTime}
} else {
// 创建上个月、当前月份及未来几个月（使用 UTC 时区，与分表逻辑保持一致）
// 默认：上个月(-1)、当前月(0)、未来2个月(1,2)，共4个月
now := time.Now().UTC()
currentMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)
// 从 -1 开始（上个月），到 monthsFlag-2（未来几个月）
// 例如：monthsFlag=4 时，创建：上个月(-1)、当前月(0)、未来1个月(1)、未来2个月(2)
startOffset := -1 // 从上个月开始
for i := startOffset; i < monthsFlag-1; i++ {
month := currentMonth.AddDate(0, i, 0)
months = append(months, month)
}
}
createdCount := 0
skippedCount := 0
for _, month := range months {
tableName := utils.GetShardingTableName("payments", month)
// 创建支付记录分表
if facades.Schema().HasTable(tableName) {
ctx.Info(fmt.Sprintf("分表 %s 已存在，跳过", tableName))
skippedCount++
} else {
if err := migrations.CreatePaymentsShardingTable(tableName); err != nil {
errorlog.Record(context.Background(), "sharding", "创建支付记录分表失败", map[string]any{
"table_name": tableName,
"error": err.Error(),
}, "创建支付记录分表 %s 失败: %v", tableName, err)
return fmt.Errorf("创建支付记录分表 %s 失败: %v", tableName, err)
}
ctx.Info(fmt.Sprintf("✓ 创建分表: %s", tableName))
createdCount++
}
}
ctx.Info(fmt.Sprintf("\n完成！创建了 %d 个分表，跳过了 %d 个已存在的分表", createdCount, skippedCount))
return nil
}
app/console/commands/create_token.go
+187
View File
@@ -0,0 +1,187 @@
package commands
import (
"errors"
"strconv"
"strings"
"time"
"github.com/goravel/framework/contracts/console"
"github.com/goravel/framework/contracts/console/command"
"github.com/goravel/framework/facades"
"goravel/app/models"
"goravel/app/services"
"goravel/app/utils"
)
// # Admin用户 - 1小时后过期
// go run . artisan token:create 1 --expires=1h
// # 7天后过期
// go run . artisan token:create 1 --expires=7d
// # 使用简写
// go run . artisan token:create 1 -e=1h
// # 指定token名称
// go run . artisan token:create 1 --expires=7d --name=api-token
// # 创建永久token（不设置expires）
// go run . artisan token:create 1
type CreateToken struct {
}
// Signature The name and signature of the console command.
func (receiver *CreateToken) Signature() string {
return "token:create"
}
// Description The console command description.
func (receiver *CreateToken) Description() string {
return "为指定管理员创建token（永久或指定过期时间）"
}
// Extend The console command extend.
func (receiver *CreateToken) Extend() command.Extend {
return command.Extend{
Category: "token",
Flags: []command.Flag{
&command.StringFlag{
Name: "expires",
Aliases: []string{"e"},
Usage: "过期时间，格式：1h（1小时）、24h（24小时）、7d（7天）等，不设置则创建永久token",
},
&command.StringFlag{
Name: "name",
Aliases: []string{"n"},
Value: "console-token",
Usage: "token名称",
},
},
}
}
// Handle Execute the console command.
func (receiver *CreateToken) Handle(ctx console.Context) error {
// 获取用户标识（ID或用户名）
userIdentifier := ctx.Argument(0)
if userIdentifier == "" {
return errors.New("请提供用户ID或用户名")
}
// 用户类型固定为 admin
userType := "admin"
// 获取过期时间选项
expiresStr := ctx.Option("expires")
var expiresAt *time.Time
if expiresStr != "" {
// 解析过期时间
duration, err := parseDuration(expiresStr)
if err != nil {
return errors.New("过期时间格式错误，请使用：1h（1小时）、24h（24小时）、7d（7天）等格式")
}
exp := time.Now().Add(duration)
expiresAt = &exp
}
// 如果 expiresStr 为空，expiresAt 为 nil，表示永久token
// 获取token名称
tokenName := ctx.Option("name")
if tokenName == "" {
tokenName = "console-token"
}
// 查询管理员
var admin models.Admin
var err error
// 尝试作为ID解析
if id, parseErr := strconv.ParseUint(userIdentifier, 10, 32); parseErr == nil {
// 作为ID查询
err = facades.Orm().Query().Where("id", uint(id)).First(&admin)
} else {
// 作为用户名查询
err = facades.Orm().Query().Where("username", userIdentifier).First(&admin)
}
if err != nil {
return errors.New("管理员不存在")
}
userID := admin.ID
username := admin.Username
// 创建token
tokenService := services.NewTokenServiceImpl()
// 命令行创建token时，浏览器、IP、操作系统信息为空
plainToken, accessToken, err := tokenService.CreateToken(userType, userID, tokenName, expiresAt, "", "", "", "")
if err != nil {
return errors.New("创建token失败: " + err.Error())
}
// 输出结果
ctx.Info("Token创建成功！")
ctx.Line("")
ctx.Info("用户信息:")
ctx.Line(" 类型: " + userType)
ctx.Line(" ID: " + strconv.FormatUint(uint64(userID), 10))
ctx.Line(" 用户名: " + username)
ctx.Line("")
ctx.Info("Token信息:")
ctx.Line(" 名称: " + accessToken.Name)
if accessToken.ExpiresAt != nil {
ctx.Line(" 过期时间: " + accessToken.ExpiresAt.Format(utils.DateTimeFormat))
} else {
ctx.Line(" 过期时间: 永久有效")
}
ctx.Line(" 创建时间: " + accessToken.CreatedAt.Format(utils.DateTimeFormat))
ctx.Line("")
ctx.Warning("请妥善保管以下token，它只会显示一次：")
ctx.Line("")
ctx.Line(plainToken)
ctx.Line("")
return nil
}
// parseDuration 解析时间字符串，支持 h（小时）、d（天）、m（分钟）等格式
func parseDuration(s string) (time.Duration, error) {
// 移除空格
s = strings.TrimSpace(s)
if len(s) == 0 {
return 0, errors.New("时间字符串为空")
}
// 获取最后一个字符作为单位
unit := s[len(s)-1:]
valueStr := s[:len(s)-1]
// 解析数值
value, err := strconv.ParseFloat(valueStr, 64)
if err != nil {
return 0, err
}
// 根据单位转换为duration
switch unit {
case "m", "M":
// 分钟
return time.Duration(value) * time.Minute, nil
case "h", "H":
// 小时
return time.Duration(value) * time.Hour, nil
case "d", "D":
// 天
return time.Duration(value) * 24 * time.Hour, nil
case "w", "W":
// 周
return time.Duration(value) * 7 * 24 * time.Hour, nil
default:
// 尝试直接解析为duration（如 "1h30m"）
return time.ParseDuration(s)
}
}
app/console/commands/generate_test_orders.go
+510
View File
@@ -0,0 +1,510 @@
package commands
import (
"fmt"
"math/rand"
"strconv"
"sync"
"time"
"github.com/goravel/framework/contracts/console"
"github.com/goravel/framework/contracts/console/command"
"github.com/goravel/framework/facades"
"github.com/goravel/framework/support/carbon"
"github.com/oklog/ulid/v2"
"goravel/app/models"
"goravel/app/services"
"goravel/app/utils"
)
// GenerateTestOrders 生成测试订单数据
type GenerateTestOrders struct {
shardingService services.ShardingService
}
// Signature The name and signature of the console command.
func (receiver *GenerateTestOrders) Signature() string {
return "order:generate-test-data"
}
// # 使用默认10个并发协程
// go run . artisan order:generate-test-data --count=1000000
// # 使用20个并发协程（更快）
// go run . artisan order:generate-test-data --count=1000000 --workers=20
// # 根据服务器性能调整并发数
// go run . artisan order:generate-test-data --count=1000000 --workers=50 --batch-size=2000
// Description The console command description.
func (receiver *GenerateTestOrders) Description() string {
return "生成订单测试数据（用于测试订单导出等功能）"
}
// Extend The console command extend.
func (receiver *GenerateTestOrders) Extend() command.Extend {
return command.Extend{
Category: "order",
Flags: []command.Flag{
&command.IntFlag{
Name: "count",
Aliases: []string{"c"},
Value: 1000000,
Usage: "要生成的订单数量（默认：1000000）",
},
&command.IntFlag{
Name: "batch-size",
Aliases: []string{"b"},
Value: 1000,
Usage: "批量插入的大小（默认：1000）",
},
&command.StringFlag{
Name: "start-date",
Aliases: []string{"s"},
Usage: "开始日期（格式：YYYY-MM-DD，默认：当前月份的第一天）",
},
&command.StringFlag{
Name: "end-date",
Aliases: []string{"e"},
Usage: "结束日期（格式：YYYY-MM-DD，默认：当前日期）",
},
&command.IntFlag{
Name: "min-user-id",
Value: 1,
Usage: "最小用户ID（默认：1）",
},
&command.IntFlag{
Name: "max-user-id",
Value: 1000,
Usage: "最大用户ID（默认：1000）",
},
&command.IntFlag{
Name: "workers",
Aliases: []string{"w"},
Value: 10,
Usage: "并发工作协程数量（默认：10）",
},
},
}
}
// Handle Execute the console command.
func (receiver *GenerateTestOrders) Handle(ctx console.Context) error {
receiver.shardingService = services.NewShardingService()
// 获取参数
count := ctx.OptionInt("count")
batchSize := ctx.OptionInt("batch-size")
startDateStr := ctx.Option("start-date")
endDateStr := ctx.Option("end-date")
minUserID := ctx.OptionInt("min-user-id")
maxUserID := ctx.OptionInt("max-user-id")
workers := ctx.OptionInt("workers")
// 解析日期
var startDate, endDate time.Time
var err error
if startDateStr != "" {
startDate, err = utils.ParseDate(startDateStr)
if err != nil {
return fmt.Errorf("开始日期格式错误，请使用 YYYY-MM-DD 格式: %v", err)
}
} else {
// 默认：当前月份的第一天
now := time.Now().UTC()
startDate = time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)
}
if endDateStr != "" {
endDate, err = utils.ParseDate(endDateStr)
if err != nil {
return fmt.Errorf("结束日期格式错误，请使用 YYYY-MM-DD 格式: %v", err)
}
// 设置为当天的最后一秒
endDate = time.Date(endDate.Year(), endDate.Month(), endDate.Day(), 23, 59, 59, 999999999, time.UTC)
} else {
// 默认：当前时间
endDate = time.Now().UTC()
}
if startDate.After(endDate) {
return fmt.Errorf("开始日期不能晚于结束日期")
}
// 验证用户ID范围
if minUserID <= 0 || maxUserID <= 0 || minUserID > maxUserID {
return fmt.Errorf("用户ID范围无效")
}
ctx.Info("开始生成测试订单数据...")
ctx.Line("")
ctx.Info(fmt.Sprintf("订单数量: %s", formatNumber(count)))
ctx.Info(fmt.Sprintf("批量大小: %s", formatNumber(batchSize)))
ctx.Info(fmt.Sprintf("并发协程: %d", workers))
ctx.Info(fmt.Sprintf("开始日期: %s", utils.FormatDate(startDate)))
ctx.Info(fmt.Sprintf("结束日期: %s", utils.FormatDate(endDate)))
ctx.Info(fmt.Sprintf("用户ID范围: %d - %d", minUserID, maxUserID))
ctx.Line("")
// 计算时间范围（秒）
timeRange := endDate.Sub(startDate).Seconds()
if timeRange <= 0 {
return fmt.Errorf("时间范围无效")
}
// 生成订单状态列表
statuses := []string{"pending", "paid", "cancelled"}
// 生成商品列表（用于订单详情）
products := []struct {
ID uint
Name string
Price float64
}{
{1, "商品A", 99.99},
{2, "商品B", 199.99},
{3, "商品C", 299.99},
{4, "商品D", 49.99},
{5, "商品E", 149.99},
{6, "商品F", 79.99},
{7, "商品G", 249.99},
{8, "商品H", 89.99},
}
// 开始生成
startTime := time.Now()
totalInserted := 0
batches := (count + batchSize - 1) / batchSize // 向上取整
// 使用随机种子
rand.Seed(time.Now().UnixNano())
for range batches {
// 计算本批次要插入的数量
remaining := count - totalInserted
currentBatchSize := min(remaining, batchSize)
// 准备批量数据
orders := make([]models.Order, 0, currentBatchSize)
for i := range currentBatchSize {
// 随机生成订单时间（在指定时间范围内）
randomSeconds := rand.Float64() * timeRange
orderTime := startDate.Add(time.Duration(randomSeconds) * time.Second)
// 生成订单号
yearMonth := orderTime.Format("200601")
ulidStr := ulid.MustNew(ulid.Timestamp(orderTime), ulid.DefaultEntropy()).String()
orderNo := fmt.Sprintf("ORD%s%s", yearMonth, ulidStr)
// 随机生成用户ID
userID := uint(minUserID + rand.Intn(maxUserID-minUserID+1))
// 随机生成订单金额（10.00 - 9999.99）
amount := 10.0 + rand.Float64()*(9999.99-10.0)
// 随机选择订单状态
status := statuses[rand.Intn(len(statuses))]
// 将time.Time转换为时间字符串
timeStr := utils.FormatDateTime(orderTime)
// 创建订单对象用于后续处理
order := models.Order{
OrderNo: orderNo,
UserID: userID,
Amount: amount,
Status: status,
Remark: fmt.Sprintf("测试订单-%d", totalInserted+i+1),
}
// 设置CreatedAt用于分表计算（使用carbon.NewDateTime）
order.CreatedAt = carbon.NewDateTime(carbon.Parse(timeStr))
order.UpdatedAt = carbon.NewDateTime(carbon.Parse(timeStr))
// 存储订单对象
orders = append(orders, order)
}
// 按分表分组插入订单，同时准备订单数据map
ordersByTable := make(map[string][]models.Order)
orderDataByTable := make(map[string][]map[string]any)
for _, order := range orders {
// 从carbon.DateTime获取time.Time用于分表
timeStr := order.CreatedAt.ToDateTimeString()
orderTime, _ := utils.ParseDateTimeUTC(timeStr)
tableName := utils.GetShardingTableName("orders", orderTime)
ordersByTable[tableName] = append(ordersByTable[tableName], order)
// 准备订单数据map
orderData := map[string]any{
"order_no": order.OrderNo,
"user_id": order.UserID,
"amount": order.Amount,
"status": order.Status,
"remark": order.Remark,
"created_at": timeStr,
"updated_at": timeStr,
}
orderDataByTable[tableName] = append(orderDataByTable[tableName], orderData)
}
// 并发插入订单
var wg sync.WaitGroup
var mu sync.Mutex
workerChan := make(chan struct{}, workers) // 控制并发数量
errChan := make(chan error, len(ordersByTable))
orderIDMap := make(map[string]uint) // 订单号到ID的映射（线程安全）
for tableName, tableOrders := range ordersByTable {
wg.Add(1)
workerChan <- struct{}{} // 获取工作协程
go func(tn string, tos []models.Order, odl []map[string]any) {
defer wg.Done()
defer func() { <-workerChan }() // 释放工作协程
// 确保分表存在
if err := receiver.shardingService.EnsureShardingTable(tn, "orders"); err != nil {
errChan <- fmt.Errorf("确保分表 %s 存在失败: %v", tn, err)
return
}
// 分批插入订单
for i := 0; i < len(odl); i += 100 {
end := min(i+100, len(odl))
batchData := odl[i:end]
// 批量插入
for j := range batchData {
if err := facades.Orm().Query().Table(tn).Create(batchData[j]); err != nil {
errChan <- fmt.Errorf("插入订单失败: %v", err)
return
}
}
// 批量查询ID（优化：减少查询次数）
orderNos := make([]string, 0, len(batchData))
for j := range batchData {
if orderNo, ok := batchData[j]["order_no"].(string); ok {
orderNos = append(orderNos, orderNo)
}
}
// 批量查询订单ID
if len(orderNos) > 0 {
var insertedOrders []models.Order
if err := facades.Orm().Query().Table(tn).Where("order_no IN ?", orderNos).Find(&insertedOrders); err == nil {
// 更新全局订单ID映射
mu.Lock()
for k := range insertedOrders {
orderIDMap[insertedOrders[k].OrderNo] = insertedOrders[k].ID
}
mu.Unlock()
}
}
}
}(tableName, tableOrders, orderDataByTable[tableName])
}
// 等待所有goroutine完成
go func() {
wg.Wait()
close(errChan)
}()
// 检查错误
for err := range errChan {
if err != nil {
return err
}
}
// 更新所有订单的ID
mu.Lock()
for tableName := range ordersByTable {
for i := range ordersByTable[tableName] {
if id, exists := orderIDMap[ordersByTable[tableName][i].OrderNo]; exists {
ordersByTable[tableName][i].ID = id
}
}
}
mu.Unlock()
// 为每个订单生成订单详情
orderDetailsMap := make(map[string][]models.OrderDetail) // key: tableName, value: details
for _, tableOrders := range ordersByTable {
for i := range tableOrders {
order := &tableOrders[i]
// 从carbon.DateTime获取time.Time用于分表
timeStr := order.CreatedAt.ToDateTimeString()
orderTime, _ := utils.ParseDateTimeUTC(timeStr)
detailTableName := utils.GetShardingTableName("order_details", orderTime)
// 确保订单详情分表存在
if err := receiver.shardingService.EnsureShardingTable(detailTableName, "order_details"); err != nil {
return fmt.Errorf("确保分表 %s 存在失败: %v", detailTableName, err)
}
// 随机生成1-3个商品
productCount := 1 + rand.Intn(3)
details := make([]models.OrderDetail, 0, productCount)
for j := 0; j < productCount; j++ {
product := products[rand.Intn(len(products))]
quantity := 1 + rand.Intn(5) // 1-5个
subtotal := product.Price * float64(quantity)
detail := models.OrderDetail{
OrderID: order.ID,
ProductID: product.ID,
ProductName: product.Name,
Price: product.Price,
Quantity: quantity,
Subtotal: subtotal,
}
// 设置CreatedAt和UpdatedAt（使用订单的创建时间）
detail.CreatedAt = order.CreatedAt
detail.UpdatedAt = order.CreatedAt
details = append(details, detail)
}
// 将详情添加到对应分表的列表中
if _, exists := orderDetailsMap[detailTableName]; !exists {
orderDetailsMap[detailTableName] = make([]models.OrderDetail, 0)
}
orderDetailsMap[detailTableName] = append(orderDetailsMap[detailTableName], details...)
}
}
// 并发插入订单详情
detailWg := sync.WaitGroup{}
detailErrChan := make(chan error, len(orderDetailsMap))
for detailTableName, details := range orderDetailsMap {
if len(details) == 0 {
continue
}
detailWg.Add(1)
workerChan <- struct{}{} // 获取工作协程
go func(dtn string, dets []models.OrderDetail) {
defer detailWg.Done()
defer func() { <-workerChan }() // 释放工作协程
// 确保订单详情分表存在
if err := receiver.shardingService.EnsureShardingTable(dtn, "order_details"); err != nil {
detailErrChan <- fmt.Errorf("确保分表 %s 存在失败: %v", dtn, err)
return
}
// 分批插入订单详情
for i := 0; i < len(dets); i += 100 {
end := min(i+100, len(dets))
batchDetails := dets[i:end]
// 批量插入
for j := range batchDetails {
// 准备详情数据map
// 安全处理CreatedAt，如果为nil则使用当前时间
var createdAtStr string
if batchDetails[j].CreatedAt != nil && !batchDetails[j].CreatedAt.IsZero() {
createdAtStr = batchDetails[j].CreatedAt.ToDateTimeString()
} else {
// 如果CreatedAt为nil，使用当前时间
createdAtStr = utils.FormatDateTime(time.Now().UTC())
}
detailData := map[string]any{
"order_id": batchDetails[j].OrderID,
"product_id": batchDetails[j].ProductID,
"product_name": batchDetails[j].ProductName,
"price": batchDetails[j].Price,
"quantity": batchDetails[j].Quantity,
"subtotal": batchDetails[j].Subtotal,
"created_at": createdAtStr,
"updated_at": createdAtStr,
}
if err := facades.Orm().Query().Table(dtn).Create(detailData); err != nil {
detailErrChan <- fmt.Errorf("插入订单详情失败: %v", err)
return
}
}
}
}(detailTableName, details)
}
// 等待所有详情插入完成
go func() {
detailWg.Wait()
close(detailErrChan)
}()
// 检查错误
for err := range detailErrChan {
if err != nil {
return err
}
}
totalInserted += currentBatchSize
// 显示进度
progress := float64(totalInserted) / float64(count) * 100
elapsed := time.Since(startTime)
rate := float64(totalInserted) / elapsed.Seconds()
remainingCount := count - totalInserted
eta := time.Duration(float64(remainingCount)/rate) * time.Second
ctx.Info(fmt.Sprintf("进度: %s/%s (%.2f%%) | 已用时间: %s | 速度: %.0f 条/秒 | 预计剩余: %s",
formatNumber(totalInserted),
formatNumber(count),
progress,
formatDuration(elapsed),
rate,
formatDuration(eta),
))
}
// 完成
elapsed := time.Since(startTime)
ctx.Line("")
ctx.Info(fmt.Sprintf("✅ 成功生成 %s 条订单数据！", formatNumber(totalInserted)))
ctx.Info(fmt.Sprintf("总耗时: %s", formatDuration(elapsed)))
ctx.Info(fmt.Sprintf("平均速度: %.0f 条/秒", float64(totalInserted)/elapsed.Seconds()))
return nil
}
// formatNumber 格式化数字（添加千位分隔符）
func formatNumber(n int) string {
str := strconv.Itoa(n)
if len(str) <= 3 {
return str
}
result := ""
for i, c := range str {
if i > 0 && (len(str)-i)%3 == 0 {
result += ","
}
result += string(c)
}
return result
}
// formatDuration 格式化时间间隔
func formatDuration(d time.Duration) string {
if d < time.Minute {
return fmt.Sprintf("%.0f秒", d.Seconds())
} else if d < time.Hour {
return fmt.Sprintf("%.0f分钟", d.Minutes())
} else {
hours := int(d.Hours())
minutes := int(d.Minutes()) % 60
return fmt.Sprintf("%d小时%d分钟", hours, minutes)
}
}
app/console/commands/generate_test_payments.go
+366
View File
@@ -0,0 +1,366 @@
package commands
import (
"context"
"fmt"
"math/rand"
"strconv"
"sync"
"time"
"github.com/goravel/framework/contracts/console"
"github.com/goravel/framework/contracts/console/command"
"github.com/goravel/framework/facades"
"github.com/oklog/ulid/v2"
"goravel/app/models"
"goravel/app/services"
"goravel/app/utils"
"goravel/app/utils/errorlog"
"goravel/database/migrations"
)
// GenerateTestPayments 生成测试支付记录数据
type GenerateTestPayments struct {
paymentService services.PaymentService
}
// Signature The name and signature of the console command.
func (receiver *GenerateTestPayments) Signature() string {
return "payment:generate-test-data"
}
// # 使用默认10个并发协程
// go run . artisan payment:generate-test-data --count=1000000
// # 使用20个并发协程（更快）
// go run . artisan payment:generate-test-data --count=1000000 --workers=20
// # 根据服务器性能调整并发数
// go run . artisan payment:generate-test-data --count=1000000 --workers=50 --batch-size=2000
// Description The console command description.
func (receiver *GenerateTestPayments) Description() string {
return "生成支付记录测试数据（用于测试支付记录导出等功能）"
}
// Extend The console command extend.
func (receiver *GenerateTestPayments) Extend() command.Extend {
return command.Extend{
Category: "payment",
Flags: []command.Flag{
&command.IntFlag{
Name: "count",
Aliases: []string{"c"},
Value: 10000,
Usage: "生成的记录数量",
},
&command.IntFlag{
Name: "workers",
Aliases: []string{"w"},
Value: 10,
Usage: "并发协程数",
},
&command.IntFlag{
Name: "batch-size",
Aliases: []string{"b"},
Value: 1000,
Usage: "每个批次插入的记录数",
},
&command.StringFlag{
Name: "start-date",
Aliases: []string{"s"},
Value: "",
Usage: "开始日期（格式：2006-01-02），默认为3个月前",
},
&command.StringFlag{
Name: "end-date",
Aliases: []string{"e"},
Value: "",
Usage: "结束日期（格式：2006-01-02），默认为当前时间",
},
},
}
}
// Handle Execute the console command.
func (receiver *GenerateTestPayments) Handle(ctx console.Context) error {
count := ctx.OptionInt("count")
workers := ctx.OptionInt("workers")
batchSize := ctx.OptionInt("batch-size")
startDateStr := ctx.Option("start-date")
endDateStr := ctx.Option("end-date")
// 参数验证
if count <= 0 {
return fmt.Errorf("记录数量必须大于0")
}
if workers <= 0 {
return fmt.Errorf("并发协程数必须大于0")
}
if batchSize <= 0 {
return fmt.Errorf("批次大小必须大于0")
}
// 解析日期范围
var startDate, endDate time.Time
var err error
if startDateStr != "" {
startDate, err = time.ParseInLocation("2006-01-02", startDateStr, time.UTC)
if err != nil {
return fmt.Errorf("开始日期格式错误，应为 YYYY-MM-DD 格式: %v", err)
}
} else {
startDate = time.Now().UTC().AddDate(0, -3, 0) // 默认3个月前
}
if endDateStr != "" {
endDate, err = time.ParseInLocation("2006-01-02", endDateStr, time.UTC)
if err != nil {
return fmt.Errorf("结束日期格式错误，应为 YYYY-MM-DD 格式: %v", err)
}
} else {
endDate = time.Now().UTC() // 默认当前时间
}
if startDate.After(endDate) {
return fmt.Errorf("开始日期不能晚于结束日期")
}
ctx.Info(fmt.Sprintf("开始生成 %d 条支付记录测试数据", count))
ctx.Info(fmt.Sprintf("时间范围: %s 至 %s", startDate.Format("2006-01-02"), endDate.Format("2006-01-02")))
ctx.Info(fmt.Sprintf("并发协程数: %d, 批次大小: %d", workers, batchSize))
// 获取支付方式列表
paymentMethods, err := receiver.getPaymentMethods()
if err != nil {
return fmt.Errorf("获取支付方式失败: %v", err)
}
if len(paymentMethods) == 0 {
return fmt.Errorf("没有找到支付方式，请先创建支付方式")
}
// 计算每个协程的工作量
totalBatches := (count + batchSize - 1) / batchSize
batchesPerWorker := totalBatches / workers
remainder := totalBatches % workers
startTime := time.Now()
var wg sync.WaitGroup
var mu sync.Mutex
var successCount int64
var errorCount int64
// 启动工作协程
for i := 0; i < workers; i++ {
wg.Add(1)
go func(workerID int) {
defer wg.Done()
// 计算当前协程的批次范围
batchStart := i * batchesPerWorker
batchEnd := batchStart + batchesPerWorker
if i < remainder {
batchEnd++
}
for batchID := batchStart; batchID < batchEnd; batchID++ {
// 计算当前批次的记录数
remainingCount := count - batchID*batchSize
currentBatchSize := batchSize
if remainingCount < batchSize {
currentBatchSize = remainingCount
}
if currentBatchSize <= 0 {
break
}
// 生成批次数据
err := receiver.generateBatch(currentBatchSize, startDate, endDate, paymentMethods)
mu.Lock()
if err != nil {
errorCount++
ctx.Error(fmt.Sprintf("Worker %d 批次 %d 失败: %v", workerID, batchID, err))
} else {
successCount += int64(currentBatchSize)
ctx.Info(fmt.Sprintf("Worker %d 批次 %d 完成，生成 %d 条记录", workerID, batchID, currentBatchSize))
}
mu.Unlock()
}
}(i)
}
wg.Wait()
// 输出结果
duration := time.Since(startTime)
ctx.Info(fmt.Sprintf("\n✅ 完成！"))
ctx.Info(fmt.Sprintf("总耗时: %v", duration))
ctx.Info(fmt.Sprintf("成功生成: %d 条记录", successCount))
ctx.Info(fmt.Sprintf("失败批次: %d", errorCount))
if successCount > 0 {
ctx.Info(fmt.Sprintf("平均速度: %.0f 条/秒", float64(successCount)/duration.Seconds()))
}
return nil
}
// getPaymentMethods 获取支付方式列表
func (receiver *GenerateTestPayments) getPaymentMethods() ([]models.PaymentMethod, error) {
var paymentMethods []models.PaymentMethod
err := facades.Orm().Query().Model(&models.PaymentMethod{}).Where("is_active", true).Find(&paymentMethods)
return paymentMethods, err
}
// generateBatch 生成一批支付记录
func (receiver *GenerateTestPayments) generateBatch(count int, startDate, endDate time.Time, paymentMethods []models.PaymentMethod) error {
payments := make([]models.Payment, 0, count)
for i := 0; i < count; i++ {
// 随机生成时间
randomTime := randomTimeInRange(startDate, endDate)
// 随机选择支付方式
paymentMethod := paymentMethods[rand.Intn(len(paymentMethods))]
// 随机生成订单号（模拟已存在的订单）
orderNo := fmt.Sprintf("ORD%s%s", randomTime.Format("20060102"), strconv.Itoa(rand.Intn(999999)))
// 随机生成用户ID
userID := uint(rand.Intn(10000) + 1)
// 随机生成金额（0.01-10000.00）
amount := float64(rand.Intn(1000000)+1) / 100
// 随机生成状态
statuses := []string{"pending", "paid", "failed", "cancelled"}
status := statuses[rand.Intn(len(statuses))]
// 生成支付单号
paymentNo := fmt.Sprintf("PAY%s%s", randomTime.Format("20060102"), ulid.Make().String())
payment := models.Payment{
PaymentNo: paymentNo,
OrderNo: orderNo,
PaymentMethodID: paymentMethod.ID,
UserID: userID,
Amount: amount,
Status: status,
}
// 根据状态设置其他字段
switch status {
case "paid":
payment.PayTime = &randomTime
payment.ThirdPartyNo = fmt.Sprintf("TXN%s%d", randomTime.Format("20060102150405"), rand.Intn(999999))
case "failed":
payment.FailReason = "支付超时"
}
payments = append(payments, payment)
}
// 确保分表存在（使用最早的时间）
tableName := utils.GetShardingTableName("payments", startDate)
if !facades.Schema().HasTable(tableName) {
if err := receiver.ensurePaymentShardingTableExists(startDate); err != nil {
return fmt.Errorf("创建分表失败: %v", err)
}
}
// 批量插入（按分表分组）
tableGroups := make(map[string][]models.Payment)
for _, payment := range payments {
// 从支付单号提取日期
if len(payment.PaymentNo) >= 11 {
dateStr := payment.PaymentNo[3:11] // 提取日期部分 YYYYMMDD
if parsedTime, err := time.Parse("20060102", dateStr); err == nil {
tableName := utils.GetShardingTableName("payments", parsedTime)
tableGroups[tableName] = append(tableGroups[tableName], payment)
}
}
}
// 分表插入
for tableName, tablePayments := range tableGroups {
// 确保分表存在
if !facades.Schema().HasTable(tableName) {
// 从表名解析月份
if len(tableName) > 8 {
monthStr := tableName[len(tableName)-6:] // 提取 YYYYMM
if parsedTime, err := time.Parse("200601", monthStr); err == nil {
if err := receiver.ensurePaymentShardingTableExists(parsedTime); err != nil {
return fmt.Errorf("创建分表 %s 失败: %v", tableName, err)
}
}
}
}
// 转换为 map 以便批量插入
paymentMaps := make([]map[string]any, len(tablePayments))
for i, payment := range tablePayments {
// 从支付单号解析创建时间
var createdAt time.Time
if len(payment.PaymentNo) >= 11 {
dateStr := payment.PaymentNo[3:11] // 提取日期部分 YYYYMMDD
if parsedTime, err := time.Parse("20060102", dateStr); err == nil {
// 添加随机时分秒
createdAt = parsedTime.Add(time.Duration(rand.Intn(86400)) * time.Second)
}
}
if createdAt.IsZero() {
createdAt = time.Now().UTC()
}
paymentMaps[i] = map[string]any{
"payment_no": payment.PaymentNo,
"order_no": payment.OrderNo,
"payment_method_id": payment.PaymentMethodID,
"user_id": payment.UserID,
"amount": payment.Amount,
"status": payment.Status,
"third_party_no": payment.ThirdPartyNo,
"pay_time": payment.PayTime,
"fail_reason": payment.FailReason,
"remark": payment.Remark,
"created_at": createdAt,
"updated_at": createdAt,
}
}
if err := facades.Orm().Query().Table(tableName).Create(paymentMaps); err != nil {
errorlog.Record(context.Background(), "payment", "批量插入支付记录失败", map[string]any{
"table_name": tableName,
"count": len(paymentMaps),
"error": err.Error(),
}, "批量插入支付记录失败: %v", err)
return fmt.Errorf("插入分表 %s 失败: %v", tableName, err)
}
}
return nil
}
// randomTimeInRange 生成指定时间范围内的随机时间
func randomTimeInRange(start, end time.Time) time.Time {
delta := end.Sub(start)
sec := rand.Int63n(int64(delta.Seconds()))
return start.Add(time.Duration(sec) * time.Second)
}
// ensurePaymentShardingTableExists 确保支付记录分表存在
func (receiver *GenerateTestPayments) ensurePaymentShardingTableExists(paymentTime time.Time) error {
tableName := utils.GetShardingTableName("payments", paymentTime)
if !facades.Schema().HasTable(tableName) {
// 使用迁移函数创建分表
if err := migrations.CreatePaymentsShardingTable(tableName); err != nil {
return fmt.Errorf("创建支付记录分表失败: %v", err)
}
}
return nil
}
app/console/commands/optimize_tables.go
+121
View File
@@ -0,0 +1,121 @@
package commands
import (
"fmt"
"strings"
"github.com/goravel/framework/contracts/console"
"github.com/goravel/framework/contracts/console/command"
"github.com/goravel/framework/facades"
)
type OptimizeTables struct {
}
func (r *OptimizeTables) Signature() string {
return "db:optimize-tables"
}
func (r *OptimizeTables) Description() string {
// # MySQL: OPTIMIZE TABLE（整理碎片，回收空间）
// # PostgreSQL: VACUUM (ANALYZE)（清理死元组并更新统计信息）
//
// # 传参方式：直接传表名（可多个）
// go run . artisan db:optimize-tables payments
// go run . artisan db:optimize-tables payments orders_202601 order_details_202601
//
// # 选项方式：--tables= 逗号分隔
// go run . artisan db:optimize-tables --tables=payments,orders_202601
//
// # PostgreSQL 重度回收空间（风险高：会锁表，且耗时长）
// go run . artisan db:optimize-tables payments --full=true
//
// # 帮助
// go run . artisan db:optimize-tables --help
return "优化表（MySQL: OPTIMIZE TABLE; PostgreSQL: VACUUM）"
}
func (r *OptimizeTables) Extend() command.Extend {
return command.Extend{
Category: "db",
Flags: []command.Flag{
&command.StringFlag{
Name: "tables",
Aliases: []string{"t"},
Usage: "要优化的表名列表（逗号分隔），也可以直接用参数方式传入多个表名",
},
&command.BoolFlag{
Name: "full",
Value: false,
Usage: "PostgreSQL 是否使用 VACUUM FULL（更重，可能锁表，默认 false）",
},
},
}
}
func (r *OptimizeTables) Handle(ctx console.Context) error {
dbConnection := strings.ToLower(facades.Config().GetString("database.default", "sqlite"))
full := ctx.OptionBool("full")
var tables []string
// 1) 从 --tables 读取（逗号分隔）
tablesFlag := strings.TrimSpace(ctx.Option("tables"))
if tablesFlag != "" {
parts := strings.Split(tablesFlag, ",")
for _, p := range parts {
t := strings.TrimSpace(p)
if t != "" {
tables = append(tables, t)
}
}
}
// 2) 从参数读取（db:optimize-tables table1 table2 ...）
for i := 0; ; i++ {
arg := strings.TrimSpace(ctx.Argument(i))
if arg == "" {
break
}
tables = append(tables, arg)
}
if len(tables) == 0 {
return fmt.Errorf("请提供要优化的表名，例如：go run . artisan db:optimize-tables payments 或使用 --tables=payments,orders_202601")
}
execOptimize := func(table string) error {
var sql string
switch dbConnection {
case "mysql":
sql = fmt.Sprintf("OPTIMIZE TABLE `%s`", table)
case "postgres":
if full {
sql = fmt.Sprintf("VACUUM (FULL, ANALYZE) %s", table)
} else {
sql = fmt.Sprintf("VACUUM (ANALYZE) %s", table)
}
default:
return fmt.Errorf("unsupported database: %s", dbConnection)
}
if _, err := facades.Orm().Query().Exec(sql); err != nil {
return err
}
ctx.Info("✓ " + sql)
return nil
}
ctx.Info("开始执行优化...")
for _, table := range tables {
if facades.Schema().HasTable(table) {
if err := execOptimize(table); err != nil {
return fmt.Errorf("optimize %s failed: %v", table, err)
}
}
}
ctx.Info("完成")
return nil
}
app/console/commands/queue_clear.go
+292
View File
@@ -0,0 +1,292 @@
package commands
import (
"context"
"fmt"
"os"
"strings"
"github.com/goravel/framework/contracts/console"
"github.com/goravel/framework/contracts/console/command"
"github.com/goravel/framework/facades"
"goravel/app/utils"
"goravel/app/utils/errorlog"
)
type QueueClear struct {
}
// Signature The name and signature of the console command.
func (r *QueueClear) Signature() string {
return "queue:clear"
}
// Description The console command description.
func (r *QueueClear) Description() string {
return "清理队列中的任务（仅支持 Redis 驱动）"
}
// Extend The console command extend.
func (r *QueueClear) Extend() command.Extend {
return command.Extend{
Category: "queue",
Flags: []command.Flag{
&command.StringFlag{
Name: "queue",
Aliases: []string{"q"},
Usage: "队列名称（可选，默认清理默认队列）",
},
&command.StringFlag{
Name: "connection",
Aliases: []string{"c"},
Usage: "队列连接名称（可选，默认使用默认连接）",
},
&command.BoolFlag{
Name: "force",
Usage: "强制清理，不提示确认",
},
},
}
}
// Handle Execute the console command.
func (r *QueueClear) Handle(ctx console.Context) error {
queueName := ctx.Option("queue")
connectionName := ctx.Option("connection")
// 布尔标志：检查命令行参数中是否包含 --force
// 在 Goravel 框架中，BoolFlag 存在时 ctx.Option 返回空字符串，所以需要检查命令行参数
force := r.hasForceFlag()
if connectionName == "" {
connectionName = facades.Config().GetString("queue.default", "sync")
}
// 判断队列驱动类型
driver := facades.Config().GetString(fmt.Sprintf("queue.connections.%s.driver", connectionName), "")
// 检查是否是 Redis 驱动
isRedis := r.isRedisDriver(connectionName)
if !isRedis {
if driver == "sync" {
ctx.Info("同步驱动：任务立即执行，无需清理")
return nil
}
if driver == "database" {
ctx.Warning("数据库驱动暂不支持清理命令，请直接操作数据库 jobs 表")
return nil
}
ctx.Warning(fmt.Sprintf("驱动 %s 暂不支持清理命令", driver))
return nil
}
// Redis 驱动：清理队列
defaultQueue := facades.Config().GetString(fmt.Sprintf("queue.connections.%s.queue", connectionName), "default")
if queueName == "" {
queueName = defaultQueue
}
// 获取 Redis 连接名称
redisConnectionName := r.getRedisConnectionName(connectionName)
if redisConnectionName == "" {
ctx.Warning("无法确定 Redis 连接名称")
return nil
}
// 查询当前队列统计
stats, err := r.getRedisQueueStats(redisConnectionName, connectionName, queueName)
if err != nil {
ctx.Error(fmt.Sprintf("查询队列统计失败: %v", err))
return err
}
totalCount := stats.Pending + stats.Reserved + stats.Delayed
if totalCount == 0 {
ctx.Info(fmt.Sprintf("队列 '%s' 中没有任何任务", queueName))
return nil
}
// 显示当前统计
ctx.Info(fmt.Sprintf("队列 '%s' 当前状态：", queueName))
ctx.Info(fmt.Sprintf(" 待执行任务: %d", stats.Pending))
ctx.Info(fmt.Sprintf(" 正在执行任务: %d", stats.Reserved))
ctx.Info(fmt.Sprintf(" 延迟任务: %d", stats.Delayed))
ctx.Info(fmt.Sprintf(" 总计: %d", totalCount))
ctx.Info("")
// 确认清理
if !force {
ctx.Warning(fmt.Sprintf("警告：此操作将删除队列 '%s' 中的所有任务（共 %d 个）", queueName, totalCount))
ctx.Info("如果确定要继续，请使用 --force 参数")
ctx.Info(fmt.Sprintf(" go run . artisan queue:clear --queue=%s --connection=%s --force", queueName, connectionName))
return nil
}
// 执行清理
ctx.Info("开始清理队列...")
redisClient, err := utils.GetRedisClient(redisConnectionName)
if err != nil {
ctx.Error(fmt.Sprintf("获取 Redis 客户端失败: %v", err))
return err
}
// 注意：使用公共 Redis 客户端池，不需要手动关闭
ctxRedis := context.Background()
clearedCount := int64(0)
// 清理待执行队列
pendingKey := r.redisQueueKey(connectionName, queueName)
pendingLen, _ := redisClient.LLen(ctxRedis, pendingKey).Result()
if pendingLen > 0 {
if err := redisClient.Del(ctxRedis, pendingKey).Err(); err != nil {
ctx.Error(fmt.Sprintf("清理待执行队列失败: %v", err))
} else {
clearedCount += pendingLen
ctx.Info(fmt.Sprintf("已清理待执行队列: %d 个任务", pendingLen))
}
}
// 清理正在执行队列
reservedKey := r.redisReservedKey(connectionName, queueName)
reservedLen, _ := redisClient.ZCard(ctxRedis, reservedKey).Result()
if reservedLen > 0 {
if err := redisClient.Del(ctxRedis, reservedKey).Err(); err != nil {
ctx.Error(fmt.Sprintf("清理正在执行队列失败: %v", err))
} else {
clearedCount += reservedLen
ctx.Info(fmt.Sprintf("已清理正在执行队列: %d 个任务", reservedLen))
}
}
// 清理延迟队列
delayedKey := r.redisDelayedKey(connectionName, queueName)
delayedLen, _ := redisClient.ZCard(ctxRedis, delayedKey).Result()
if delayedLen > 0 {
if err := redisClient.Del(ctxRedis, delayedKey).Err(); err != nil {
ctx.Error(fmt.Sprintf("清理延迟队列失败: %v", err))
} else {
clearedCount += delayedLen
ctx.Info(fmt.Sprintf("已清理延迟队列: %d 个任务", delayedLen))
}
}
ctx.Info(fmt.Sprintf("队列清理完成！共清理 %d 个任务", clearedCount))
return nil
}
// isRedisDriver 判断是否是 Redis 驱动
func (r *QueueClear) isRedisDriver(connectionName string) bool {
via := facades.Config().Get(fmt.Sprintf("queue.connections.%s.via", connectionName))
return via != nil || strings.Contains(connectionName, "redis")
}
// getRedisConnectionName 从队列连接配置中获取 Redis 连接名称
func (r *QueueClear) getRedisConnectionName(queueConnectionName string) string {
connection := facades.Config().GetString(fmt.Sprintf("queue.connections.%s.connection", queueConnectionName), "default")
if strings.Contains(queueConnectionName, "redis") {
redisHost := facades.Config().GetString(fmt.Sprintf("database.redis.%s.host", queueConnectionName), "")
if redisHost != "" {
return queueConnectionName
}
}
return connection
}
// getRedisQueueStats 获取 Redis 队列统计信息
func (r *QueueClear) getRedisQueueStats(redisConnectionName, queueConnectionName, queueName string) (*RedisQueueStatsInfo, error) {
redisClient, err := utils.GetRedisClient(redisConnectionName)
if err != nil {
errorlog.Record(context.Background(), "queue", "获取 Redis 客户端失败", map[string]any{
"connection": redisConnectionName,
"error": err.Error(),
}, "获取 Redis 客户端失败: %v", err)
return nil, fmt.Errorf("获取 Redis 客户端失败: %v", err)
}
// 注意：使用公共 Redis 客户端池，不需要手动关闭
ctx := context.Background()
stats := &RedisQueueStatsInfo{}
pendingKey := r.redisQueueKey(queueConnectionName, queueName)
pendingLen, err := redisClient.LLen(ctx, pendingKey).Result()
if err != nil {
errorlog.Record(context.Background(), "queue", "查询待执行队列失败", map[string]any{
"queue_name": queueName,
"key": pendingKey,
"error": err.Error(),
}, "查询待执行队列失败: %v", err)
return nil, fmt.Errorf("查询待执行队列失败: %v", err)
}
stats.Pending = pendingLen
reservedKey := r.redisReservedKey(queueConnectionName, queueName)
reservedLen, err := redisClient.ZCard(ctx, reservedKey).Result()
if err != nil {
errorlog.Record(context.Background(), "queue", "查询正在执行队列失败", map[string]any{
"queue_name": queueName,
"key": reservedKey,
"error": err.Error(),
}, "查询正在执行队列失败: %v", err)
return nil, fmt.Errorf("查询正在执行队列失败: %v", err)
}
stats.Reserved = reservedLen
delayedKey := r.redisDelayedKey(queueConnectionName, queueName)
delayedLen, err := redisClient.ZCard(ctx, delayedKey).Result()
if err != nil {
errorlog.Record(context.Background(), "queue", "查询延迟队列失败", map[string]any{
"queue_name": queueName,
"key": delayedKey,
"error": err.Error(),
}, "查询延迟队列失败: %v", err)
return nil, fmt.Errorf("查询延迟队列失败: %v", err)
}
stats.Delayed = delayedLen
// 失败任务：从数据库 failed_jobs 表查询
var failedCount int64
if queueName != "" {
failedCount, err = facades.Orm().Query().Table("failed_jobs").
Where("queue = ?", queueName).
Count()
} else {
failedCount, err = facades.Orm().Query().Table("failed_jobs").Count()
}
if err != nil {
stats.Failed = 0
} else {
stats.Failed = failedCount
}
stats.Total = stats.Pending + stats.Reserved
return stats, nil
}
// redisQueueKey Goravel Redis queue key format:
// {appName}_queues:{queueConnection}_{queue}
func (r *QueueClear) redisQueueKey(queueConnectionName, queueName string) string {
appName := facades.Config().GetString("app.name", "goravel")
return fmt.Sprintf("%s_queues:%s_%s", appName, queueConnectionName, queueName)
}
func (r *QueueClear) redisReservedKey(queueConnectionName, queueName string) string {
return fmt.Sprintf("%s:reserved", r.redisQueueKey(queueConnectionName, queueName))
}
func (r *QueueClear) redisDelayedKey(queueConnectionName, queueName string) string {
return fmt.Sprintf("%s:delayed", r.redisQueueKey(queueConnectionName, queueName))
}
// hasForceFlag 检查命令行参数中是否包含 --force 标志
func (r *QueueClear) hasForceFlag() bool {
for _, arg := range os.Args {
if arg == "--force" || arg == "-force" {
return true
}
}
return false
}
app/console/commands/queue_peek.go
+369
View File
@@ -0,0 +1,369 @@
package commands
import (
"context"
"encoding/json"
"fmt"
"math"
"os"
"strings"
"time"
"github.com/goravel/framework/contracts/console"
"github.com/goravel/framework/contracts/console/command"
"github.com/goravel/framework/facades"
"goravel/app/utils"
"goravel/app/utils/errorlog"
)
type QueuePeek struct{}
// Signature The name and signature of the console command.
func (r *QueuePeek) Signature() string {
return "queue:peek"
}
// ./main artisan queue:peek --connection=redis --queue=long-running --state=all --limit=5 --full
func (r *QueuePeek) Description() string {
return "查看队列中前 N 条任务内容（支持 Redis/database），用于排查“导出”等任务到底投递了什么"
}
// Extend The console command extend.
func (r *QueuePeek) Extend() command.Extend {
return command.Extend{
Category: "queue",
Flags: []command.Flag{
&command.StringFlag{
Name: "queue",
Aliases: []string{"q"},
Usage: "队列名称（可选；导出任务一般在 long-running）",
},
&command.StringFlag{
Name: "connection",
Aliases: []string{"c"},
Usage: "队列连接名称（可选，默认使用默认连接）",
},
&command.StringFlag{
Name: "state",
Aliases: []string{"s"},
Usage: "查看队列状态：pending|reserved|delayed|all（默认 pending）",
},
&command.IntFlag{
Name: "limit",
Aliases: []string{"l"},
Value: 10,
Usage: "最多显示多少条（默认 10）",
},
&command.BoolFlag{
Name: "raw",
Usage: "输出原始 payload（不做 JSON 美化/摘要）",
},
&command.BoolFlag{
Name: "full",
Usage: "不截断输出（默认会截断到 500 字符）",
},
},
}
}
// Handle Execute the console command.
func (r *QueuePeek) Handle(ctx console.Context) error {
queueName := ctx.Option("queue")
connectionName := ctx.Option("connection")
state := strings.TrimSpace(strings.ToLower(ctx.Option("state")))
limit := ctx.OptionInt("limit")
raw := r.hasFlag("--raw")
full := r.hasFlag("--full")
if limit <= 0 {
limit = 10
}
if state == "" {
state = "pending"
}
if connectionName == "" {
connectionName = facades.Config().GetString("queue.default", "sync")
}
ctx.Info(fmt.Sprintf("队列连接: %s", connectionName))
if queueName != "" {
ctx.Info(fmt.Sprintf("队列名称: %s", queueName))
}
ctx.Info(fmt.Sprintf("查看状态: %s, limit=%d, raw=%v, full=%v", state, limit, raw, full))
ctx.Info("")
driver := facades.Config().GetString(fmt.Sprintf("queue.connections.%s.driver", connectionName), "")
isRedis := r.isRedisDriver(connectionName)
if isRedis {
defaultQueue := facades.Config().GetString(fmt.Sprintf("queue.connections.%s.queue", connectionName), "default")
if queueName == "" {
queueName = defaultQueue
}
redisConnectionName := r.getRedisConnectionName(connectionName)
if redisConnectionName == "" {
ctx.Warning("无法确定 Redis 连接名称")
return nil
}
redisClient, err := utils.GetRedisClient(redisConnectionName)
if err != nil {
errorlog.Record(context.Background(), "queue", "获取 Redis 客户端失败", map[string]any{
"connection": redisConnectionName,
"error": err.Error(),
}, "获取 Redis 客户端失败: %v", err)
return fmt.Errorf("获取 Redis 客户端失败: %v", err)
}
c := context.Background()
printPayload := func(prefix, payload string) {
ctx.Info(prefix)
ctx.Info(r.renderPayload(payload, raw, full))
ctx.Info("")
}
if state == "pending" || state == "all" {
key := r.redisQueueKey(connectionName, queueName)
values, err := redisClient.LRange(c, key, 0, int64(limit-1)).Result()
if err != nil {
return fmt.Errorf("读取 pending 队列失败: %v", err)
}
ctx.Info("═══════════════════════════════════════")
ctx.Info(fmt.Sprintf("Redis pending: key=%s, count(shown)=%d", key, len(values)))
ctx.Info("═══════════════════════════════════════")
if len(values) == 0 {
ctx.Info("（空）")
ctx.Info("")
}
for i, v := range values {
printPayload(fmt.Sprintf("[%d] pending", i+1), v)
}
}
if state == "reserved" || state == "all" {
key := r.redisReservedKey(connectionName, queueName)
zs, err := redisClient.ZRangeWithScores(c, key, 0, int64(limit-1)).Result()
if err != nil {
return fmt.Errorf("读取 reserved 队列失败: %v", err)
}
ctx.Info("═══════════════════════════════════════")
ctx.Info(fmt.Sprintf("Redis reserved(ZSET): key=%s, count(shown)=%d", key, len(zs)))
ctx.Info("═══════════════════════════════════════")
if len(zs) == 0 {
ctx.Info("（空）")
ctx.Info("")
}
for i, z := range zs {
member, _ := z.Member.(string)
scoreInfo := r.formatScoreAsTime(z.Score)
printPayload(fmt.Sprintf("[%d] reserved score=%v%s", i+1, z.Score, scoreInfo), member)
}
}
if state == "delayed" || state == "all" {
key := r.redisDelayedKey(connectionName, queueName)
zs, err := redisClient.ZRangeWithScores(c, key, 0, int64(limit-1)).Result()
if err != nil {
return fmt.Errorf("读取 delayed 队列失败: %v", err)
}
ctx.Info("═══════════════════════════════════════")
ctx.Info(fmt.Sprintf("Redis delayed(ZSET): key=%s, count(shown)=%d", key, len(zs)))
ctx.Info("═══════════════════════════════════════")
if len(zs) == 0 {
ctx.Info("（空）")
ctx.Info("")
}
for i, z := range zs {
member, _ := z.Member.(string)
scoreInfo := r.formatScoreAsTime(z.Score)
printPayload(fmt.Sprintf("[%d] delayed score=%v%s", i+1, z.Score, scoreInfo), member)
}
}
return nil
}
if driver == "sync" {
ctx.Info("同步驱动：任务立即执行，无队列数据")
return nil
}
if driver != "database" {
ctx.Warning(fmt.Sprintf("驱动 %s 暂不支持 peek 查看", driver))
return nil
}
// database driver
q := facades.Orm().Query().Table("jobs").Select("id", "queue", "payload", "attempts", "reserved_at", "available_at", "created_at")
now := time.Now()
switch state {
case "pending":
q = q.Where("reserved_at IS NULL").Where("available_at", "<=", now)
case "reserved":
q = q.Where("reserved_at IS NOT NULL")
case "delayed":
q = q.Where("reserved_at IS NULL").Where("available_at", ">", now)
case "all":
// no filter
default:
ctx.Warning("state 参数只支持 pending|reserved|delayed|all")
return nil
}
if queueName != "" {
q = q.Where("queue", "=", queueName)
}
var rows []map[string]any
if err := q.OrderByDesc("id").Limit(limit).Get(&rows); err != nil {
return fmt.Errorf("查询 jobs 表失败: %v", err)
}
ctx.Info("═══════════════════════════════════════")
ctx.Info(fmt.Sprintf("Database jobs: state=%s, count(shown)=%d", state, len(rows)))
ctx.Info("═══════════════════════════════════════")
if len(rows) == 0 {
ctx.Info("（空）")
return nil
}
for i, row := range rows {
id := row["id"]
qn := row["queue"]
attempts := row["attempts"]
payload, _ := row["payload"].(string)
ctx.Info(fmt.Sprintf("[%d] id=%v queue=%v attempts=%v", i+1, id, qn, attempts))
ctx.Info(r.renderPayload(payload, raw, full))
ctx.Info("")
}
return nil
}
func (r *QueuePeek) isRedisDriver(connectionName string) bool {
via := facades.Config().Get(fmt.Sprintf("queue.connections.%s.via", connectionName))
return via != nil || strings.Contains(connectionName, "redis")
}
func (r *QueuePeek) getRedisConnectionName(queueConnectionName string) string {
connection := facades.Config().GetString(fmt.Sprintf("queue.connections.%s.connection", queueConnectionName), "default")
if strings.Contains(queueConnectionName, "redis") {
redisHost := facades.Config().GetString(fmt.Sprintf("database.redis.%s.host", queueConnectionName), "")
if redisHost != "" {
return queueConnectionName
}
}
return connection
}
// redisQueueKey Goravel Redis queue key format:
// {appName}_queues:{queueConnection}_{queue}
func (r *QueuePeek) redisQueueKey(queueConnectionName, queueName string) string {
appName := facades.Config().GetString("app.name", "goravel")
return fmt.Sprintf("%s_queues:%s_%s", appName, queueConnectionName, queueName)
}
func (r *QueuePeek) redisReservedKey(queueConnectionName, queueName string) string {
return fmt.Sprintf("%s:reserved", r.redisQueueKey(queueConnectionName, queueName))
}
func (r *QueuePeek) redisDelayedKey(queueConnectionName, queueName string) string {
return fmt.Sprintf("%s:delayed", r.redisQueueKey(queueConnectionName, queueName))
}
func (r *QueuePeek) renderPayload(payload string, raw, full bool) string {
if raw {
return r.maybeTruncate(payload, full)
}
trimmed := strings.TrimSpace(payload)
if trimmed == "" {
return "（空 payload）"
}
// best-effort JSON pretty + add a tiny summary if we can detect job/signature fields
var obj any
if err := json.Unmarshal([]byte(trimmed), &obj); err == nil {
summary := r.summarizePayload(obj)
pretty, _ := json.MarshalIndent(obj, "", " ")
out := string(pretty)
if summary != "" {
out = summary + "\n" + out
}
return r.maybeTruncate(out, full)
}
// not JSON, fallback to raw
return r.maybeTruncate(payload, full)
}
func (r *QueuePeek) summarizePayload(obj any) string {
m, ok := obj.(map[string]any)
if !ok {
return ""
}
// Goravel/Laravel-like payloads often include these keys (best-effort)
candidates := []string{"job", "name", "signature", "displayName", "command"}
for _, k := range candidates {
if v, ok := m[k]; ok {
if s, ok := v.(string); ok && s != "" {
return fmt.Sprintf("摘要: %s=%s", k, s)
}
}
}
// sometimes job info nested
if data, ok := m["data"].(map[string]any); ok {
for _, k := range candidates {
if v, ok := data[k]; ok {
if s, ok := v.(string); ok && s != "" {
return fmt.Sprintf("摘要: data.%s=%s", k, s)
}
}
}
}
return ""
}
func (r *QueuePeek) maybeTruncate(s string, full bool) string {
if full {
return s
}
const max = 500
if len(s) <= max {
return s
}
return s[:max] + "\n...(truncated, use --full to show all)"
}
func (r *QueuePeek) hasFlag(flag string) bool {
for _, arg := range os.Args {
if arg == flag {
return true
}
}
return false
}
func (r *QueuePeek) formatScoreAsTime(score float64) string {
// Redis ZSET score for delayed jobs is commonly a unix timestamp (seconds).
if math.IsNaN(score) || math.IsInf(score, 0) {
return ""
}
sec := int64(score)
// heuristic: 10-digit seconds timestamp range
if sec < 1000000000 || sec > 5000000000 {
return ""
}
t := time.Unix(sec, 0).Local()
return fmt.Sprintf(" (%s)", t.Format(utils.DateTimeFormat))
}
app/console/commands/queue_stats.go
+524
View File
@@ -0,0 +1,524 @@
package commands
import (
"context"
"fmt"
"strings"
"time"
"github.com/goravel/framework/contracts/console"
"github.com/goravel/framework/contracts/console/command"
"github.com/goravel/framework/facades"
"github.com/samber/lo"
"goravel/app/utils"
"goravel/app/utils/errorlog"
)
type QueueStats struct {
}
// Signature The name and signature of the console command.
func (r *QueueStats) Signature() string {
return "queue:stats"
}
// ./main artisan queue:stats --connection=redis --queue=long-running
func (r *QueueStats) Description() string {
return "查询队列统计信息，显示待执行、正在执行和失败任务数量"
}
// Extend The console command extend.
func (r *QueueStats) Extend() command.Extend {
return command.Extend{
Category: "queue",
Flags: []command.Flag{
&command.StringFlag{
Name: "queue",
Aliases: []string{"q"},
Usage: "队列名称（可选，用于筛选特定队列）",
},
&command.StringFlag{
Name: "connection",
Aliases: []string{"c"},
Usage: "队列连接名称（可选，默认使用默认连接）",
},
},
}
}
// Handle Execute the console command.
func (r *QueueStats) Handle(ctx console.Context) error {
queueName := ctx.Option("queue")
connectionName := ctx.Option("connection")
if connectionName == "" {
connectionName = facades.Config().GetString("queue.default", "sync")
}
ctx.Info(fmt.Sprintf("队列连接: %s", connectionName))
if queueName != "" {
ctx.Info(fmt.Sprintf("队列名称: %s", queueName))
}
ctx.Info("")
// 判断队列驱动类型
driver := facades.Config().GetString(fmt.Sprintf("queue.connections.%s.driver", connectionName), "")
ctx.Info(fmt.Sprintf("驱动类型: %s", driver))
// 检查是否是 Redis 驱动（custom driver with via）
isRedis := r.isRedisDriver(connectionName)
if isRedis {
// Redis 驱动：通过 Redis 客户端查询队列大小
defaultQueue := facades.Config().GetString(fmt.Sprintf("queue.connections.%s.queue", connectionName), "default")
originalQueueName := queueName
queueNameForStats := queueName
if queueNameForStats == "" {
queueNameForStats = defaultQueue
}
// 获取 Redis 连接名称（从队列配置中获取）
redisConnectionName := r.getRedisConnectionName(connectionName)
if redisConnectionName == "" {
ctx.Warning("无法确定 Redis 连接名称")
return nil
}
// 查询 Redis 队列统计
stats, err := r.getRedisQueueStats(redisConnectionName, connectionName, queueNameForStats)
if err != nil {
ctx.Error(fmt.Sprintf("查询 Redis 队列统计失败: %v", err))
ctx.Info("提示：请确保 Redis 连接配置正确且 Redis 服务正在运行")
return err
}
totalCount := stats.Pending + stats.Reserved
// 显示统计信息
ctx.Info("═══════════════════════════════════════")
ctx.Info("队列统计信息 (Redis)")
ctx.Info("═══════════════════════════════════════")
ctx.Info(fmt.Sprintf("队列名称: %s", queueNameForStats))
ctx.Info(fmt.Sprintf("待执行任务: %d", stats.Pending))
ctx.Info(fmt.Sprintf("正在执行任务: %d", stats.Reserved))
ctx.Info(fmt.Sprintf("延迟任务: %d", stats.Delayed))
ctx.Info(fmt.Sprintf("失败任务: %d", stats.Failed))
ctx.Info(fmt.Sprintf("总任务数: %d", totalCount))
ctx.Info("═══════════════════════════════════════")
// 提示信息
if stats.Pending > 0 {
ctx.Info("")
ctx.Warning(fmt.Sprintf("提示：队列中有 %d 个待执行任务", stats.Pending))
ctx.Info("")
ctx.Info("如果需要处理这些任务：")
ctx.Info(" 1. 启动主程序（main.go 中会自动启动队列 Worker）")
ctx.Info(" go run .")
ctx.Info(" 2. 确保 Worker 监听正确的队列名称和连接")
ctx.Info(" 3. 确保任务已正确注册到 QueueServiceProvider")
ctx.Info("")
ctx.Info("如果不需要这些任务，可以清理队列：")
ctx.Info(" 使用 Redis 客户端执行以下命令清理队列：")
appName := facades.Config().GetString("app.name", "goravel")
baseKey := r.redisQueueKey(connectionName, queueNameForStats)
ctx.Info(fmt.Sprintf(" # app.name=%s, queue.connection=%s, queue=%s", appName, connectionName, queueNameForStats))
ctx.Info(fmt.Sprintf(" redis-cli DEL %s", baseKey))
ctx.Info(fmt.Sprintf(" redis-cli DEL %s:reserved", baseKey))
ctx.Info(fmt.Sprintf(" redis-cli DEL %s:delayed", baseKey))
ctx.Info(" 或者使用命令：go run . artisan queue:clear --queue=" + queueNameForStats)
}
// 如果未指定队列名称，显示按队列分组的统计
if originalQueueName == "" {
ctx.Info("")
ctx.Info("按队列分组统计:")
byQueue, err := r.getRedisStatsByQueue(redisConnectionName, connectionName)
if err != nil {
ctx.Warning(fmt.Sprintf("获取按队列分组统计失败: %v", err))
} else {
if len(byQueue) == 0 {
ctx.Info(" 暂无队列数据")
} else {
for qName, qStats := range byQueue {
ctx.Info(fmt.Sprintf(" 队列 [%s]:", qName))
ctx.Info(fmt.Sprintf(" 待执行: %d, 正在执行: %d, 延迟: %d, 失败: %d, 总计: %d",
qStats.Pending, qStats.Reserved, qStats.Delayed, qStats.Failed, qStats.Total))
}
}
}
}
return nil
}
if driver == "sync" {
ctx.Info("同步驱动：任务立即执行，无队列数据")
return nil
}
if driver != "database" {
ctx.Warning(fmt.Sprintf("驱动 %s 暂不支持统计查询", driver))
return nil
}
// Database 驱动：查询 jobs 表
var pendingCount, reservedCount int64
var err error
// 查询待执行任务数（available_at <= now 且 reserved_at 为 null）
pendingQuery := facades.Orm().Query().Table("jobs").
Where("available_at", "<=", time.Now()).
Where("reserved_at IS NULL")
// 查询正在执行任务数（reserved_at 不为 null）
reservedQuery := facades.Orm().Query().Table("jobs").
Where("reserved_at IS NOT NULL")
// 如果指定了队列名称，添加筛选条件
if queueName != "" {
pendingQuery = pendingQuery.Where("queue", "=", queueName)
reservedQuery = reservedQuery.Where("queue", "=", queueName)
}
pendingCount, err = pendingQuery.Count()
if err != nil {
ctx.Error(fmt.Sprintf("查询待执行任务数失败: %v", err))
return err
}
reservedCount, err = reservedQuery.Count()
if err != nil {
ctx.Error(fmt.Sprintf("查询正在执行任务数失败: %v", err))
return err
}
// 查询失败任务数（从 failed_jobs 表）
failedQuery := facades.Orm().Query().Table("failed_jobs")
if queueName != "" {
failedQuery = failedQuery.Where("queue", "=", queueName)
}
failedCount, err := failedQuery.Count()
if err != nil {
ctx.Error(fmt.Sprintf("查询失败任务数失败: %v", err))
return err
}
totalCount := pendingCount + reservedCount
// 显示统计信息
ctx.Info("═══════════════════════════════════════")
ctx.Info("队列统计信息")
ctx.Info("═══════════════════════════════════════")
ctx.Info(fmt.Sprintf("待执行任务: %d", pendingCount))
ctx.Info(fmt.Sprintf("正在执行任务: %d", reservedCount))
ctx.Info(fmt.Sprintf("失败任务: %d", failedCount))
ctx.Info(fmt.Sprintf("总任务数: %d", totalCount))
ctx.Info("═══════════════════════════════════════")
// 如果未指定队列名称，显示按队列分组的统计
if queueName == "" {
ctx.Info("")
ctx.Info("按队列分组统计:")
byQueue, err := r.getStatsByQueue()
if err != nil {
ctx.Warning(fmt.Sprintf("获取按队列分组统计失败: %v", err))
} else {
if len(byQueue) == 0 {
ctx.Info(" 暂无队列数据")
} else {
for qName, stats := range byQueue {
ctx.Info(fmt.Sprintf(" 队列 [%s]:", qName))
ctx.Info(fmt.Sprintf(" 待执行: %d, 正在执行: %d, 失败: %d, 总计: %d",
stats.Pending, stats.Reserved, stats.Failed, stats.Total))
}
}
}
}
return nil
}
// isRedisDriver 判断是否是 Redis 驱动
func (r *QueueStats) isRedisDriver(connectionName string) bool {
via := facades.Config().Get(fmt.Sprintf("queue.connections.%s.via", connectionName))
return via != nil || strings.Contains(connectionName, "redis")
}
// QueueStatsInfo 队列统计信息
type QueueStatsInfo struct {
Pending int64
Reserved int64
Failed int64
Total int64
}
// RedisQueueStatsInfo Redis 队列统计信息
type RedisQueueStatsInfo struct {
Pending int64
Reserved int64
Delayed int64
Failed int64
Total int64
}
// getStatsByQueue 按队列分组获取统计信息
func (r *QueueStats) getStatsByQueue() (map[string]QueueStatsInfo, error) {
// 获取所有队列名称
var queues []string
err := facades.Orm().Query().Table("jobs").
Select("DISTINCT queue").
Pluck("queue", &queues)
if err != nil {
return nil, err
}
// 获取失败任务的队列名称
var failedQueues []string
err = facades.Orm().Query().Table("failed_jobs").
Select("DISTINCT queue").
Pluck("queue", &failedQueues)
if err != nil {
return nil, err
}
// 合并队列名称
queueMap := make(map[string]bool)
for _, q := range queues {
queueMap[q] = true
}
for _, q := range failedQueues {
queueMap[q] = true
}
result := make(map[string]QueueStatsInfo)
now := time.Now()
for qName := range queueMap {
// 待执行任务数
pendingCount, _ := facades.Orm().Query().Table("jobs").
Where("queue", "=", qName).
Where("available_at", "<=", now).
Where("reserved_at IS NULL").
Count()
// 正在执行任务数
reservedCount, _ := facades.Orm().Query().Table("jobs").
Where("queue", "=", qName).
Where("reserved_at IS NOT NULL").
Count()
// 失败任务数
failedCount, _ := facades.Orm().Query().Table("failed_jobs").
Where("queue", "=", qName).
Count()
result[qName] = QueueStatsInfo{
Pending: pendingCount,
Reserved: reservedCount,
Failed: failedCount,
Total: pendingCount + reservedCount,
}
}
return result, nil
}
// getRedisConnectionName 从队列连接配置中获取 Redis 连接名称
func (r *QueueStats) getRedisConnectionName(queueConnectionName string) string {
// 从队列配置中获取 connection 字段
connection := facades.Config().GetString(fmt.Sprintf("queue.connections.%s.connection", queueConnectionName), "default")
// 如果队列连接名称包含 redis，尝试使用它
if strings.Contains(queueConnectionName, "redis") {
// 检查 Redis 配置中是否存在对应的连接
redisHost := facades.Config().GetString(fmt.Sprintf("database.redis.%s.host", queueConnectionName), "")
if redisHost != "" {
return queueConnectionName
}
}
// 默认使用 default
return connection
}
// getRedisQueueStats 获取 Redis 队列统计信息
func (r *QueueStats) getRedisQueueStats(redisConnectionName, queueConnectionName, queueName string) (*RedisQueueStatsInfo, error) {
// 使用公共 Redis 客户端
redisClient, err := utils.GetRedisClient(redisConnectionName)
if err != nil {
errorlog.Record(context.Background(), "queue", "获取 Redis 客户端失败", map[string]any{
"connection": redisConnectionName,
"error": err.Error(),
}, "获取 Redis 客户端失败: %v", err)
return nil, fmt.Errorf("获取 Redis 客户端失败: %v", err)
}
// 注意：使用公共 Redis 客户端池，不需要手动关闭
ctx := context.Background()
stats := &RedisQueueStatsInfo{}
// Goravel Redis driver:
// pending: {app}_queues:{queueConnection}_{queue} (List)
// reserved: {app}_queues:{queueConnection}_{queue}:reserved (ZSET)
// delayed: {app}_queues:{queueConnection}_{queue}:delayed (ZSET)
// 注意：这里的 queueConnectionName 是队列连接名（例如 redis），不是 redis client connection（default）
baseKey := r.redisQueueKey(queueConnectionName, queueName)
pendingKey := baseKey
pendingLen, err := redisClient.LLen(ctx, pendingKey).Result()
if err != nil {
errorlog.Record(context.Background(), "queue", "查询待执行队列失败", map[string]any{
"queue_name": queueName,
"key": pendingKey,
"error": err.Error(),
}, "查询待执行队列失败: %v", err)
return nil, fmt.Errorf("查询待执行队列失败: %v", err)
}
stats.Pending = pendingLen
// 正在执行队列：{baseKey}:reserved (ZSET)
reservedKey := fmt.Sprintf("%s:reserved", baseKey)
reservedLen, err := redisClient.ZCard(ctx, reservedKey).Result()
if err != nil {
errorlog.Record(context.Background(), "queue", "查询正在执行队列失败", map[string]any{
"queue_name": queueName,
"key": reservedKey,
"error": err.Error(),
}, "查询正在执行队列失败: %v", err)
return nil, fmt.Errorf("查询正在执行队列失败: %v", err)
}
stats.Reserved = reservedLen
// 延迟队列：{baseKey}:delayed (ZSET)
delayedKey := fmt.Sprintf("%s:delayed", baseKey)
delayedLen, err := redisClient.ZCard(ctx, delayedKey).Result()
if err != nil {
errorlog.Record(context.Background(), "queue", "查询延迟队列失败", map[string]any{
"queue_name": queueName,
"key": delayedKey,
"error": err.Error(),
}, "查询延迟队列失败: %v", err)
return nil, fmt.Errorf("查询延迟队列失败: %v", err)
}
stats.Delayed = delayedLen
// 调试信息：显示 Redis 键的实际值（可选，用于排查问题）
// 可以查看队列中的实际内容
if pendingLen > 0 {
// 查看队列中的第一个任务（不移除）
firstTask, _ := redisClient.LIndex(ctx, pendingKey, 0).Result()
if firstTask != "" {
// 只显示前100个字符，避免输出过长
if len(firstTask) > 100 {
firstTask = firstTask[:100] + "..."
}
}
}
// 失败任务：从数据库 failed_jobs 表查询（Redis 队列的失败任务也存储在数据库中）
var failedCount int64
if queueName != "" {
failedCount, err = facades.Orm().Query().Table("failed_jobs").
Where("queue = ?", queueName).
Count()
} else {
failedCount, err = facades.Orm().Query().Table("failed_jobs").Count()
}
if err != nil {
// 失败任务查询失败不影响其他统计
stats.Failed = 0
} else {
stats.Failed = failedCount
}
stats.Total = stats.Pending + stats.Reserved
return stats, nil
}
// getRedisStatsByQueue 按队列分组获取 Redis 队列统计信息
func (r *QueueStats) getRedisStatsByQueue(redisConnectionName, queueConnectionName string) (map[string]*RedisQueueStatsInfo, error) {
// 使用公共 Redis 客户端
redisClient, err := utils.GetRedisClient(redisConnectionName)
if err != nil {
errorlog.Record(context.Background(), "queue", "获取 Redis 客户端失败", map[string]any{
"connection": redisConnectionName,
"error": err.Error(),
}, "获取 Redis 客户端失败: %v", err)
return nil, fmt.Errorf("获取 Redis 客户端失败: %v", err)
}
// 注意：使用公共 Redis 客户端池，不需要手动关闭
ctx := context.Background()
result := make(map[string]*RedisQueueStatsInfo)
// 查找所有队列键（{app}_queues:{queueConnection}_*）
appName := facades.Config().GetString("app.name", "goravel")
prefix := fmt.Sprintf("%s_queues:%s_", appName, queueConnectionName)
pattern := prefix + "*"
keys, err := redisClient.Keys(ctx, pattern).Result()
if err != nil {
errorlog.Record(context.Background(), "queue", "查找队列键失败", map[string]any{
"pattern": pattern,
"error": err.Error(),
}, "查找队列键失败: %v", err)
return nil, fmt.Errorf("查找队列键失败: %v", err)
}
// 提取队列名称（排除 reserved 和 delayed 键）
queueNames := lo.FilterMap(keys, func(key string, _ int) (string, bool) {
// 跳过 reserved 和 delayed 键（ZSET）
if strings.HasSuffix(key, ":reserved") || strings.HasSuffix(key, ":delayed") {
return "", false
}
// 必须以 prefix 开头
if !strings.HasPrefix(key, prefix) {
return "", false
}
after := strings.TrimPrefix(key, prefix)
if after == "" {
return "", false
}
return after, true
})
// 去重队列名称
queueMap := lo.SliceToMap(lo.Uniq(queueNames), func(queueName string) (string, bool) {
return queueName, true
})
// 如果没有找到队列键，尝试从失败任务表中获取队列名称
if len(queueMap) == 0 {
var failedQueues []string
err = facades.Orm().Query().Table("failed_jobs").
Select("DISTINCT queue").
Pluck("queue", &failedQueues)
if err == nil {
validQueues := lo.Filter(failedQueues, func(q string, _ int) bool {
return q != ""
})
queueMap = lo.SliceToMap(validQueues, func(queueName string) (string, bool) {
return queueName, true
})
}
}
// 为每个队列获取统计信息
for queueName := range queueMap {
stats, err := r.getRedisQueueStats(redisConnectionName, queueConnectionName, queueName)
if err != nil {
// 单个队列查询失败不影响其他队列
continue
}
result[queueName] = stats
}
return result, nil
}
// redisQueueKey Goravel Redis queue key format:
// {appName}_queues:{queueConnection}_{queue}
func (r *QueueStats) redisQueueKey(queueConnectionName, queueName string) string {
appName := facades.Config().GetString("app.name", "goravel")
return fmt.Sprintf("%s_queues:%s_%s", appName, queueConnectionName, queueName)
}
app/console/kernel.go
+46
View File
@@ -0,0 +1,46 @@
package console
import (
"github.com/goravel/framework/contracts/console"
"github.com/goravel/framework/contracts/schedule"
"github.com/goravel/framework/facades"
"goravel/app/console/commands"
)
type Kernel struct {
}
func (kernel *Kernel) Schedule() []schedule.Event {
return []schedule.Event{
// 每天凌晨2点执行（北京时间），清理6个月前的日志
// 北京时间 02:00 = UTC 18:00（前一天）
facades.Schedule().Command("app:clear-logs").DailyAt("18:00").OnOneServer(),
// 每天凌晨3点执行（北京时间），清理3天前的分片文件
// 北京时间 03:00 = UTC 19:00（前一天）
facades.Schedule().Command("app:clear-chunks").DailyAt("19:00").OnOneServer(),
// 每天凌晨3点30分执行（UTC 19:30 / 北京时间 03:30）, 分析表（更新统计信息）
facades.Schedule().Command("db:analyze-stats").DailyAt("19:30").OnOneServer(),
// 每月1号凌晨1点执行（UTC时间），创建下个月的订单分表
facades.Schedule().Command("order:create-sharding-tables").Monthly().OnOneServer(),
// 每月1号凌晨1点30分执行（UTC时间），创建下个月的支付记录分表
facades.Schedule().Command("payment:create-sharding-tables").Monthly().OnOneServer(),
}
}
func (kernel *Kernel) Commands() []console.Command {
return []console.Command{
&commands.ClearLogs{},
&commands.ClearChunks{},
&commands.CreateToken{},
&commands.QueueStats{},
&commands.QueueClear{},
&commands.QueuePeek{},
commands.NewCreateOrderShardingTables(),
commands.NewCreatePaymentShardingTables(),
&commands.GenerateTestOrders{},
&commands.GenerateTestPayments{},
&commands.AnalyzeStats{},
&commands.OptimizeTables{},
}
}