Compare commits

14 Commits

Author SHA1 Message Date
cb2db256d4 modify scripts 2025-07-21 18:25:17 +08:00
1dda30cd75 modify scripts 2025-07-18 20:43:50 +08:00
29bd03b61f modify scripts 2025-07-17 12:02:36 +08:00
9b28efd2c9 modify scripts 2025-07-17 11:46:55 +08:00
77bb81b477 modify scripts 2025-07-12 11:38:07 +08:00
bacbac32a3 modify scripts 2025-07-11 19:10:34 +08:00
1493bc736c modify scripts 2025-07-11 19:07:53 +08:00
1b79e21b76 modify scripts 2025-07-11 17:39:30 +08:00
77ac61144d modify redis 2025-07-09 18:22:01 +08:00
447eebf39d modify conf 2025-03-14 18:49:58 +08:00
f64d03a19e modify go files 2025-03-14 18:25:56 +08:00
d54035d44b modify some scripts 2025-03-14 15:39:00 +08:00
a3dadecfc8 modify scripts 2025-03-14 15:27:03 +08:00
089cb11f8c modify files. 2025-03-14 03:46:41 +00:00
30 changed files with 705 additions and 446 deletions

35
.gitignore vendored Normal file
View File

@ -0,0 +1,35 @@
# 忽略日志文件
log/
logs/
*.log
# 忽略数据文件
tools/data/*
# 忽略编译后的二进制文件
bin/*
obj/
*.exe
*.out
*.o
# 排除 bin/ 目录下的脚本(不忽略)
!bin/*.sh
!bin/*.py
# 若 bin/ 目录下有子目录,且需要保留子目录中的脚本,可添加:
!bin/**/ # 不忽略 bin/ 下的子目录
!bin/**/*.sh # 保留子目录中的 .sh 脚本
!bin/**/*.py # 保留子目录中的 .py 脚本
# 忽略依赖文件
vendor/
node_modules/
# 忽略系统文件
.DS_Store
Thumbs.db
# 忽略 IDE 配置文件
.vscode/
.idea/

48
Makefile Normal file
View File

@ -0,0 +1,48 @@
# 设置变量
PROJECT_NAME := aigrammar
SRC_DIR := src
BIN_DIR := bin
CONF_DIR := conf
OUTPUT := $(BIN_DIR)/$(PROJECT_NAME)
# Go 编译参数
GO := go
GOFLAGS := -mod=readonly
GOFILES := $(shell find $(SRC_DIR) -name '*.go')
# 默认目标: 构建可执行文件
all: build
# 编译代码
build: $(GOFILES)
@mkdir -p $(BIN_DIR)
$(GO) build -o $(OUTPUT) ./$(SRC_DIR)/...
@echo "✅ Build complete: $(OUTPUT)"
# 运行程序
run: build
$(OUTPUT) --config=$(CONF_DIR)/config.toml
# 清理编译生成的二进制文件
clean:
rm -rf $(BIN_DIR)/$(PROJECT_NAME)
@echo "🗑 Cleaned up $(BIN_DIR)/$(PROJECT_NAME)"
# 格式化 Go 代码
fmt:
$(GO) fmt ./$(SRC_DIR)/...
# 整理 go.mod清理无用依赖
tidy:
$(GO) mod tidy
# 显示帮助信息
help:
@echo "Usage: make [target]"
@echo "Targets:"
@echo " all - 编译 (默认目标)"
@echo " build - 编译 Go 代码"
@echo " run - 运行编译后的程序"
@echo " clean - 删除 bin 目录下生成的可执行文件"
@echo " fmt - 格式化 Go 代码"
@echo " tidy - 整理 go.mod 依赖"

BIN
aigrammar

Binary file not shown.

41
bin/deploy.sh Executable file
View File

@ -0,0 +1,41 @@
#!/bin/bash
# 远程服务器信息
REMOTE_USER="ubuntu"
REMOTE_HOST="170.106.191.35"
REMOTE_DIR="/usr/local/aigrammar"
BACKUP_DIR="$REMOTE_DIR/backup"
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
# 配置文件环境(默认生产环境)
ENV=${1:-prod} # 运行时可以传入 `dev` 或 `prod`
CONFIG_FILE="conf/config.${ENV}.toml"
# 检查本地文件是否存在
if [[ ! -f "bin/aigrammar" || ! -f "bin/service.sh" || ! -f "$CONFIG_FILE" ]]; then
echo "❌ 关键文件不存在,请检查 bin/aigrammar, bin/service.sh, $CONFIG_FILE"
exit 1
fi
# 远程创建必要目录(如果不存在)
echo "🔹 确保远程目录结构完整..."
ssh $REMOTE_USER@$REMOTE_HOST "mkdir -p $REMOTE_DIR/{bin,conf,log,backup}"
# 备份远程服务器的旧文件
echo "📂 备份远程服务器文件..."
ssh $REMOTE_USER@$REMOTE_HOST "mkdir -p $BACKUP_DIR && \
[ -f $REMOTE_DIR/bin/aigrammar ] && mv $REMOTE_DIR/bin/aigrammar $BACKUP_DIR/aigrammar_$TIMESTAMP || true && \
[ -f $REMOTE_DIR/bin/service.sh ] && mv $REMOTE_DIR/bin/service.sh $BACKUP_DIR/service_$TIMESTAMP.sh || true && \
[ -f $REMOTE_DIR/conf/config.toml ] && mv $REMOTE_DIR/conf/config.toml $BACKUP_DIR/config_$TIMESTAMP.toml || true"
# 复制文件到远程服务器(保持目录结构)
echo "📤 复制文件到远程服务器..."
scp bin/aigrammar $REMOTE_USER@$REMOTE_HOST:$REMOTE_DIR/bin/
scp bin/service.sh $REMOTE_USER@$REMOTE_HOST:$REMOTE_DIR/bin/
scp $CONFIG_FILE $REMOTE_USER@$REMOTE_HOST:$REMOTE_DIR/conf/config.toml
# 远程执行 restart
echo "🔄 远程重启服务..."
ssh $REMOTE_USER@$REMOTE_HOST "cd $REMOTE_DIR/bin && chmod +x service.sh && ./service.sh restart"
echo "✅ 发布完成!"

58
bin/service.sh Executable file
View File

@ -0,0 +1,58 @@
#!/bin/bash
# 确保脚本使用一个参数
if [ $# -ne 1 ]; then
echo "Usage: $0 {start|stop|restart}"
exit 1
fi
# 获取当前脚本所在目录
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
program_name="aigrammar"
program_path="$script_dir/$program_name"
# 配置和日志目录
config_file="$script_dir/../conf/config.toml"
log_dir="$script_dir/../log"
log_file="$log_dir/output.log"
pid_file="$script_dir/${program_name}.pid"
# 确保日志目录存在
mkdir -p "$log_dir"
case "$1" in
start)
echo "Starting $program_name..."
if [ -f "$pid_file" ] && kill -0 $(cat "$pid_file") 2>/dev/null; then
echo "$program_name is already running."
exit 1
fi
nohup "$program_path" --config="$config_file" > "$log_file" 2>&1 &
echo $! > "$pid_file"
echo "$program_name started with PID $(cat "$pid_file")."
;;
stop)
echo "Stopping $program_name..."
if [ -f "$pid_file" ]; then
kill -TERM $(cat "$pid_file") 2>/dev/null && rm -f "$pid_file"
echo "$program_name stopped."
else
echo "No PID file found, attempting pkill..."
pkill -f "$program_name" && echo "$program_name stopped."
fi
;;
restart)
echo "Restarting $program_name..."
$0 stop
sleep 2
$0 start
;;
*)
echo "Unknown command: $1"
echo "Usage: $0 {start|stop|restart}"
exit 1
;;
esac

75
bin/stat.sh Executable file
View File

@ -0,0 +1,75 @@
#!/bin/bash
# Configuration
LOG_FILE="/usr/local/aigrammar/log/app.log"
PY_SCRIPT="/home/ubuntu/projects/devops/tools/send_to_wecom.py"
TODAY=$(date +%Y-%m-%d)
YESTERDAY=$(date -d "yesterday" +%Y-%m-%d)
FILTERED_LOG="${LOG_FILE%.log}_${YESTERDAY}.log"
# Check if log file exists
if [ ! -f "$LOG_FILE" ]; then
echo "Error: Log file not found - $LOG_FILE" >&2
exit 1
fi
# Check if Python script exists and is executable
if [ ! -f "$PY_SCRIPT" ]; then
echo "Error: Python script not found or not executable - $PY_SCRIPT" >&2
exit 1
fi
# Create filtered log file
echo "Filtering yesterday's logs..."
#grep "$TODAY" "$LOG_FILE" > "$FILTERED_LOG"
grep "$YESTERDAY" "$LOG_FILE" > "$FILTERED_LOG"
# Check if filtered log has content
if [ ! -s "$FILTERED_LOG" ]; then
echo "Warning: No logs found for $YESTERDAY"
content="Date: $YESTERDAY\nNo log records found"
python3 "$PY_SCRIPT" "$content"
exit 0
fi
echo "Filtered log created: $FILTERED_LOG"
# Statistics
echo "Calculating statistics..."
request_total=$(grep -E "\"level\":\"info\"" "$FILTERED_LOG" | grep "\"func\":\"main.parseToken\"" | wc -l)
grammar_total=$(grep -v "\"level\":\"debug\"" "$FILTERED_LOG" | grep "\"func\":\"main.GrammarHandler\"" | wc -l)
translate_total=$(grep -v "\"level\":\"debug\"" "$FILTERED_LOG" | grep "\"func\":\"main.TranslateHandler\"" | wc -l)
words_total=$(grep -v "\"level\":\"debug\"" "$FILTERED_LOG" | grep "\"func\":\"main.WordsHandler\"" | wc -l)
free_limit_total=$(grep -E "\"level\":\"warn\"" "$FILTERED_LOG" | grep "\"func\":\"main.queryUserBenefits\"" | wc -l)
error_total=$(grep "\"level\":\"error\"" "$FILTERED_LOG" | wc -l)
# Generate content to send
content="Date: $YESTERDAY
Total Requests: $request_total
Grammar Check Requests: $grammar_total
Translation Requests: $translate_total
Words Requests: $words_total
Free Quota Exceeded: $free_limit_total
Total Errors: $error_total"
# Print statistics
echo "===== Statistics ====="
echo -e "$content"
# Send to WeCom
echo "Sending to WeChat Work..."
python3 "$PY_SCRIPT" "$content"
# Check result
if [ $? -eq 0 ]; then
echo "Successfully sent to WeChat Work"
else
echo "Error: Failed to send to WeChat Work" >&2
exit 1
fi
# remove tmp logfile
rm -rf "$FILTERED_LOG"
echo "Script execution completed"
exit 0

46
commit.sh Executable file
View File

@ -0,0 +1,46 @@
#!/bin/bash
# 确保脚本有执行权限(只需执行一次)
# chmod +x git_commit.sh
# 检查是否在 Git 仓库内
if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
echo "❌ 当前目录不是 Git 仓库,请先执行 git init"
exit 1
fi
# 获取 commit message
commit_msg="$1"
# 如果没有提供 commit message提示用户输入
if [ -z "$commit_msg" ]; then
commit_msg="modify scripts"
#read -p "请输入 commit message: " commit_msg
#if [ -z "$commit_msg" ]; then
# echo "❌ 提交信息不能为空!"
# exit 1
#fi
fi
# 添加所有更改
git add .
if [ $? -ne 0 ]; then
echo "❌ git add 失败!"
exit 1
fi
# 提交更改
git commit -m "$commit_msg"
if [ $? -ne 0 ]; then
echo "❌ git commit 失败!"
exit 1
fi
# 推送到远程仓库
git push -u origin master
if [ $? -ne 0 ]; then
echo "❌ git push 失败!请检查远程仓库设置。"
exit 1
fi
echo "✅ 代码提交成功!"

27
conf/config.prod.toml Normal file
View File

@ -0,0 +1,27 @@
[base]
jwt_secret = "mCTf-JhNRnhaaGJy_x"
bind_addr = ":8090"
[log]
echo_log_file = "../log/echo.log"
log_file = "../log/app.log"
max_size = 500
max_backups = 3
max_age = 28
compress = true
level = "debug"
[azure_openai]
endpoint = "https://grammar.openai.azure.com/"
keys = "8b68c235b737488ab9a99983a14f8cca,0274ccde58aa47b189f0d13349885ad3"
gpt4_model = "gpt4"
gpt35_model = "gpt35"
[database]
mysql_conn = "localhost:3306"
mysql_user = "devops"
mysql_pass = "b5hs945wXjHr"
redis_conn = "127.0.0.1:6379"
redis_pass = "cK4dC3mN7"

View File

@ -4,8 +4,8 @@ jwt_secret = "mCTf-JhNRnhaaGJy_x"
bind_addr = ":80"
[log]
echo_log_file = "logs/echo.log"
log_file = "logs/app.log"
echo_log_file = "../log/echo.log"
log_file = "../log/app.log"
max_size = 500
max_backups = 3
max_age = 28
@ -21,7 +21,9 @@ gpt35_model = "gpt35"
[database]
mysql_conn = "172.18.0.3:3306"
mysql_conn = "testdb:3306"
mysql_user = "root"
mysql_pass = "mysqlpw"
redis_conn = "172.18.0.2:6379"
#redis_conn = "172.18.0.4:6379"
redis_conn = "redis:6379"
redis_pass = "cK4dC3mN7"

35
gitignore Normal file
View File

@ -0,0 +1,35 @@
# 忽略日志文件
log/
logs/
*.log
# 忽略数据文件
tools/data/*
# 忽略编译后的二进制文件
bin/*
obj/
*.exe
*.out
*.o
# 排除 bin/ 目录下的脚本(不忽略)
!bin/*.sh
!bin/*.py
# 若 bin/ 目录下有子目录,且需要保留子目录中的脚本,可添加:
!bin/**/ # 不忽略 bin/ 下的子目录
!bin/**/*.sh # 保留子目录中的 .sh 脚本
!bin/**/*.py # 保留子目录中的 .py 脚本
# 忽略依赖文件
vendor/
node_modules/
# 忽略系统文件
.DS_Store
Thumbs.db
# 忽略 IDE 配置文件
.vscode/
.idea/

File diff suppressed because one or more lines are too long

View File

@ -1,67 +0,0 @@
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-01T19:05:10+08:00","uri":"/pub/iap/callback","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-01T19:24:23+08:00","uri":"/pub/iap/callback","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-01T19:24:38+08:00","uri":"/internal/iap/history","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-02T16:48:49+08:00","uri":"/iap/verify","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-02T16:54:57+08:00","uri":"/iap/verify","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":500,"time":"2024-07-02T17:20:21+08:00","uri":"/iap/verify","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-02T17:24:08+08:00","uri":"/pub/iap/callback","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-02T17:29:08+08:00","uri":"/pub/iap/callback","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-02T17:30:30+08:00","uri":"/pub/iap/callback","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-02T17:37:40+08:00","uri":"/grammar/feedback","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":500,"time":"2024-07-02T18:21:48+08:00","uri":"/iap/verify","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":500,"time":"2024-07-02T18:22:29+08:00","uri":"/iap/verify","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-03T11:52:24+08:00","uri":"/iap/verify","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-03T11:53:23+08:00","uri":"/iap/verify","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-03T18:29:58+08:00","uri":"/user/get","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":500,"time":"2024-07-04T08:46:11+08:00","uri":"/user/get","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":500,"time":"2024-07-04T09:03:49+08:00","uri":"/user/get","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T09:05:34+08:00","uri":"/user/get","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T09:12:22+08:00","uri":"/user/get","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T09:36:49+08:00","uri":"/user/get","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T11:10:46+08:00","uri":"/iap/verify","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:15:31+08:00","uri":"/user/get","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:21:29+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:21:33+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:24:02+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:24:02+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:26:46+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:26:48+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:29:29+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:29:31+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:32:41+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:32:43+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:37:03+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:37:04+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:39:05+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:39:05+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:42:24+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:42:25+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:45:15+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:45:16+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:48:36+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:48:37+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:51:16+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:51:18+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:53:33+08:00","uri":"/user/get","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:54:18+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:54:19+08:00","uri":"/iap/verify","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T17:57:38+08:00","uri":"/user/get","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T19:07:40+08:00","uri":"/grammar/translate","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T19:08:08+08:00","uri":"/grammar/grammar","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-04T19:10:37+08:00","uri":"/grammar/words","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-05T08:37:44+08:00","uri":"/user/get","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-05T08:50:41+08:00","uri":"/grammar/grammar","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-05T08:56:16+08:00","uri":"/grammar/grammar","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-05T08:58:28+08:00","uri":"/grammar/words","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-05T08:59:33+08:00","uri":"/grammar/translate","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"GET","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-05T09:03:06+08:00","uri":"/internal/user/rights","user_agent":""}
{"level":"info","method":"GET","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-05T09:05:35+08:00","uri":"/internal/user/rights?ID=10004","user_agent":""}
{"level":"fatal","msg":"Failed to start server{error 26 0 listen tcp :80: bind: address already in use}","time":"2024-07-05T09:32:29+08:00"}
{"level":"info","method":"GET","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-05T09:33:56+08:00","uri":"/internal/user/rights/reset?ID=10004","user_agent":""}
{"level":"info","method":"GET","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-05T09:34:05+08:00","uri":"/internal/user/rights?ID=10004","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-05T09:34:14+08:00","uri":"/grammar/translate","user_agent":"AIGrammar/1.0 (com.easyprompts.aigrammar; build:1; iOS 16.7.8) Alamofire/5.9.1"}
{"level":"info","method":"GET","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-07-05T09:45:11+08:00","uri":"/internal/user/rights?ID=10004","user_agent":""}
{"level":"info","method":"GET","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-08-01T17:49:18+08:00","uri":"/internal/user/rights?ID=10004","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-08-01T17:49:38+08:00","uri":"/grammar/words","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-08-01T18:56:25+08:00","uri":"/grammar/words","user_agent":""}
{"level":"info","method":"POST","msg":"request log","remote_ip":"192.168.65.1","status":200,"time":"2024-08-01T18:56:35+08:00","uri":"/grammar/feedback","user_agent":""}

View File

@ -1,39 +0,0 @@
#!/bin/bash
# 确保脚本使用一个参数
if [ $# -ne 1 ]; then
echo "Usage: $0 {start|stop|restart}"
exit 1
fi
# 定义程序的目录路径
program_dir="/usr/local/aigrammar"
program_name="aigrammar"
log_file="output.log"
case "$1" in
start)
echo "Starting $program_name..."
cd $program_dir
nohup ./$program_name > $log_file 2>&1 &
echo "$program_name started."
;;
stop)
echo "Stopping $program_name..."
killall $program_name
echo "$program_name stopped."
;;
restart)
echo "Restarting $program_name..."
killall $program_name
sleep 2
cd $program_dir
nohup ./$program_name > $log_file 2>&1 &
echo "$program_name restarted."
;;
*)
echo "Unknown command: $1"
echo "Usage: $0 {start|stop|restart}"
exit 1
;;
esac

View File

@ -1,93 +0,0 @@
CREATE TABLE aigrammar.`user` (
ID INT UNSIGNED DEFAULT 10000 auto_increment NOT NULL,
UserID varchar(100) NULL COMMENT 'UserID',
UserName varchar(100) NULL COMMENT 'username',
DeviceID varchar(256) NULL COMMENT 'DeviceID',
RegChannel varchar(100) NULL COMMENT 'Email, Apple, Google',
OpenID varchar(100) NULL COMMENT 'ID from other channels',
RegTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP NULL,
UpdateTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP NULL,
CONSTRAINT user_pk PRIMARY KEY (ID)
)
ENGINE=InnoDB
DEFAULT CHARSET=utf8mb4
AUTO_INCREMENT 10000
COLLATE=utf8mb4_0900_ai_ci;
ALTER TABLE aigrammar.`user` MODIFY COLUMN UserID varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT '' COMMENT 'UserID';
ALTER TABLE aigrammar.`user` MODIFY COLUMN UserName varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT '' COMMENT 'username';
ALTER TABLE aigrammar.`user` MODIFY COLUMN RegChannel varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT '' COMMENT 'Email, Apple, Google';
ALTER TABLE aigrammar.`user` MODIFY COLUMN OpenID varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT '' COMMENT 'ID from other channels';
ALTER TABLE aigrammar.`user` MODIFY COLUMN DeviceID varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT '' COMMENT 'DeviceID';
CREATE TABLE aigrammar.vip (
ID INT UNSIGNED NOT NULL,
IsVIP INT DEFAULT 0 NULL COMMENT '1-VIP; 0-not vip',
AppStore varchar(100) DEFAULT 'apple' NULL COMMENT 'apple;google',
ProductID varchar(100) NULL,
ProductType varchar(100) NULL COMMENT 'yearly;monthly;weekly;',
Environment varchar(100) NULL COMMENT 'prod;sandbox',
PurchaseDate TIMESTAMP NULL,
Price INT NULL,
Currency varchar(100) NULL,
Storefront varchar(100) NULL COMMENT 'USA',
ExpDate TIMESTAMP NULL,
AutoRenew INT NULL COMMENT '1-yes;0-no',
OriginalTransactionID varchar(100) NULL COMMENT 'applestore originalTransactionId',
CONSTRAINT vip_pk PRIMARY KEY (ID)
)
ENGINE=InnoDB
DEFAULT CHARSET=utf8mb4
COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE aigrammar.order_log (
LogID INT UNSIGNED auto_increment NOT NULL,
AppStore varchar(100) NULL COMMENT 'apple;google',
NotificationType varchar(100) NULL,
Subtype varchar(100) NULL,
Environment varchar(100) NULL COMMENT 'product;sandbox',
AppAccountToken varchar(100) NULL,
CreateTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP NULL,
TransactionInfo TEXT NULL,
RenewalInfo TEXT NULL,
Payload TEXT NULL,
CONSTRAINT oder_log_pk PRIMARY KEY (LogID)
)
ENGINE=InnoDB
DEFAULT CHARSET=utf8mb4
COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE aigrammar.product (
ProductID varchar(100) NOT NULL,
AppStore varchar(100) DEFAULT 'apple' NOT NULL,
Duration INT DEFAULT 0 NULL COMMENT '订阅天数',
ProductName varchar(100) NULL COMMENT '自定义商品名称',
Price INT NULL COMMENT '定价,分',
Currency varchar(100) NULL COMMENT '币种',
CONSTRAINT product_pk PRIMARY KEY (ProductID,AppStore)
)
ENGINE=InnoDB
DEFAULT CHARSET=utf8mb4
COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE aigrammar.feedback_log (
LogID INT UNSIGNED auto_increment NOT NULL,
AppStore varchar(100) NULL,
Product varchar(100) NULL,
`Input` TEXT NULL,
`Output` TEXT NULL,
`Result` varchar(100) NULL,
UserID INT UNSIGNED NULL,
CreateTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP NULL,
CONSTRAINT feedback_log_pk PRIMARY KEY (LogID)
)
ENGINE=InnoDB
DEFAULT CHARSET=utf8mb4
COLLATE=utf8mb4_0900_ai_ci;

View File

@ -31,6 +31,7 @@ type DataBaseConfig struct {
// 在 Go 中,只有首字母大写的字段才能被外部包(如 viper访问。
MysqlConn string `mapstructure:"mysql_conn"`
RedisConn string `mapstructure:"redis_conn"`
RedisPass string `mapstructure:"redis_pass"`
MysqlUser string `mapstructure:"mysql_user"`
MysqlPass string `mapstructure:"mysql_pass"`
}
@ -61,16 +62,13 @@ var initError error
func GetConfigManager() (*ConfigManager, error) {
once.Do(func() {
instance = &ConfigManager{}
initError = instance.initConfig()
// initError = instance.initConfig(configFile)
})
return instance, initError
}
func (cm *ConfigManager) initConfig() error {
viper.SetConfigName("config")
viper.SetConfigType("toml")
viper.AddConfigPath(".")
func (cm *ConfigManager) initConfig(configFile string) error {
viper.SetConfigFile(configFile)
if err := viper.ReadInConfig(); err != nil {
return err
}

View File

@ -53,7 +53,7 @@ func initDBManager(dbManager *DBManager, config *ConfigManager) error {
// 初始化 Redis 连接
rdb := redis.NewClient(&redis.Options{
Addr: config.GetDatabaseConfig().RedisConn,
Password: "", // no password set
Password: config.GetDatabaseConfig().RedisPass,
DB: 0, // use default DB
PoolSize: 10, // 连接池大小
})

View File

View File

@ -15,12 +15,17 @@ import (
"github.com/natefinch/lumberjack"
"github.com/sirupsen/logrus"
"go.uber.org/zap"
"github.com/spf13/pflag"
)
// 私有变量只能在main包内访问
var jwtSigningKey []byte
func main() {
// 使用 pflag 解析命令行参数
configFile := pflag.String("config", "../conf/config.toml", "Path to the configuration file")
pflag.Parse()
//检查时区配置,需要包含东八区
_, err := time.LoadLocation(KEY_LOCAL_TIMEZONE)
if err != nil {
@ -35,6 +40,12 @@ func main() {
os.Exit(1) // Exit the program with an error code
}
errconf := configManager.initConfig(*configFile)
if errconf != nil {
fmt.Fprintf(os.Stderr, "Failed to load config: %v\n", errconf)
os.Exit(1) // Exit the program with an error code
}
logconfig := configManager.GetLogConfig()
initLogger(logconfig.LogFile, logconfig.MaxSize, logconfig.MaxBackups, logconfig.MaxAge, logconfig.Compress, logconfig.Level) // 初始化全局日志
defer logger.Sync() // 刷到磁盘
@ -152,7 +163,7 @@ func parseToken(c echo.Context, auth string) (interface{}, error) {
}
if claims, ok := token.Claims.(*jwtCustomClaims); ok && token.Valid {
logger.Info("claims: ", zap.Any("claims", claims))
logger.Info("claims: ", zap.Int("GID", claims.GID), zap.String("DeviceID", claims.DeviceID), zap.Int64("Exp1", claims.Exp1))
// 判断token有效期
if time.Now().Unix() > claims.Exp1 {
return nil, echo.NewHTTPError(http.StatusUnauthorized, "Token expired")

View File

@ -141,7 +141,8 @@ func queryUserBenefits(c echo.Context) (bool, error) {
db, _ := GetDBManager()
var vip int
err := db.MySQL.QueryRow("SELECT IsVIP FROM vip WHERE ID = ?", ID).Scan(&vip)
// 查询 vip 表因为可能VIP过期所以要加上时间戳的判断。这里的服务器是东八区时间。
err := db.MySQL.QueryRow("SELECT IsVIP FROM vip WHERE ID = ? AND ExpDate >= ?", ID, time.Now()).Scan(&vip)
if err == sql.ErrNoRows {
// 非VIP查询redis的免费次数
db, _ := GetDBManager()
@ -152,6 +153,9 @@ func queryUserBenefits(c echo.Context) (bool, error) {
return false, err
} else {
logger.Debug("CheckAndDecrement", zap.Int("ID", ID), zap.String("timeZone", timeZone), zap.Int("secondsFromGMT", secondsFromGMT), zap.Int("status", status))
if status != 0 {
logger.Warn("user beyond limit.", zap.Int("ID", ID), zap.String("timeZone", timeZone), zap.Int("secondsFromGMT", secondsFromGMT), zap.Int("status", status))
}
return status == 0, nil
}
} else if err != nil {
@ -160,6 +164,7 @@ func queryUserBenefits(c echo.Context) (bool, error) {
}
if vip == 1 {
logger.Debug("queryUserBenefits", zap.Int("ID", ID), zap.String("timeZone", timeZone), zap.Int("secondsFromGMT", secondsFromGMT), zap.Int("isvip", 1))
return true, nil
}
return false, nil
@ -192,16 +197,24 @@ func UpdateOrderByVerify(ID int, AppAcountToken string, OriginTransID string, tr
// 如果是Sandbox交易那么直接使用Transaction中的过期时间注意匹配时区
if strings.EqualFold(string(transantion.Environment), "Sandbox") {
nextDay = time.Unix(transantion.ExpiresDate/1000, 0).In(time.Local)
logger.Debug("Sandbox ExpireDate", zap.Any("ExpireDate", nextDay), zap.Any("NowDate", currentTime))
logger.Debug("Sandbox ExpireDate", zap.Int("ID", ID), zap.Any("ExpireDate", nextDay), zap.Any("NowDate", currentTime))
} else {
logger.Debug("Production ExpireDate", zap.Int("ID", ID), zap.Any("ExpireDate", nextDay), zap.Any("NowDate", currentTime))
}
// TODO: transaction.TransactionReason 有新购和续费,需要区分;同一个购买或者续费事件,可能有通知多次,需要排重
var tmpID int
errDup := db.MySQL.QueryRow("SELECT ID from vip where TransactionID = ? and OriginalTransactionID = ? and IsVip = 1 and ExpDate > ?", transantion.TransactionID, transantion.OriginalTransactionId, currentTime).Scan(&tmpID)
if errDup != sql.ErrNoRows {
// 不为空有两种可能一是请求重复了比如用户发起了Restore另一种可能是用户换了设备甚至是UserID这时候就得处理把原来的ID会员拿掉换到新的上面来。
if tmpID != ID {
logger.Warn("duplicate request, but different ID", zap.Int("tblID", tmpID), zap.Int("ReqID", ID), zap.String("AppAcountToken", AppAcountToken), zap.String("OriginTransID", OriginTransID), zap.String("TransactionID", transantion.TransactionID))
return nil
} else {
// 表示重复了,可以直接返回
logger.Info("duplicate request", zap.Int("ID", ID), zap.String("AppAcountToken", AppAcountToken), zap.String("OriginTransID", OriginTransID), zap.String("TransactionID", transantion.TransactionID))
return nil
}
} else if errDup != nil {
logger.Info("prepare to insert record", zap.Int("ID", ID), zap.String("AppAcountToken", AppAcountToken), zap.String("OriginTransID", OriginTransID))
// 这里不返回,继续尝试更新。
@ -213,15 +226,40 @@ func UpdateOrderByVerify(ID int, AppAcountToken string, OriginTransID string, tr
ON DUPLICATE KEY UPDATE
IsVip = 1, AppStore = ?, ProductID = ?, ProductType = ?, Environment = ?, Price = ?, Currency = ?, Storefront = ?, PurchaseDate = ?, ExpDate = ?, AutoRenew = ?, OriginalTransactionID = ? , TransactionID = ?, AppAccountToken = ?, TransactionReason = ? `
_, err2 := db.MySQL.Exec(sql,
// 开始事务
tx, err := db.MySQL.Begin()
if err != nil {
logger.Error("Mysql Transaction error.")
return nil
}
// 写入订阅信息
_, err2 := tx.Exec(sql,
ID, APPSTORE, transantion.ProductID, ProductType, transantion.Environment, Price, Currency, transantion.Storefront, currentTime, nextDay, 1, OriginTransID, transantion.TransactionID, transantion.AppAccountToken, transantion.TransactionReason,
APPSTORE, transantion.ProductID, ProductType, transantion.Environment, Price, Currency, transantion.Storefront, currentTime, nextDay, 1, OriginTransID, transantion.TransactionID, transantion.AppAccountToken, transantion.TransactionReason)
if err2 != nil {
logger.Error("UpdateOrderByVerify", zap.Error(err), zap.Int("ID", ID), zap.String("AppAcountToken", AppAcountToken), zap.String("OriginTransID", OriginTransID))
logger.Error("UpdateOrderByVerify", zap.Error(err2), zap.Int("ID", ID), zap.String("AppAcountToken", AppAcountToken), zap.String("OriginTransID", OriginTransID))
tx.Rollback()
return err2
}
// 如果是订阅信息转移到了另外一个ID上需要把旧的ID 的vip去掉。
if tmpID != ID {
// 更新vip表tmpID 行的 ExpDate 设置为 currentTime
_, err3 := tx.Exec("update vip set ExpDate = ? where ID = ? ", currentTime, tmpID)
if err3 != nil {
logger.Error("UpdateOrderByVerify update error", zap.Error(err3), zap.Int("ID", tmpID), zap.String("AppAcountToken", AppAcountToken), zap.String("OriginTransID", OriginTransID))
tx.Rollback()
return err3
}
}
err = tx.Commit()
if err != nil {
logger.Error("tx.Commit", zap.Error(err))
return err
}
return nil
}

272
tools/puzzle.py Normal file
View File

@ -0,0 +1,272 @@
'''
词库来自: https://diginoodles.com/projects/eowl
'''
import os
import json
import random
import time
import logging
import argparse
from collections import defaultdict
from pathlib import Path
from openai import AzureOpenAI
endpoint = "https://grammar.openai.azure.com/"
model_name = "gpt-4o"
deployment = "gpt4"
subscription_key = "8b68c235b737488ab9a99983a14f8cca"
api_version = "2024-12-01-preview"
client = AzureOpenAI(
api_version=api_version,
azure_endpoint=endpoint,
api_key=subscription_key,
)
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s: %(message)s",
handlers=[logging.StreamHandler()]
)
BASE_DIR = './data'
WORDS_DIR = f"{BASE_DIR}/EOWL-v1.1.2/LF Delimited Format"
RESULT_DIR = f"{BASE_DIR}/result"
os.makedirs(RESULT_DIR, exist_ok=True)
TEMP_FILE = f"{BASE_DIR}/temp_words.txt"
batch_words_size = 100
def find_words_files(folder):
txt_files = []
for f in Path(folder).glob("*.txt"):
if "Words" in f.name:
txt_files.append(f)
return txt_files
def collect_words(files):
words_set = set()
for file in files:
with open(file, 'r', encoding='utf-8') as f:
for line in f:
word = line.strip()
if len(word) >= 3:
words_set.add(word)
return list(words_set)
def write_temp(words):
with open(TEMP_FILE, 'w', encoding='utf-8') as f:
for word in words:
f.write(word + '\n')
def read_batches(batch_size=batch_words_size):
with open(TEMP_FILE, 'r', encoding='utf-8') as f:
words = [line.strip() for line in f if line.strip()]
for i in range(0, len(words), batch_size):
yield words[i:i+batch_size]
'''Please respond with pure JSON only, without any formatting or explanations.'''
def build_prompt(words):
word_list = ", ".join(words)
prompt = f"""
Please analyze the following list of English words and do the following:
1. Classify each word into a theme (like Animals, Plants, Materials, Body Parts, Clothes & Accessories, Food & Drinks, Places, Transportation, Sports, Colors, Numbers, Emotions, Tools, People & Occupations, etc.).
2. Identify the part of speech of each word (verb, noun, adjective, etc.).
3. Mark the frequency of usage of each word in everyday English as High, Medium, or Low.
4. Identify words with the same word root and group them.
For each word, return a JSON array where each item is an object with these keys:
- w: the word
- t: theme (like Animals, Tools, etc.)
- p: part of speech (noun, verb, etc.)
- f: frequency (Low/Medium/High)
- s: same root group (array of words with the same root)
Respond with PURE JSON ONLY, without markdown or explanations.
Here are the words:
{word_list}
"""
return prompt
def call_openai_with_retry(prompt, retries=3, delay=5):
for attempt in range(retries):
try:
response = client.chat.completions.create(
messages=[
{"role": "system", "content": "You are an expert English linguist and lexicographer."},
{"role": "user", "content": prompt}
],
max_tokens=16000,
temperature=0.7,
top_p=1.0,
model=deployment
)
#return response.choices[0].message.content.strip()
text = response.choices[0].message.content.strip()
# 如果还有 ```json 开头的,去掉
if text.startswith("```json"):
text = text[7:-3].strip()
return text
except Exception as e:
logging.warning(f"OpenAI request failed (attempt {attempt+1}): {e}")
time.sleep(delay)
logging.error("OpenAI request failed after all retries.")
return None
def save_result(index, req, resp, is_json):
matched = True if is_json and len(req) == len(resp) else False
flag = "json" if is_json else "txt"
match_str = "matched" if matched else 'notmatch'
filename = f"{RESULT_DIR}/{str(index).zfill(5)}_{match_str}_{flag}.json"
data = {
'req_len': len(req),
'rsp_len': len(resp) if is_json else 0,
'match':matched,
'req': req,
'rsp': resp
}
with open(filename, 'w', encoding='utf-8') as f:
json.dump(data, f, indent=2, ensure_ascii=False)
logging.info(f"Saved result to {filename}")
def process_folder(folder):
files = find_words_files(folder)
logging.info(f"Found {len(files)} files to process.")
words = collect_words(files)
logging.info(f"Collected {len(words)} unique words.")
write_temp(words)
for idx, batch in enumerate(read_batches(), 1):
logging.info(f"Processing batch {idx} with {len(batch)} words")
prompt = build_prompt(batch)
resp_text = call_openai_with_retry(prompt)
if resp_text is None:
save_result(idx, batch, "Failed to get response", False)
continue
try:
resp_json = json.loads(resp_text)
save_result(idx, batch, resp_json, True)
except json.JSONDecodeError:
logging.warning(f"Batch {idx} response is not valid JSON.")
save_result(idx, batch, resp_text, False)
time.sleep(2) # 每批之间暂停
# redo逻辑
def redo_results():
files = sorted(Path(RESULT_DIR).glob('*.json'))
for f in files:
if 'matched' in f.name:
continue
logging.info(f"Redoing {f}")
try:
with open(f, 'r', encoding='utf-8') as fp:
data = json.load(fp)
words = data.get("req")
if not words:
logging.warning(f"No req in {f}")
continue
prompt = build_prompt(words)
resp_text = call_openai_with_retry(prompt)
if resp_text is None:
logging.warning(f"Failed to get response: {f}")
continue
try:
resp_json = json.loads(resp_text)
if len(words) == len(resp_json):
logging.info(f"get correct response. rewrite file. {f}")
f.unlink()
save_result(int(f.name[:5]), words, resp_json, True)
else:
logging.warning(f"response not complete: {f}, req len: {len(words)}, rsp len: {len(resp_json)}")
except json.JSONDecodeError:
logging.warning(f"response is not valid JSON: {f}")
time.sleep(2) # 每批之间暂停
except Exception as e:
logging.error(f"Error processing {f}: {e}")
# 检测是否无重复字母
def has_no_repeated_letters(word):
return len(set(word)) == len(word)
def generate_wordlist():
"""
从 RESULT_DIR 下的 matched 文件中提取无重复字母的单词,并按 f 分类写入 words_{f}.txt
"""
word_map = defaultdict(list)
all_words = set()
# 优化写法:先筛选再排序
matched_files = []
for file in os.scandir(RESULT_DIR):
# 同上的过滤条件
if (file.is_file()
and file.name.endswith('.json')
and 'matched' in file.name
and len(file.name) >= 5
and file.name[:5].isdigit()):
matched_files.append(file)
for file in sorted(matched_files, key=lambda f: int(f.name[:5])):
if 'matched' not in file.name:
continue
with open(file.path, 'r', encoding='utf-8') as f:
data = json.load(f)
rsp = data.get('rsp', [])
for item in rsp:
word = item.get('w')
freq = item.get('f')
if word and freq and has_no_repeated_letters(word):
word_map[freq].append(word)
all_words.add(word)
# 写入文件
for freq, words in word_map.items():
filename = os.path.join(RESULT_DIR, f'words_{freq}.txt')
with open(filename, 'w', encoding='utf-8') as f:
for word in words:
f.write(word + '\n')
logging.info(f'✅ 写入完成: {filename} ({len(words)} 个单词)')
# 写全量
filename = os.path.join(RESULT_DIR, 'wordlist.txt')
with open(filename, 'w', encoding='utf-8') as f:
for word in all_words:
f.write(word + '\n')
logging.info(f'✅ 写入完成: {filename} ({len(all_words)} 个单词)')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('cmd', help='执行的命令: init / redo / gen')
args = parser.parse_args()
if args.cmd == 'init':
process_folder(WORDS_DIR)
elif args.cmd == 'redo':
redo_results()
elif args.cmd == 'gen':
generate_wordlist()
else:
print("❌ 未知命令,请使用: all / redo / gen")
if __name__ == '__main__':
main()