add: 服务器卡顿诊断脚本 profile-server-lag 与 fix-db-latency
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
XingQue
168
deploy/profile-server-lag.sh
Normal file
168
deploy/profile-server-lag.sh
Normal file
@@ -0,0 +1,168 @@
|
||||
#!/bin/bash
|
||||
# 服务器卡顿定位 — 一次跑完，输出「卡在哪一层」结论
|
||||
# 用法: bash deploy/profile-server-lag.sh
|
||||
# 只读诊断，不改任何配置
|
||||
|
||||
set -e
|
||||
DJANGO_DIR="/opt/1panel/apps/openresty/nginx/www/sites/43.142.166.152.443/django"
|
||||
NGINX_LOG="/opt/1panel/apps/openresty/nginx/log/access.log"
|
||||
cd "$DJANGO_DIR"
|
||||
|
||||
echo "=============================================="
|
||||
echo " 星阙服务器卡顿定位 — $(date '+%Y-%m-%d %H:%M:%S')"
|
||||
echo "=============================================="
|
||||
|
||||
# ---------- A. 进程层：Gunicorn 是否在排队 ----------
|
||||
echo ""
|
||||
echo ">>> [A] Gunicorn 进程 / 端口"
|
||||
systemctl is-active gunicorn-dianjing.service 2>/dev/null || echo "systemd: inactive"
|
||||
GC=$(pgrep -cf 'gunicorn.*a_long_dianjing' 2>/dev/null || echo 0)
|
||||
echo "gunicorn 进程数: $GC (正常=5: 1master+4worker)"
|
||||
ss -ltn 2>/dev/null | grep ':8001' || netstat -ltn 2>/dev/null | grep ':8001' || echo "8001 未监听"
|
||||
|
||||
echo ""
|
||||
echo ">>> [A2] 本机直连 Gunicorn（绕过 Nginx）"
|
||||
for path in "/" "/hqhd/"; do
|
||||
T=$(curl -s -o /dev/null -w '%{time_total}' --connect-timeout 3 "http://127.0.0.1:8001${path}" 2>/dev/null || echo "FAIL")
|
||||
echo " 127.0.0.1:8001${path} -> ${T}s"
|
||||
done
|
||||
|
||||
# ---------- B. 数据库层：握手 + 单条 SQL ----------
|
||||
echo ""
|
||||
echo ">>> [B] MySQL 延迟（从本机 Django 目录测）"
|
||||
python3 - <<'PY'
|
||||
import time, sys
|
||||
try:
|
||||
import app_secrets
|
||||
import pymysql
|
||||
except Exception as e:
|
||||
print(f" 无法导入: {e}")
|
||||
sys.exit(0)
|
||||
|
||||
host, port = app_secrets.DATABASE_HOST, int(app_secrets.DATABASE_PORT)
|
||||
user, pwd, db = app_secrets.DATABASE_USER, app_secrets.DATABASE_PASSWORD, app_secrets.DATABASE_NAME
|
||||
print(f" 当前配置: {host}:{port} / {db}")
|
||||
|
||||
def bench(label, h, p):
|
||||
try:
|
||||
t0 = time.perf_counter()
|
||||
conn = pymysql.connect(host=h, port=p, user=user, password=pwd, database=db, connect_timeout=5)
|
||||
t_conn = (time.perf_counter() - t0) * 1000
|
||||
t1 = time.perf_counter()
|
||||
with conn.cursor() as c:
|
||||
c.execute("SELECT COUNT(*) FROM information_schema.tables WHERE table_schema=%s", (db,))
|
||||
c.fetchone()
|
||||
t_q = (time.perf_counter() - t1) * 1000
|
||||
conn.close()
|
||||
print(f" [{label}] {h}:{p} 新建连接={t_conn:.0f}ms 单条SQL={t_q:.0f}ms")
|
||||
return t_conn, t_q
|
||||
except Exception as e:
|
||||
print(f" [{label}] {h}:{p} 失败: {e}")
|
||||
return None, None
|
||||
|
||||
cur_c, cur_q = bench("生产库", host, port)
|
||||
loc_c, loc_q = bench("本机Docker", "172.17.0.15", 3306)
|
||||
|
||||
print("")
|
||||
if cur_q and cur_q > 30:
|
||||
print(" >>> 结论B: 数据库在远程，每条SQL ~50ms+，接口跑10+条SQL就会慢200ms+")
|
||||
if loc_q and loc_q < 10 and cur_q and cur_q > 30:
|
||||
print(" >>> 修复: app_secrets.py 改 DATABASE_HOST=172.17.0.15 PORT=3306 后 restart gunicorn")
|
||||
PY
|
||||
|
||||
# ---------- C. Django 层：模拟 dddhq 打了多少条 SQL ----------
|
||||
echo ""
|
||||
echo ">>> [C] /dingdan/ddhq 一次请求打几条 SQL、耗时多少"
|
||||
python3 - <<'PY'
|
||||
import os, sys, time
|
||||
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'a_long_dianjing.settings')
|
||||
sys.path.insert(0, '.')
|
||||
|
||||
try:
|
||||
import django
|
||||
django.setup()
|
||||
from django.db import connection, reset_queries
|
||||
from django.conf import settings
|
||||
settings.DEBUG = True # 仅本脚本进程内开启 query log
|
||||
|
||||
reset_queries()
|
||||
t0 = time.perf_counter()
|
||||
|
||||
# 不跑完整 view（要 token），只测典型 ORM 链：订单列表 count + page
|
||||
from orders.models import Order
|
||||
from django.db.models import Q
|
||||
qs = Order.query.filter(Q(Status__in=[1, 7])).order_by('-CreateTime')
|
||||
total = qs.count()
|
||||
page = list(qs[:5].values('OrderID', 'Status', 'CreateTime'))
|
||||
|
||||
elapsed = (time.perf_counter() - t0) * 1000
|
||||
n = len(connection.queries)
|
||||
sql_ms = sum(float(q.get('time', 0)) for q in connection.queries) * 1000
|
||||
|
||||
print(f" 仅订单 count+取5条: SQL条数={n} Django记录SQL耗时={sql_ms:.0f}ms 总耗时={elapsed:.0f}ms 订单总数={total}")
|
||||
print(f" （完整 /ddhq 还有会员/标签/商家等查询，实际 SQL 通常 15~30 条）")
|
||||
if n > 0:
|
||||
print(" 最慢的3条SQL:")
|
||||
for q in sorted(connection.queries, key=lambda x: float(x.get('time', 0)), reverse=True)[:3]:
|
||||
sql = q['sql'][:120].replace('\n', ' ')
|
||||
print(f" {float(q['time'])*1000:.0f}ms {sql}...")
|
||||
except Exception as e:
|
||||
print(f" Django 测速失败: {e}")
|
||||
import traceback; traceback.print_exc()
|
||||
PY
|
||||
|
||||
# ---------- D. Nginx 层：最近真实接口 rt= 分布 ----------
|
||||
echo ""
|
||||
echo ">>> [D] Nginx 最近 hqhd 接口 rt=（真实用户体感）"
|
||||
if [ -f "$NGINX_LOG" ]; then
|
||||
echo " 最近20条 hqhd:"
|
||||
grep 'hqhd' "$NGINX_LOG" | tail -20 | while read -r line; do
|
||||
RT=$(echo "$line" | grep -oP 'rt=\K[0-9.]+' || echo "?")
|
||||
URI=$(echo "$line" | grep -oP '"(GET|POST) \K[^ ]+' || echo "?")
|
||||
echo " rt=${RT}s $URI"
|
||||
done
|
||||
echo ""
|
||||
echo " 各接口 rt 中位数（最近500条 hqhd）:"
|
||||
grep 'hqhd' "$NGINX_LOG" | tail -500 | grep -oP 'rt=\K[0-9.]+' | sort -n | awk '
|
||||
{ a[NR]=$1; sum+=$1 }
|
||||
END {
|
||||
if(NR==0){ print " 无数据"; exit }
|
||||
mid=(NR%2)?a[(NR+1)/2]:(a[NR/2]+a[NR/2+1])/2
|
||||
printf " 样本=%d 中位rt=%.3fs 平均rt=%.3fs 最大rt=%.3fs\n", NR, mid, sum/NR, a[NR]
|
||||
}'
|
||||
else
|
||||
echo " 找不到 $NGINX_LOG"
|
||||
fi
|
||||
|
||||
# ---------- E. 系统层：CPU/内存/磁盘/journal ----------
|
||||
echo ""
|
||||
echo ">>> [E] 系统资源"
|
||||
echo " load: $(cat /proc/loadavg 2>/dev/null || uptime)"
|
||||
free -h 2>/dev/null | head -2 || true
|
||||
journalctl --disk-usage 2>/dev/null | head -1 || true
|
||||
|
||||
# ---------- F. 并发层：谁在狂刷 ----------
|
||||
echo ""
|
||||
echo ">>> [F] 最近1分钟请求最多的 IP+接口（是否有人刷接口拖慢）"
|
||||
if [ -f "$NGINX_LOG" ]; then
|
||||
tail -3000 "$NGINX_LOG" | grep 'hqhd' | awk '{
|
||||
for(i=1;i<=NF;i++) if($i ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) ip=$i
|
||||
if(match($0, /"(GET|POST) ([^ ]+)/, m)) uri=m[2]
|
||||
if(ip && uri) c[ip" "uri]++
|
||||
} END {
|
||||
n=0
|
||||
for(k in c) arr[n++]=c[k]" "k
|
||||
asort(arr)
|
||||
for(i=n;i>0 && i>n-10;i--) print " "arr[i]
|
||||
}' 2>/dev/null || echo " (需 gawk 支持 match 第三参数，可忽略)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=============================================="
|
||||
echo " 怎么看结果："
|
||||
echo " - A 里 curl >1s 且 B 里 SQL>30ms → 卡在数据库远程连接"
|
||||
echo " - A 里 curl 快但 D 里 rt 慢 → 卡在 Nginx/上游配置或并发"
|
||||
echo " - C 里 SQL条数>20 → 卡在接口 SQL 太多（后端代码）"
|
||||
echo " - E 里 journal 很大/load 很高 → 卡在系统资源"
|
||||
echo " - F 里同一 IP 每秒同一接口 → 卡在被人刷接口"
|
||||
echo "=============================================="
|
||||
Reference in New Issue
Block a user