349 lines
12 KiB
Python
349 lines
12 KiB
Python
"""GitMirror 服务入口
|
||
|
||
把 git.gvsds.com/GVSDS/* 实时镜像到 github.com/GVSDS/*：
|
||
|
||
- POST /hooks/gitea Gitea 组织级 webhook（HMAC-SHA256 验签）→ 入队同步
|
||
- POST /backfill 手动全量回填（X-Admin-Token）
|
||
- POST /sync/<repo> 手动同步单仓（X-Admin-Token）
|
||
- GET /status 镜像状态（X-Admin-Token）
|
||
- GET /healthz 健康检查
|
||
|
||
后台：工作线程池执行镜像；定时线程按 SyncIntervalMinutes 做全量兜底。
|
||
|
||
用法：
|
||
python app.py serve
|
||
python app.py backfill
|
||
python app.py sync <repo>
|
||
python app.py register-webhook --url https://mirror.example.com/hooks/gitea
|
||
python app.py status
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import hmac
|
||
import json
|
||
import logging
|
||
import os
|
||
import queue
|
||
import sys
|
||
import threading
|
||
import time
|
||
|
||
from flask import Flask, jsonify, request
|
||
|
||
from mirror import (
|
||
GiteaClient,
|
||
GithubClient,
|
||
MirrorConfig,
|
||
MirrorError,
|
||
MirrorManager,
|
||
MirrorStore,
|
||
)
|
||
|
||
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
||
CONFIG_PATH = os.environ.get('GITMIRROR_CONFIG') or os.path.join(BASE_DIR, 'config.ini')
|
||
|
||
logger = logging.getLogger('gitmirror')
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 全局组件（CLI 与 HTTP 共用）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
Cfg = MirrorConfig(CONFIG_PATH)
|
||
Gitea = GiteaClient(Cfg.GiteaBaseUrl, Cfg.GiteaOrg, Cfg.GiteaToken)
|
||
Github = GithubClient(Cfg.GithubOrg, Cfg.GithubToken)
|
||
Store = MirrorStore(Cfg.DbFile)
|
||
Manager = MirrorManager(Cfg, Gitea, Github, Store)
|
||
|
||
|
||
class SyncWorker:
|
||
"""去重的同步队列 + 工作线程
|
||
|
||
失败处理：丢弃本地镜像并把该仓库**追加到队尾**（不阻塞其它仓库）。
|
||
每个仓库在单轮回填中最多重排 MAX_REQUEUE 次，成功即清零；
|
||
每 15 分钟的定时回填会重置预算，继续兜底。
|
||
"""
|
||
|
||
MAX_REQUEUE = 10
|
||
|
||
def __init__(self, Workers: int):
|
||
self.Queue: queue.Queue[tuple[str, dict | None]] = queue.Queue()
|
||
self._Pending: set[str] = set()
|
||
self._Guard = threading.Lock()
|
||
self._Workers = Workers
|
||
self._Threads: list[threading.Thread] = []
|
||
self._Requeues: dict[str, int] = {}
|
||
|
||
def Start(self) -> None:
|
||
for Index in range(self._Workers):
|
||
Thread = threading.Thread(target=self._Run, name=f'GitMirrorWorker-{Index}', daemon=True)
|
||
Thread.start()
|
||
self._Threads.append(Thread)
|
||
|
||
def Put(self, RepoName: str, Repo: dict | None = None) -> bool:
|
||
"""入队；已在队列中的同名仓库不重复入队"""
|
||
with self._Guard:
|
||
if RepoName in self._Pending:
|
||
return False
|
||
self._Pending.add(RepoName)
|
||
self.Queue.put((RepoName, Repo))
|
||
return True
|
||
|
||
def PendingCount(self) -> int:
|
||
with self._Guard:
|
||
return len(self._Pending)
|
||
|
||
def ResetRequeueBudget(self) -> None:
|
||
"""新一轮回填：清空重排计数"""
|
||
with self._Guard:
|
||
self._Requeues.clear()
|
||
|
||
def _Run(self) -> None:
|
||
while True:
|
||
RepoName, Repo = self.Queue.get()
|
||
Requeue = False
|
||
try:
|
||
Requeue = self._SyncOnce(RepoName, Repo)
|
||
except Exception: # 兜底，避免工作线程退出
|
||
logger.exception('同步任务异常：%s', RepoName)
|
||
finally:
|
||
with self._Guard:
|
||
self._Pending.discard(RepoName)
|
||
self.Queue.task_done()
|
||
if Requeue:
|
||
self.Put(RepoName, Repo)
|
||
|
||
def _SyncOnce(self, RepoName: str, Repo: dict | None) -> bool:
|
||
"""执行一次同步；返回 True 表示需要重新排到队尾"""
|
||
try:
|
||
Info = Repo or Gitea.GetRepo(RepoName)
|
||
if not Info:
|
||
raise MirrorError(f'Gitea 中不存在仓库 {RepoName}')
|
||
Manager.Sync(Info)
|
||
with self._Guard:
|
||
self._Requeues.pop(RepoName, None)
|
||
return False
|
||
except Exception as Error:
|
||
logger.warning('同步失败 %s：%s', RepoName, Error)
|
||
# 丢弃可能损坏的本地镜像（大仓 HTTP/2 断流后无法续传）
|
||
Manager.DropLocal(RepoName)
|
||
with self._Guard:
|
||
Count = self._Requeues.get(RepoName, 0) + 1
|
||
self._Requeues[RepoName] = Count
|
||
if Count <= self.MAX_REQUEUE:
|
||
Store.SetStatus(RepoName, Status='pending', LastError=str(Error)[:1000])
|
||
logger.info('已排到队尾（第 %s 次重排）：%s', Count, RepoName)
|
||
return True
|
||
Store.SetStatus(RepoName, Status='failed', LastError=str(Error)[:1000])
|
||
logger.error('超过重排上限，标记失败 %s：%s', RepoName, Error)
|
||
return False
|
||
|
||
|
||
Worker = SyncWorker(Cfg.Concurrency)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 任务：全量回填 / 删除处理
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def Backfill() -> dict:
|
||
"""枚举组织全部仓库并入队（幂等）"""
|
||
Worker.ResetRequeueBudget()
|
||
Repos = Gitea.ListRepos()
|
||
Queued = 0
|
||
for Repo in Repos:
|
||
Name = Repo.get('name') or ''
|
||
if not Name:
|
||
continue
|
||
Store.SetStatus(Name, SourcePrivate=int(bool(Repo.get('private', True))))
|
||
if Worker.Put(Name, Repo):
|
||
Queued += 1
|
||
logger.info('全量回填：共 %s 个仓库，新入队 %s 个', len(Repos), Queued)
|
||
return {'Total': len(Repos), 'Queued': Queued}
|
||
|
||
|
||
def HandleRepoDeleted(RepoName: str) -> None:
|
||
try:
|
||
Manager.HandleRepoDeleted(RepoName)
|
||
except Exception:
|
||
logger.exception('处理源仓库删除失败：%s', RepoName)
|
||
|
||
|
||
class Scheduler:
|
||
"""定时兜底：每 SyncIntervalMinutes 一次全量回填（防 webhook 丢失）"""
|
||
|
||
def __init__(self, IntervalMinutes: int, RunOnStart: bool):
|
||
self.Interval = max(1, IntervalMinutes) * 60
|
||
self.RunOnStart = RunOnStart
|
||
self._Stop = threading.Event()
|
||
|
||
def Start(self) -> None:
|
||
threading.Thread(target=self._Run, name='GitMirrorScheduler', daemon=True).start()
|
||
|
||
def _Run(self) -> None:
|
||
if self.RunOnStart:
|
||
try:
|
||
Backfill()
|
||
except Exception:
|
||
logger.exception('启动回填失败')
|
||
while not self._Stop.wait(self.Interval):
|
||
try:
|
||
Backfill()
|
||
except Exception:
|
||
logger.exception('定时回填失败')
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Flask
|
||
# ---------------------------------------------------------------------------
|
||
|
||
App = Flask(__name__)
|
||
|
||
|
||
def _VerifySignature(RawBody: bytes, Signature: str | None) -> bool:
|
||
if not Signature:
|
||
return False
|
||
Expected = hmac.new(Cfg.WebhookSecret.encode('utf-8'), RawBody, hashlib.sha256).hexdigest()
|
||
return hmac.compare_digest(Expected, Signature)
|
||
|
||
|
||
def _IsAdmin() -> bool:
|
||
return bool(Cfg.AdminToken) and request.headers.get('X-Admin-Token') == Cfg.AdminToken
|
||
|
||
|
||
@App.get('/healthz')
|
||
def Healthz():
|
||
return jsonify({'ok': True, 'pending': Worker.PendingCount(), 'gitea': Cfg.GiteaOrg, 'github': Cfg.GithubOrg})
|
||
|
||
|
||
@App.post('/hooks/gitea')
|
||
def GiteaHook():
|
||
Raw = request.get_data() or b''
|
||
if not _VerifySignature(Raw, request.headers.get('X-Gitea-Signature')):
|
||
return jsonify({'code': 1, 'msg': '签名校验失败'}), 401
|
||
Event = request.headers.get('X-Gitea-Event', '')
|
||
Payload = request.get_json(silent=True) or {}
|
||
Repo = Payload.get('repository') or {}
|
||
RepoName = (Repo.get('name') or '').strip()
|
||
if not RepoName:
|
||
return jsonify({'code': 0, 'msg': 'ignored'}), 202
|
||
|
||
Action = (Payload.get('action') or '').strip()
|
||
if Event == 'repository' and Action == 'deleted':
|
||
threading.Thread(target=HandleRepoDeleted, args=(RepoName,), daemon=True).start()
|
||
return jsonify({'code': 0, 'msg': 'repo deleted handled'}), 202
|
||
|
||
Worker.Put(RepoName, Repo)
|
||
logger.info('webhook 入队 event=%s action=%s repo=%s', Event, Action, RepoName)
|
||
return jsonify({'code': 0, 'msg': 'queued'}), 202
|
||
|
||
|
||
@App.post('/backfill')
|
||
def BackfillRoute():
|
||
if not _IsAdmin():
|
||
return jsonify({'code': 1, 'msg': 'unauthorized'}), 401
|
||
return jsonify({'code': 0, 'data': Backfill()})
|
||
|
||
|
||
@App.post('/sync/<RepoName>')
|
||
def SyncRoute(RepoName: str):
|
||
if not _IsAdmin():
|
||
return jsonify({'code': 1, 'msg': 'unauthorized'}), 401
|
||
Queued = Worker.Put(RepoName)
|
||
return jsonify({'code': 0, 'data': {'RepoName': RepoName, 'Queued': Queued}})
|
||
|
||
|
||
@App.get('/status')
|
||
def StatusRoute():
|
||
if not _IsAdmin():
|
||
return jsonify({'code': 1, 'msg': 'unauthorized'}), 401
|
||
return jsonify({'code': 0, 'data': {'Pending': Worker.PendingCount(), 'Repos': Store.List()}})
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# CLI
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def _SetupLogging() -> None:
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format='%(asctime)s [%(levelname)s] %(name)s: %(message)s',
|
||
)
|
||
|
||
|
||
def _CmdServe() -> None:
|
||
Worker.Start()
|
||
Scheduler(Cfg.SyncIntervalMinutes, Cfg.BackfillOnStart).Start()
|
||
logger.info('GitMirror 启动：%s -> github.com/%s，每 %s 分钟兜底，webhook secret 已就绪',
|
||
f'{Cfg.GiteaBaseUrl}/{Cfg.GiteaOrg}', Cfg.GithubOrg, Cfg.SyncIntervalMinutes)
|
||
logger.info('监听 %s:%s（webhook 路径 /hooks/gitea）', Cfg.ServerHost, Cfg.ServerPort)
|
||
App.run(host=Cfg.ServerHost, port=Cfg.ServerPort, threaded=True)
|
||
|
||
|
||
def _CmdBackfill() -> None:
|
||
Result = Backfill()
|
||
Worker.Start()
|
||
logger.info('已入队 %s 个仓库，等待完成（Ctrl+C 可提前退出）', Result['Queued'])
|
||
while Worker.PendingCount() > 0:
|
||
time.sleep(1)
|
||
logger.info('回填结束')
|
||
|
||
|
||
def _CmdSync(RepoName: str) -> None:
|
||
Info = Gitea.GetRepo(RepoName)
|
||
if not Info:
|
||
raise MirrorError(f'Gitea 中不存在仓库 {RepoName}')
|
||
Manager.Sync(Info)
|
||
logger.info('同步完成：%s', RepoName)
|
||
|
||
|
||
def _CmdRegisterWebhook(Url: str) -> None:
|
||
Action = Gitea.RegisterWebhook(Url, Cfg.WebhookSecret)
|
||
logger.info('组织 webhook 已 %s：%s（events: push/create/delete/repository）', Action, Url)
|
||
logger.info('Webhook Secret：%s', Cfg.WebhookSecret)
|
||
|
||
|
||
def _CmdStatus() -> None:
|
||
print(json.dumps({'Pending': Worker.PendingCount(), 'Repos': Store.List()}, ensure_ascii=False, indent=2))
|
||
|
||
|
||
def main() -> int:
|
||
_SetupLogging()
|
||
Parser = argparse.ArgumentParser(description='Gitea → GitHub 仓库镜像服务')
|
||
Sub = Parser.add_subparsers(dest='command')
|
||
|
||
Sub.add_parser('serve', help='启动 HTTP 服务（webhook + 定时兜底）')
|
||
Sub.add_parser('backfill', help='全量回填一次')
|
||
SyncParser = Sub.add_parser('sync', help='同步单个仓库')
|
||
SyncParser.add_argument('repo')
|
||
HookParser = Sub.add_parser('register-webhook', help='注册/更新 Gitea 组织级 webhook')
|
||
HookParser.add_argument('--url', required=True, help='本服务对外地址，如 https://mirror.example.com/hooks/gitea')
|
||
Sub.add_parser('status', help='打印镜像状态')
|
||
|
||
Args = Parser.parse_args()
|
||
Command = Args.command or 'serve'
|
||
try:
|
||
if Command == 'serve':
|
||
_CmdServe()
|
||
elif Command == 'backfill':
|
||
_CmdBackfill()
|
||
elif Command == 'sync':
|
||
_CmdSync(Args.repo)
|
||
elif Command == 'register-webhook':
|
||
_CmdRegisterWebhook(Args.url)
|
||
elif Command == 'status':
|
||
_CmdStatus()
|
||
else:
|
||
Parser.print_help()
|
||
return 2
|
||
except MirrorError as Error:
|
||
logger.error('%s', Error)
|
||
return 1
|
||
return 0
|
||
|
||
|
||
if __name__ == '__main__':
|
||
sys.exit(main())
|