bc29118a6e
商品/专区/规则/轮播分别进 shangpin/shangpintupian、zhuanqu、guize、a_long/lunbo；库只存相对路径。 Co-authored-by: Cursor <cursoragent@cursor.com>
416 lines
17 KiB
Python
416 lines
17 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""
|
||
将原 UFO 站点商品目录灌入星阙，且只写 club_id=ufo。
|
||
|
||
默认 dry-run（只打印计划，不写库）。真正写入必须加 --apply。
|
||
默认按全局类型名「三角洲」**全名精确匹配**找 id（不含这三字的一律不要）。
|
||
找到唯一一条后：
|
||
1) ClubShangpinLeixingConfig 给 club=ufo 上架该类型
|
||
2) ShangpinZhuanqu / Shangpin 的 leixing_id 挂这个全局类型 id，且 club_id=ufo
|
||
|
||
服务器上（复制整段）:
|
||
|
||
cd /opt/1panel/apps/openresty/nginx/www/sites/43.142.166.152.443/django
|
||
git pull origin main
|
||
./venv/bin/python manage.py import_ufo_catalog
|
||
./venv/bin/python manage.py import_ufo_catalog --apply
|
||
./venv/bin/python manage.py import_ufo_catalog --apply --with-banners
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import mimetypes
|
||
import os
|
||
import re
|
||
import ssl
|
||
import uuid
|
||
from decimal import Decimal, InvalidOperation
|
||
from io import BytesIO
|
||
from pathlib import Path
|
||
from urllib.request import Request, urlopen
|
||
|
||
from django.core.management.base import BaseCommand, CommandError
|
||
from django.db import transaction
|
||
|
||
TARGET_CLUB = 'ufo'
|
||
DEFAULT_LEIXING_NAME = '三角洲' # 全名精确匹配，不模糊
|
||
DEFAULT_CATALOG = Path(__file__).resolve().parents[2] / 'data' / 'ufo_catalog.json'
|
||
|
||
|
||
class Command(BaseCommand):
|
||
help = '导入 UFO 商品/专区到 club_id=ufo（默认 dry-run；类型名默认精确匹配「三角洲」）'
|
||
|
||
def add_arguments(self, parser):
|
||
parser.add_argument('--apply', action='store_true', help='真正写库（仅 ufo）')
|
||
parser.add_argument('--catalog', type=str, default='', help='catalog.json 路径')
|
||
parser.add_argument(
|
||
'--list-leixing',
|
||
action='store_true',
|
||
help='只列出全局商品类型 id/名称',
|
||
)
|
||
parser.add_argument(
|
||
'--leixing-id',
|
||
type=int,
|
||
default=0,
|
||
help='可选：直接指定全局类型主键；不传则按 --leixing-name 精确匹配',
|
||
)
|
||
parser.add_argument(
|
||
'--leixing-name',
|
||
type=str,
|
||
default=DEFAULT_LEIXING_NAME,
|
||
help='全局 ShangpinLeixing.jieshao 全名精确匹配，默认：三角洲',
|
||
)
|
||
parser.add_argument('--with-banners', action='store_true', help='同时导入点单主轮播+第二横幅')
|
||
parser.add_argument('--skip-oss', action='store_true', help='不上传 OSS，保留原 URL（应急）')
|
||
parser.add_argument('--club-id', type=str, default=TARGET_CLUB, help='仅允许 ufo')
|
||
|
||
def handle(self, *args, **options):
|
||
from products.models import ShangpinLeixing
|
||
|
||
if options.get('list_leixing'):
|
||
self.stdout.write('全局商品类型清单（ShangpinLeixing）：')
|
||
for lx in ShangpinLeixing.query.order_by('id'):
|
||
mark = ' <<-- 精确名=三角洲' if (lx.jieshao or '') == DEFAULT_LEIXING_NAME else ''
|
||
self.stdout.write(
|
||
f' id={lx.id} name={lx.jieshao!r} '
|
||
f'shenhezhuangtai={lx.shenhezhuangtai} paixu={lx.paixu}{mark}'
|
||
)
|
||
self.stdout.write(self.style.NOTICE(
|
||
'默认会精确匹配 name==「三角洲」；也可 --leixing-id=... 覆盖。\n'
|
||
'俱乐部上架表 ClubShangpinLeixingConfig 会在 --apply 时给 ufo 自动开启该类型。'
|
||
))
|
||
return
|
||
|
||
club_id = (options.get('club_id') or '').strip()
|
||
if club_id != TARGET_CLUB:
|
||
raise CommandError(
|
||
f'拒绝执行：只允许 club_id={TARGET_CLUB}，你传的是 {club_id!r}'
|
||
)
|
||
|
||
catalog_path = Path(options['catalog']).expanduser() if options['catalog'] else DEFAULT_CATALOG
|
||
if not catalog_path.is_file():
|
||
raise CommandError(f'找不到目录文件: {catalog_path}')
|
||
|
||
data = json.loads(catalog_path.read_text(encoding='utf-8'))
|
||
products = data.get('products') or []
|
||
zones = data.get('zones') or []
|
||
banners = data.get('banners') or []
|
||
banners2 = data.get('banners2') or []
|
||
apply = bool(options['apply'])
|
||
with_banners = bool(options['with_banners'])
|
||
skip_oss = bool(options['skip_oss'])
|
||
|
||
from jituan.models import Club, ClubShangpinLeixingConfig
|
||
from products.models import Shangpin, ShangpinZhuanqu
|
||
|
||
if not Club.query.filter(club_id=TARGET_CLUB).exists():
|
||
raise CommandError(f'俱乐部 {TARGET_CLUB} 不存在，请先 create_club ufo')
|
||
|
||
leixing = self._resolve_leixing(
|
||
ShangpinLeixing,
|
||
leixing_id=int(options.get('leixing_id') or 0),
|
||
leixing_name=(options.get('leixing_name') or '').strip(),
|
||
)
|
||
|
||
self.stdout.write(self.style.NOTICE(
|
||
f'目标俱乐部={TARGET_CLUB} 类型={leixing.jieshao!r}(id={leixing.id}) '
|
||
f'shenhezhuangtai={leixing.shenhezhuangtai} '
|
||
f'专区={len(zones)} 商品={len(products)} banners={len(banners)}+{len(banners2)} '
|
||
f'mode={"APPLY" if apply else "DRY-RUN"}'
|
||
))
|
||
if leixing.shenhezhuangtai == 2:
|
||
raise CommandError(
|
||
f'类型 id={leixing.id}「{leixing.jieshao}」是审核专用，拒绝挂载'
|
||
)
|
||
expect_name = (options.get('leixing_name') or DEFAULT_LEIXING_NAME).strip()
|
||
if (leixing.jieshao or '') != expect_name and not int(options.get('leixing_id') or 0):
|
||
raise CommandError(
|
||
f'安全拦截：解析到的类型名是「{leixing.jieshao}」，'
|
||
f'与要求的精确名「{expect_name}」不一致。'
|
||
)
|
||
|
||
self.stdout.write(
|
||
f'将写入：ClubShangpinLeixingConfig(club=ufo, leixing_id={leixing.id}, is_enabled=True)；'
|
||
f'专区/商品 leixing_id={leixing.id}, club_id=ufo'
|
||
)
|
||
|
||
for z in zones:
|
||
self.stdout.write(f' 专区: {z.get("name")} goods={len(z.get("goods_ids") or [])}')
|
||
for p in products[:5]:
|
||
self.stdout.write(
|
||
f' 商品样例: [{p.get("source_id")}] {p.get("title")} ¥{p.get("price")} → {p.get("zone_name")}'
|
||
)
|
||
if len(products) > 5:
|
||
self.stdout.write(f' ... 其余 {len(products) - 5} 个商品省略')
|
||
|
||
if not apply:
|
||
self.stdout.write(self.style.WARNING(
|
||
'当前 dry-run，未写库。确认无误后：\n'
|
||
' ./venv/bin/python manage.py import_ufo_catalog --apply\n'
|
||
' ./venv/bin/python manage.py import_ufo_catalog --apply --with-banners'
|
||
))
|
||
return
|
||
|
||
stats = {
|
||
'leixing_enabled': 0,
|
||
'zones_created': 0,
|
||
'zones_updated': 0,
|
||
'products_created': 0,
|
||
'products_updated': 0,
|
||
'banners': 0,
|
||
'oss_ok': 0,
|
||
'oss_fail': 0,
|
||
}
|
||
|
||
def to_oss(url: str, prefix: str) -> str:
|
||
"""下载原图 → 上传到本站 OSS → 返回相对路径（与后台 AddProduct 一致）。"""
|
||
if not url:
|
||
return ''
|
||
if skip_oss:
|
||
# 应急：允许写完整 URL；正式导入不要加 --skip-oss
|
||
return url
|
||
if not str(url).startswith('http'):
|
||
return url
|
||
path = self._upload_remote(url, prefix)
|
||
if not path:
|
||
stats['oss_fail'] += 1
|
||
raise CommandError(f'OSS 上传失败，已中止以免库里写入外站链接: {url}')
|
||
stats['oss_ok'] += 1
|
||
return path
|
||
|
||
with transaction.atomic():
|
||
cfg, created = ClubShangpinLeixingConfig.query.get_or_create(
|
||
club_id=TARGET_CLUB,
|
||
leixing_id=leixing.id,
|
||
defaults={
|
||
'is_enabled': True,
|
||
'paixu': int(leixing.paixu or 0),
|
||
'tupian_url': '',
|
||
},
|
||
)
|
||
if not cfg.is_enabled:
|
||
cfg.is_enabled = True
|
||
cfg.save(update_fields=['is_enabled', 'UpdateTime'])
|
||
stats['leixing_enabled'] += 1
|
||
elif created:
|
||
stats['leixing_enabled'] += 1
|
||
|
||
zone_id_map = {}
|
||
for idx, z in enumerate(zones):
|
||
name = (z.get('name') or '').strip()
|
||
if not name:
|
||
continue
|
||
src_id = int(z.get('source_id') or 0)
|
||
thumb = to_oss(z.get('thumb_url') or '', 'shangpin/zhuanqu')
|
||
row = ShangpinZhuanqu.query.filter(
|
||
club_id=TARGET_CLUB,
|
||
mingzi=name,
|
||
leixing_id=leixing.id,
|
||
dianpu__isnull=True,
|
||
).first()
|
||
if row:
|
||
row.tupian_url = thumb or row.tupian_url
|
||
row.shenhezhuangtai = 1
|
||
row.paixu = max(len(zones) - idx, 0)
|
||
row.save(update_fields=['tupian_url', 'shenhezhuangtai', 'paixu', 'UpdateTime'])
|
||
stats['zones_updated'] += 1
|
||
else:
|
||
row = ShangpinZhuanqu.query.create(
|
||
mingzi=name,
|
||
leixing_id=leixing.id,
|
||
club_id=TARGET_CLUB,
|
||
tupian_url=thumb or '',
|
||
shenhezhuangtai=1,
|
||
paixu=max(len(zones) - idx, 0),
|
||
dianpu=None,
|
||
dianpu_leixing=None,
|
||
)
|
||
stats['zones_created'] += 1
|
||
if src_id:
|
||
zone_id_map[src_id] = row.id
|
||
zone_id_map[name] = row.id
|
||
|
||
for p in products:
|
||
title = (p.get('title') or '').strip()
|
||
if not title:
|
||
continue
|
||
zone_name = (p.get('zone_name') or p.get('target_zhuanqu_name') or '').strip()
|
||
cat2 = int(p.get('cat2_id') or 0)
|
||
zhuanqu_id = zone_id_map.get(cat2) or zone_id_map.get(zone_name)
|
||
if not zhuanqu_id:
|
||
self.stderr.write(f'跳过无专区商品: {title} zone={zone_name}')
|
||
continue
|
||
|
||
price = self._decimal(p.get('price'))
|
||
intro = (p.get('intro') or '').strip()
|
||
xuzhi = (p.get('buy_tips_text') or '').strip()
|
||
if p.get('rise_reason'):
|
||
xuzhi = ((xuzhi + '\n\n') if xuzhi else '') + f"加价说明：{p.get('rise_reason')}"
|
||
|
||
pics = list(p.get('pics_url') or [])
|
||
thumb = p.get('thumb_url') or (pics[0] if pics else '')
|
||
tupian = to_oss(thumb, 'shangpin/shangpintupian')
|
||
rule_imgs = list(p.get('rule_images_url') or [])
|
||
guize = to_oss(rule_imgs[0], 'shangpin/guize') if rule_imgs else ''
|
||
|
||
weigh = int(p.get('weigh') or p.get('home_weigh') or 0)
|
||
on_shelf = int(p.get('status') or 1) == 1
|
||
is_raffle = int(p.get('raffle_id') or 0) > 0
|
||
|
||
row = Shangpin.query.filter(
|
||
club_id=TARGET_CLUB,
|
||
biaoqian=title,
|
||
zhuanqu_id=zhuanqu_id,
|
||
dianpu__isnull=True,
|
||
).first()
|
||
|
||
fields = dict(
|
||
biaoqian=title,
|
||
jiage=price,
|
||
kucun=99999,
|
||
leixing_id=leixing.id,
|
||
zhuanqu_id=zhuanqu_id,
|
||
jieshao=intro,
|
||
xiadan_xuzhi=xuzhi[:5000] if xuzhi else '',
|
||
guize_tupian=guize or '',
|
||
tupian_url=tupian or '',
|
||
yaoqiuleixing=leixing.yaoqiuleixing,
|
||
huiyuan_id=leixing.huiyuan_id,
|
||
yongjin=leixing.yongjin,
|
||
paixu=weigh,
|
||
shenhezhuangtai=1,
|
||
shangjia_zhuangtai=on_shelf,
|
||
club_id=TARGET_CLUB,
|
||
shi_zhuanpan=bool(is_raffle),
|
||
dianpu=None,
|
||
dianpu_leixing=None,
|
||
)
|
||
|
||
if row:
|
||
for k, v in fields.items():
|
||
setattr(row, k, v)
|
||
row.save()
|
||
stats['products_updated'] += 1
|
||
else:
|
||
Shangpin.query.create(
|
||
zhenshi_xiaoliang=0,
|
||
duiwai_xiaoliang=0,
|
||
**fields,
|
||
)
|
||
stats['products_created'] += 1
|
||
|
||
if with_banners:
|
||
from config.models import Lunbo
|
||
from jituan.services.display_config import (
|
||
LUNBO_PAGE_ACCEPT_BANNER2,
|
||
LUNBO_PAGE_ACCEPT_ORDER,
|
||
lunbo_image_type_for_page,
|
||
)
|
||
|
||
def upsert_lunbo(page_key, items):
|
||
image_type = lunbo_image_type_for_page(page_key)
|
||
Lunbo.query.filter(
|
||
club_id=TARGET_CLUB,
|
||
page_key=page_key,
|
||
ImageType=image_type,
|
||
).delete()
|
||
for b in items:
|
||
url = to_oss(b.get('url') or '', 'a_long/lunbo')
|
||
if not url:
|
||
continue
|
||
Lunbo.query.create(
|
||
club_id=TARGET_CLUB,
|
||
page_key=page_key,
|
||
ImageURL=url,
|
||
ImageType=image_type,
|
||
)
|
||
stats['banners'] += 1
|
||
|
||
upsert_lunbo(LUNBO_PAGE_ACCEPT_ORDER, banners)
|
||
upsert_lunbo(LUNBO_PAGE_ACCEPT_BANNER2, banners2)
|
||
|
||
self.stdout.write(self.style.SUCCESS(
|
||
'完成（仅 club=ufo）: '
|
||
f"类型上架={stats['leixing_enabled']} "
|
||
f"专区+={stats['zones_created']}/改={stats['zones_updated']} "
|
||
f"商品+={stats['products_created']}/改={stats['products_updated']} "
|
||
f"轮播={stats['banners']} "
|
||
f"OSS成功={stats['oss_ok']} 失败={stats['oss_fail']}"
|
||
))
|
||
|
||
def _resolve_leixing(self, ShangpinLeixing, leixing_id: int, leixing_name: str):
|
||
"""优先 id；否则 jieshao 全名精确匹配且必须唯一。不做 icontains。"""
|
||
if leixing_id:
|
||
try:
|
||
lx = ShangpinLeixing.query.get(id=leixing_id)
|
||
except ShangpinLeixing.DoesNotExist as exc:
|
||
raise CommandError(f'类型 id={leixing_id} 不存在') from exc
|
||
return lx
|
||
|
||
name = (leixing_name or DEFAULT_LEIXING_NAME).strip() or DEFAULT_LEIXING_NAME
|
||
rows = list(
|
||
ShangpinLeixing.query.filter(jieshao=name)
|
||
.exclude(shenhezhuangtai=2)
|
||
.order_by('id')
|
||
)
|
||
if not rows:
|
||
near = list(
|
||
ShangpinLeixing.query.filter(jieshao__icontains=name).order_by('id')[:20]
|
||
)
|
||
hint = '\n'.join(
|
||
f' id={x.id} name={x.jieshao!r} shenhezhuangtai={x.shenhezhuangtai}'
|
||
for x in near
|
||
) or ' （无近似）'
|
||
raise CommandError(
|
||
f'找不到 jieshao 精确等于「{name}」且非审核专用的类型。库中近似：\n{hint}\n'
|
||
'请先在后台把类型名改成正好「三角洲」，或 --leixing-id=... 指定。'
|
||
)
|
||
if len(rows) > 1:
|
||
detail = '\n'.join(
|
||
f' id={x.id} name={x.jieshao!r} shenhezhuangtai={x.shenhezhuangtai}'
|
||
for x in rows
|
||
)
|
||
raise CommandError(
|
||
f'「{name}」精确匹配到多条，拒绝自动选，避免挂错：\n{detail}'
|
||
)
|
||
return rows[0]
|
||
|
||
@staticmethod
|
||
def _decimal(v) -> Decimal:
|
||
try:
|
||
return Decimal(str(v if v is not None else '0'))
|
||
except (InvalidOperation, TypeError, ValueError):
|
||
return Decimal('0')
|
||
|
||
@staticmethod
|
||
def _upload_remote(url: str, prefix: str) -> str:
|
||
"""返回 OSS 相对 key；失败返回空串。与后台一致：DB 只存相对路径。"""
|
||
from datetime import datetime
|
||
|
||
from utils.oss_utils import upload_to_oss
|
||
|
||
try:
|
||
ctx = ssl.create_default_context()
|
||
req = Request(url, headers={'User-Agent': 'UFO-Import/1.0'})
|
||
with urlopen(req, timeout=60, context=ctx) as resp:
|
||
raw = resp.read()
|
||
ctype = resp.headers.get('Content-Type') or ''
|
||
except Exception: # noqa: BLE001
|
||
return ''
|
||
|
||
ext = mimetypes.guess_extension((ctype or '').split(';')[0].strip()) or ''
|
||
if not ext or ext == '.jpe':
|
||
m = re.search(r'\.(jpg|jpeg|png|gif|webp|bmp)(?:\?|$)', url, re.I)
|
||
ext = ('.' + m.group(1).lower()) if m else '.jpg'
|
||
if ext == '.jpeg':
|
||
ext = '.jpg'
|
||
|
||
ts = datetime.now().strftime('%Y%m%d%H%M%S')
|
||
rel = f"{prefix.strip('/')}/{ts}_{uuid.uuid4().hex[:8]}{ext}"
|
||
bio = BytesIO(raw)
|
||
bio.name = os.path.basename(rel)
|
||
ok = upload_to_oss(bio, rel)
|
||
if not ok:
|
||
return ''
|
||
return rel # 关键：只返回相对路径，不返回完整 CDN URL
|