#!/usr/bin/env python
"""
正方元智能广告工作台 - 服务守护进程
功能：
1. 启动全部后端服务（proxy_server 最后启动）
2. 每 10 秒检测各服务端口健康状态
3. 发现崩溃自动重启
4. 日志记录到 guardian.log
5. 打开门户页面

用法：
    python service_guardian.py          # 启动守护 + 自动启动所有服务
    python service_guardian.py --watch  # 仅守护已运行的服务（不自动启动）
"""
import os
import sys
import time
import json
import socket
import signal
import atexit
import logging
import threading
import subprocess
import webbrowser
from datetime import datetime
from pathlib import Path

# ===== 配置 =====
PORTAL_DIR = Path(__file__).parent.resolve()
LOG_FILE = PORTAL_DIR / "guardian.log"
PID_FILE = PORTAL_DIR / "guardian.pid"
RESTART_SIGNAL_FILE = PORTAL_DIR / "restart_signal.json"
RESTART_RESULT_FILE = PORTAL_DIR / "restart_result.json"
PORTAL_URL = "http://192.168.5.250:9000/portal.html"
CHECK_INTERVAL = 10  # 健康检查间隔（秒）
STARTUP_TIMEOUT = 120  # 服务启动超时（秒）

# 服务定义：name, port, check_path, startup_cmd
# startup_cmd 中的 %PORTAL_DIR% 会被替换为实际路径
SERVICES = [
    {
        "name": "Ollama",
        "port": 11434,
        "check_path": "/api/tags",
        "required": True,
        "startup": {
            "exe": "ollama",
            "args": ["serve"],
        }
    },
    {
        "name": "ComfyUI",
        "port": 8188,
        "check_path": "/system_stats",
        "check_method": "GET",
        "required": True,
        "cooldown": 120,
        "startup": {
            "env_extra": {"PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True"},
            "exe": r"C:\AI\ComfyUI_windows_portable\python_embeded\python.exe",
            "args": ["-s", "ComfyUI\\main.py", "--windows-standalone-build",
                     "--listen", "0.0.0.0", "--port", "8188",
                     "--enable-cors-header",
                     "--reserve-vram", "2.0",
                     "--disable-smart-memory"],
            "cwd": r"C:\AI\ComfyUI_windows_portable",
        }
    },
    {
        "name": "IOPaint",
        "port": 8080,
        "check_path": "/",
        "check_method": "GET",  # IOPaint 可能不支持 HEAD
        "required": True,
        "startup": {
            "exe": r"C:\Program Files\Python311\python.exe",
            "args": ["-m", "iopaint", "start", "--model", "cv2",
                     "--enable-remove-bg",
                     "--remove-bg-device", "cuda",
                     "--device", "cuda", "--port", "8080", "--host", "0.0.0.0"],
        }
    },
    # 数字人 v2（Duix + GPT-SoVITS + RMBG）
    # 注意：必须在 GPT-SoVITS 之后启动；WSL2/Duix 未安装时自动跳过
    # ⚠️ 必须使用 ComfyUI Python 3.13 加载 .pyc 文件（Python 3.11 会段错误）
    {
        "name": "数字人v2",
        "port": 8766,
        "check_path": "/api/digital-human/v2/status",
        "check_method": "GET",
        "required": True,  # 核心业务，必须运行
        "cooldown": 60,  # GPU 模型加载需 40-60s
        "startup": {
            "exe": r"C:\AI\ComfyUI_windows_portable\python_embeded\python.exe",
            "args": ["dh_v2_startup.py"],
            "cwd": str(PORTAL_DIR),
        }
    },
    # Duix.Avatar WSL2 服务 (视频合成 :8383)
    {
        "name": "Duix.Avatar",
        "port": 8383,
        "check_path": "/",
        "check_method": "GET",
        "required": False,
        "cooldown": 120,
        "is_wsl": True,
        "wsl_distro": "duix-avatar",
        "startup": {
            "exe": "wsl",
            "args": ["-d", "duix-avatar", "--", "bash", "/mnt/d/AI-Ad-Agent/scripts/run-avatar.sh"],
        }
    },
    # Duix Fish-Speech WSL2 服务 (TTS :18180)
    {
        "name": "Duix.TTS",
        "port": 18180,
        "check_path": "/",
        "check_method": "GET",
        "required": False,
        "cooldown": 120,
        "is_wsl": True,
        "wsl_distro": "duix-fish",
        "startup": {
            "exe": "wsl",
            "args": ["-d", "duix-fish", "--", "bash", "/mnt/d/AI-Ad-Agent/scripts/run-fish.sh"],
        }
    },
    # Duix Fun-ASR WSL2 服务 (ASR :10095)
    {
        "name": "Duix.ASR",
        "port": 10095,
        "check_path": "/",
        "check_method": "GET",
        "required": False,
        "cooldown": 120,
        "is_wsl": True,
        "wsl_distro": "duix-asr",
        "startup": {
            "exe": "wsl",
            "args": ["-d", "duix-asr", "--", "bash", "/mnt/d/AI-Ad-Agent/scripts/run-asr.sh"],
        }
    },
    # GPT-SoVITS 声音克隆（数字人 v2 依赖，必须在数字人之前启动）
    {
        "name": "GPT-SoVITS",
        "port": 9880,
        "check_path": "/docs",
        "check_method": "GET",
        "required": True,
        "cooldown": 180,  # 模型加载 CUDA 初始化需要 2-3 分钟
        "startup": {
            "exe": r"C:\AI\gptsovits_env\Scripts\python.exe",
            "args": ["api_v2.py", "-c", "GPT_SoVITS/configs/tts_infer.yaml", "-a", "0.0.0.0", "-p", "9880"],
            "cwd": r"D:\AI-Ad-Agent\GPT-SoVITS",
            "env_extra": {
                "PYTORCH_CUDA_ALLOC_CONF": "max_split_size_mb:256",
            }
        }
    },
    {
        "name": "代理服务器",
        "port": 9000,
        "check_path": "/api/health",
        "check_method": "GET",  # 使用轻量健康检查端点，避免被长请求阻塞
        "required": True,
        "cooldown": 30,  # 代理重启后 30 秒冷却期
        "startup": {
            "exe": r"C:\Program Files\Python311\python.exe",
            "args": ["proxy_server.py"],
            "cwd": str(PORTAL_DIR),
        }
    },
]

# ===== 日志 =====
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s [%(levelname)s] %(message)s',
    handlers=[
        logging.FileHandler(LOG_FILE, encoding='utf-8'),
        logging.StreamHandler(sys.stdout)
    ]
)
logger = logging.getLogger("Guardian")

# ===== 进程管理 =====
_lock_fd = None  # PID 文件锁的文件句柄
processes = {}  # name -> subprocess.Popen
crash_counts = {}  # name -> int
service_start_times = {}  # name -> timestamp（启动时间，用于冷却期判断）
COOLDOWN_SECONDS = 60  # 服务启动后冷却期（秒），在此期间不触发重启


def check_port(port, check_path="/", method='GET'):
    """检测端口是否响应 HTTP 请求（超时10秒，适应大模型加载场景）
    
    特殊处理：
    - HTTP 4xx (如 404): 服务在线（只是路径不对）→ 返回 True
    - HTTP 5xx: 服务异常 → 返回 False  
    - 连接错误: TCP 回退检测（兼容 WebSocket 服务如 ASR）
    """
    import urllib.request
    import urllib.error
    import socket
    url = f"http://127.0.0.1:{port}{check_path}"
    try:
        req = urllib.request.Request(url, method=method)
        with urllib.request.urlopen(req, timeout=10) as resp:
            return resp.status < 500
    except urllib.error.HTTPError as he:
        # 4xx 错误（如 404）说明服务在线，只是路径不对
        ok = he.code < 500
        if not ok:
            logger.debug(f"check_port({port}) FAIL: HTTP {he.code}")
        return ok
    except (urllib.error.URLError, ConnectionRefusedError, ConnectionResetError,
            OSError, TimeoutError) as e:
        # 连接失败 → TCP 回退（兼容 WebSocket 服务）
        try:
            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            sock.settimeout(5)
            result = sock.connect_ex(('127.0.0.1', port))
            sock.close()
            if result == 0:
                return True  # TCP 可达，WebSocket 等服务在线
            else:
                logger.debug(f"check_port({port}) FAIL: TCP connect failed (code={result})")
                return False
        except Exception as e2:
            logger.debug(f"check_port({port}) FAIL: {type(e2).__name__}: {e2}")
            return False
    except Exception as e:
        logger.debug(f"check_port({port}) FAIL: {type(e).__name__}: {e}")
        return False


def check_ollama():
    """Ollama 特殊检查"""
    try:
        import urllib.request
        url = "http://127.0.0.1:11434/api/tags"
        req = urllib.request.Request(url, method='GET')
        with urllib.request.urlopen(req, timeout=3) as resp:
            return resp.status == 200
    except Exception:
        return False


def is_port_in_use(port):
    """检测端口是否被占用"""
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
        try:
            s.settimeout(1)
            s.connect(('127.0.0.1', port))
            return True
        except:
            return False


def start_service(svc):
    """启动单个服务。
    
    启动前会检查端口是否已被监听，避免重复启动。
    如果端口已被占用（无论服务是否健康），都跳过启动，
    交由健康检查循环中的 restart 逻辑处理。
    """
    name = svc["name"]
    port = svc["port"]
    
    # 端口去重：如果端口已被监听，跳过启动
    if is_port_in_use(port):
        logger.info(f"[{name}] 端口 {port} 已被占用，跳过启动")
        # 标记为"已运行"状态（即使不健康也避免重复启动）
        return "skipped"
    
    startup = svc["startup"]
    exe = startup["exe"]
    args = startup.get("args", [])
    cwd = startup.get("cwd", None)
    
    # WSL2 服务前置检查
    # 注意: wsl.exe --list --quiet 输出为 UTF-16LE 编码（无BOM）
    # 必须用 encoding='utf-16-le' 解码，否则会得到 \x00 间隔的乱码
    if svc.get("is_wsl", False):
        import shutil
        wsl_exe = shutil.which("wsl.exe")
        if not wsl_exe:
            logger.warning(f"[{svc['name']}] wsl.exe 未找到，跳过启动")
            return None
        distro = svc.get("wsl_distro", "")
        try:
            result = subprocess.run(
                ["wsl.exe", "--list", "--quiet"],
                capture_output=True, encoding='utf-16-le', timeout=5
            )
            if distro and distro not in result.stdout:
                logger.warning(f"[{svc['name']}] WSL 发行版 '{distro}' 未找到，跳过启动")
                return None
        except Exception as e:
            logger.warning(f"[{svc['name']}] 无法检查 WSL 发行版: {e}")
            return None
    elif svc["name"] == "数字人v2":
        # 数字人 v2 也需要检查 Duix.Avatar
        import shutil
        wsl_exe = shutil.which("wsl.exe")
        if not wsl_exe:
            logger.warning(f"[{svc['name']}] wsl.exe 未找到，跳过启动（WSL2 未安装）")
            return None
        try:
            result = subprocess.run(
                ["wsl.exe", "--list", "--quiet"],
                capture_output=True, encoding='utf-16-le', timeout=5
            )
            if "duix-avatar" not in result.stdout:
                logger.warning(f"[{svc['name']}] WSL 发行版 'duix-avatar' 未找到，跳过启动")
                return None
        except Exception as e:
            logger.warning(f"[{svc['name']}] 无法检查 WSL 发行版: {e}")
    
    # 检查可执行文件是否存在
    if not os.path.exists(exe):
        # 尝试 PATH 查找
        import shutil
        found = shutil.which(exe)
        if found:
            exe = found
        else:
            logger.error(f"[{svc['name']}] 可执行文件不存在: {exe}")
            return None
    
    cmd = [exe] + args
    logger.info(f"[{svc['name']}] 启动: {' '.join(cmd)}")
    
    try:
        env = os.environ.copy()
        # IOPaint 需要 HF 镜像来下载模型
        if svc["name"] == "IOPaint":
            env["HF_ENDPOINT"] = "https://hf-mirror.com"
        # 应用服务自定义环境变量
        env_extra = startup.get("env_extra", {})
        for k, v in env_extra.items():
            env[k] = v
        
        # WSL2 服务需要独立进程组，避免父进程退出时被终止
        if svc.get("is_wsl", False):
            # v7.4: 移除 DETACHED_PROCESS（会为每个 wsl.exe 创建新的控制台窗口=4个CMD！）
            # 改用 CREATE_NO_WINDOW + STARTF_USESHOWWINDOW/SW_HIDE 来隐藏窗口
            # CREATE_NEW_PROCESS_GROUP (0x200) 确保独立进程组，父进程退出不杀子进程
            flags = 0
            startupinfo = None
            if sys.platform == 'win32':
                flags = subprocess.CREATE_NO_WINDOW | 0x00000200  # CREATE_NEW_PROCESS_GROUP
                startupinfo = subprocess.STARTUPINFO()
                startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
                startupinfo.wShowWindow = subprocess.SW_HIDE
            proc = subprocess.Popen(
                cmd,
                cwd=cwd,
                stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL,
                env=env,
                creationflags=flags,
                startupinfo=startupinfo,
            )
        else:
            proc = subprocess.Popen(
                cmd,
                cwd=cwd,
                stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL,
                env=env,
                creationflags=subprocess.CREATE_NO_WINDOW if sys.platform == 'win32' else 0,
            )
        processes[svc["name"]] = proc
        service_start_times[svc["name"]] = time.time()  # 记录启动时间
        return proc
    except Exception as e:
        logger.error(f"[{svc['name']}] 启动失败: {e}")
        return None


def stop_service(name):
    """停止服务进程"""
    proc = processes.get(name)
    if proc:
        try:
            proc.terminate()
            proc.wait(timeout=5)
        except:
            try:
                proc.kill()
            except:
                pass
        del processes[name]
        logger.info(f"[{name}] 已停止")


def restart_service(svc):
    """重启服务"""
    name = svc["name"]
    crash_counts[name] = crash_counts.get(name, 0) + 1
    count = crash_counts[name]
    
    logger.warning(f"[{name}] 第 {count} 次重启...")
    
    # 先杀掉旧进程
    stop_service(name)
    time.sleep(2)
    
    # 清理端口（Windows）- 只清理 LISTEN 状态的端口（忽略 TIME_WAIT）
    if sys.platform == 'win32':
        time.sleep(2)  # 给端口释放时间
        import re
        try:
            result = subprocess.run(
                f'netstat -ano | findstr :{svc["port"]}',
                shell=True, capture_output=True, text=True, timeout=10
            )
            killed_pids = set()
            for line in result.stdout.strip().split('\n'):
                if f':{svc["port"]}' in line and 'LISTENING' in line.upper():
                    parts = line.strip().split()
                    if len(parts) >= 5:
                        pid = parts[-1]
                        if pid.isdigit() and pid not in killed_pids:
                            subprocess.run(f'taskkill /F /PID {pid}', shell=True,
                                          capture_output=True, timeout=5)
                            killed_pids.add(pid)
                            logger.info(f"[{name}] 已终止占用端口的进程 PID={pid}")
        except:
            pass
        time.sleep(1)
    
    # 重新启动
    proc = start_service(svc)
    if proc and proc != "skipped":
        crash_counts[name] = 0  # 成功后重置计数
        logger.info(f"[{name}] 重启成功")
    elif proc == "skipped":
        logger.info(f"[{name}] 端口已被占用，视为已在运行")
        crash_counts[name] = 0
    else:
        logger.error(f"[{name}] 重启失败！")
    
    return proc


def start_all_services(only_required=False):
    """按依赖顺序启动所有服务
    
    启动顺序：Ollama -> ComfyUI -> IOPaint -> GPT-SoVITS -> 数字人v2 -> Duix服务 -> 代理服务器
    代理服务器必须最后启动，因为需要其他服务已经在线
    """
    # 过滤掉代理服务器，先启动其他服务
    non_proxy = [s for s in SERVICES if s["name"] != "代理服务器"]
    proxy_svc = next((s for s in SERVICES if s["name"] == "代理服务器"), None)
    
    # 可选过滤
    if only_required:
        to_start = [s for s in non_proxy if s["required"]]
    else:
        to_start = [s for s in non_proxy]
    
    logger.info("=" * 60)
    logger.info("  正方元智能广告工作台 - 服务启动")
    logger.info(f"  时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    logger.info(f"  核心服务: {len(to_start)} 个")
    logger.info("=" * 60)
    
    started_count = 0
    failed_services = []
    
    for svc in to_start:
        name = svc["name"]
        
        # 检查是否已在运行
        if name == "Ollama":
            already_running = check_ollama()
        else:
            method = svc.get("check_method", "GET")
            already_running = check_port(svc["port"], svc["check_path"], method=method)
        
        if already_running:
            logger.info(f"[{name}] 已在运行 (端口 {svc['port']})")
            started_count += 1
            continue
        
        # 启动服务
        logger.info(f"[{name}] 启动中...")
        proc = start_service(svc)
        
        if proc is None:
            failed_services.append(name)
            continue
        if proc == "skipped":
            # 端口已被占用，视为"已运行"
            logger.info(f"[{name}] 端口已被占用，视为已在运行")
            started_count += 1
            continue
        
        # 等待服务就绪
        # WSL2 服务模型加载需要更长时间（最多 300 秒）
        if svc.get("is_wsl", False):
            wait_seconds = 300
        else:
            wait_seconds = STARTUP_TIMEOUT
        logger.info(f"[{name}] 等待就绪 (最多 {wait_seconds}秒)...")
        ready = False
        for i in range(wait_seconds):
            time.sleep(1)
            if name == "Ollama":
                ready = check_ollama()
            else:
                method = svc.get("check_method", "GET")
                ready = check_port(svc["port"], svc["check_path"], method=method)
            
            if ready:
                logger.info(f"[{name}] 已就绪 ({i+1}秒)")
                started_count += 1
                break
            
            if (i + 1) % 10 == 0:
                logger.info(f"[{name}] 仍在等待... ({i+1}秒)")
        
        if not ready:
            if svc.get("required", True):
                logger.warning(f"[{name}] 启动超时 ({wait_seconds}秒)")
                failed_services.append(name)
            else:
                logger.info(f"[{name}] 未就绪（可选服务，继续执行）")
    
    # 最后启动代理服务器
    if proxy_svc and (not only_required or proxy_svc["required"]):
        name = proxy_svc["name"]
        method = proxy_svc.get("check_method", "GET")
        if check_port(proxy_svc["port"], proxy_svc["check_path"], method=method):
            logger.info(f"[{name}] 已在运行 (端口 {proxy_svc['port']})")
        else:
            logger.info(f"[{name}] 启动中...")
            proc = start_service(proxy_svc)
            if proc:
                # 等待代理服务器就绪
                for i in range(10):
                    time.sleep(1)
                    if check_port(proxy_svc["port"], "/portal.html"):
                        logger.info(f"[{name}] 已就绪 ({i+1}秒)")
                        break
    
    logger.info("=" * 60)
    logger.info(f"  启动完成: {started_count}/{len(to_start)} 核心服务在线")
    if failed_services:
        logger.warning(f"  失败服务: {', '.join(failed_services)}")
    logger.info("=" * 60)
    
    return started_count, failed_services
    

def restart_all_services():
    """重启所有服务（先停后起）"""
    logger.info("=" * 60)
    logger.info("  收到重启信号 — 正在重启所有服务...")
    logger.info("=" * 60)

    # 1. 停止所有运行中的服务（逆序，代理服务器先停）
    logger.info("[重启] 第1步：停止所有服务...")
    running_names = list(processes.keys())
    # 逆序停止：代理服务器先停，避免它把请求转发到正在停止的服务
    running_names.reverse()
    for name in running_names:
        logger.info(f"[重启] 停止 {name}...")
        stop_service(name)
        time.sleep(1)

    # 2. 等待端口释放
    logger.info("[重启] 第2步：等待端口释放（5秒）...")
    time.sleep(5)

    # 3. 清理僵尸进程（端口仍被占用的情况）
    logger.info("[重启] 第3步：清理残留进程...")
    for svc in SERVICES:
        port = svc["port"]
        if sys.platform == 'win32':
            try:
                result = subprocess.run(
                    f'netstat -ano | findstr :{port}',
                    shell=True, capture_output=True, text=True, timeout=5
                )
                for line in result.stdout.strip().split('\n'):
                    if f':{port}' in line and 'LISTENING' in line.upper():
                        parts = line.strip().split()
                        if len(parts) >= 5:
                            pid = parts[-1]
                            if pid.isdigit():
                                subprocess.run(f'taskkill /F /PID {pid}',
                                              shell=True, capture_output=True, timeout=5)
                                logger.info(f"[重启] 已清理残留进程 {pid} (端口 {port})")
            except:
                pass

    # 4. 重新启动所有服务
    logger.info("[重启] 第4步：重新启动所有服务...")
    global service_start_times
    service_start_times.clear()  # 清除启动时间记录，强制重新启动
    start_all_services(only_required=False)

    # 5. 写入重启结果文件（供前端查询）
    try:
        result_data = {
            'success': True,
            'timestamp': time.time(),
            'message': '所有服务已重启完成'
        }
        with open(RESTART_RESULT_FILE, 'w', encoding='utf-8') as f:
            json.dump(result_data, f, ensure_ascii=False)
    except:
        pass

    logger.info("=" * 60)
    logger.info("  所有服务重启完成！")
    logger.info("=" * 60)


def health_check():
    """对所有服务进行健康检查，返回不健康的服务列表"""
    unhealthy = []
    
    for svc in SERVICES:
        name = svc["name"]
        
        # 检查所有服务（不跳过任何服务）
        # 检查
        if name == "Ollama":
            healthy = check_ollama()
        else:
            method = svc.get("check_method", "GET")
            healthy = check_port(svc["port"], svc["check_path"], method=method)
        
        if not healthy:
            unhealthy.append(svc)
    
    return unhealthy


def guardian_loop():
    """主守护循环"""
    logger.info("守护进程已启动，开始监控服务状态...")
    logger.info(f"检查间隔: {CHECK_INTERVAL} 秒")

    consecutive_failures = {}  # name -> count

    while True:
        try:
            time.sleep(CHECK_INTERVAL)

            # ===== 重启信号检测 =====
            if os.path.exists(RESTART_SIGNAL_FILE):
                try:
                    with open(RESTART_SIGNAL_FILE, 'r', encoding='utf-8') as f:
                        signal_data = json.load(f)
                    os.remove(RESTART_SIGNAL_FILE)  # 删除信号文件
                    logger.info(f"[重启信号] 收到重启请求: {signal_data.get('requested_by', 'unknown')}")
                    restart_all_services()
                    # 重启后跳过本次健康检查（服务正在启动中）
                    consecutive_failures.clear()
                    continue
                except Exception as e:
                    logger.error(f"[重启信号] 处理失败: {e}")
                    try:
                        os.remove(RESTART_SIGNAL_FILE)
                    except:
                        pass

            unhealthy = health_check()
            
            if not unhealthy:
                # 所有服务正常，重置计数
                consecutive_failures.clear()
                continue
            
            for svc in unhealthy:
                name = svc["name"]
                
                # 冷却期检查：服务刚启动不久，给它时间加载
                start_time = service_start_times.get(name, 0)
                svc_cooldown = svc.get("cooldown", COOLDOWN_SECONDS)
                cooldown_remaining = svc_cooldown - (time.time() - start_time)
                if cooldown_remaining > 0:
                    logger.info(f"[{name}] 冷却中 ({cooldown_remaining:.0f}秒后开始健康检查)")
                    continue
                
                consecutive_failures[name] = consecutive_failures.get(name, 0) + 1
                
                
                # 连续2次检查失败才重启（避免网络抖动误判）
                # 数字人v2 合成长文本时可能暂时响应慢，需要更高的容错阈值
                # v8.0: 代理服务器和数字人v2处理长请求时可能暂时响应慢，需要更高容错
                restart_threshold = 6 if name in ("数字人v2", "代理服务器") else 2
                if consecutive_failures[name] >= restart_threshold:
                    logger.warning(f"[{name}] 连续 {consecutive_failures[name]} 次无响应，正在重启...")
                    
                    if svc["required"]:
                        restart_service(svc)
                        consecutive_failures[name] = 0
                    else:
                        logger.info(f"[{name}] (可选服务) 已离线")
        
        except KeyboardInterrupt:
            raise
        except Exception as e:
            logger.error(f"守护循环异常: {e}", exc_info=True)


def cleanup():
    """退出时清理所有子进程"""
    logger.info("正在停止所有服务...")
    for name in list(processes.keys()):
        stop_service(name)
    logger.info("所有服务已停止")


def open_portal():
    """打开门户页面"""
    try:
        # 等待代理服务器就绪
        for i in range(15):
            if check_port(9000, "/portal.html", method="GET"):
                webbrowser.open(PORTAL_URL)
                logger.info(f"门户页面已打开: {PORTAL_URL}")
                return True
            time.sleep(1)
        logger.warning(f"代理服务器未就绪，无法打开门户")
        return False
    except Exception as e:
        logger.error(f"打开门户失败: {e}")
        return False


def acquire_pid_lock():
    """获取 PID 文件锁，防止多个守护进程同时运行。
    
    使用文件锁 (Windows: msvcrt, Linux: fcntl) 确保原子性，
    避免竞态条件导致多个守护进程同时启动。
    """
    global _lock_fd
    try:
        # 以读写模式打开 PID 文件（文件锁需要可写 fd）
        _lock_fd = open(str(PID_FILE), 'a+')
        
        if sys.platform == 'win32':
            import msvcrt
            try:
                msvcrt.locking(_lock_fd.fileno(), msvcrt.LK_NBLCK, 1)
            except OSError:
                # 文件被其他进程锁定 → 已有守护进程运行
                _lock_fd.close()
                logger.error("已有守护进程运行（文件锁被占用），退出")
                sys.exit(1)
        else:
            import fcntl
            try:
                fcntl.flock(_lock_fd.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
            except OSError:
                _lock_fd.close()
                logger.error("已有守护进程运行（文件锁被占用），退出")
                sys.exit(1)
        
        # 写入当前 PID
        _lock_fd.seek(0)
        _lock_fd.truncate()
        _lock_fd.write(str(os.getpid()))
        _lock_fd.flush()
        
        # atexit 清理（正常退出时释放锁）
        def cleanup_lock():
            global _lock_fd
            try:
                if _lock_fd:
                    PID_FILE.unlink(missing_ok=True)
                    _lock_fd.close()
                    _lock_fd = None
            except:
                pass
        
        atexit.register(cleanup_lock)
        
        # 额外的守护检查：每 30 秒验证自己是否仍是 PID 持有者
        def periodic_lock_check():
            global _lock_fd
            while True:
                time.sleep(30)
                try:
                    if _lock_fd is None or _lock_fd.closed:
                        logger.error("PID 锁丢失，退出守护")
                        os._exit(1)
                except:
                    pass
        
        t = threading.Thread(target=periodic_lock_check, daemon=True)
        t.start()
        
        return True
    except SystemExit:
        raise
    except Exception as e:
        logger.warning(f"PID 锁获取异常: {e}")
        return True  # 不因锁失败而阻止启动


def main():
    import argparse
    parser = argparse.ArgumentParser(description='正方元智能广告工作台 - 服务守护进程')
    parser.add_argument('--watch', action='store_true', help='仅守护已运行的服务（不自动启动）')
    parser.add_argument('--no-browser', action='store_true', help='不自动打开浏览器')
    parser.add_argument('--all', action='store_true', help='启动所有服务（包括可选服务）')
    args = parser.parse_args()
    
    # PID 文件锁：防止多个守护进程同时运行
    acquire_pid_lock()
    
    # 注册退出处理
    atexit.register(cleanup)
    signal.signal(signal.SIGINT, lambda sig, frame: sys.exit(0))
    signal.signal(signal.SIGTERM, lambda sig, frame: sys.exit(0))
    
    if args.watch:
        # 仅守护模式
        logger.info("守护模式：监控已运行的服务")
    else:
        # 启动所有服务
        only_required = not args.all
        start_all_services(only_required=only_required)
    
    # 打开门户
    if not args.no_browser:
        threading.Thread(target=open_portal, daemon=True).start()
    
    # 进入守护循环
    try:
        guardian_loop()
    except KeyboardInterrupt:
        logger.info("\n收到中断信号，正在退出...")


if __name__ == '__main__':
    main()