74 lines
2.3 KiB
Python
74 lines
2.3 KiB
Python
|
#!/usr/bin/env python3
|
|||
|
|
"""
|
|||
|
|
监控Phase 4B训练状态
|
|||
|
|
定期检查分割性能是否改善
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import time
|
|||
|
|
import subprocess
|
|||
|
|
|
|||
|
|
def check_training_status():
|
|||
|
|
"""检查训练状态"""
|
|||
|
|
try:
|
|||
|
|
# 检查进程
|
|||
|
|
result = subprocess.run(
|
|||
|
|
"ps aux | grep python | grep -v grep | wc -l",
|
|||
|
|
shell=True, capture_output=True, text=True
|
|||
|
|
)
|
|||
|
|
process_count = int(result.stdout.strip())
|
|||
|
|
|
|||
|
|
print(f"🖥️ Python进程数量: {process_count}")
|
|||
|
|
|
|||
|
|
if process_count < 10:
|
|||
|
|
print("❌ 训练进程异常")
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
# 检查最新日志
|
|||
|
|
try:
|
|||
|
|
with open('phase4b_training_v2.log', 'r') as f:
|
|||
|
|
lines = f.readlines()[-10:]
|
|||
|
|
for line in lines:
|
|||
|
|
if 'divider/dice:' in line:
|
|||
|
|
# 提取dice loss值
|
|||
|
|
import re
|
|||
|
|
match = re.search(r'loss/map/divider/dice: ([0-9.]+)', line)
|
|||
|
|
if match:
|
|||
|
|
dice_loss = float(match.group(1))
|
|||
|
|
dice_coeff = 1 - dice_loss
|
|||
|
|
print(f"Dice Loss: {dice_loss:.4f}")
|
|||
|
|
print(f"Dice系数: {dice_coeff:.4f}")
|
|||
|
|
if dice_coeff > 0.5:
|
|||
|
|
print("🎉 分割性能显著改善！")
|
|||
|
|
elif dice_coeff > 0.1:
|
|||
|
|
print("✅ 分割性能有所改善")
|
|||
|
|
else:
|
|||
|
|
print("⚠️ 分割性能仍需改善")
|
|||
|
|
return True
|
|||
|
|
except FileNotFoundError:
|
|||
|
|
print("📄 日志文件不存在")
|
|||
|
|
|
|||
|
|
return True
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
print(f"❌ 检查失败: {e}")
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
def main():
|
|||
|
|
print("🔍 开始监控Phase 4B训练状态...")
|
|||
|
|
print("=" * 50)
|
|||
|
|
|
|||
|
|
for i in range(60): # 监控60次，每次间隔30秒
|
|||
|
|
print(f"\n🔄 第 {i+1}/60 次检查 (每30秒)")
|
|||
|
|
print(f"时间: {time.strftime('%H:%M:%S')}")
|
|||
|
|
|
|||
|
|
if not check_training_status():
|
|||
|
|
print("❌ 训练异常，停止监控")
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
time.sleep(30)
|
|||
|
|
|
|||
|
|
print("\n🏁 监控完成")
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
main()
|