[{"data":1,"prerenderedAt":2915},["ShallowReactive",2],{"\u002F2026-05-10-fa-007-phase-done":3,"\u002F2026-05-10-fa-007-phase-done-rel":1126},{"id":4,"title":5,"body":6,"column":1109,"date":1110,"description":12,"extension":1111,"hero_image":1112,"meta":1113,"navigation":297,"path":1114,"seo":1115,"series_id":939,"severity":1112,"stem":1116,"summary":1117,"tags":1118,"__hash__":1125},"posts\u002F2026-05-10-FA-007-phase-Done假报成功.md","同一个弹窗，我修了三次",{"type":7,"value":8,"toc":1093},"minimark",[9,13,16,19,24,27,173,176,213,216,223,245,248,251,262,358,361,364,403,414,446,466,477,527,530,533,566,569,572,578,652,663,670,746,749,752,757,811,816,857,860,864,869,872,955,958,969,976,979,985,988,991,994,1000,1007,1010,1013,1016,1034,1037,1040,1057,1060,1063,1066,1080,1083,1086,1089],[10,11,12],"p",{},"同一个弹窗，我已经修过三次了。",[10,14,15],{},"0.14.x 到 1.0.0，双机测试中看到的症状是一个反复循环的升级 modal：\"升级到 1.0.0\"，点更新 panel 关闭，然后立刻被拉起，modal 又来。每 15~20 秒一个 \"OTA applied to 1.0.0\" 的成功记录堆进 events log，但 cache 版本从来没动过，还是 0.14.18。",[10,17,18],{},"前面 FA-005 和 FA-006 各修了一处。FA-005 是 admin 端的版本优先级，FA-006 是 panel 进程的 onFinished 回调。两处都修对了，dead loop 依然死循环，说明不是单一根因。",[20,21,23],"h2",{"id":22},"现象三个不一致的版本号","现象：三个不一致的版本号",[10,25,26],{},"state.json（位于 launcher 工作目录）里写着三个版本号：",[28,29,34],"pre",{"className":30,"code":31,"language":32,"meta":33,"style":33},"language-json shiki shiki-themes github-light github-dark","{\n  \"phase\": \"done\",\n  \"version\": \"0.14.15\",\n  \"previous_version\": \"0.14.14\",\n  \"local_apply\": {\n    \"applied_version\": \"0.14.15\",\n    \"completed_dirs\": [\"version.json\", \"WakouPanel.exe\"]\n  },\n  \"signed_manifest\": { \"version\": \"1.0.0\", \"force_update\": true }\n}\n","json","",[35,36,37,46,63,76,89,98,110,131,137,167],"code",{"__ignoreMap":33},[38,39,42],"span",{"class":40,"line":41},"line",1,[38,43,45],{"class":44},"sVt8B","{\n",[38,47,49,53,56,60],{"class":40,"line":48},2,[38,50,52],{"class":51},"sj4cs","  \"phase\"",[38,54,55],{"class":44},": ",[38,57,59],{"class":58},"sZZnC","\"done\"",[38,61,62],{"class":44},",\n",[38,64,66,69,71,74],{"class":40,"line":65},3,[38,67,68],{"class":51},"  \"version\"",[38,70,55],{"class":44},[38,72,73],{"class":58},"\"0.14.15\"",[38,75,62],{"class":44},[38,77,79,82,84,87],{"class":40,"line":78},4,[38,80,81],{"class":51},"  \"previous_version\"",[38,83,55],{"class":44},[38,85,86],{"class":58},"\"0.14.14\"",[38,88,62],{"class":44},[38,90,92,95],{"class":40,"line":91},5,[38,93,94],{"class":51},"  \"local_apply\"",[38,96,97],{"class":44},": {\n",[38,99,101,104,106,108],{"class":40,"line":100},6,[38,102,103],{"class":51},"    \"applied_version\"",[38,105,55],{"class":44},[38,107,73],{"class":58},[38,109,62],{"class":44},[38,111,113,116,119,122,125,128],{"class":40,"line":112},7,[38,114,115],{"class":51},"    \"completed_dirs\"",[38,117,118],{"class":44},": [",[38,120,121],{"class":58},"\"version.json\"",[38,123,124],{"class":44},", ",[38,126,127],{"class":58},"\"WakouPanel.exe\"",[38,129,130],{"class":44},"]\n",[38,132,134],{"class":40,"line":133},8,[38,135,136],{"class":44},"  },\n",[38,138,140,143,146,149,151,154,156,159,161,164],{"class":40,"line":139},9,[38,141,142],{"class":51},"  \"signed_manifest\"",[38,144,145],{"class":44},": { ",[38,147,148],{"class":51},"\"version\"",[38,150,55],{"class":44},[38,152,153],{"class":58},"\"1.0.0\"",[38,155,124],{"class":44},[38,157,158],{"class":51},"\"force_update\"",[38,160,55],{"class":44},[38,162,163],{"class":51},"true",[38,165,166],{"class":44}," }\n",[38,168,170],{"class":40,"line":169},10,[38,171,172],{"class":44},"}\n",[10,174,175],{},"state.json 中记录了三个不同的版本号：",[177,178,179,186,192,199,205,210],"ul",{},[180,181,182,185],"li",{},[35,183,184],{},"local_apply.applied_version = 0.14.15","（上一轮 OTA 标记为已安装的版本）",[180,187,188,191],{},[35,189,190],{},"signed_manifest.version = 1.0.0","（本轮要安装的目标版本）",[180,193,194,195,198],{},"cache 实际的 ",[35,196,197],{},"version.json = 0.14.18","（当前运行的实际版本，通过热部署写过）",[180,200,201,204],{},[35,202,203],{},"applied_version = 0.14.15","（上一轮 OTA 标记装好的）",[180,206,207,209],{},[35,208,190],{},"（这一轮要装的目标）",[180,211,212],{},"cache 实际版本 = 0.14.18（热部署后的真实版本）",[10,214,215],{},"三个数不一样。phase 显示 done，应该表示\"安装完成\"，但完成状态对应 0.14.15，既不是当前版本 0.14.18，也不是目标版本 1.0.0。",[10,217,218,219,222],{},"events log 里是这样的（路径 ",[35,220,221],{},"%LOCALAPPDATA%\\Microsoft\\WakouAI\\Update\\\u003Cserial-hash>\\ota-events.jsonl","）：",[28,224,228],{"className":225,"code":226,"language":227,"meta":33,"style":33},"language-jsonl shiki shiki-themes github-light github-dark","{\"ts\":\"...14:55:10\",\"source\":\"restart_loop\",\"outcome\":\"succeeded\",\"message\":\"OTA applied to 1.0.0\",\"version_target\":\"1.0.0\"}\n{\"ts\":\"...14:55:30\",\"source\":\"restart_loop\",\"outcome\":\"succeeded\",\"message\":\"OTA applied to 1.0.0\",\"version_target\":\"1.0.0\"}\n{\"ts\":\"...14:55:45\",\"source\":\"restart_loop\",\"outcome\":\"succeeded\",\"message\":\"OTA applied to 1.0.0\",\"version_target\":\"1.0.0\"}\n","jsonl",[35,229,230,235,240],{"__ignoreMap":33},[38,231,232],{"class":40,"line":41},[38,233,234],{},"{\"ts\":\"...14:55:10\",\"source\":\"restart_loop\",\"outcome\":\"succeeded\",\"message\":\"OTA applied to 1.0.0\",\"version_target\":\"1.0.0\"}\n",[38,236,237],{"class":40,"line":48},[38,238,239],{},"{\"ts\":\"...14:55:30\",\"source\":\"restart_loop\",\"outcome\":\"succeeded\",\"message\":\"OTA applied to 1.0.0\",\"version_target\":\"1.0.0\"}\n",[38,241,242],{"class":40,"line":65},[38,243,244],{},"{\"ts\":\"...14:55:45\",\"source\":\"restart_loop\",\"outcome\":\"succeeded\",\"message\":\"OTA applied to 1.0.0\",\"version_target\":\"1.0.0\"}\n",[10,246,247],{},"35 秒三次\"成功\"，一条路径，同一目标版本。正常 OTA 应该装一遍就停，重复成功是链路在假报。",[20,249,250],{"id":250},"根因分析",[10,252,253,254,257,258,261],{},"代码在 ",[35,255,256],{},"launcher\u002Fsrc\u002Fota_main_chain.rs"," 的 ",[35,259,260],{},"run_full_apply_chain"," 函数（550+ 行）。这个函数按顺序执行 Download、Verify、Apply、Finalization 四个 stage。每个 stage 前都有 phase 条件判断：",[28,263,267],{"className":264,"code":265,"language":266,"meta":33,"style":33},"language-rust shiki shiki-themes github-light github-dark","\u002F\u002F Stage A: Download\nif matches!(state.phase, Phase::Idle | Phase::Downloading) { \n    \u002F\u002F 执行下载\n    state.phase = Phase::Verifying;\n}\n\n\u002F\u002F Stage B: Verify + Extract\nif matches!(state.phase, Phase::Downloading | Phase::Verifying) { \n    \u002F\u002F 执行验证和解包\n    state.phase = Phase::ReadyToApply;\n}\n\n\u002F\u002F Stage C: Apply\nif matches!(state.phase, Phase::ReadyToApply | Phase::LocalApplying) { \n    \u002F\u002F 执行部署\n    state.phase = Phase::Done;\n}\n","rust",[35,268,269,274,279,284,289,293,299,304,309,314,319,324,329,335,341,347,353],{"__ignoreMap":33},[38,270,271],{"class":40,"line":41},[38,272,273],{},"\u002F\u002F Stage A: Download\n",[38,275,276],{"class":40,"line":48},[38,277,278],{},"if matches!(state.phase, Phase::Idle | Phase::Downloading) { \n",[38,280,281],{"class":40,"line":65},[38,282,283],{},"    \u002F\u002F 执行下载\n",[38,285,286],{"class":40,"line":78},[38,287,288],{},"    state.phase = Phase::Verifying;\n",[38,290,291],{"class":40,"line":91},[38,292,172],{},[38,294,295],{"class":40,"line":100},[38,296,298],{"emptyLinePlaceholder":297},true,"\n",[38,300,301],{"class":40,"line":112},[38,302,303],{},"\u002F\u002F Stage B: Verify + Extract\n",[38,305,306],{"class":40,"line":133},[38,307,308],{},"if matches!(state.phase, Phase::Downloading | Phase::Verifying) { \n",[38,310,311],{"class":40,"line":139},[38,312,313],{},"    \u002F\u002F 执行验证和解包\n",[38,315,316],{"class":40,"line":169},[38,317,318],{},"    state.phase = Phase::ReadyToApply;\n",[38,320,322],{"class":40,"line":321},11,[38,323,172],{},[38,325,327],{"class":40,"line":326},12,[38,328,298],{"emptyLinePlaceholder":297},[38,330,332],{"class":40,"line":331},13,[38,333,334],{},"\u002F\u002F Stage C: Apply\n",[38,336,338],{"class":40,"line":337},14,[38,339,340],{},"if matches!(state.phase, Phase::ReadyToApply | Phase::LocalApplying) { \n",[38,342,344],{"class":40,"line":343},15,[38,345,346],{},"    \u002F\u002F 执行部署\n",[38,348,350],{"class":40,"line":349},16,[38,351,352],{},"    state.phase = Phase::Done;\n",[38,354,356],{"class":40,"line":355},17,[38,357,172],{},[10,359,360],{},"这个设计支持 resume：网络中断或进程崩溃后下次启动可以从 phase 决定从哪一步恢复。合理的思路。",[10,362,363],{},"问题出在 caller 对 Done 状态的处理：",[28,365,367],{"className":264,"code":366,"language":266,"meta":33,"style":33},"run_full_apply_chain(state_base, &manifest, ...)?;\nlet final_state = state_machine::load(state_base)?;\nif final_state.phase == Phase::Done {\n    Ok(RunOutcome::UpdatedToVersion(manifest.version))   \u002F\u002F ← 假成功，只看 phase\n} else {\n    Ok(RunOutcome::Resumed(final_state.phase))\n}\n",[35,368,369,374,379,384,389,394,399],{"__ignoreMap":33},[38,370,371],{"class":40,"line":41},[38,372,373],{},"run_full_apply_chain(state_base, &manifest, ...)?;\n",[38,375,376],{"class":40,"line":48},[38,377,378],{},"let final_state = state_machine::load(state_base)?;\n",[38,380,381],{"class":40,"line":65},[38,382,383],{},"if final_state.phase == Phase::Done {\n",[38,385,386],{"class":40,"line":78},[38,387,388],{},"    Ok(RunOutcome::UpdatedToVersion(manifest.version))   \u002F\u002F ← 假成功，只看 phase\n",[38,390,391],{"class":40,"line":91},[38,392,393],{},"} else {\n",[38,395,396],{"class":40,"line":100},[38,397,398],{},"    Ok(RunOutcome::Resumed(final_state.phase))\n",[38,400,401],{"class":40,"line":112},[38,402,172],{},[10,404,405,406,409,410,413],{},"当进入 ",[35,407,408],{},"Phase::Done"," 后，链路到达末尾，清掉标记文件就返回。关键问题在于调用方 ",[35,411,412],{},"run_update_only_chain_inner"," 对 Done 状态的处理：",[28,415,416],{"className":264,"code":366,"language":266,"meta":33,"style":33},[35,417,418,422,426,430,434,438,442],{"__ignoreMap":33},[38,419,420],{"class":40,"line":41},[38,421,373],{},[38,423,424],{"class":40,"line":48},[38,425,378],{},[38,427,428],{"class":40,"line":65},[38,429,383],{},[38,431,432],{"class":40,"line":78},[38,433,388],{},[38,435,436],{"class":40,"line":91},[38,437,393],{},[38,439,440],{"class":40,"line":100},[38,441,398],{},[38,443,444],{"class":40,"line":112},[38,445,172],{},[10,447,448,449,452,453,456,457,461,462,465],{},"这段代码的逻辑是：如果 ",[35,450,451],{},"phase==Done","，就返回 ",[35,454,455],{},"UpdatedToVersion","（表示\"成功升级到目标版本\"）。它",[458,459,460],"strong",{},"完全依赖 phase 字段","来判断成功，",[458,463,464],{},"不校验实际产物","。",[10,467,468,469,472,473,476],{},"现场状态是 ",[35,470,471],{},"phase=Done"," 但 ",[35,474,475],{},"applied_version=0.14.15 != target=1.0.0","。当 OTA chain 重新运行时：",[478,479,480,486,496,504,512,515],"ol",{},[180,481,482,483,485],{},"进入 ",[35,484,260],{},"，加载 state，phase 已经是 Done",[180,487,488,489,492,493],{},"Download stage 检查 ",[35,490,491],{},"if matches!(phase, Idle | Downloading)"," → ",[458,494,495],{},"不匹配，跳过",[180,497,498,499,492,502],{},"Verify stage 检查 ",[35,500,501],{},"if matches!(phase, Downloading | Verifying)",[458,503,495],{},[180,505,506,507,492,510],{},"Apply stage 检查 ",[35,508,509],{},"if matches!(phase, ReadyToApply | LocalApplying)",[458,511,495],{},[180,513,514],{},"所有 stage 都被跳过，函数走到末尾清 marker 返回",[180,516,517,518,492,521],{},"caller 看到 ",[35,519,520],{},"final_state.phase == Done",[458,522,523,524],{},"返回 ",[35,525,526],{},"UpdatedToVersion(1.0.0)",[10,528,529],{},"没有任何实际的 download、verify、apply 操作发生。cache 仍是 0.14.18。applied_version 仍是 0.14.15。但链路声称\"已经升级到 1.0.0\"。",[10,531,532],{},"这个循环完全在 OTA chain 内部转圈：",[478,534,535,541,544,551,554,557,560,563],{},[180,536,537,538,540],{},"restart-loop 收到 ",[35,539,526],{},"，记进 events log \"OTA applied to 1.0.0\"，认为任务完成",[180,542,543],{},"restart-loop 重启 panel",[180,545,546,547,550],{},"panel 启动执行 ",[35,548,549],{},"ota_check_update","，通过 wrapper 询问后端",[180,552,553],{},"后端告诉 panel：ga 版本是 1.0.0",[180,555,556],{},"panel 本地 cache 还是 0.14.18（因为 apply 根本没执行），对比 1.0.0 ≠ 0.14.18",[180,558,559],{},"panel 决策 ForceUpdate，弹出\"升级到 1.0.0\"modal",[180,561,562],{},"用户看到 modal，再点一次\"立即更新\"",[180,564,565],{},"回到步骤 1，循环",[10,567,568],{},"launcher 的 boot 自愈路径（FA-006 修的 G15\u002FG16）也救不了，因为每一轮都是从 phase=Done 这个\"合法\"状态出发，boot 根本不会被触发。",[20,570,571],{"id":571},"修复与验证",[10,573,574,575,577],{},"chain 入口新增版本校验（",[35,576,260],{}," 开头）：",[28,579,581],{"className":264,"code":580,"language":266,"meta":33,"style":33},"let stale_done = state.phase == Phase::Done\n    && state\n        .local_apply\n        .as_ref()\n        .and_then(|la| la.applied_version.as_deref())\n        .map(|applied| applied != manifest.version)\n        .unwrap_or(false);\nif stale_done {\n    eprintln!(\n        \"[ota_main_chain] G18: state.phase=Done but applied_version={} != target={}; resetting to Idle\",\n        applied, manifest.version\n    );\n    state_machine::transition(state_base, &mut state, Phase::Idle)?;\n}\n",[35,582,583,588,593,598,603,608,613,618,623,628,633,638,643,648],{"__ignoreMap":33},[38,584,585],{"class":40,"line":41},[38,586,587],{},"let stale_done = state.phase == Phase::Done\n",[38,589,590],{"class":40,"line":48},[38,591,592],{},"    && state\n",[38,594,595],{"class":40,"line":65},[38,596,597],{},"        .local_apply\n",[38,599,600],{"class":40,"line":78},[38,601,602],{},"        .as_ref()\n",[38,604,605],{"class":40,"line":91},[38,606,607],{},"        .and_then(|la| la.applied_version.as_deref())\n",[38,609,610],{"class":40,"line":100},[38,611,612],{},"        .map(|applied| applied != manifest.version)\n",[38,614,615],{"class":40,"line":112},[38,616,617],{},"        .unwrap_or(false);\n",[38,619,620],{"class":40,"line":133},[38,621,622],{},"if stale_done {\n",[38,624,625],{"class":40,"line":139},[38,626,627],{},"    eprintln!(\n",[38,629,630],{"class":40,"line":169},[38,631,632],{},"        \"[ota_main_chain] G18: state.phase=Done but applied_version={} != target={}; resetting to Idle\",\n",[38,634,635],{"class":40,"line":321},[38,636,637],{},"        applied, manifest.version\n",[38,639,640],{"class":40,"line":326},[38,641,642],{},"    );\n",[38,644,645],{"class":40,"line":331},[38,646,647],{},"    state_machine::transition(state_base, &mut state, Phase::Idle)?;\n",[38,649,650],{"class":40,"line":337},[38,651,172],{},[10,653,654,655,658,659,662],{},"如果 phase 是 Done 但 applied_version 与目标版本不一致，强制调 ",[35,656,657],{},"transition(_, _, Phase::Idle)","。这个转移函数会自动清空 download、local_apply、usb_sync、signed_manifest 这些子结构（已有单测 ",[35,660,661],{},"transition_done_to_idle_clears_substructures"," 验证），然后 chain 后续所有 stage 都能从干净的 Idle 状态开始正常执行。",[10,664,665,666,669],{},"同时在 ",[35,667,668],{},"main.rs::boot_self_heal_ota_state"," 里也加这条检查（这个函数在 boot 启动时 Stage 5 和 Stage 6 之间调用）：",[28,671,673],{"className":264,"code":672,"language":266,"meta":33,"style":33},"let cache_ver = version_file::read(cache_root.join(\"version.json\")).unwrap_or_default();\nlet stale_done = state.phase == Phase::Done\n    && state.local_apply.as_ref()\n        .and_then(|la| la.applied_version.as_deref())\n        .map(|av| !cache_ver.is_empty() && av != cache_ver.as_str())\n        .unwrap_or(false);\nif stuck || stale_done {\n    state.phase = Phase::Idle;\n    state.cancellable = true;\n    state.download = None;\n    state.local_apply = None;\n    state.usb_sync = None;\n    state.signed_manifest = None;\n    state_machine::save(...)?;\n}\n",[35,674,675,680,684,689,693,698,702,707,712,717,722,727,732,737,742],{"__ignoreMap":33},[38,676,677],{"class":40,"line":41},[38,678,679],{},"let cache_ver = version_file::read(cache_root.join(\"version.json\")).unwrap_or_default();\n",[38,681,682],{"class":40,"line":48},[38,683,587],{},[38,685,686],{"class":40,"line":65},[38,687,688],{},"    && state.local_apply.as_ref()\n",[38,690,691],{"class":40,"line":78},[38,692,607],{},[38,694,695],{"class":40,"line":91},[38,696,697],{},"        .map(|av| !cache_ver.is_empty() && av != cache_ver.as_str())\n",[38,699,700],{"class":40,"line":100},[38,701,617],{},[38,703,704],{"class":40,"line":112},[38,705,706],{},"if stuck || stale_done {\n",[38,708,709],{"class":40,"line":133},[38,710,711],{},"    state.phase = Phase::Idle;\n",[38,713,714],{"class":40,"line":139},[38,715,716],{},"    state.cancellable = true;\n",[38,718,719],{"class":40,"line":169},[38,720,721],{},"    state.download = None;\n",[38,723,724],{"class":40,"line":321},[38,725,726],{},"    state.local_apply = None;\n",[38,728,729],{"class":40,"line":326},[38,730,731],{},"    state.usb_sync = None;\n",[38,733,734],{"class":40,"line":331},[38,735,736],{},"    state.signed_manifest = None;\n",[38,738,739],{"class":40,"line":337},[38,740,741],{},"    state_machine::save(...)?;\n",[38,743,744],{"class":40,"line":343},[38,745,172],{},[10,747,748],{},"这样即使没经过 chain 直接启动系统，boot 早期也能检测出\"applied_version 跟 cache 不一致\"的脏 Done 状态并重置。",[10,750,751],{},"复测设计了两条路径，对应 G18（chain 入口）和 G16（boot 启动）：",[10,753,754],{},[458,755,756],{},"路径 A：通过 OTA chain 验证 G18",[478,758,759,765,768,771,777,783,790,793,796,799,802,808],{},[180,760,761,762],{},"客户机上手动编辑 state.json：设置 ",[35,763,764],{},"phase=done + applied_version=0.14.15",[180,766,767],{},"cache 保持 0.14.18（这时应该已经是通过热部署或之前的 OTA 装的）",[180,769,770],{},"启动正常的 OTA 流程（比如通过 admin 端推送新版本 1.0.0）",[180,772,773,774,776],{},"chain 执行到 ",[35,775,260],{}," 开头，触发 G18 检查",[180,778,779,780],{},"代码检测 ",[35,781,782],{},"state.phase==Done && applied_version=0.14.15 != manifest.version=1.0.0",[180,784,785,786,789],{},"G18 强制调 ",[35,787,788],{},"transition(..., Phase::Idle)"," 清掉残留状态",[180,791,792],{},"后续 stage 从干净的 Idle 开始执行",[180,794,795],{},"Download → Verify → Apply 三个阶段真正跑完",[180,797,798],{},"cache version.json 真的变成 1.0.0",[180,800,801],{},"local_apply.applied_version 更新为 1.0.0",[180,803,804,805],{},"panel 重启，ota_check 对比 cache(1.0.0) == ga(1.0.0) → NoUpdate → ",[458,806,807],{},"不弹 modal",[180,809,810],{},"events log 中这一轮只有一条 \"OTA applied to 1.0.0\" 成功记录，没有重复",[10,812,813],{},[458,814,815],{},"路径 B：通过 boot 启动验证 G16 扩展",[478,817,818,821,824,827,830,833,839,842,845,848,851,854],{},[180,819,820],{},"手动编写 state.json 再次制造脏状态：phase=done，applied_version=0.14.15",[180,822,823],{},"cache 故意设为 0.14.18（模拟热部署后的状态）",[180,825,826],{},"关闭 panel，让 launcher 完全退出",[180,828,829],{},"重新启动系统（VBS → launcher → Stage 1-5 初始化）",[180,831,832],{},"在 Stage 5 和 Stage 6 之间，boot_self_heal_ota_state 被调用",[180,834,835,836],{},"G16 检测到 stale_done：",[35,837,838],{},"phase==Done && applied_version=0.14.15 != cache_version=0.14.18",[180,840,841],{},"G16 强制清掉这个脏状态：设置 phase 为 Idle，清空所有子字段",[180,843,844],{},"进入 Stage 6 的 OTA check",[180,846,847],{},"read_app_version 读到 cache 的 0.14.18，与 ga 版本 1.0.0 比对不一致",[180,849,850],{},"ota_check 决策 ForceUpdate(1.0.0)，写 marker，panel 弹 modal",[180,852,853],{},"用户点\"立即更新\"，走正常的 OTA 流程，最终装成 1.0.0",[180,855,856],{},"再次重启后不再弹 modal",[10,858,859],{},"两条路径都验证了修复的有效性：G18 拦截了 chain 内的假成功，G16 则保证了即使跳过 chain 直接启动系统，脏状态也会被清掉。",[20,861,863],{"id":862},"为什么修了这么多次这一次改了什么","为什么修了这么多次，这一次改了什么",[865,866,868],"h3",{"id":867},"同一症状三处已知根因","同一症状，三处已知根因",[10,870,871],{},"回顾整个 OTA 死循环系列，这是一个典型的\"多根因叠加\"的故障：",[873,874,875,897],"table",{},[876,877,878],"thead",{},[879,880,881,885,888,891,894],"tr",{},[882,883,884],"th",{},"编号",[882,886,887],{},"故障名",[882,889,890],{},"根因",[882,892,893],{},"修复位置",[882,895,896],{},"防护作用",[898,899,900,918,935],"tbody",{},[879,901,902,906,909,912,915],{},[903,904,905],"td",{},"FA-005",[903,907,908],{},"版本号优先级",[903,910,911],{},"admin 端推送的 ga 版本没有考虑热部署导致 cache 版本提前的情况",[903,913,914],{},"admin wrapper 的版本比对逻辑",[903,916,917],{},"防止\"本地已新，后端说旧\"导致的错误降级",[879,919,920,923,926,929,932],{},[903,921,922],{},"FA-006",[903,924,925],{},"panel 不退出",[903,927,928],{},"panel 在 onFinished 回调用 reload() 而非 exit()，launcher worker 进程继续运行导致重复 spawn",[903,930,931],{},"panel 端的 onFinished 回调",[903,933,934],{},"防止多个 worker 进程叠加争夺 state.lock",[879,936,937,940,943,946,952],{},[903,938,939],{},"FA-007",[903,941,942],{},"本篇，phase 假终态",[903,944,945],{},"OTA chain 把终态只用 phase=Done 表示，不校验 applied_version 是否匹配目标版本",[903,947,948,949,951],{},"chain 入口 ",[35,950,260],{}," 的版本校验",[903,953,954],{},"防止残留的\"上一轮成功\"状态在目标版本不同时假报成功",[10,956,957],{},"单独看每一个根因的影响：",[177,959,960,963,966],{},[180,961,962],{},"FA-005 单独发生：panel 看到\"版本已是 ga，不用升\"，用户见不到 modal，系统无害",[180,964,965],{},"FA-006 单独发生：worker 进程可能叠加，但第一个 worker 拿到 state.lock，后续的失败重试，最终还能装上新版本，不会死循环",[180,967,968],{},"FA-007 单独发生：如果每次启动 state 都是干净的 Idle，这个 bug 也不会触发，系统照常运行",[10,970,971,972,975],{},"但当这几个条件",[458,973,974],{},"同时出现","时，它们形成了一个完整的死循环链。FA-006 修完后，0.14.18 的 OTA 看起来可以跑通。然而升级到 1.0.0 时，由于某些原因（可能是 0.14.17 到 1.0.0 的架构变更），state.json 中残留了一个\"applied_version=0.14.15\"的 Done 状态。这时 FA-007 就被触发了：chain 看到 Done，所有 stage 跳过，假报成功，触发 FA-005 的对比逻辑，最终弹 modal，形成死循环。",[865,977,978],{"id":978},"这一次改了什么",[10,980,981,982],{},"FA-007 要修的是：",[458,983,984],{},"任何时候进入 chain，都必须先校验 applied_version 与目标版本是否匹配。如果不匹配，不能相信之前的 Done 状态，必须强制重置到 Idle 重新执行。",[10,986,987],{},"这个修复摧毁了死循环链中的\"假成功报告\"这一环。即使状态机上还有其他缺陷（如 FA-005，或尚未发现的残留问题），由于 chain 不再假报成功，它们也更难找到触发的机会。修完这一层也不能断言链路上不存在第四个根因——只能说已知的三个都各自被堵上了。每一处修复都切断了死循环的一条传播路径，修完 FA-005 + FA-006 + FA-007 之后，形成完整死循环的条件才真正被破坏。",[20,989,990],{"id":990},"防回归",[865,992,993],{"id":993},"代码层面",[10,995,996,997,999],{},"单元测试 ",[35,998,661],{}," 已覆盖 Done → Idle 的转移路径，验证状态重置后所有相关字段都被清空。",[10,1001,1002,1003,1006],{},"schema 兼容性：applied_version 字段在 v4 版本的 review 阶段引入，旧版本 launcher 写的 state.json 中可能缺失这个字段。G18 的代码使用了 Rust 的 ",[35,1004,1005],{},"and_then().map().unwrap_or()"," 链，确保字段缺失时返回默认值，不触发假检测。",[10,1008,1009],{},"worker 路径兼容：ota_worker 不经过 run_full_apply_chain，而是走独立的 phase1_download_and_verify 函数。但因为 G16 在 boot 阶段早期就会重置脏状态，worker 进来时的 state 已经干净，无需额外改动。",[865,1011,1012],{"id":1012},"运维观察",[10,1014,1015],{},"events log 是诊断 FA-007 的重要信号。具体特征：",[177,1017,1018,1025,1028,1031],{},[180,1019,1020,1021,1024],{},"同一个 ",[35,1022,1023],{},"source","（如 restart_loop）",[180,1026,1027],{},"在短时间内（\u003C 1 分钟）",[180,1029,1030],{},"多次（≥ 3 次）出现 \"outcome\":\"succeeded\"",[180,1032,1033],{},"但实际 cache\u002Fversion.json 没有变化",[10,1035,1036],{},"这个组合高度可疑，可以作为自动化巡检的告警触发条件。正常的 OTA 应该一次装完（除非是主动的 resume 路径），重复成功通常意味着链路在假报。",[865,1038,1039],{"id":1039},"兼容性总结",[177,1041,1042,1045,1048,1051,1054],{},[180,1043,1044],{},"旧版本 launcher 的 state.json 兼容：缺失的 applied_version 字段不会误触 G18 检测",[180,1046,1047],{},"G15\u002FG16\u002FG17（来自 FA-006 的 launcher 防御）全部保留，不冲突",[180,1049,1050],{},"panel 端无变更，G13\u002FG14（FA-006 修的 onFinished 退出逻辑）仍在",[180,1052,1053],{},"worker 进程的 phase1_download_and_verify 逻辑不变，不经过 run_full_apply_chain，但因为 G16 在 boot 阶段早期就会重置脏状态，worker 进来时的 state 已经干净，无需额外改动",[180,1055,1056],{},"构建脚本、CI 流程无需改动",[20,1058,1059],{"id":1059},"关键设计原则",[10,1061,1062],{},"状态机的终态判定不能只依赖枚举值，必须验证实际产物。没有这层校验，\"上一轮残留\"的终态会让整个执行链路跳过，转化成假成功报告，最终驱动上层重复触发同一个已经\"完成\"的操作。",[10,1064,1065],{},"对于 phase-conditional 执行链：",[478,1067,1068,1071,1074,1077],{},[180,1069,1070],{},"链的入口做合法性检查：当前 phase 是否与执行目标兼容",[180,1072,1073],{},"残留终态（如 phase=Done）不能盲目信任，必须校验是否指向正确版本",[180,1075,1076],{},"校验失败强制重置，不假设\"上一轮可能对的\"",[180,1078,1079],{},"任何状态转移都应该有明确的幂等性保证——重复执行同一状态转移应该得到相同结果",[10,1081,1082],{},"后续扩展 OTA 协议时，新增 phase 和 RunOutcome 都必须纳入 chain entrypoint 的合法性检查框架内。这是分布式系统设计中的普遍问题，不仅限于 OTA 场景。",[20,1084,1085],{"id":1085},"总结",[10,1087,1088],{},"状态机的终态不能只由阶段枚举表示，必须同时校验实际产物。没有这层校验，\"上一轮残留\"的终态会让整个执行链路跳过，转化成假成功报告，最终驱动上层重复触发同一个已经\"完成\"的操作。这也说明了为什么同一个死循环症状需要改多处——每一处修复都在阻止不同的\"跨越\"路径达到假终态，改完 FA-005 + FA-006 + FA-007 之后才能真正阻断死循环的形成。",[1090,1091,1092],"style",{},"html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":33,"searchDepth":48,"depth":48,"links":1094},[1095,1096,1097,1098,1102,1107,1108],{"id":22,"depth":48,"text":23},{"id":250,"depth":48,"text":250},{"id":571,"depth":48,"text":571},{"id":862,"depth":48,"text":863,"children":1099},[1100,1101],{"id":867,"depth":65,"text":868},{"id":978,"depth":65,"text":978},{"id":990,"depth":48,"text":990,"children":1103},[1104,1105,1106],{"id":993,"depth":65,"text":993},{"id":1012,"depth":65,"text":1012},{"id":1039,"depth":65,"text":1039},{"id":1059,"depth":48,"text":1059},{"id":1085,"depth":48,"text":1085},"故障档案","2026-05-10","md",null,{},"\u002F2026-05-10-fa-007-phase-done",{"title":5,"description":12},"2026-05-10-FA-007-phase-Done假报成功","状态机的终态只由 phase 表示，不校验实际产物导致假报成功。修复是入口处检测 applied_version 与目标版本不一致时强制重置。",[1119,1120,1121,1122,1123,1124],"OTA","状态机","Rust","Windows","死循环","版本管理","sovovJBp96cwFJqXF6aE7Ou9pUEpg-z1Fegj6dl8ABM",[1127,1533,1992],{"id":1128,"title":1129,"body":1130,"column":1109,"date":1518,"description":1134,"extension":1111,"hero_image":1112,"meta":1519,"navigation":297,"path":1520,"seo":1521,"series_id":1522,"severity":1112,"stem":1523,"summary":1524,"tags":1525,"__hash__":1532},"posts\u002F2026-07-23-FA-018-共享存储的subPath陷阱.md","一个 PVC 装所有用户，当时看不出有什么问题",{"type":7,"value":1131,"toc":1509},[1132,1135,1138,1142,1145,1148,1155,1200,1203,1213,1217,1223,1226,1237,1240,1244,1247,1250,1253,1271,1274,1281,1284,1299,1303,1306,1309,1320,1331,1334,1337,1340,1402,1405,1412,1415,1418,1444,1447,1458,1461,1464,1503,1506],[10,1133,1134],{},"平台从 Docker Swarm 迁移到 Kubernetes，存储架构做了一个看似理想的调整：用共享 PVC + subPath 替代原来的\"每用户独占 PVC\"。方案优势显而易见——管理简单（1 个 PVC 对 574 个）、自动扩展、成本低。当时看不出有什么缺点。",[10,1136,1137],{},"但实施中遇到了四个隐蔽的问题。",[20,1139,1141],{"id":1140},"权限错配bind-型数据的属主陷阱","权限错配：bind 型数据的属主陷阱",[10,1143,1144],{},"旧系统中，用户数据通过两种方式持久化：bind mount 和 Docker volume。迁移时需要从这两个来源完整导出数据。",[10,1146,1147],{},"对于 bind mount 型数据，文件属主通常是 10001（或其他固定 UID）。当我登上旧集群的宿主机，以普通用户身份尝试读取这些文件时，只能看到 13 个文件。权限拒绝。同一目录里其实有 4778 个文件，但大多数因为属主权限不匹配而不可见。",[10,1149,1150,1151,1154],{},"只有用 sudo 才能完整读到。所以迁移脚本必须用 ",[35,1152,1153],{},"sudo tar"," 来打包：",[28,1156,1160],{"className":1157,"code":1158,"language":1159,"meta":33,"style":33},"language-bash shiki shiki-themes github-light github-dark","sudo tar -czf \u002Ftmp\u002Fmig\u002F${uid}\u002Fdata.tgz \\\n  -C \u002Fmnt\u002Fyun-claw\u002Fusers\u002F${uid} .\n","bash",[35,1161,1162,1186],{"__ignoreMap":33},[38,1163,1164,1168,1171,1174,1177,1180,1183],{"class":40,"line":41},[38,1165,1167],{"class":1166},"sScJk","sudo",[38,1169,1170],{"class":58}," tar",[38,1172,1173],{"class":51}," -czf",[38,1175,1176],{"class":58}," \u002Ftmp\u002Fmig\u002F",[38,1178,1179],{"class":44},"${uid}",[38,1181,1182],{"class":58},"\u002Fdata.tgz",[38,1184,1185],{"class":51}," \\\n",[38,1187,1188,1191,1194,1197],{"class":40,"line":48},[38,1189,1190],{"class":51},"  -C",[38,1192,1193],{"class":58}," \u002Fmnt\u002Fyun-claw\u002Fusers\u002F",[38,1195,1196],{"class":44},"${uid} ",[38,1198,1199],{"class":58},".\n",[10,1201,1202],{},"这不是脚本设计的问题，而是 Kubernetes 环境下容器权限与宿主权限映射的一个陷阱。容器内运行的网关进程可能以容器用户身份运行，但宿主上的文件属主是不同的 UID。当两个权限体系接不上，访问就会失败。",[10,1204,1205,1206,1209,1210,1212],{},"新方案中，用户数据通过 subPath 挂到容器的 ",[35,1207,1208],{},"\u002Fdata"," 目录。如果 subPath 指向的目录是由 Kubernetes 在宿主机上创建的，属主可能是 root 或其他用户，容器内进程（以容器指定的 UID 运行）仍然会遭遇权限问题。这个风险需要在 Dockerfile 和容器启动时明确处理：容器内应该预先创建 ",[35,1211,1208],{}," 并设置正确的属主，或者使用 securityContext 的 fsGroup 或 runAsUser 来强制权限。",[20,1214,1216],{"id":1215},"subpath-目录创建的时机与属主处理","subPath 目录创建的时机与属主处理",[10,1218,1219,1220],{},"Kubernetes 对 subPath 的处理有个微妙之处：如果挂载的 subPath 目录在宿主机上不存在，kubelet 会自动创建它。但 ",[38,1221,1222],{},"待补：这个自动创建的目录的属主是什么？是 root 还是其他用户？容器的 securityContext 能否保证挂载后的权限符合预期？",[10,1224,1225],{},"设计文档中没有明确说明这一点。在实施前，需要验证：",[177,1227,1228,1231,1234],{},[180,1229,1230],{},"首次 subPath 目录不存在时，kubelet 创建它并将其属主设置为谁",[180,1232,1233],{},"容器的 securityContext（fsGroup、runAsUser）是否能在挂载时生效",[180,1235,1236],{},"是否应该预先在共享 PVC 上创建所有 subPath 目录并设置好属主，而不是依赖 Kubernetes 的隐式创建",[10,1238,1239],{},"没有明确的答案，就容易踩坑。一个保险做法是在 destroy 用户账户时清空 subPath 目录，同时在 provision 时让一个 init-container 验证或修复目录属主，确保容器进程有写权限。",[20,1241,1243],{"id":1242},"单-pvc-中的节点级故障域","单 PVC 中的节点级故障域",[10,1245,1246],{},"这是最严重的问题，也是与 FA-015（节点假 Ready）的关联点。",[10,1248,1249],{},"当一个节点的 NFS 客户端或到 SFS 的网络链路发生 hang 时，所有依赖共享 PVC 的 Pod 都会受影响。这不是共享 PVC 本身的问题，而是故障隔离的问题。",[10,1251,1252],{},"具体现象：",[177,1254,1255,1262,1265,1268],{},[180,1256,1257,1258,1261],{},"如果 NFS 挂载使用了硬 mount（",[35,1259,1260],{},"hard"," 参数是 NFS 的默认值），而网络故障或存储服务中断，NFS 客户端会无限重试",[180,1263,1264],{},"重试期间，所有试图访问 NFS 的进程都会被阻塞在 I\u002FO 上（D 状态，不可中断）",[180,1266,1267],{},"如果 kubelet 或 containerd 的某个操作（如创建容器的卷挂载步骤）陷入 I\u002FO 阻塞，整个节点就会冻死",[180,1269,1270],{},"该节点上的所有实例（无论是否在访问存储）都会因为无法创建或删除 Pod 而故障",[10,1272,1273],{},"这与独立 PVC 的效果截然不同。旧架构中，每个用户有独立 PVC，意味着如果某个 PVC 对应的存储有问题，只会影响这一个用户。其他用户的 PVC 可能分散在不同的存储卷甚至不同的节点上，相对独立。",[10,1275,1276,1277,1280],{},"新架构下，单个共享 PVC 承载所有用户，一旦这个 PVC 对应的网络链路或挂载出问题，",[458,1278,1279],{},"同节点上所有实例都会被殃及","。如果该节点碰巧堆积了 50 个实例，一次节点级的存储 hang 就会导致 50 个实例集体故障，且外表看起来是\"节点 Ready，但实例卡住\"——kubelet 的心跳正常，Pod 状态却卡在 ContainerCreating 或 Terminating。",[10,1282,1283],{},"这正是 FA-015 在 2026-06-05 观察到的现象：节点假 Ready，但大量实例网关无响应。防护措施包括：",[177,1285,1286,1293,1296],{},[180,1287,1288,1289,1292],{},"NFS 挂载参数改为软 mount（",[35,1290,1291],{},"soft,timeo=100,retrans=3","），让 I\u002FO 超时而不是无限等待",[180,1294,1295],{},"节点加健康检测，检查 ContainerCreating 堆积数和网关探测失败率，及早发现节点冻死",[180,1297,1298],{},"实例分散部署，用 topologySpreadConstraints 避免单节点堆积",[20,1300,1302],{"id":1301},"无-per-subpath-硬配额","无 per-subPath 硬配额",[10,1304,1305],{},"共享存储的架构决定了无法在存储层面按 subPath 限制用户配额。SFS（及 NFS 一般）没有 per-directory 的硬配额功能，只能在整个卷级别限制。",[10,1307,1308],{},"这意味着：",[177,1310,1311,1314,1317],{},[180,1312,1313],{},"无法阻止某个用户的数据膨胀而挤占其他用户的空间",[180,1315,1316],{},"无法在存储层面实现\"超额用户的写入失败\"",[180,1318,1319],{},"只能在应用层进行监控、告警和逻辑控制",[10,1321,1322,1323,1326,1327,1330],{},"新系统的方案是应用层监控：定期 ",[35,1324,1325],{},"du -sb \u002Fdata"," 扫描每个用户的目录大小，落库到 Instance 表，",[35,1328,1329],{},"\u002Fstatus"," 接口返回超额标志位。前端据此提示用户\"存储已满\"。这是纯监控，不阻断。如果用户无视提示继续写入，直到共享卷真的满了，才会出现\"所有用户集体写入失败\"的惨淡局面。",[10,1332,1333],{},"这个限制是方案选择的代价。",[20,1335,1336],{"id":1336},"为什么仍然选了这个方案",[10,1338,1339],{},"尽管有这些问题，平台仍然采用了共享 PVC + subPath 的方案。原因是对比了替代方案：",[873,1341,1342,1358],{},[876,1343,1344],{},[879,1345,1346,1349,1352,1355],{},[882,1347,1348],{},"方案",[882,1350,1351],{},"优点",[882,1353,1354],{},"缺点",[882,1356,1357],{},"故障隔离",[898,1359,1360,1374,1388],{},[879,1361,1362,1365,1368,1371],{},[903,1363,1364],{},"独立 PVC（旧架构）",[903,1366,1367],{},"天然的每用户隔离；故障域清晰",[903,1369,1370],{},"管理复杂（574 个 PVC）；扩展性差；成本高",[903,1372,1373],{},"✓ 最好",[879,1375,1376,1379,1382,1385],{},[903,1377,1378],{},"共享 PVC + subPath（新架构）",[903,1380,1381],{},"管理简单（1 个 PVC）；自动扩展；成本低",[903,1383,1384],{},"故障隔离性差；无硬配额；权限管理复杂",[903,1386,1387],{},"✗ 较差",[879,1389,1390,1393,1396,1399],{},[903,1391,1392],{},"对象存储（S3\u002FOSS）",[903,1394,1395],{},"真正的多租户隔离；天然分布",[903,1397,1398],{},"延迟高；成本更高；应用改造大",[903,1400,1401],{},"✓ 最好，但代价大",[10,1403,1404],{},"独立 PVC 的管理开销是关键问题。574 个用户意味着 574 个 PVC 对象、574 条 PV 绑定、574 个存储卷。每次实例创建、删除或迁移都要涉及 PVC 的生命周期管理。扩展到 5000 用户时，这种开销会成为瓶颈。对象存储虽然隔离性最好，但需要应用层改造（兼容 S3 API、处理延迟、调整备份策略），而且成本更高。",[10,1406,1407,1408,1411],{},"共享 PVC 方案的核心优势是",[458,1409,1410],{},"运维简洁","：增删用户只需改 subPath（一行 YAML），不涉及存储层操作。代价是故障隔离性从\"用户级\"降到\"节点级\"。",[20,1413,1414],{"id":1414},"适用边界",[10,1416,1417],{},"这个方案适合以下场景：",[177,1419,1420,1426,1432,1438],{},[180,1421,1422,1425],{},[458,1423,1424],{},"用户数量有上限","（几百到几千）。用户过多时，共享卷的单点压力会成问题。",[180,1427,1428,1431],{},[458,1429,1430],{},"用户数据量可控","（单用户通常 GB 级）。如果单用户数据量达 TB，一次 du 扫描会拖累整体。",[180,1433,1434,1437],{},[458,1435,1436],{},"可以接受节点级故障隔离","。只要网络和存储配置足够稳定（硬化 NFS 参数、冗余链路），节点 hang 的概率不会很高。",[180,1439,1440,1443],{},[458,1441,1442],{},"能够实施应用层监控和告警","。无硬配额，就必须有实时监控。",[10,1445,1446],{},"不适合的场景包括：",[177,1448,1449,1452,1455],{},[180,1450,1451],{},"超大规模用户（万级以上）",[180,1453,1454],{},"用户数据特别不均衡（少数用户占大头）的场景",[180,1456,1457],{},"对故障隔离要求极高的系统（比如金融交易）",[20,1459,1460],{"id":1460},"防护与调整",[10,1462,1463],{},"基于这四个问题，实施中做了以下调整：",[478,1465,1466,1475,1485,1491,1497],{},[180,1467,1468,1471,1472,1474],{},[458,1469,1470],{},"权限处理","：容器 Dockerfile 中预先创建 ",[35,1473,1208],{}," 并设置正确的属主；启动脚本检查并修复权限。",[180,1476,1477,1480,1481,1484],{},[458,1478,1479],{},"NFS 参数硬化","：StorageClass 的挂载选项改为 ",[35,1482,1483],{},"soft,timeo=100,retrans=3,intr","，避免硬 mount 导致节点冻。",[180,1486,1487,1490],{},[458,1488,1489],{},"节点健康检测","：cron 每 60 秒检查各节点的 ContainerCreating 堆积和网关探测失败率，及早发现故障。",[180,1492,1493,1496],{},[458,1494,1495],{},"实例分散","：StatefulSet 加 topologySpreadConstraints，避免单节点堆积 50+ 个实例。",[180,1498,1499,1502],{},[458,1500,1501],{},"应用层配额","：\u002Fstatus 接口返回 diskUsedMi 和 overQuota 标志位，前端告警用户。",[10,1504,1505],{},"这些措施不能消除风险，但可以大幅降低故障概率和影响范围。",[1090,1507,1508],{},"html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":33,"searchDepth":48,"depth":48,"links":1510},[1511,1512,1513,1514,1515,1516,1517],{"id":1140,"depth":48,"text":1141},{"id":1215,"depth":48,"text":1216},{"id":1242,"depth":48,"text":1243},{"id":1301,"depth":48,"text":1302},{"id":1336,"depth":48,"text":1336},{"id":1414,"depth":48,"text":1414},{"id":1460,"depth":48,"text":1460},"2026-07-23",{},"\u002F2026-07-23-fa-018-subpath",{"title":1129,"description":1134},"FA-018","2026-07-23-FA-018-共享存储的subPath陷阱","K8s 共享 PVC + subPath 方案在实施中暴露的四个陷阱：权限不匹配、目录创建时机、节点级故障域、无 per-subPath 配额。",[1526,1527,1528,1529,1530,1531,1357],"Kubernetes","存储","subPath","PVC","NFS","权限","4pJJyzbII4OOc9jJS74HUpUfyVc9CfxdE5ovso8ylSE",{"id":1534,"title":1535,"body":1536,"column":1109,"date":1976,"description":1977,"extension":1111,"hero_image":1112,"meta":1978,"navigation":297,"path":1979,"seo":1980,"series_id":1981,"severity":1112,"stem":1982,"summary":1983,"tags":1984,"__hash__":1991},"posts\u002F2026-07-11-FA-017-长任务异步化.md","三个超时机制，一起把长任务掐死了",{"type":7,"value":1537,"toc":1968},[1538,1548,1551,1554,1560,1566,1572,1579,1585,1589,1592,1595,1603,1606,1617,1623,1627,1630,1637,1643,1646,1661,1665,1668,1679,1728,1743,1750,1754,1757,1760,1778,1781,1789,1792,1807,1810,1813,1959,1962,1965],[10,1539,1540,1541,1544,1545,465],{},"数字人口播的「分析」功能从视频提取口播文稿、分镜脚本、结构卖点，需要下载视频、压缩、调 Gemini 视觉 API，耗时从几十秒到几分钟。最初同步实现：前端调 ",[35,1542,1543],{},"POST \u002Fapi\u002Fworkflow\u002Fdub\u002Fanalyze-parsed","，后端直接跑完再返回。这个方案暴露的问题不是单点，而是",[458,1546,1547],{},"三个独立的缺陷叠加",[10,1549,1550],{},"网关超时、CDN 空闲切断、客户端重试——这三个没有一个是\"bug\"，每个都是合理的系统设计。但组合在一起就是连锁灾难：客户端看不到结果（超时或连接断），认为失败了重试，后端却已经扣过费了；或者分析其实已跑完，但因为响应发不出去，重试又跑了一遍。退款时无法幂等判断，某些用户最终被多扣了好几倍。",[10,1552,1553],{},"具体来说，三个问题各自是什么：",[10,1555,1556,1559],{},[458,1557,1558],{},"第一个问题是网关超时","。云平台的入口网关（API Gateway）有一个固定的读超时，通常设为几十秒。当后端分析耗时超过这个阈值，网关主动断连接，客户端收到 504 或连接重置，但后端的分析进程并不知道客户端已经走了，继续跑完整个流程。如果分析结果的退款逻辑放在 try-catch 的 finally 里，此时 HTTP 响应已发不出去，客户就看到\"分析失败\"，但款已经扣了。",[10,1561,1562,1565],{},[458,1563,1564],{},"第二个问题是 CDN 空闲切断","。不少 CDN 和负载均衡层有这样的策略：如果一条 HTTP 连接在某个时间段内没有数据往返，就认为它已死而主动关闭。这与 FA-013 那次踩到的问题一样——长连接因为没有心跳数据而被 CDN 中间件切断，导致请求被突然中断。这个机制在分析请求上也会造成麻烦：如果分析用时较长，连接就可能被切，响应发不出去。",[10,1567,1568,1571],{},[458,1569,1570],{},"第三个问题是客户端重试导致重复扣费","。当客户端没有收到响应（网关超时或 CDN 切断），一般会重试这个请求。问题在于这次重试是一条完全独立的请求，后端会把它当作新任务处理，再次扣费。而且如果第一次的分析实际上已经跑完了，就会出现\"分析跑了两遍、扣费扣了两遍、用户只看到了一个结果\"的情况。如果第一次分析中途被 CDN 切了，第二次重试重新开始，那扣费可能是三倍。而退款的时候，因为业务流程上没有幂等设计，很难追溯哪个 operationId 应该被退，哪个不应该。",[10,1573,1574,1575,1578],{},"这三个问题是",[458,1576,1577],{},"叠加的","，不是独立的。它们共同指向一个根本的架构问题。",[10,1580,1581,1582,465],{},"根本原因一个：",[458,1583,1584],{},"不该让耗时任务挂在 HTTP 连接上",[20,1586,1588],{"id":1587},"异步任务是必需的不是可选的","异步任务是必需的，不是可选的",[10,1590,1591],{},"解决方案很简单：把分析改成异步任务模式。",[10,1593,1594],{},"前端提交分析请求直接返回任务 ID（HTTP 耗时极短：校验参数、建数据库记录、预扣费）。后端后台 worker 跑分析重逻辑，完成后持久化结果。前端轮询查询状态，得到结果后停止。",[28,1596,1601],{"className":1597,"code":1599,"language":1600},[1598],"language-text","客户端              后端\n  |                 |\n  +--提交---------->|\n  |             创建任务，返回 ID\n  |\u003C--返回 ID-----+\n  |                 |（后台运行分析）\n  |                 |\n  +--轮询--------->|\n  |             查询状态\n  |\u003C--返回状态---+\n  |                 |\n  |（重复轮询）\n  +--轮询--------->|\n  |             查询状态\n  |\u003C--返回结果---+\n  |                 |\n","text",[35,1602,1599],{"__ignoreMap":33},[10,1604,1605],{},"好处显而易见：",[177,1607,1608,1611,1614],{},[180,1609,1610],{},"单次 HTTP 请求的耗时从分钟级降到秒级，不会触发网关或 CDN 的超时。",[180,1612,1613],{},"后端分析的运行生命周期与 HTTP 连接完全解耦。即使连接在中途被切，已经启动的分析会继续在后台跑，结果最终还是会被保存。",[180,1615,1616],{},"客户端重试不会产生新的分析任务。只要实现幂等判重，同一个请求在短时间内重复提交也只会产生一个任务。",[10,1618,1619,1620,465],{},"但异步化不是万灵药，它把问题转移到了三个新的维度上：",[458,1621,1622],{},"幂等、持久化、失败语义",[20,1624,1626],{"id":1625},"幂等同一请求只产生一个任务","幂等：同一请求只产生一个任务",[10,1628,1629],{},"假设客户端因为网络抖动，在短时间内连续发了两次\"开始分析\"请求，后端收到两条独立的 HTTP 请求。如果没有幂等设计，就会创建两个任务、扣两次费。",[10,1631,1632,1633,1636],{},"幂等的实现最直接的办法是在",[458,1634,1635],{},"提交阶段就判重","：为这个用户、这个资源设置一个\"只有一个进行中的分析任务\"的约束。素材中的设计是这样做的：",[28,1638,1641],{"className":1639,"code":1640,"language":1600},[1598],"if 该用户已有 status=running && kind=analyze_parsed 的任务:\n    return 409 Conflict（分析已在进行中）\n",[35,1642,1640],{"__ignoreMap":33},[10,1644,1645],{},"这样双击提交同一个视频，第一次返回 201 + taskId，第二次返回 409，客户端看到 409 就知道不要再建新任务，拿之前返回的 ID 去轮询。",[10,1647,1648,1649,1652,1653,1656,1657,1660],{},"另一个幂等层次是",[458,1650,1651],{},"计费操作的幂等","。预扣费的时候用一个全局唯一的 ",[35,1654,1655],{},"operationId","（比如 ",[35,1658,1659],{},"dub-analyze:{uuid}","），这个 ID 关联了这次预扣。如果扣费 API 被重复调用，因为 operationId 重复，系统只会扣一次。退款时也用同一个 operationId，保证退款与预扣能正确配对。",[20,1662,1664],{"id":1663},"持久化进程重启后任务不丢","持久化：进程重启后任务不丢",[10,1666,1667],{},"后台分析 worker 可能因为发版、机器重启、容器被杀等原因在中途退出。如果任务的状态只保存在内存里，进程一死任务就丢了。",[10,1669,1670,1671,1674,1675,1678],{},"持久化的关键是",[458,1672,1673],{},"任务状态表","。素材中复用了既有的 ",[35,1676,1677],{},"SkyhumanTask"," 表，包含字段：",[177,1680,1681,1687,1693,1698,1704,1710,1716,1722],{},[180,1682,1683,1686],{},[35,1684,1685],{},"id",": 任务 ID",[180,1688,1689,1692],{},[35,1690,1691],{},"status",": 进度（running \u002F completed \u002F failed）",[180,1694,1695,1697],{},[35,1696,1655],{},": 幂等 ID",[180,1699,1700,1703],{},[35,1701,1702],{},"chargedPoints",": 预扣费用",[180,1705,1706,1709],{},[35,1707,1708],{},"resultPayload",": 分析结果的 JSON",[180,1711,1712,1715],{},[35,1713,1714],{},"error",": 失败原因",[180,1717,1718,1721],{},[35,1719,1720],{},"updatedAt",": 最后更新时间戳",[180,1723,1724,1727],{},[35,1725,1726],{},"sourceObjectKey",": 源视频对象路径（用于幂等判重和重试）",[10,1729,1730,1731,1734,1735,1738,1739,1742],{},"任务创建时插一条 ",[35,1732,1733],{},"status=running"," 的记录。后台 worker 定期通过 heartbeat（每 30 秒做一次空 update，仅触碰 updatedAt）来证明自己还活着。分析完成时更新为 ",[35,1736,1737],{},"status=completed, resultPayload=...","；失败时更新为 ",[35,1740,1741],{},"status=failed, error=..."," 并触发退款。",[10,1744,1745,1746,1749],{},"这样即使 worker 进程突然死了，任务记录还在数据库里。新启起的 worker 可以扫描表里的 ",[35,1747,1748],{},"status=running && updatedAt 过期"," 的任务，判断它们已经没有 worker 在跑了，执行清理逻辑。",[20,1751,1753],{"id":1752},"失败语义区分任务失败和查询失败","失败语义：区分\"任务失败\"和\"查询失败\"",[10,1755,1756],{},"异步设计引入了一个新的错误维度：客户端查询任务状态时得到的 404，可能是\"这个任务 ID 不存在\"（用户输错了），也可能是\"任务 ID 之前存在但已被清理\"（进程清理了过期数据）。这两种情况的含义很不一样。",[10,1758,1759],{},"更重要的是，前端轮询收到的错误需要区分是否应该重试：",[177,1761,1762,1772],{},[180,1763,1764,1767,1768,1771],{},[458,1765,1766],{},"任务本身失败","（",[35,1769,1770],{},"status=failed, error=\"Gemini API 返回错误\"","）：这种情况不该重试，因为重试也会失败。应该直接向用户展示失败信息，并根据 operationId 触发退款。",[180,1773,1774,1777],{},[458,1775,1776],{},"查询接口失败","（500、网络超时）：这种情况应该重试查询，因为任务本身可能还在继续跑。",[10,1779,1780],{},"素材中的设计把这两类区分得很清楚：",[177,1782,1783,1786],{},[180,1784,1785],{},"任务最终的状态（completed 或 failed）持久化到数据库，客户端查询会拿到明确的业务语义。",[180,1787,1788],{},"查询接口的 HTTP 错误（5xx）是技术层故障，客户端应该重试。",[10,1790,1791],{},"同时，后端的 reaper 机制（定期扫表清理超期任务）也遵循这个语义：",[177,1793,1794,1800],{},[180,1795,1796,1797,1799],{},"如果一个 ",[35,1798,1733],{}," 的任务超过 90 秒没有 heartbeat 更新，说明 worker 已死，reaper 会强制置为 failed 并退款。",[180,1801,1802,1803,1806],{},"但这个强制置为 failed 不应该抛错，因为此时可能有其他进程也想更新这个任务。更新语句应该带条件：",[35,1804,1805],{},"UPDATE SkyhumanTask SET status=failed WHERE id=? AND status=running","，这样如果 reaper 和其他 worker 同时操作，只有一方会成功。",[20,1808,1809],{"id":1809},"具体实现模式",[10,1811,1812],{},"数字人口播的分析异步化采用了这样的模式：",[478,1814,1815,1859,1906,1929],{},[180,1816,1817,1820,1821],{},[458,1818,1819],{},"提交阶段","（同步）：",[177,1822,1823,1826,1829,1835,1841,1848,1854],{},[180,1824,1825],{},"校验用户和资源",[180,1827,1828],{},"判重：是否已有进行中的任务（409）",[180,1830,1831,1834],{},[35,1832,1833],{},"getObject"," 拉视频，检测时长（若≤0 返 400 不扣费）",[180,1836,1837,1840],{},[35,1838,1839],{},"chargeResource"," 预扣费用，获得 operationId",[180,1842,1843,1844,1847],{},"创建 ",[35,1845,1846],{},"SkyhumanTask{ status:running, operationId, sourceObjectKey }"," 记录",[180,1849,1850,1853],{},[35,1851,1852],{},"scheduleTask"," 丢到后台队列",[180,1855,523,1856],{},[35,1857,1858],{},"{ taskId }",[180,1860,1861,1864,1865],{},[458,1862,1863],{},"后台运行阶段","：",[177,1866,1867,1870,1873,1880,1886,1896,1903],{},[180,1868,1869],{},"Worker 取出任务记录",[180,1871,1872],{},"开启 heartbeat 定时器（每 30s update 一次 updatedAt）",[180,1874,1875,1876,1879],{},"执行 ",[35,1877,1878],{},"analyzeDubVisionOnly","（下载→压缩→调用 Gemini→解析）",[180,1881,1882,1883],{},"成功：",[35,1884,1885],{},"update( status:completed, resultPayload )",[180,1887,1888,1889,1892,1893],{},"失败：",[35,1890,1891],{},"update( status:failed, error )"," + ",[35,1894,1895],{},"refundResource(operationId)",[180,1897,1898,1899,1902],{},"所有 update 都带条件 ",[35,1900,1901],{},"WHERE id=? AND status=running","（防被 reaper 覆盖）",[180,1904,1905],{},"Finally 清理 heartbeat 定时器",[180,1907,1908,1911,1912],{},[458,1909,1910],{},"查询阶段","（前端轮询）：",[177,1913,1914,1923,1926],{},[180,1915,1916,1919,1920],{},[35,1917,1918],{},"GET \u002Fapi\u002Fworkflow\u002Fdub\u002Ftasks\u002F:id"," 返回 ",[35,1921,1922],{},"{ status, resultPayload, error }",[180,1924,1925],{},"若 status=completed 或 failed，停止轮询",[180,1927,1928],{},"若查询接口返回 5xx，则后续重试",[180,1930,1931,1934,1935],{},[458,1932,1933],{},"清理阶段","（reaper）：",[177,1936,1937,1940,1947,1950,1956],{},[180,1938,1939],{},"后台定期扫表",[180,1941,1942,1943,1946],{},"对于 ",[35,1944,1945],{},"status=running && updatedAt > 90s"," 的任务，判定 worker 已死",[180,1948,1949],{},"若 providerTaskId（这里没有）存在，调用上游的 finalize API",[180,1951,1952,1953,1955],{},"若不存在，直接 ",[35,1954,1895],{}," + 置 failed",[180,1957,1958],{},"本设计无 providerTaskId，所以 reaper 自动走退款分支，无需改动 reaper 代码",[10,1960,1961],{},"这个模式的关键是三道防线：提交时判重（409）、后台心跳（定期 touch）、reaper 兜底（自动退款）。任何一个环节出问题，都有后续环节来补救。",[20,1963,1964],{"id":1964},"防线缺一不可",[10,1966,1967],{},"长耗时任务不能挂在 HTTP 连接上不只是超时问题，而是连接脆弱性与重复处理的共谋。异步化表面是\"返回 ID、后台跑\"这一步，真正的难点在三个维度的同时保证：幂等（不重复扣费）、持久化（不丢数据）、失败语义（不破坏重试逻辑）。缺一就会在某个场景暴露。",{"title":33,"searchDepth":48,"depth":48,"links":1969},[1970,1971,1972,1973,1974,1975],{"id":1587,"depth":48,"text":1588},{"id":1625,"depth":48,"text":1626},{"id":1663,"depth":48,"text":1664},{"id":1752,"depth":48,"text":1753},{"id":1809,"depth":48,"text":1809},{"id":1964,"depth":48,"text":1964},"2026-07-11","数字人口播的「分析」功能从视频提取口播文稿、分镜脚本、结构卖点，需要下载视频、压缩、调 Gemini 视觉 API，耗时从几十秒到几分钟。最初同步实现：前端调 POST \u002Fapi\u002Fworkflow\u002Fdub\u002Fanalyze-parsed，后端直接跑完再返回。这个方案暴露的问题不是单点，而是三个独立的缺陷叠加。",{},"\u002F2026-07-11-fa-017",{"title":1535,"description":1977},"FA-017","2026-07-11-FA-017-长任务异步化","长耗时分析请求（几十秒到分钟级）用同步 HTTP 导致网关超时、CDN切断、客户端重试重复扣费。改异步任务需同时解决幂等、持久化、失败语义。",[1985,1986,1987,1988,1989,1990],"异步任务","HTTP超时","CDN","幂等","任务队列","微服务","ARFDvBMGX3tw-E3Rj_gw1RYEjrykplO6u1dxtdS7Uto",{"id":1993,"title":1994,"body":1995,"column":1109,"date":2900,"description":2901,"extension":1111,"hero_image":1112,"meta":2902,"navigation":297,"path":2903,"seo":2904,"series_id":2905,"severity":1112,"stem":2906,"summary":2907,"tags":2908,"__hash__":2914},"posts\u002F2026-06-17-FA-016-WS永久断开三条路径.md","日志里没有重连记录——它根本没在重连",{"type":7,"value":1996,"toc":2888},[1997,2008,2015,2022,2025,2028,2032,2039,2116,2122,2190,2201,2205,2208,2330,2341,2345,2351,2406,2409,2412,2415,2430,2433,2436,2439,2446,2695,2705,2708,2718,2731,2741,2744,2747,2754,2805,2808,2811,2814,2817,2820,2865,2882,2885],[10,1998,1999,2000,2003,2004,2007],{},"从 2026-05-11 开始，多台便携包客户反馈 WebSocket 断线后面板卡死。网关进程（",[35,2001,2002],{},"openclaw_gateway.exe","）明确在运行，",[35,2005,2006],{},"\u002Fhealth"," 返回 200，但面板显示\"已停止重连，请手动刷新\"——用户必须手动重启才能恢复。",[10,2009,2010,2011,2014],{},"关键线索在控制台：看不到任何 ",[35,2012,2013],{},"[ws] 计划重连"," 的日志。",[10,2016,2017,2018,2021],{},"通常断线后客户端会频繁尝试重连，日志里应该是满屏的重连记录。没有日志意味着问题不在重连失败，而在",[458,2019,2020],{},"压根没启动重连机制","。说明客户端进入了某个永久休眠状态。",[20,2023,2024],{"id":2024},"三条独立的永久断开路径",[10,2026,2027],{},"代码审查发现了三条完全不同的、独立的永久断开路径。任意一条命中，就会停止调度任何重连计时器。",[865,2029,2031],{"id":2030},"路径-a凭据刷新异常","路径 A：凭据刷新异常",[10,2033,2034,2035,2038],{},"在 ",[35,2036,2037],{},"_scheduleReconnect"," 方法末尾，每次普通重连都会调用：",[28,2040,2044],{"className":2041,"code":2042,"language":2043,"meta":33,"style":33},"language-javascript shiki shiki-themes github-light github-dark","this._reconnectTimer = setTimeout(() => {\n  if (!this._intentionalClose) {\n    this._refreshCredentialsAndReconnect(0)\n  }\n}, delay)\n","javascript",[35,2045,2046,2070,2086,2106,2111],{"__ignoreMap":33},[38,2047,2048,2051,2054,2058,2061,2064,2067],{"class":40,"line":41},[38,2049,2050],{"class":51},"this",[38,2052,2053],{"class":44},"._reconnectTimer ",[38,2055,2057],{"class":2056},"szBVR","=",[38,2059,2060],{"class":1166}," setTimeout",[38,2062,2063],{"class":44},"(() ",[38,2065,2066],{"class":2056},"=>",[38,2068,2069],{"class":44}," {\n",[38,2071,2072,2075,2078,2081,2083],{"class":40,"line":48},[38,2073,2074],{"class":2056},"  if",[38,2076,2077],{"class":44}," (",[38,2079,2080],{"class":2056},"!",[38,2082,2050],{"class":51},[38,2084,2085],{"class":44},"._intentionalClose) {\n",[38,2087,2088,2091,2094,2097,2100,2103],{"class":40,"line":65},[38,2089,2090],{"class":51},"    this",[38,2092,2093],{"class":44},".",[38,2095,2096],{"class":1166},"_refreshCredentialsAndReconnect",[38,2098,2099],{"class":44},"(",[38,2101,2102],{"class":51},"0",[38,2104,2105],{"class":44},")\n",[38,2107,2108],{"class":40,"line":78},[38,2109,2110],{"class":44},"  }\n",[38,2112,2113],{"class":40,"line":91},[38,2114,2115],{"class":44},"}, delay)\n",[10,2117,2118,2119,2121],{},"而 ",[35,2120,2096],{}," 是异步方法，catch 分支没有任何后续处理：",[28,2123,2125],{"className":2041,"code":2124,"language":2043,"meta":33,"style":33},"} catch (e) {\n  console.error('[ws] 刷新凭据失败:', e)\n  this._setConnected(false, 'error', `凭据刷新失败: ${e}`)\n}\n",[35,2126,2127,2138,2153,2186],{"__ignoreMap":33},[38,2128,2129,2132,2135],{"class":40,"line":41},[38,2130,2131],{"class":44},"} ",[38,2133,2134],{"class":2056},"catch",[38,2136,2137],{"class":44}," (e) {\n",[38,2139,2140,2143,2145,2147,2150],{"class":40,"line":48},[38,2141,2142],{"class":44},"  console.",[38,2144,1714],{"class":1166},[38,2146,2099],{"class":44},[38,2148,2149],{"class":58},"'[ws] 刷新凭据失败:'",[38,2151,2152],{"class":44},", e)\n",[38,2154,2155,2158,2160,2163,2165,2168,2170,2173,2175,2178,2181,2184],{"class":40,"line":65},[38,2156,2157],{"class":51},"  this",[38,2159,2093],{"class":44},[38,2161,2162],{"class":1166},"_setConnected",[38,2164,2099],{"class":44},[38,2166,2167],{"class":51},"false",[38,2169,124],{"class":44},[38,2171,2172],{"class":58},"'error'",[38,2174,124],{"class":44},[38,2176,2177],{"class":58},"`凭据刷新失败: ${",[38,2179,2180],{"class":44},"e",[38,2182,2183],{"class":58},"}`",[38,2185,2105],{"class":44},[38,2187,2188],{"class":40,"line":78},[38,2189,172],{"class":44},[10,2191,2192,2193,2196,2197,2200],{},"如果 ",[35,2194,2195],{},"api.readOpenclawConfig()"," 或 ",[35,2198,2199],{},"api.autoPairDevice()"," 抛错——磁盘 IO 抖、Rust 端处理器忙、Tauri IPC 队列阻塞——这次重连就永久终止。便携磁盘的 IO 抖动 + 配置文件读写争抢时最容易发生。",[865,2202,2204],{"id":2203},"路径-b认证失败后的强制关闭","路径 B：认证失败后的强制关闭",[10,2206,2207],{},"当 WebSocket 收到 1008 unauthorized 响应时的处理逻辑：",[28,2209,2211],{"className":2041,"code":2210,"language":2043,"meta":33,"style":33},"if (this._authRetryCount \u003C 2) {\n  this._authRetryCount++\n  this._refreshCredentialsAndReconnect()\n  return\n}\nthis._setConnected(false, 'auth_failed', `认证失败: ${e.reason}。请检查 Gateway Token 配置。`)\nthis._intentionalClose = true   \u002F\u002F ← 永久关闭标记\nthis._flushPending()\nreturn\n",[35,2212,2213,2234,2244,2255,2260,2264,2298,2314,2325],{"__ignoreMap":33},[38,2214,2215,2218,2220,2222,2225,2228,2231],{"class":40,"line":41},[38,2216,2217],{"class":2056},"if",[38,2219,2077],{"class":44},[38,2221,2050],{"class":51},[38,2223,2224],{"class":44},"._authRetryCount ",[38,2226,2227],{"class":2056},"\u003C",[38,2229,2230],{"class":51}," 2",[38,2232,2233],{"class":44},") {\n",[38,2235,2236,2238,2241],{"class":40,"line":48},[38,2237,2157],{"class":51},[38,2239,2240],{"class":44},"._authRetryCount",[38,2242,2243],{"class":2056},"++\n",[38,2245,2246,2248,2250,2252],{"class":40,"line":65},[38,2247,2157],{"class":51},[38,2249,2093],{"class":44},[38,2251,2096],{"class":1166},[38,2253,2254],{"class":44},"()\n",[38,2256,2257],{"class":40,"line":78},[38,2258,2259],{"class":2056},"  return\n",[38,2261,2262],{"class":40,"line":91},[38,2263,172],{"class":44},[38,2265,2266,2268,2270,2272,2274,2276,2278,2281,2283,2286,2288,2290,2293,2296],{"class":40,"line":100},[38,2267,2050],{"class":51},[38,2269,2093],{"class":44},[38,2271,2162],{"class":1166},[38,2273,2099],{"class":44},[38,2275,2167],{"class":51},[38,2277,124],{"class":44},[38,2279,2280],{"class":58},"'auth_failed'",[38,2282,124],{"class":44},[38,2284,2285],{"class":58},"`认证失败: ${",[38,2287,2180],{"class":44},[38,2289,2093],{"class":58},[38,2291,2292],{"class":44},"reason",[38,2294,2295],{"class":58},"}。请检查 Gateway Token 配置。`",[38,2297,2105],{"class":44},[38,2299,2300,2302,2305,2307,2310],{"class":40,"line":112},[38,2301,2050],{"class":51},[38,2303,2304],{"class":44},"._intentionalClose ",[38,2306,2057],{"class":2056},[38,2308,2309],{"class":51}," true",[38,2311,2313],{"class":2312},"sJ8bj","   \u002F\u002F ← 永久关闭标记\n",[38,2315,2316,2318,2320,2323],{"class":40,"line":133},[38,2317,2050],{"class":51},[38,2319,2093],{"class":44},[38,2321,2322],{"class":1166},"_flushPending",[38,2324,2254],{"class":44},[38,2326,2327],{"class":40,"line":139},[38,2328,2329],{"class":2056},"return\n",[10,2331,2332,2333,2336,2337,2340],{},"设置 ",[35,2334,2335],{},"_intentionalClose = true"," 之后，所有重连逻辑都被 ",[35,2338,2339],{},"if (!this._intentionalClose)"," 短路。即使只是 Gateway 重启窗口碰好出现 token 短暂不匹配，也会一次性把客户端打死，必须刷新页面才能恢复。",[865,2342,2344],{"id":2343},"路径-c快速重连配额耗尽","路径 C：快速重连配额耗尽",[10,2346,2347,2348,1864],{},"定义了常数 ",[35,2349,2350],{},"MAX_RECONNECT_ATTEMPTS = 60",[28,2352,2354],{"className":2041,"code":2353,"language":2043,"meta":33,"style":33},"if (this._reconnectAttempts >= MAX_RECONNECT_ATTEMPTS) {\n  this._setConnected(false, 'error', `连接失败，已停止重连。请手动刷新页面重试。`)\n  return\n}\n",[35,2355,2356,2375,2398,2402],{"__ignoreMap":33},[38,2357,2358,2360,2362,2364,2367,2370,2373],{"class":40,"line":41},[38,2359,2217],{"class":2056},[38,2361,2077],{"class":44},[38,2363,2050],{"class":51},[38,2365,2366],{"class":44},"._reconnectAttempts ",[38,2368,2369],{"class":2056},">=",[38,2371,2372],{"class":51}," MAX_RECONNECT_ATTEMPTS",[38,2374,2233],{"class":44},[38,2376,2377,2379,2381,2383,2385,2387,2389,2391,2393,2396],{"class":40,"line":48},[38,2378,2157],{"class":51},[38,2380,2093],{"class":44},[38,2382,2162],{"class":1166},[38,2384,2099],{"class":44},[38,2386,2167],{"class":51},[38,2388,124],{"class":44},[38,2390,2172],{"class":58},[38,2392,124],{"class":44},[38,2394,2395],{"class":58},"`连接失败，已停止重连。请手动刷新页面重试。`",[38,2397,2105],{"class":44},[38,2399,2400],{"class":40,"line":65},[38,2401,2259],{"class":2056},[38,2403,2404],{"class":40,"line":78},[38,2405,172],{"class":44},[10,2407,2408],{},"客户机晚上挂机，Gateway 因为便携磁盘 GC 或 Windows 休眠争抢资源短暂掉线，客户端尝试 60 次仍未成功重连，就永久停摆。早上用户回来面板已成死链。",[20,2410,2411],{"id":2411},"为什么三条缺陷同时存在",[10,2413,2414],{},"这是逐步累加的历史债：",[177,2416,2417,2424,2427],{},[180,2418,2419,2420,2423],{},"路径 B 是在修复\"避免无限自动配对循环\"时添加的保护，但用错了对象——",[35,2421,2422],{},"_intentionalClose=true"," 本来是用户主动断开的语义，不该用在被动失败上",[180,2425,2426],{},"路径 C 是早期\"避免无穷重试\"的防护，但 60 次后完全放弃而不留任何复活路径是绝对错误",[180,2428,2429],{},"路径 A 是在\"凭据 reload\"改动时把所有重连都改走 refreshCredentials，没留意 catch 分支已经成了终态",[10,2431,2432],{},"三条各自独立、都能单独打死客户端，组合在一起命中率非常高。",[20,2434,2435],{"id":2435},"修复方案",[10,2437,2438],{},"核心思想是永不彻底放弃。任何\"快速重连配额耗尽\"的分支都转入慢轮询，留出窗口让用户改配置或等待 Gateway 恢复后能自动复连。",[10,2440,2441,2442,2445],{},"新增辅助方法 ",[35,2443,2444],{},"_schedulePoll(delayMs, kind)"," 作为慢轮询的触发器：",[28,2447,2449],{"className":2041,"code":2448,"language":2043,"meta":33,"style":33},"const AUTH_RETRY_LIMIT = 2\nconst SLOW_POLL_DELAY_AUTH = 60_000      \u002F\u002F 认证持续失败：1 分钟探一次\nconst SLOW_POLL_DELAY_GENERAL = 300_000  \u002F\u002F 一般持续失败：5 分钟探一次\n\n_schedulePoll(delayMs, kind) {\n  this._clearReconnectTimer()\n  this._reconnectAttempts = 0\n  if (kind === 'auth') this._authRetryCount = 0\n  this._reconnectState = 'scheduled'\n  this._pendingReconnect = true\n  this._reconnectTimer = setTimeout(() => {\n    this._reconnectTimer = null\n    if (this._intentionalClose) return\n    this._reconnectState = 'attempting'\n    if (kind === 'auth') {\n      this._refreshCredentialsAndReconnect(0)\n    } else {\n      this._doConnect()\n    }\n  }, delayMs)\n}\n",[35,2450,2451,2465,2480,2495,2499,2507,2518,2529,2553,2565,2577,2593,2604,2618,2629,2641,2656,2666,2678,2684,2690],{"__ignoreMap":33},[38,2452,2453,2456,2459,2462],{"class":40,"line":41},[38,2454,2455],{"class":2056},"const",[38,2457,2458],{"class":51}," AUTH_RETRY_LIMIT",[38,2460,2461],{"class":2056}," =",[38,2463,2464],{"class":51}," 2\n",[38,2466,2467,2469,2472,2474,2477],{"class":40,"line":48},[38,2468,2455],{"class":2056},[38,2470,2471],{"class":51}," SLOW_POLL_DELAY_AUTH",[38,2473,2461],{"class":2056},[38,2475,2476],{"class":51}," 60_000",[38,2478,2479],{"class":2312},"      \u002F\u002F 认证持续失败：1 分钟探一次\n",[38,2481,2482,2484,2487,2489,2492],{"class":40,"line":65},[38,2483,2455],{"class":2056},[38,2485,2486],{"class":51}," SLOW_POLL_DELAY_GENERAL",[38,2488,2461],{"class":2056},[38,2490,2491],{"class":51}," 300_000",[38,2493,2494],{"class":2312},"  \u002F\u002F 一般持续失败：5 分钟探一次\n",[38,2496,2497],{"class":40,"line":78},[38,2498,298],{"emptyLinePlaceholder":297},[38,2500,2501,2504],{"class":40,"line":91},[38,2502,2503],{"class":1166},"_schedulePoll",[38,2505,2506],{"class":44},"(delayMs, kind) {\n",[38,2508,2509,2511,2513,2516],{"class":40,"line":100},[38,2510,2157],{"class":51},[38,2512,2093],{"class":44},[38,2514,2515],{"class":1166},"_clearReconnectTimer",[38,2517,2254],{"class":44},[38,2519,2520,2522,2524,2526],{"class":40,"line":112},[38,2521,2157],{"class":51},[38,2523,2366],{"class":44},[38,2525,2057],{"class":2056},[38,2527,2528],{"class":51}," 0\n",[38,2530,2531,2533,2536,2539,2542,2545,2547,2549,2551],{"class":40,"line":133},[38,2532,2074],{"class":2056},[38,2534,2535],{"class":44}," (kind ",[38,2537,2538],{"class":2056},"===",[38,2540,2541],{"class":58}," 'auth'",[38,2543,2544],{"class":44},") ",[38,2546,2050],{"class":51},[38,2548,2224],{"class":44},[38,2550,2057],{"class":2056},[38,2552,2528],{"class":51},[38,2554,2555,2557,2560,2562],{"class":40,"line":139},[38,2556,2157],{"class":51},[38,2558,2559],{"class":44},"._reconnectState ",[38,2561,2057],{"class":2056},[38,2563,2564],{"class":58}," 'scheduled'\n",[38,2566,2567,2569,2572,2574],{"class":40,"line":169},[38,2568,2157],{"class":51},[38,2570,2571],{"class":44},"._pendingReconnect ",[38,2573,2057],{"class":2056},[38,2575,2576],{"class":51}," true\n",[38,2578,2579,2581,2583,2585,2587,2589,2591],{"class":40,"line":321},[38,2580,2157],{"class":51},[38,2582,2053],{"class":44},[38,2584,2057],{"class":2056},[38,2586,2060],{"class":1166},[38,2588,2063],{"class":44},[38,2590,2066],{"class":2056},[38,2592,2069],{"class":44},[38,2594,2595,2597,2599,2601],{"class":40,"line":326},[38,2596,2090],{"class":51},[38,2598,2053],{"class":44},[38,2600,2057],{"class":2056},[38,2602,2603],{"class":51}," null\n",[38,2605,2606,2609,2611,2613,2616],{"class":40,"line":331},[38,2607,2608],{"class":2056},"    if",[38,2610,2077],{"class":44},[38,2612,2050],{"class":51},[38,2614,2615],{"class":44},"._intentionalClose) ",[38,2617,2329],{"class":2056},[38,2619,2620,2622,2624,2626],{"class":40,"line":337},[38,2621,2090],{"class":51},[38,2623,2559],{"class":44},[38,2625,2057],{"class":2056},[38,2627,2628],{"class":58}," 'attempting'\n",[38,2630,2631,2633,2635,2637,2639],{"class":40,"line":343},[38,2632,2608],{"class":2056},[38,2634,2535],{"class":44},[38,2636,2538],{"class":2056},[38,2638,2541],{"class":58},[38,2640,2233],{"class":44},[38,2642,2643,2646,2648,2650,2652,2654],{"class":40,"line":349},[38,2644,2645],{"class":51},"      this",[38,2647,2093],{"class":44},[38,2649,2096],{"class":1166},[38,2651,2099],{"class":44},[38,2653,2102],{"class":51},[38,2655,2105],{"class":44},[38,2657,2658,2661,2664],{"class":40,"line":355},[38,2659,2660],{"class":44},"    } ",[38,2662,2663],{"class":2056},"else",[38,2665,2069],{"class":44},[38,2667,2669,2671,2673,2676],{"class":40,"line":2668},18,[38,2670,2645],{"class":51},[38,2672,2093],{"class":44},[38,2674,2675],{"class":1166},"_doConnect",[38,2677,2254],{"class":44},[38,2679,2681],{"class":40,"line":2680},19,[38,2682,2683],{"class":44},"    }\n",[38,2685,2687],{"class":40,"line":2686},20,[38,2688,2689],{"class":44},"  }, delayMs)\n",[38,2691,2693],{"class":40,"line":2692},21,[38,2694,172],{"class":44},[10,2696,2697,2698,2700,2701,2704],{},"慢轮询命中后失败会重新进入 ",[35,2699,2037],{}," 快速重连周期（因为 ",[35,2702,2703],{},"_reconnectAttempts"," 重置为 0），相当于\"快速 60 次 → 慢一次 → 快速 60 次 → 慢一次\"的循环，永不放弃。",[10,2706,2707],{},"三条修复并行：",[10,2709,2710,2713,2714,2717],{},[458,2711,2712],{},"Fix A","：普通重连直接走 ",[35,2715,2716],{},"_doConnect()","，凭据刷新隔离到认证失败分支，避免配置读取错误牵连普通重连。",[10,2719,2720,2723,2724,2727,2728,2730],{},[458,2721,2722],{},"Fix B","：认证失败耗尽后转入 ",[35,2725,2726],{},"_schedulePoll('auth')","，不再设置 ",[35,2729,2422],{},"，给认证恢复留出 60 秒的探测窗口。",[10,2732,2733,2736,2737,2740],{},[458,2734,2735],{},"Fix C","：重连次数超过 MAX_RECONNECT_ATTEMPTS 后转入 ",[35,2738,2739],{},"_schedulePoll('general')","，设置 UI 状态为\"连接持续失败，300 秒后重试\"而不是终态。",[10,2742,2743],{},"慢轮询失败后重新进入快速重连周期，形成\"快速 60 次 → 慢一次 → 快速 60 次\"的循环，永不放弃。",[20,2745,2746],{"id":2746},"防回归验证",[10,2748,2749,2750,2753],{},"新增 ",[35,2751,2752],{},"wakou-full\u002Fsrc\u002Flib\u002Fws-client.slow-poll.test.js","，覆盖 7 个 case：",[177,2755,2756,2764,2776,2779,2788,2795,2800],{},[180,2757,2758,2759,2761,2762],{},"Fix A：普通重连命中 ",[35,2760,2675],{},"，不调用 ",[35,2763,2096],{},[180,2765,2766,2767,2770,2771,2773,2774],{},"Fix C：MAX_RECONNECT_ATTEMPTS 后状态是 ",[35,2768,2769],{},"reconnecting","（非终态 ",[35,2772,1714],{},"），5 分钟后触发 ",[35,2775,2675],{},[180,2777,2778],{},"Fix C：第二轮慢轮询失败后还能再调度快速重连",[180,2780,2781,2782,2784,2785],{},"Fix B：",[35,2783,2726],{}," 等待 60 秒调用 ",[35,2786,2787],{},"_refreshCredentialsAndReconnect(0)",[180,2789,2781,2790,2792,2793],{},[35,2791,2739],{}," 等待后调用 ",[35,2794,2675],{},[180,2796,2797,2799],{},[35,2798,2422],{}," 时慢轮询应跳过",[180,2801,2802,2804],{},[35,2803,2096],{}," 抛错路径调度慢轮询",[10,2806,2807],{},"测试全部通过（7\u002F7）。",[20,2809,2810],{"id":2810},"外部因素的叠加",[10,2812,2813],{},"这个时期同步发生了另一个外部问题：CDN 对 WebSocket 空闲连接会在 4-9 分钟后强制切断。客户端收到连接中断后根据指数退避策略重连，如果恰好命中上述三条路径之一，就进入永久断开状态。这解释了为什么故障特别在长时间挂机后高发——空闲足够长，CDN 必定切断，而后续重连很容易落入某条缺陷路径。两个问题各自独立，但组合效果是\"挂机过夜必死\"。",[20,2815,2816],{"id":2816},"排查验证",[10,2818,2819],{},"故障修复后，可以用以下方式验证重连状态：",[28,2821,2823],{"className":2041,"code":2822,"language":2043,"meta":33,"style":33},"__clawpanelWsClient.getConnectionInfo()\n\u002F\u002F {\n\u002F\u002F   connected: false,\n\u002F\u002F   reconnectState: 'scheduled' | 'attempting',\n\u002F\u002F   reconnectAttempts: 0~60,\n\u002F\u002F   ...\n\u002F\u002F }\n",[35,2824,2825,2835,2840,2845,2850,2855,2860],{"__ignoreMap":33},[38,2826,2827,2830,2833],{"class":40,"line":41},[38,2828,2829],{"class":44},"__clawpanelWsClient.",[38,2831,2832],{"class":1166},"getConnectionInfo",[38,2834,2254],{"class":44},[38,2836,2837],{"class":40,"line":48},[38,2838,2839],{"class":2312},"\u002F\u002F {\n",[38,2841,2842],{"class":40,"line":65},[38,2843,2844],{"class":2312},"\u002F\u002F   connected: false,\n",[38,2846,2847],{"class":40,"line":78},[38,2848,2849],{"class":2312},"\u002F\u002F   reconnectState: 'scheduled' | 'attempting',\n",[38,2851,2852],{"class":40,"line":91},[38,2853,2854],{"class":2312},"\u002F\u002F   reconnectAttempts: 0~60,\n",[38,2856,2857],{"class":40,"line":100},[38,2858,2859],{"class":2312},"\u002F\u002F   ...\n",[38,2861,2862],{"class":40,"line":112},[38,2863,2864],{"class":2312},"\u002F\u002F }\n",[10,2866,2867,2870,2871,2874,2875,2870,2878,2881],{},[35,2868,2869],{},"reconnectState='scheduled'"," 且 ",[35,2872,2873],{},"reconnectAttempts=0"," 表示在慢轮询窗口正常工作；",[35,2876,2877],{},"reconnectState='idle'",[35,2879,2880],{},"connected=false"," 是老的永久断开路径。",[10,2883,2884],{},"重连机制里避免静默终态是最基本的要求。任何异常路径都必须有可见的状态提示和后续操作入口，否则用户端看到的就是死机。这次故障的根本教训是，不能让客户端在任何情况下进入\"不可恢复\"的状态而没有任何提示。慢轮询的引入给了所有失败情景一个\"最后的机会\"，即使前面的快速重连机制彻底耗尽了，用户等待足够长的时间后系统仍有自动恢复的可能。",[1090,2886,2887],{},"html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .szBVR, html code.shiki .szBVR{--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sJ8bj, html code.shiki .sJ8bj{--shiki-default:#6A737D;--shiki-dark:#6A737D}",{"title":33,"searchDepth":48,"depth":48,"links":2889},[2890,2895,2896,2897,2898,2899],{"id":2024,"depth":48,"text":2024,"children":2891},[2892,2893,2894],{"id":2030,"depth":65,"text":2031},{"id":2203,"depth":65,"text":2204},{"id":2343,"depth":65,"text":2344},{"id":2411,"depth":48,"text":2411},{"id":2435,"depth":48,"text":2435},{"id":2746,"depth":48,"text":2746},{"id":2810,"depth":48,"text":2810},{"id":2816,"depth":48,"text":2816},"2026-06-17","从 2026-05-11 开始，多台便携包客户反馈 WebSocket 断线后面板卡死。网关进程（openclaw_gateway.exe）明确在运行，\u002Fhealth 返回 200，但面板显示\"已停止重连，请手动刷新\"——用户必须手动重启才能恢复。",{},"\u002F2026-06-17-fa-016-ws",{"title":1994,"description":2901},"FA-016","2026-06-17-FA-016-WS永久断开三条路径","客户端重连逻辑中三处独立的\"静默终止\"缺陷导致 WS 永久断开，任何一条路径命中就不再调度重连计时器。",[2909,2910,2911,2912,2913],"WebSocket","Node.js","客户端重连机制","异常处理","便携包","qaISKpAwjYnhiSdxEMYYnJtJ-A8QaOq-djOEJ0W9-bM",1785406912385]