[{"data":1,"prerenderedAt":2541},["ShallowReactive",2],{"\u002F2026-06-17-fa-016-ws":3,"\u002F2026-06-17-fa-016-ws-rel":969},{"id":4,"title":5,"body":6,"column":951,"date":952,"description":953,"extension":954,"hero_image":955,"meta":956,"navigation":539,"path":957,"seo":958,"series_id":959,"severity":955,"stem":960,"summary":961,"tags":962,"__hash__":968},"posts\u002F2026-06-17-FA-016-WS永久断开三条路径.md","日志里没有重连记录——它根本没在重连",{"type":7,"value":8,"toc":939},"minimark",[9,22,29,37,41,44,49,56,145,151,223,234,238,241,367,378,382,389,444,447,450,453,470,473,476,479,486,745,755,758,768,781,791,794,797,804,855,858,861,864,867,870,915,932,935],[10,11,12,13,17,18,21],"p",{},"从 2026-05-11 开始，多台便携包客户反馈 WebSocket 断线后面板卡死。网关进程（",[14,15,16],"code",{},"openclaw_gateway.exe","）明确在运行，",[14,19,20],{},"\u002Fhealth"," 返回 200，但面板显示\"已停止重连，请手动刷新\"——用户必须手动重启才能恢复。",[10,23,24,25,28],{},"关键线索在控制台：看不到任何 ",[14,26,27],{},"[ws] 计划重连"," 的日志。",[10,30,31,32,36],{},"通常断线后客户端会频繁尝试重连，日志里应该是满屏的重连记录。没有日志意味着问题不在重连失败，而在",[33,34,35],"strong",{},"压根没启动重连机制","。说明客户端进入了某个永久休眠状态。",[38,39,40],"h2",{"id":40},"三条独立的永久断开路径",[10,42,43],{},"代码审查发现了三条完全不同的、独立的永久断开路径。任意一条命中，就会停止调度任何重连计时器。",[45,46,48],"h3",{"id":47},"路径-a凭据刷新异常","路径 A：凭据刷新异常",[10,50,51,52,55],{},"在 ",[14,53,54],{},"_scheduleReconnect"," 方法末尾，每次普通重连都会调用：",[57,58,63],"pre",{"className":59,"code":60,"language":61,"meta":62,"style":62},"language-javascript shiki shiki-themes github-light github-dark","this._reconnectTimer = setTimeout(() => {\n  if (!this._intentionalClose) {\n    this._refreshCredentialsAndReconnect(0)\n  }\n}, delay)\n","javascript","",[14,64,65,95,112,133,139],{"__ignoreMap":62},[66,67,70,74,78,82,86,89,92],"span",{"class":68,"line":69},"line",1,[66,71,73],{"class":72},"sj4cs","this",[66,75,77],{"class":76},"sVt8B","._reconnectTimer ",[66,79,81],{"class":80},"szBVR","=",[66,83,85],{"class":84},"sScJk"," setTimeout",[66,87,88],{"class":76},"(() ",[66,90,91],{"class":80},"=>",[66,93,94],{"class":76}," {\n",[66,96,98,101,104,107,109],{"class":68,"line":97},2,[66,99,100],{"class":80},"  if",[66,102,103],{"class":76}," (",[66,105,106],{"class":80},"!",[66,108,73],{"class":72},[66,110,111],{"class":76},"._intentionalClose) {\n",[66,113,115,118,121,124,127,130],{"class":68,"line":114},3,[66,116,117],{"class":72},"    this",[66,119,120],{"class":76},".",[66,122,123],{"class":84},"_refreshCredentialsAndReconnect",[66,125,126],{"class":76},"(",[66,128,129],{"class":72},"0",[66,131,132],{"class":76},")\n",[66,134,136],{"class":68,"line":135},4,[66,137,138],{"class":76},"  }\n",[66,140,142],{"class":68,"line":141},5,[66,143,144],{"class":76},"}, delay)\n",[10,146,147,148,150],{},"而 ",[14,149,123],{}," 是异步方法，catch 分支没有任何后续处理：",[57,152,154],{"className":59,"code":153,"language":61,"meta":62,"style":62},"} catch (e) {\n  console.error('[ws] 刷新凭据失败:', e)\n  this._setConnected(false, 'error', `凭据刷新失败: ${e}`)\n}\n",[14,155,156,167,184,218],{"__ignoreMap":62},[66,157,158,161,164],{"class":68,"line":69},[66,159,160],{"class":76},"} ",[66,162,163],{"class":80},"catch",[66,165,166],{"class":76}," (e) {\n",[66,168,169,172,175,177,181],{"class":68,"line":97},[66,170,171],{"class":76},"  console.",[66,173,174],{"class":84},"error",[66,176,126],{"class":76},[66,178,180],{"class":179},"sZZnC","'[ws] 刷新凭据失败:'",[66,182,183],{"class":76},", e)\n",[66,185,186,189,191,194,196,199,202,205,207,210,213,216],{"class":68,"line":114},[66,187,188],{"class":72},"  this",[66,190,120],{"class":76},[66,192,193],{"class":84},"_setConnected",[66,195,126],{"class":76},[66,197,198],{"class":72},"false",[66,200,201],{"class":76},", ",[66,203,204],{"class":179},"'error'",[66,206,201],{"class":76},[66,208,209],{"class":179},"`凭据刷新失败: ${",[66,211,212],{"class":76},"e",[66,214,215],{"class":179},"}`",[66,217,132],{"class":76},[66,219,220],{"class":68,"line":135},[66,221,222],{"class":76},"}\n",[10,224,225,226,229,230,233],{},"如果 ",[14,227,228],{},"api.readOpenclawConfig()"," 或 ",[14,231,232],{},"api.autoPairDevice()"," 抛错——磁盘 IO 抖、Rust 端处理器忙、Tauri IPC 队列阻塞——这次重连就永久终止。便携磁盘的 IO 抖动 + 配置文件读写争抢时最容易发生。",[45,235,237],{"id":236},"路径-b认证失败后的强制关闭","路径 B：认证失败后的强制关闭",[10,239,240],{},"当 WebSocket 收到 1008 unauthorized 响应时的处理逻辑：",[57,242,244],{"className":59,"code":243,"language":61,"meta":62,"style":62},"if (this._authRetryCount \u003C 2) {\n  this._authRetryCount++\n  this._refreshCredentialsAndReconnect()\n  return\n}\nthis._setConnected(false, 'auth_failed', `认证失败: ${e.reason}。请检查 Gateway Token 配置。`)\nthis._intentionalClose = true   \u002F\u002F ← 永久关闭标记\nthis._flushPending()\nreturn\n",[14,245,246,267,277,288,293,297,332,349,361],{"__ignoreMap":62},[66,247,248,251,253,255,258,261,264],{"class":68,"line":69},[66,249,250],{"class":80},"if",[66,252,103],{"class":76},[66,254,73],{"class":72},[66,256,257],{"class":76},"._authRetryCount ",[66,259,260],{"class":80},"\u003C",[66,262,263],{"class":72}," 2",[66,265,266],{"class":76},") {\n",[66,268,269,271,274],{"class":68,"line":97},[66,270,188],{"class":72},[66,272,273],{"class":76},"._authRetryCount",[66,275,276],{"class":80},"++\n",[66,278,279,281,283,285],{"class":68,"line":114},[66,280,188],{"class":72},[66,282,120],{"class":76},[66,284,123],{"class":84},[66,286,287],{"class":76},"()\n",[66,289,290],{"class":68,"line":135},[66,291,292],{"class":80},"  return\n",[66,294,295],{"class":68,"line":141},[66,296,222],{"class":76},[66,298,300,302,304,306,308,310,312,315,317,320,322,324,327,330],{"class":68,"line":299},6,[66,301,73],{"class":72},[66,303,120],{"class":76},[66,305,193],{"class":84},[66,307,126],{"class":76},[66,309,198],{"class":72},[66,311,201],{"class":76},[66,313,314],{"class":179},"'auth_failed'",[66,316,201],{"class":76},[66,318,319],{"class":179},"`认证失败: ${",[66,321,212],{"class":76},[66,323,120],{"class":179},[66,325,326],{"class":76},"reason",[66,328,329],{"class":179},"}。请检查 Gateway Token 配置。`",[66,331,132],{"class":76},[66,333,335,337,340,342,345],{"class":68,"line":334},7,[66,336,73],{"class":72},[66,338,339],{"class":76},"._intentionalClose ",[66,341,81],{"class":80},[66,343,344],{"class":72}," true",[66,346,348],{"class":347},"sJ8bj","   \u002F\u002F ← 永久关闭标记\n",[66,350,352,354,356,359],{"class":68,"line":351},8,[66,353,73],{"class":72},[66,355,120],{"class":76},[66,357,358],{"class":84},"_flushPending",[66,360,287],{"class":76},[66,362,364],{"class":68,"line":363},9,[66,365,366],{"class":80},"return\n",[10,368,369,370,373,374,377],{},"设置 ",[14,371,372],{},"_intentionalClose = true"," 之后，所有重连逻辑都被 ",[14,375,376],{},"if (!this._intentionalClose)"," 短路。即使只是 Gateway 重启窗口碰好出现 token 短暂不匹配，也会一次性把客户端打死，必须刷新页面才能恢复。",[45,379,381],{"id":380},"路径-c快速重连配额耗尽","路径 C：快速重连配额耗尽",[10,383,384,385,388],{},"定义了常数 ",[14,386,387],{},"MAX_RECONNECT_ATTEMPTS = 60","：",[57,390,392],{"className":59,"code":391,"language":61,"meta":62,"style":62},"if (this._reconnectAttempts >= MAX_RECONNECT_ATTEMPTS) {\n  this._setConnected(false, 'error', `连接失败，已停止重连。请手动刷新页面重试。`)\n  return\n}\n",[14,393,394,413,436,440],{"__ignoreMap":62},[66,395,396,398,400,402,405,408,411],{"class":68,"line":69},[66,397,250],{"class":80},[66,399,103],{"class":76},[66,401,73],{"class":72},[66,403,404],{"class":76},"._reconnectAttempts ",[66,406,407],{"class":80},">=",[66,409,410],{"class":72}," MAX_RECONNECT_ATTEMPTS",[66,412,266],{"class":76},[66,414,415,417,419,421,423,425,427,429,431,434],{"class":68,"line":97},[66,416,188],{"class":72},[66,418,120],{"class":76},[66,420,193],{"class":84},[66,422,126],{"class":76},[66,424,198],{"class":72},[66,426,201],{"class":76},[66,428,204],{"class":179},[66,430,201],{"class":76},[66,432,433],{"class":179},"`连接失败，已停止重连。请手动刷新页面重试。`",[66,435,132],{"class":76},[66,437,438],{"class":68,"line":114},[66,439,292],{"class":80},[66,441,442],{"class":68,"line":135},[66,443,222],{"class":76},[10,445,446],{},"客户机晚上挂机，Gateway 因为便携磁盘 GC 或 Windows 休眠争抢资源短暂掉线，客户端尝试 60 次仍未成功重连，就永久停摆。早上用户回来面板已成死链。",[38,448,449],{"id":449},"为什么三条缺陷同时存在",[10,451,452],{},"这是逐步累加的历史债：",[454,455,456,464,467],"ul",{},[457,458,459,460,463],"li",{},"路径 B 是在修复\"避免无限自动配对循环\"时添加的保护，但用错了对象——",[14,461,462],{},"_intentionalClose=true"," 本来是用户主动断开的语义，不该用在被动失败上",[457,465,466],{},"路径 C 是早期\"避免无穷重试\"的防护，但 60 次后完全放弃而不留任何复活路径是绝对错误",[457,468,469],{},"路径 A 是在\"凭据 reload\"改动时把所有重连都改走 refreshCredentials，没留意 catch 分支已经成了终态",[10,471,472],{},"三条各自独立、都能单独打死客户端，组合在一起命中率非常高。",[38,474,475],{"id":475},"修复方案",[10,477,478],{},"核心思想是永不彻底放弃。任何\"快速重连配额耗尽\"的分支都转入慢轮询，留出窗口让用户改配置或等待 Gateway 恢复后能自动复连。",[10,480,481,482,485],{},"新增辅助方法 ",[14,483,484],{},"_schedulePoll(delayMs, kind)"," 作为慢轮询的触发器：",[57,487,489],{"className":59,"code":488,"language":61,"meta":62,"style":62},"const AUTH_RETRY_LIMIT = 2\nconst SLOW_POLL_DELAY_AUTH = 60_000      \u002F\u002F 认证持续失败：1 分钟探一次\nconst SLOW_POLL_DELAY_GENERAL = 300_000  \u002F\u002F 一般持续失败：5 分钟探一次\n\n_schedulePoll(delayMs, kind) {\n  this._clearReconnectTimer()\n  this._reconnectAttempts = 0\n  if (kind === 'auth') this._authRetryCount = 0\n  this._reconnectState = 'scheduled'\n  this._pendingReconnect = true\n  this._reconnectTimer = setTimeout(() => {\n    this._reconnectTimer = null\n    if (this._intentionalClose) return\n    this._reconnectState = 'attempting'\n    if (kind === 'auth') {\n      this._refreshCredentialsAndReconnect(0)\n    } else {\n      this._doConnect()\n    }\n  }, delayMs)\n}\n",[14,490,491,505,520,535,541,549,560,571,595,607,620,637,649,664,676,689,705,716,728,734,740],{"__ignoreMap":62},[66,492,493,496,499,502],{"class":68,"line":69},[66,494,495],{"class":80},"const",[66,497,498],{"class":72}," AUTH_RETRY_LIMIT",[66,500,501],{"class":80}," =",[66,503,504],{"class":72}," 2\n",[66,506,507,509,512,514,517],{"class":68,"line":97},[66,508,495],{"class":80},[66,510,511],{"class":72}," SLOW_POLL_DELAY_AUTH",[66,513,501],{"class":80},[66,515,516],{"class":72}," 60_000",[66,518,519],{"class":347},"      \u002F\u002F 认证持续失败：1 分钟探一次\n",[66,521,522,524,527,529,532],{"class":68,"line":114},[66,523,495],{"class":80},[66,525,526],{"class":72}," SLOW_POLL_DELAY_GENERAL",[66,528,501],{"class":80},[66,530,531],{"class":72}," 300_000",[66,533,534],{"class":347},"  \u002F\u002F 一般持续失败：5 分钟探一次\n",[66,536,537],{"class":68,"line":135},[66,538,540],{"emptyLinePlaceholder":539},true,"\n",[66,542,543,546],{"class":68,"line":141},[66,544,545],{"class":84},"_schedulePoll",[66,547,548],{"class":76},"(delayMs, kind) {\n",[66,550,551,553,555,558],{"class":68,"line":299},[66,552,188],{"class":72},[66,554,120],{"class":76},[66,556,557],{"class":84},"_clearReconnectTimer",[66,559,287],{"class":76},[66,561,562,564,566,568],{"class":68,"line":334},[66,563,188],{"class":72},[66,565,404],{"class":76},[66,567,81],{"class":80},[66,569,570],{"class":72}," 0\n",[66,572,573,575,578,581,584,587,589,591,593],{"class":68,"line":351},[66,574,100],{"class":80},[66,576,577],{"class":76}," (kind ",[66,579,580],{"class":80},"===",[66,582,583],{"class":179}," 'auth'",[66,585,586],{"class":76},") ",[66,588,73],{"class":72},[66,590,257],{"class":76},[66,592,81],{"class":80},[66,594,570],{"class":72},[66,596,597,599,602,604],{"class":68,"line":363},[66,598,188],{"class":72},[66,600,601],{"class":76},"._reconnectState ",[66,603,81],{"class":80},[66,605,606],{"class":179}," 'scheduled'\n",[66,608,610,612,615,617],{"class":68,"line":609},10,[66,611,188],{"class":72},[66,613,614],{"class":76},"._pendingReconnect ",[66,616,81],{"class":80},[66,618,619],{"class":72}," true\n",[66,621,623,625,627,629,631,633,635],{"class":68,"line":622},11,[66,624,188],{"class":72},[66,626,77],{"class":76},[66,628,81],{"class":80},[66,630,85],{"class":84},[66,632,88],{"class":76},[66,634,91],{"class":80},[66,636,94],{"class":76},[66,638,640,642,644,646],{"class":68,"line":639},12,[66,641,117],{"class":72},[66,643,77],{"class":76},[66,645,81],{"class":80},[66,647,648],{"class":72}," null\n",[66,650,652,655,657,659,662],{"class":68,"line":651},13,[66,653,654],{"class":80},"    if",[66,656,103],{"class":76},[66,658,73],{"class":72},[66,660,661],{"class":76},"._intentionalClose) ",[66,663,366],{"class":80},[66,665,667,669,671,673],{"class":68,"line":666},14,[66,668,117],{"class":72},[66,670,601],{"class":76},[66,672,81],{"class":80},[66,674,675],{"class":179}," 'attempting'\n",[66,677,679,681,683,685,687],{"class":68,"line":678},15,[66,680,654],{"class":80},[66,682,577],{"class":76},[66,684,580],{"class":80},[66,686,583],{"class":179},[66,688,266],{"class":76},[66,690,692,695,697,699,701,703],{"class":68,"line":691},16,[66,693,694],{"class":72},"      this",[66,696,120],{"class":76},[66,698,123],{"class":84},[66,700,126],{"class":76},[66,702,129],{"class":72},[66,704,132],{"class":76},[66,706,708,711,714],{"class":68,"line":707},17,[66,709,710],{"class":76},"    } ",[66,712,713],{"class":80},"else",[66,715,94],{"class":76},[66,717,719,721,723,726],{"class":68,"line":718},18,[66,720,694],{"class":72},[66,722,120],{"class":76},[66,724,725],{"class":84},"_doConnect",[66,727,287],{"class":76},[66,729,731],{"class":68,"line":730},19,[66,732,733],{"class":76},"    }\n",[66,735,737],{"class":68,"line":736},20,[66,738,739],{"class":76},"  }, delayMs)\n",[66,741,743],{"class":68,"line":742},21,[66,744,222],{"class":76},[10,746,747,748,750,751,754],{},"慢轮询命中后失败会重新进入 ",[14,749,54],{}," 快速重连周期（因为 ",[14,752,753],{},"_reconnectAttempts"," 重置为 0），相当于\"快速 60 次 → 慢一次 → 快速 60 次 → 慢一次\"的循环，永不放弃。",[10,756,757],{},"三条修复并行：",[10,759,760,763,764,767],{},[33,761,762],{},"Fix A","：普通重连直接走 ",[14,765,766],{},"_doConnect()","，凭据刷新隔离到认证失败分支，避免配置读取错误牵连普通重连。",[10,769,770,773,774,777,778,780],{},[33,771,772],{},"Fix B","：认证失败耗尽后转入 ",[14,775,776],{},"_schedulePoll('auth')","，不再设置 ",[14,779,462],{},"，给认证恢复留出 60 秒的探测窗口。",[10,782,783,786,787,790],{},[33,784,785],{},"Fix C","：重连次数超过 MAX_RECONNECT_ATTEMPTS 后转入 ",[14,788,789],{},"_schedulePoll('general')","，设置 UI 状态为\"连接持续失败，300 秒后重试\"而不是终态。",[10,792,793],{},"慢轮询失败后重新进入快速重连周期，形成\"快速 60 次 → 慢一次 → 快速 60 次\"的循环，永不放弃。",[38,795,796],{"id":796},"防回归验证",[10,798,799,800,803],{},"新增 ",[14,801,802],{},"wakou-full\u002Fsrc\u002Flib\u002Fws-client.slow-poll.test.js","，覆盖 7 个 case：",[454,805,806,814,826,829,838,845,850],{},[457,807,808,809,811,812],{},"Fix A：普通重连命中 ",[14,810,725],{},"，不调用 ",[14,813,123],{},[457,815,816,817,820,821,823,824],{},"Fix C：MAX_RECONNECT_ATTEMPTS 后状态是 ",[14,818,819],{},"reconnecting","（非终态 ",[14,822,174],{},"），5 分钟后触发 ",[14,825,725],{},[457,827,828],{},"Fix C：第二轮慢轮询失败后还能再调度快速重连",[457,830,831,832,834,835],{},"Fix B：",[14,833,776],{}," 等待 60 秒调用 ",[14,836,837],{},"_refreshCredentialsAndReconnect(0)",[457,839,831,840,842,843],{},[14,841,789],{}," 等待后调用 ",[14,844,725],{},[457,846,847,849],{},[14,848,462],{}," 时慢轮询应跳过",[457,851,852,854],{},[14,853,123],{}," 抛错路径调度慢轮询",[10,856,857],{},"测试全部通过（7\u002F7）。",[38,859,860],{"id":860},"外部因素的叠加",[10,862,863],{},"这个时期同步发生了另一个外部问题：CDN 对 WebSocket 空闲连接会在 4-9 分钟后强制切断。客户端收到连接中断后根据指数退避策略重连，如果恰好命中上述三条路径之一，就进入永久断开状态。这解释了为什么故障特别在长时间挂机后高发——空闲足够长，CDN 必定切断，而后续重连很容易落入某条缺陷路径。两个问题各自独立，但组合效果是\"挂机过夜必死\"。",[38,865,866],{"id":866},"排查验证",[10,868,869],{},"故障修复后，可以用以下方式验证重连状态：",[57,871,873],{"className":59,"code":872,"language":61,"meta":62,"style":62},"__clawpanelWsClient.getConnectionInfo()\n\u002F\u002F {\n\u002F\u002F   connected: false,\n\u002F\u002F   reconnectState: 'scheduled' | 'attempting',\n\u002F\u002F   reconnectAttempts: 0~60,\n\u002F\u002F   ...\n\u002F\u002F }\n",[14,874,875,885,890,895,900,905,910],{"__ignoreMap":62},[66,876,877,880,883],{"class":68,"line":69},[66,878,879],{"class":76},"__clawpanelWsClient.",[66,881,882],{"class":84},"getConnectionInfo",[66,884,287],{"class":76},[66,886,887],{"class":68,"line":97},[66,888,889],{"class":347},"\u002F\u002F {\n",[66,891,892],{"class":68,"line":114},[66,893,894],{"class":347},"\u002F\u002F   connected: false,\n",[66,896,897],{"class":68,"line":135},[66,898,899],{"class":347},"\u002F\u002F   reconnectState: 'scheduled' | 'attempting',\n",[66,901,902],{"class":68,"line":141},[66,903,904],{"class":347},"\u002F\u002F   reconnectAttempts: 0~60,\n",[66,906,907],{"class":68,"line":299},[66,908,909],{"class":347},"\u002F\u002F   ...\n",[66,911,912],{"class":68,"line":334},[66,913,914],{"class":347},"\u002F\u002F }\n",[10,916,917,920,921,924,925,920,928,931],{},[14,918,919],{},"reconnectState='scheduled'"," 且 ",[14,922,923],{},"reconnectAttempts=0"," 表示在慢轮询窗口正常工作；",[14,926,927],{},"reconnectState='idle'",[14,929,930],{},"connected=false"," 是老的永久断开路径。",[10,933,934],{},"重连机制里避免静默终态是最基本的要求。任何异常路径都必须有可见的状态提示和后续操作入口，否则用户端看到的就是死机。这次故障的根本教训是，不能让客户端在任何情况下进入\"不可恢复\"的状态而没有任何提示。慢轮询的引入给了所有失败情景一个\"最后的机会\"，即使前面的快速重连机制彻底耗尽了，用户等待足够长的时间后系统仍有自动恢复的可能。",[936,937,938],"style",{},"html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .szBVR, html code.shiki .szBVR{--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sJ8bj, html code.shiki .sJ8bj{--shiki-default:#6A737D;--shiki-dark:#6A737D}",{"title":62,"searchDepth":97,"depth":97,"links":940},[941,946,947,948,949,950],{"id":40,"depth":97,"text":40,"children":942},[943,944,945],{"id":47,"depth":114,"text":48},{"id":236,"depth":114,"text":237},{"id":380,"depth":114,"text":381},{"id":449,"depth":97,"text":449},{"id":475,"depth":97,"text":475},{"id":796,"depth":97,"text":796},{"id":860,"depth":97,"text":860},{"id":866,"depth":97,"text":866},"故障档案","2026-06-17","从 2026-05-11 开始，多台便携包客户反馈 WebSocket 断线后面板卡死。网关进程（openclaw_gateway.exe）明确在运行，\u002Fhealth 返回 200，但面板显示\"已停止重连，请手动刷新\"——用户必须手动重启才能恢复。","md",null,{},"\u002F2026-06-17-fa-016-ws",{"title":5,"description":953},"FA-016","2026-06-17-FA-016-WS永久断开三条路径","客户端重连逻辑中三处独立的\"静默终止\"缺陷导致 WS 永久断开，任何一条路径命中就不再调度重连计时器。",[963,964,965,966,967],"WebSocket","Node.js","客户端重连机制","异常处理","便携包","qaISKpAwjYnhiSdxEMYYnJtJ-A8QaOq-djOEJ0W9-bM",[970,1382,1841],{"id":971,"title":972,"body":973,"column":951,"date":1367,"description":977,"extension":954,"hero_image":955,"meta":1368,"navigation":539,"path":1369,"seo":1370,"series_id":1371,"severity":955,"stem":1372,"summary":1373,"tags":1374,"__hash__":1381},"posts\u002F2026-07-23-FA-018-共享存储的subPath陷阱.md","一个 PVC 装所有用户，当时看不出有什么问题",{"type":7,"value":974,"toc":1358},[975,978,981,985,988,991,998,1042,1045,1055,1059,1065,1068,1079,1082,1086,1089,1092,1095,1113,1116,1123,1126,1141,1145,1148,1151,1162,1173,1176,1179,1182,1250,1253,1260,1263,1266,1292,1295,1306,1309,1312,1352,1355],[10,976,977],{},"平台从 Docker Swarm 迁移到 Kubernetes，存储架构做了一个看似理想的调整：用共享 PVC + subPath 替代原来的\"每用户独占 PVC\"。方案优势显而易见——管理简单（1 个 PVC 对 574 个）、自动扩展、成本低。当时看不出有什么缺点。",[10,979,980],{},"但实施中遇到了四个隐蔽的问题。",[38,982,984],{"id":983},"权限错配bind-型数据的属主陷阱","权限错配：bind 型数据的属主陷阱",[10,986,987],{},"旧系统中，用户数据通过两种方式持久化：bind mount 和 Docker volume。迁移时需要从这两个来源完整导出数据。",[10,989,990],{},"对于 bind mount 型数据，文件属主通常是 10001（或其他固定 UID）。当我登上旧集群的宿主机，以普通用户身份尝试读取这些文件时，只能看到 13 个文件。权限拒绝。同一目录里其实有 4778 个文件，但大多数因为属主权限不匹配而不可见。",[10,992,993,994,997],{},"只有用 sudo 才能完整读到。所以迁移脚本必须用 ",[14,995,996],{},"sudo tar"," 来打包：",[57,999,1003],{"className":1000,"code":1001,"language":1002,"meta":62,"style":62},"language-bash shiki shiki-themes github-light github-dark","sudo tar -czf \u002Ftmp\u002Fmig\u002F${uid}\u002Fdata.tgz \\\n  -C \u002Fmnt\u002Fyun-claw\u002Fusers\u002F${uid} .\n","bash",[14,1004,1005,1028],{"__ignoreMap":62},[66,1006,1007,1010,1013,1016,1019,1022,1025],{"class":68,"line":69},[66,1008,1009],{"class":84},"sudo",[66,1011,1012],{"class":179}," tar",[66,1014,1015],{"class":72}," -czf",[66,1017,1018],{"class":179}," \u002Ftmp\u002Fmig\u002F",[66,1020,1021],{"class":76},"${uid}",[66,1023,1024],{"class":179},"\u002Fdata.tgz",[66,1026,1027],{"class":72}," \\\n",[66,1029,1030,1033,1036,1039],{"class":68,"line":97},[66,1031,1032],{"class":72},"  -C",[66,1034,1035],{"class":179}," \u002Fmnt\u002Fyun-claw\u002Fusers\u002F",[66,1037,1038],{"class":76},"${uid} ",[66,1040,1041],{"class":179},".\n",[10,1043,1044],{},"这不是脚本设计的问题，而是 Kubernetes 环境下容器权限与宿主权限映射的一个陷阱。容器内运行的网关进程可能以容器用户身份运行，但宿主上的文件属主是不同的 UID。当两个权限体系接不上，访问就会失败。",[10,1046,1047,1048,1051,1052,1054],{},"新方案中，用户数据通过 subPath 挂到容器的 ",[14,1049,1050],{},"\u002Fdata"," 目录。如果 subPath 指向的目录是由 Kubernetes 在宿主机上创建的，属主可能是 root 或其他用户，容器内进程（以容器指定的 UID 运行）仍然会遭遇权限问题。这个风险需要在 Dockerfile 和容器启动时明确处理：容器内应该预先创建 ",[14,1053,1050],{}," 并设置正确的属主，或者使用 securityContext 的 fsGroup 或 runAsUser 来强制权限。",[38,1056,1058],{"id":1057},"subpath-目录创建的时机与属主处理","subPath 目录创建的时机与属主处理",[10,1060,1061,1062],{},"Kubernetes 对 subPath 的处理有个微妙之处：如果挂载的 subPath 目录在宿主机上不存在，kubelet 会自动创建它。但 ",[66,1063,1064],{},"待补：这个自动创建的目录的属主是什么？是 root 还是其他用户？容器的 securityContext 能否保证挂载后的权限符合预期？",[10,1066,1067],{},"设计文档中没有明确说明这一点。在实施前，需要验证：",[454,1069,1070,1073,1076],{},[457,1071,1072],{},"首次 subPath 目录不存在时，kubelet 创建它并将其属主设置为谁",[457,1074,1075],{},"容器的 securityContext（fsGroup、runAsUser）是否能在挂载时生效",[457,1077,1078],{},"是否应该预先在共享 PVC 上创建所有 subPath 目录并设置好属主，而不是依赖 Kubernetes 的隐式创建",[10,1080,1081],{},"没有明确的答案，就容易踩坑。一个保险做法是在 destroy 用户账户时清空 subPath 目录，同时在 provision 时让一个 init-container 验证或修复目录属主，确保容器进程有写权限。",[38,1083,1085],{"id":1084},"单-pvc-中的节点级故障域","单 PVC 中的节点级故障域",[10,1087,1088],{},"这是最严重的问题，也是与 FA-015（节点假 Ready）的关联点。",[10,1090,1091],{},"当一个节点的 NFS 客户端或到 SFS 的网络链路发生 hang 时，所有依赖共享 PVC 的 Pod 都会受影响。这不是共享 PVC 本身的问题，而是故障隔离的问题。",[10,1093,1094],{},"具体现象：",[454,1096,1097,1104,1107,1110],{},[457,1098,1099,1100,1103],{},"如果 NFS 挂载使用了硬 mount（",[14,1101,1102],{},"hard"," 参数是 NFS 的默认值），而网络故障或存储服务中断，NFS 客户端会无限重试",[457,1105,1106],{},"重试期间，所有试图访问 NFS 的进程都会被阻塞在 I\u002FO 上（D 状态，不可中断）",[457,1108,1109],{},"如果 kubelet 或 containerd 的某个操作（如创建容器的卷挂载步骤）陷入 I\u002FO 阻塞，整个节点就会冻死",[457,1111,1112],{},"该节点上的所有实例（无论是否在访问存储）都会因为无法创建或删除 Pod 而故障",[10,1114,1115],{},"这与独立 PVC 的效果截然不同。旧架构中，每个用户有独立 PVC，意味着如果某个 PVC 对应的存储有问题，只会影响这一个用户。其他用户的 PVC 可能分散在不同的存储卷甚至不同的节点上，相对独立。",[10,1117,1118,1119,1122],{},"新架构下，单个共享 PVC 承载所有用户，一旦这个 PVC 对应的网络链路或挂载出问题，",[33,1120,1121],{},"同节点上所有实例都会被殃及","。如果该节点碰巧堆积了 50 个实例，一次节点级的存储 hang 就会导致 50 个实例集体故障，且外表看起来是\"节点 Ready，但实例卡住\"——kubelet 的心跳正常，Pod 状态却卡在 ContainerCreating 或 Terminating。",[10,1124,1125],{},"这正是 FA-015 在 2026-06-05 观察到的现象：节点假 Ready，但大量实例网关无响应。防护措施包括：",[454,1127,1128,1135,1138],{},[457,1129,1130,1131,1134],{},"NFS 挂载参数改为软 mount（",[14,1132,1133],{},"soft,timeo=100,retrans=3","），让 I\u002FO 超时而不是无限等待",[457,1136,1137],{},"节点加健康检测，检查 ContainerCreating 堆积数和网关探测失败率，及早发现节点冻死",[457,1139,1140],{},"实例分散部署，用 topologySpreadConstraints 避免单节点堆积",[38,1142,1144],{"id":1143},"无-per-subpath-硬配额","无 per-subPath 硬配额",[10,1146,1147],{},"共享存储的架构决定了无法在存储层面按 subPath 限制用户配额。SFS（及 NFS 一般）没有 per-directory 的硬配额功能，只能在整个卷级别限制。",[10,1149,1150],{},"这意味着：",[454,1152,1153,1156,1159],{},[457,1154,1155],{},"无法阻止某个用户的数据膨胀而挤占其他用户的空间",[457,1157,1158],{},"无法在存储层面实现\"超额用户的写入失败\"",[457,1160,1161],{},"只能在应用层进行监控、告警和逻辑控制",[10,1163,1164,1165,1168,1169,1172],{},"新系统的方案是应用层监控：定期 ",[14,1166,1167],{},"du -sb \u002Fdata"," 扫描每个用户的目录大小，落库到 Instance 表，",[14,1170,1171],{},"\u002Fstatus"," 接口返回超额标志位。前端据此提示用户\"存储已满\"。这是纯监控，不阻断。如果用户无视提示继续写入，直到共享卷真的满了，才会出现\"所有用户集体写入失败\"的惨淡局面。",[10,1174,1175],{},"这个限制是方案选择的代价。",[38,1177,1178],{"id":1178},"为什么仍然选了这个方案",[10,1180,1181],{},"尽管有这些问题，平台仍然采用了共享 PVC + subPath 的方案。原因是对比了替代方案：",[1183,1184,1185,1204],"table",{},[1186,1187,1188],"thead",{},[1189,1190,1191,1195,1198,1201],"tr",{},[1192,1193,1194],"th",{},"方案",[1192,1196,1197],{},"优点",[1192,1199,1200],{},"缺点",[1192,1202,1203],{},"故障隔离",[1205,1206,1207,1222,1236],"tbody",{},[1189,1208,1209,1213,1216,1219],{},[1210,1211,1212],"td",{},"独立 PVC（旧架构）",[1210,1214,1215],{},"天然的每用户隔离；故障域清晰",[1210,1217,1218],{},"管理复杂（574 个 PVC）；扩展性差；成本高",[1210,1220,1221],{},"✓ 最好",[1189,1223,1224,1227,1230,1233],{},[1210,1225,1226],{},"共享 PVC + subPath（新架构）",[1210,1228,1229],{},"管理简单（1 个 PVC）；自动扩展；成本低",[1210,1231,1232],{},"故障隔离性差；无硬配额；权限管理复杂",[1210,1234,1235],{},"✗ 较差",[1189,1237,1238,1241,1244,1247],{},[1210,1239,1240],{},"对象存储（S3\u002FOSS）",[1210,1242,1243],{},"真正的多租户隔离；天然分布",[1210,1245,1246],{},"延迟高；成本更高；应用改造大",[1210,1248,1249],{},"✓ 最好，但代价大",[10,1251,1252],{},"独立 PVC 的管理开销是关键问题。574 个用户意味着 574 个 PVC 对象、574 条 PV 绑定、574 个存储卷。每次实例创建、删除或迁移都要涉及 PVC 的生命周期管理。扩展到 5000 用户时，这种开销会成为瓶颈。对象存储虽然隔离性最好，但需要应用层改造（兼容 S3 API、处理延迟、调整备份策略），而且成本更高。",[10,1254,1255,1256,1259],{},"共享 PVC 方案的核心优势是",[33,1257,1258],{},"运维简洁","：增删用户只需改 subPath（一行 YAML），不涉及存储层操作。代价是故障隔离性从\"用户级\"降到\"节点级\"。",[38,1261,1262],{"id":1262},"适用边界",[10,1264,1265],{},"这个方案适合以下场景：",[454,1267,1268,1274,1280,1286],{},[457,1269,1270,1273],{},[33,1271,1272],{},"用户数量有上限","（几百到几千）。用户过多时，共享卷的单点压力会成问题。",[457,1275,1276,1279],{},[33,1277,1278],{},"用户数据量可控","（单用户通常 GB 级）。如果单用户数据量达 TB，一次 du 扫描会拖累整体。",[457,1281,1282,1285],{},[33,1283,1284],{},"可以接受节点级故障隔离","。只要网络和存储配置足够稳定（硬化 NFS 参数、冗余链路），节点 hang 的概率不会很高。",[457,1287,1288,1291],{},[33,1289,1290],{},"能够实施应用层监控和告警","。无硬配额，就必须有实时监控。",[10,1293,1294],{},"不适合的场景包括：",[454,1296,1297,1300,1303],{},[457,1298,1299],{},"超大规模用户（万级以上）",[457,1301,1302],{},"用户数据特别不均衡（少数用户占大头）的场景",[457,1304,1305],{},"对故障隔离要求极高的系统（比如金融交易）",[38,1307,1308],{"id":1308},"防护与调整",[10,1310,1311],{},"基于这四个问题，实施中做了以下调整：",[1313,1314,1315,1324,1334,1340,1346],"ol",{},[457,1316,1317,1320,1321,1323],{},[33,1318,1319],{},"权限处理","：容器 Dockerfile 中预先创建 ",[14,1322,1050],{}," 并设置正确的属主；启动脚本检查并修复权限。",[457,1325,1326,1329,1330,1333],{},[33,1327,1328],{},"NFS 参数硬化","：StorageClass 的挂载选项改为 ",[14,1331,1332],{},"soft,timeo=100,retrans=3,intr","，避免硬 mount 导致节点冻。",[457,1335,1336,1339],{},[33,1337,1338],{},"节点健康检测","：cron 每 60 秒检查各节点的 ContainerCreating 堆积和网关探测失败率，及早发现故障。",[457,1341,1342,1345],{},[33,1343,1344],{},"实例分散","：StatefulSet 加 topologySpreadConstraints，避免单节点堆积 50+ 个实例。",[457,1347,1348,1351],{},[33,1349,1350],{},"应用层配额","：\u002Fstatus 接口返回 diskUsedMi 和 overQuota 标志位，前端告警用户。",[10,1353,1354],{},"这些措施不能消除风险，但可以大幅降低故障概率和影响范围。",[936,1356,1357],{},"html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":62,"searchDepth":97,"depth":97,"links":1359},[1360,1361,1362,1363,1364,1365,1366],{"id":983,"depth":97,"text":984},{"id":1057,"depth":97,"text":1058},{"id":1084,"depth":97,"text":1085},{"id":1143,"depth":97,"text":1144},{"id":1178,"depth":97,"text":1178},{"id":1262,"depth":97,"text":1262},{"id":1308,"depth":97,"text":1308},"2026-07-23",{},"\u002F2026-07-23-fa-018-subpath",{"title":972,"description":977},"FA-018","2026-07-23-FA-018-共享存储的subPath陷阱","K8s 共享 PVC + subPath 方案在实施中暴露的四个陷阱：权限不匹配、目录创建时机、节点级故障域、无 per-subPath 配额。",[1375,1376,1377,1378,1379,1380,1203],"Kubernetes","存储","subPath","PVC","NFS","权限","4pJJyzbII4OOc9jJS74HUpUfyVc9CfxdE5ovso8ylSE",{"id":1383,"title":1384,"body":1385,"column":951,"date":1825,"description":1826,"extension":954,"hero_image":955,"meta":1827,"navigation":539,"path":1828,"seo":1829,"series_id":1830,"severity":955,"stem":1831,"summary":1832,"tags":1833,"__hash__":1840},"posts\u002F2026-07-11-FA-017-长任务异步化.md","三个超时机制，一起把长任务掐死了",{"type":7,"value":1386,"toc":1817},[1387,1398,1401,1404,1410,1416,1422,1429,1435,1439,1442,1445,1453,1456,1467,1473,1477,1480,1487,1493,1496,1511,1515,1518,1529,1577,1592,1599,1603,1606,1609,1627,1630,1638,1641,1656,1659,1662,1808,1811,1814],[10,1388,1389,1390,1393,1394,1397],{},"数字人口播的「分析」功能从视频提取口播文稿、分镜脚本、结构卖点，需要下载视频、压缩、调 Gemini 视觉 API，耗时从几十秒到几分钟。最初同步实现：前端调 ",[14,1391,1392],{},"POST \u002Fapi\u002Fworkflow\u002Fdub\u002Fanalyze-parsed","，后端直接跑完再返回。这个方案暴露的问题不是单点，而是",[33,1395,1396],{},"三个独立的缺陷叠加","。",[10,1399,1400],{},"网关超时、CDN 空闲切断、客户端重试——这三个没有一个是\"bug\"，每个都是合理的系统设计。但组合在一起就是连锁灾难：客户端看不到结果（超时或连接断），认为失败了重试，后端却已经扣过费了；或者分析其实已跑完，但因为响应发不出去，重试又跑了一遍。退款时无法幂等判断，某些用户最终被多扣了好几倍。",[10,1402,1403],{},"具体来说，三个问题各自是什么：",[10,1405,1406,1409],{},[33,1407,1408],{},"第一个问题是网关超时","。云平台的入口网关（API Gateway）有一个固定的读超时，通常设为几十秒。当后端分析耗时超过这个阈值，网关主动断连接，客户端收到 504 或连接重置，但后端的分析进程并不知道客户端已经走了，继续跑完整个流程。如果分析结果的退款逻辑放在 try-catch 的 finally 里，此时 HTTP 响应已发不出去，客户就看到\"分析失败\"，但款已经扣了。",[10,1411,1412,1415],{},[33,1413,1414],{},"第二个问题是 CDN 空闲切断","。不少 CDN 和负载均衡层有这样的策略：如果一条 HTTP 连接在某个时间段内没有数据往返，就认为它已死而主动关闭。这与 FA-013 那次踩到的问题一样——长连接因为没有心跳数据而被 CDN 中间件切断，导致请求被突然中断。这个机制在分析请求上也会造成麻烦：如果分析用时较长，连接就可能被切，响应发不出去。",[10,1417,1418,1421],{},[33,1419,1420],{},"第三个问题是客户端重试导致重复扣费","。当客户端没有收到响应（网关超时或 CDN 切断），一般会重试这个请求。问题在于这次重试是一条完全独立的请求，后端会把它当作新任务处理，再次扣费。而且如果第一次的分析实际上已经跑完了，就会出现\"分析跑了两遍、扣费扣了两遍、用户只看到了一个结果\"的情况。如果第一次分析中途被 CDN 切了，第二次重试重新开始，那扣费可能是三倍。而退款的时候，因为业务流程上没有幂等设计，很难追溯哪个 operationId 应该被退，哪个不应该。",[10,1423,1424,1425,1428],{},"这三个问题是",[33,1426,1427],{},"叠加的","，不是独立的。它们共同指向一个根本的架构问题。",[10,1430,1431,1432,1397],{},"根本原因一个：",[33,1433,1434],{},"不该让耗时任务挂在 HTTP 连接上",[38,1436,1438],{"id":1437},"异步任务是必需的不是可选的","异步任务是必需的，不是可选的",[10,1440,1441],{},"解决方案很简单：把分析改成异步任务模式。",[10,1443,1444],{},"前端提交分析请求直接返回任务 ID（HTTP 耗时极短：校验参数、建数据库记录、预扣费）。后端后台 worker 跑分析重逻辑，完成后持久化结果。前端轮询查询状态，得到结果后停止。",[57,1446,1451],{"className":1447,"code":1449,"language":1450},[1448],"language-text","客户端              后端\n  |                 |\n  +--提交---------->|\n  |             创建任务，返回 ID\n  |\u003C--返回 ID-----+\n  |                 |（后台运行分析）\n  |                 |\n  +--轮询--------->|\n  |             查询状态\n  |\u003C--返回状态---+\n  |                 |\n  |（重复轮询）\n  +--轮询--------->|\n  |             查询状态\n  |\u003C--返回结果---+\n  |                 |\n","text",[14,1452,1449],{"__ignoreMap":62},[10,1454,1455],{},"好处显而易见：",[454,1457,1458,1461,1464],{},[457,1459,1460],{},"单次 HTTP 请求的耗时从分钟级降到秒级，不会触发网关或 CDN 的超时。",[457,1462,1463],{},"后端分析的运行生命周期与 HTTP 连接完全解耦。即使连接在中途被切，已经启动的分析会继续在后台跑，结果最终还是会被保存。",[457,1465,1466],{},"客户端重试不会产生新的分析任务。只要实现幂等判重，同一个请求在短时间内重复提交也只会产生一个任务。",[10,1468,1469,1470,1397],{},"但异步化不是万灵药，它把问题转移到了三个新的维度上：",[33,1471,1472],{},"幂等、持久化、失败语义",[38,1474,1476],{"id":1475},"幂等同一请求只产生一个任务","幂等：同一请求只产生一个任务",[10,1478,1479],{},"假设客户端因为网络抖动，在短时间内连续发了两次\"开始分析\"请求，后端收到两条独立的 HTTP 请求。如果没有幂等设计，就会创建两个任务、扣两次费。",[10,1481,1482,1483,1486],{},"幂等的实现最直接的办法是在",[33,1484,1485],{},"提交阶段就判重","：为这个用户、这个资源设置一个\"只有一个进行中的分析任务\"的约束。素材中的设计是这样做的：",[57,1488,1491],{"className":1489,"code":1490,"language":1450},[1448],"if 该用户已有 status=running && kind=analyze_parsed 的任务:\n    return 409 Conflict（分析已在进行中）\n",[14,1492,1490],{"__ignoreMap":62},[10,1494,1495],{},"这样双击提交同一个视频，第一次返回 201 + taskId，第二次返回 409，客户端看到 409 就知道不要再建新任务，拿之前返回的 ID 去轮询。",[10,1497,1498,1499,1502,1503,1506,1507,1510],{},"另一个幂等层次是",[33,1500,1501],{},"计费操作的幂等","。预扣费的时候用一个全局唯一的 ",[14,1504,1505],{},"operationId","（比如 ",[14,1508,1509],{},"dub-analyze:{uuid}","），这个 ID 关联了这次预扣。如果扣费 API 被重复调用，因为 operationId 重复，系统只会扣一次。退款时也用同一个 operationId，保证退款与预扣能正确配对。",[38,1512,1514],{"id":1513},"持久化进程重启后任务不丢","持久化：进程重启后任务不丢",[10,1516,1517],{},"后台分析 worker 可能因为发版、机器重启、容器被杀等原因在中途退出。如果任务的状态只保存在内存里，进程一死任务就丢了。",[10,1519,1520,1521,1524,1525,1528],{},"持久化的关键是",[33,1522,1523],{},"任务状态表","。素材中复用了既有的 ",[14,1526,1527],{},"SkyhumanTask"," 表，包含字段：",[454,1530,1531,1537,1543,1548,1554,1560,1565,1571],{},[457,1532,1533,1536],{},[14,1534,1535],{},"id",": 任务 ID",[457,1538,1539,1542],{},[14,1540,1541],{},"status",": 进度（running \u002F completed \u002F failed）",[457,1544,1545,1547],{},[14,1546,1505],{},": 幂等 ID",[457,1549,1550,1553],{},[14,1551,1552],{},"chargedPoints",": 预扣费用",[457,1555,1556,1559],{},[14,1557,1558],{},"resultPayload",": 分析结果的 JSON",[457,1561,1562,1564],{},[14,1563,174],{},": 失败原因",[457,1566,1567,1570],{},[14,1568,1569],{},"updatedAt",": 最后更新时间戳",[457,1572,1573,1576],{},[14,1574,1575],{},"sourceObjectKey",": 源视频对象路径（用于幂等判重和重试）",[10,1578,1579,1580,1583,1584,1587,1588,1591],{},"任务创建时插一条 ",[14,1581,1582],{},"status=running"," 的记录。后台 worker 定期通过 heartbeat（每 30 秒做一次空 update，仅触碰 updatedAt）来证明自己还活着。分析完成时更新为 ",[14,1585,1586],{},"status=completed, resultPayload=...","；失败时更新为 ",[14,1589,1590],{},"status=failed, error=..."," 并触发退款。",[10,1593,1594,1595,1598],{},"这样即使 worker 进程突然死了，任务记录还在数据库里。新启起的 worker 可以扫描表里的 ",[14,1596,1597],{},"status=running && updatedAt 过期"," 的任务，判断它们已经没有 worker 在跑了，执行清理逻辑。",[38,1600,1602],{"id":1601},"失败语义区分任务失败和查询失败","失败语义：区分\"任务失败\"和\"查询失败\"",[10,1604,1605],{},"异步设计引入了一个新的错误维度：客户端查询任务状态时得到的 404，可能是\"这个任务 ID 不存在\"（用户输错了），也可能是\"任务 ID 之前存在但已被清理\"（进程清理了过期数据）。这两种情况的含义很不一样。",[10,1607,1608],{},"更重要的是，前端轮询收到的错误需要区分是否应该重试：",[454,1610,1611,1621],{},[457,1612,1613,1616,1617,1620],{},[33,1614,1615],{},"任务本身失败","（",[14,1618,1619],{},"status=failed, error=\"Gemini API 返回错误\"","）：这种情况不该重试，因为重试也会失败。应该直接向用户展示失败信息，并根据 operationId 触发退款。",[457,1622,1623,1626],{},[33,1624,1625],{},"查询接口失败","（500、网络超时）：这种情况应该重试查询，因为任务本身可能还在继续跑。",[10,1628,1629],{},"素材中的设计把这两类区分得很清楚：",[454,1631,1632,1635],{},[457,1633,1634],{},"任务最终的状态（completed 或 failed）持久化到数据库，客户端查询会拿到明确的业务语义。",[457,1636,1637],{},"查询接口的 HTTP 错误（5xx）是技术层故障，客户端应该重试。",[10,1639,1640],{},"同时，后端的 reaper 机制（定期扫表清理超期任务）也遵循这个语义：",[454,1642,1643,1649],{},[457,1644,1645,1646,1648],{},"如果一个 ",[14,1647,1582],{}," 的任务超过 90 秒没有 heartbeat 更新，说明 worker 已死，reaper 会强制置为 failed 并退款。",[457,1650,1651,1652,1655],{},"但这个强制置为 failed 不应该抛错，因为此时可能有其他进程也想更新这个任务。更新语句应该带条件：",[14,1653,1654],{},"UPDATE SkyhumanTask SET status=failed WHERE id=? AND status=running","，这样如果 reaper 和其他 worker 同时操作，只有一方会成功。",[38,1657,1658],{"id":1658},"具体实现模式",[10,1660,1661],{},"数字人口播的分析异步化采用了这样的模式：",[1313,1663,1664,1709,1755,1778],{},[457,1665,1666,1669,1670],{},[33,1667,1668],{},"提交阶段","（同步）：",[454,1671,1672,1675,1678,1684,1690,1697,1703],{},[457,1673,1674],{},"校验用户和资源",[457,1676,1677],{},"判重：是否已有进行中的任务（409）",[457,1679,1680,1683],{},[14,1681,1682],{},"getObject"," 拉视频，检测时长（若≤0 返 400 不扣费）",[457,1685,1686,1689],{},[14,1687,1688],{},"chargeResource"," 预扣费用，获得 operationId",[457,1691,1692,1693,1696],{},"创建 ",[14,1694,1695],{},"SkyhumanTask{ status:running, operationId, sourceObjectKey }"," 记录",[457,1698,1699,1702],{},[14,1700,1701],{},"scheduleTask"," 丢到后台队列",[457,1704,1705,1706],{},"返回 ",[14,1707,1708],{},"{ taskId }",[457,1710,1711,388,1714],{},[33,1712,1713],{},"后台运行阶段",[454,1715,1716,1719,1722,1729,1735,1745,1752],{},[457,1717,1718],{},"Worker 取出任务记录",[457,1720,1721],{},"开启 heartbeat 定时器（每 30s update 一次 updatedAt）",[457,1723,1724,1725,1728],{},"执行 ",[14,1726,1727],{},"analyzeDubVisionOnly","（下载→压缩→调用 Gemini→解析）",[457,1730,1731,1732],{},"成功：",[14,1733,1734],{},"update( status:completed, resultPayload )",[457,1736,1737,1738,1741,1742],{},"失败：",[14,1739,1740],{},"update( status:failed, error )"," + ",[14,1743,1744],{},"refundResource(operationId)",[457,1746,1747,1748,1751],{},"所有 update 都带条件 ",[14,1749,1750],{},"WHERE id=? AND status=running","（防被 reaper 覆盖）",[457,1753,1754],{},"Finally 清理 heartbeat 定时器",[457,1756,1757,1760,1761],{},[33,1758,1759],{},"查询阶段","（前端轮询）：",[454,1762,1763,1772,1775],{},[457,1764,1765,1768,1769],{},[14,1766,1767],{},"GET \u002Fapi\u002Fworkflow\u002Fdub\u002Ftasks\u002F:id"," 返回 ",[14,1770,1771],{},"{ status, resultPayload, error }",[457,1773,1774],{},"若 status=completed 或 failed，停止轮询",[457,1776,1777],{},"若查询接口返回 5xx，则后续重试",[457,1779,1780,1783,1784],{},[33,1781,1782],{},"清理阶段","（reaper）：",[454,1785,1786,1789,1796,1799,1805],{},[457,1787,1788],{},"后台定期扫表",[457,1790,1791,1792,1795],{},"对于 ",[14,1793,1794],{},"status=running && updatedAt > 90s"," 的任务，判定 worker 已死",[457,1797,1798],{},"若 providerTaskId（这里没有）存在，调用上游的 finalize API",[457,1800,1801,1802,1804],{},"若不存在，直接 ",[14,1803,1744],{}," + 置 failed",[457,1806,1807],{},"本设计无 providerTaskId，所以 reaper 自动走退款分支，无需改动 reaper 代码",[10,1809,1810],{},"这个模式的关键是三道防线：提交时判重（409）、后台心跳（定期 touch）、reaper 兜底（自动退款）。任何一个环节出问题，都有后续环节来补救。",[38,1812,1813],{"id":1813},"防线缺一不可",[10,1815,1816],{},"长耗时任务不能挂在 HTTP 连接上不只是超时问题，而是连接脆弱性与重复处理的共谋。异步化表面是\"返回 ID、后台跑\"这一步，真正的难点在三个维度的同时保证：幂等（不重复扣费）、持久化（不丢数据）、失败语义（不破坏重试逻辑）。缺一就会在某个场景暴露。",{"title":62,"searchDepth":97,"depth":97,"links":1818},[1819,1820,1821,1822,1823,1824],{"id":1437,"depth":97,"text":1438},{"id":1475,"depth":97,"text":1476},{"id":1513,"depth":97,"text":1514},{"id":1601,"depth":97,"text":1602},{"id":1658,"depth":97,"text":1658},{"id":1813,"depth":97,"text":1813},"2026-07-11","数字人口播的「分析」功能从视频提取口播文稿、分镜脚本、结构卖点，需要下载视频、压缩、调 Gemini 视觉 API，耗时从几十秒到几分钟。最初同步实现：前端调 POST \u002Fapi\u002Fworkflow\u002Fdub\u002Fanalyze-parsed，后端直接跑完再返回。这个方案暴露的问题不是单点，而是三个独立的缺陷叠加。",{},"\u002F2026-07-11-fa-017",{"title":1384,"description":1826},"FA-017","2026-07-11-FA-017-长任务异步化","长耗时分析请求（几十秒到分钟级）用同步 HTTP 导致网关超时、CDN切断、客户端重试重复扣费。改异步任务需同时解决幂等、持久化、失败语义。",[1834,1835,1836,1837,1838,1839],"异步任务","HTTP超时","CDN","幂等","任务队列","微服务","ARFDvBMGX3tw-E3Rj_gw1RYEjrykplO6u1dxtdS7Uto",{"id":4,"title":5,"body":1842,"column":951,"date":952,"description":953,"extension":954,"hero_image":955,"meta":2538,"navigation":539,"path":957,"seo":2539,"series_id":959,"severity":955,"stem":960,"summary":961,"tags":2540,"__hash__":968},{"type":7,"value":1843,"toc":2526},[1844,1850,1854,1858,1860,1862,1864,1868,1922,1926,1980,1986,1988,1990,2092,2098,2100,2104,2154,2156,2158,2160,2170,2172,2174,2176,2180,2392,2398,2400,2406,2414,2420,2422,2424,2428,2466,2468,2470,2472,2474,2476,2512,2522,2524],[10,1845,12,1846,17,1848,21],{},[14,1847,16],{},[14,1849,20],{},[10,1851,24,1852,28],{},[14,1853,27],{},[10,1855,31,1856,36],{},[33,1857,35],{},[38,1859,40],{"id":40},[10,1861,43],{},[45,1863,48],{"id":47},[10,1865,51,1866,55],{},[14,1867,54],{},[57,1869,1870],{"className":59,"code":60,"language":61,"meta":62,"style":62},[14,1871,1872,1888,1900,1914,1918],{"__ignoreMap":62},[66,1873,1874,1876,1878,1880,1882,1884,1886],{"class":68,"line":69},[66,1875,73],{"class":72},[66,1877,77],{"class":76},[66,1879,81],{"class":80},[66,1881,85],{"class":84},[66,1883,88],{"class":76},[66,1885,91],{"class":80},[66,1887,94],{"class":76},[66,1889,1890,1892,1894,1896,1898],{"class":68,"line":97},[66,1891,100],{"class":80},[66,1893,103],{"class":76},[66,1895,106],{"class":80},[66,1897,73],{"class":72},[66,1899,111],{"class":76},[66,1901,1902,1904,1906,1908,1910,1912],{"class":68,"line":114},[66,1903,117],{"class":72},[66,1905,120],{"class":76},[66,1907,123],{"class":84},[66,1909,126],{"class":76},[66,1911,129],{"class":72},[66,1913,132],{"class":76},[66,1915,1916],{"class":68,"line":135},[66,1917,138],{"class":76},[66,1919,1920],{"class":68,"line":141},[66,1921,144],{"class":76},[10,1923,147,1924,150],{},[14,1925,123],{},[57,1927,1928],{"className":59,"code":153,"language":61,"meta":62,"style":62},[14,1929,1930,1938,1950,1976],{"__ignoreMap":62},[66,1931,1932,1934,1936],{"class":68,"line":69},[66,1933,160],{"class":76},[66,1935,163],{"class":80},[66,1937,166],{"class":76},[66,1939,1940,1942,1944,1946,1948],{"class":68,"line":97},[66,1941,171],{"class":76},[66,1943,174],{"class":84},[66,1945,126],{"class":76},[66,1947,180],{"class":179},[66,1949,183],{"class":76},[66,1951,1952,1954,1956,1958,1960,1962,1964,1966,1968,1970,1972,1974],{"class":68,"line":114},[66,1953,188],{"class":72},[66,1955,120],{"class":76},[66,1957,193],{"class":84},[66,1959,126],{"class":76},[66,1961,198],{"class":72},[66,1963,201],{"class":76},[66,1965,204],{"class":179},[66,1967,201],{"class":76},[66,1969,209],{"class":179},[66,1971,212],{"class":76},[66,1973,215],{"class":179},[66,1975,132],{"class":76},[66,1977,1978],{"class":68,"line":135},[66,1979,222],{"class":76},[10,1981,225,1982,229,1984,233],{},[14,1983,228],{},[14,1985,232],{},[45,1987,237],{"id":236},[10,1989,240],{},[57,1991,1992],{"className":59,"code":243,"language":61,"meta":62,"style":62},[14,1993,1994,2010,2018,2028,2032,2036,2066,2078,2088],{"__ignoreMap":62},[66,1995,1996,1998,2000,2002,2004,2006,2008],{"class":68,"line":69},[66,1997,250],{"class":80},[66,1999,103],{"class":76},[66,2001,73],{"class":72},[66,2003,257],{"class":76},[66,2005,260],{"class":80},[66,2007,263],{"class":72},[66,2009,266],{"class":76},[66,2011,2012,2014,2016],{"class":68,"line":97},[66,2013,188],{"class":72},[66,2015,273],{"class":76},[66,2017,276],{"class":80},[66,2019,2020,2022,2024,2026],{"class":68,"line":114},[66,2021,188],{"class":72},[66,2023,120],{"class":76},[66,2025,123],{"class":84},[66,2027,287],{"class":76},[66,2029,2030],{"class":68,"line":135},[66,2031,292],{"class":80},[66,2033,2034],{"class":68,"line":141},[66,2035,222],{"class":76},[66,2037,2038,2040,2042,2044,2046,2048,2050,2052,2054,2056,2058,2060,2062,2064],{"class":68,"line":299},[66,2039,73],{"class":72},[66,2041,120],{"class":76},[66,2043,193],{"class":84},[66,2045,126],{"class":76},[66,2047,198],{"class":72},[66,2049,201],{"class":76},[66,2051,314],{"class":179},[66,2053,201],{"class":76},[66,2055,319],{"class":179},[66,2057,212],{"class":76},[66,2059,120],{"class":179},[66,2061,326],{"class":76},[66,2063,329],{"class":179},[66,2065,132],{"class":76},[66,2067,2068,2070,2072,2074,2076],{"class":68,"line":334},[66,2069,73],{"class":72},[66,2071,339],{"class":76},[66,2073,81],{"class":80},[66,2075,344],{"class":72},[66,2077,348],{"class":347},[66,2079,2080,2082,2084,2086],{"class":68,"line":351},[66,2081,73],{"class":72},[66,2083,120],{"class":76},[66,2085,358],{"class":84},[66,2087,287],{"class":76},[66,2089,2090],{"class":68,"line":363},[66,2091,366],{"class":80},[10,2093,369,2094,373,2096,377],{},[14,2095,372],{},[14,2097,376],{},[45,2099,381],{"id":380},[10,2101,384,2102,388],{},[14,2103,387],{},[57,2105,2106],{"className":59,"code":391,"language":61,"meta":62,"style":62},[14,2107,2108,2124,2146,2150],{"__ignoreMap":62},[66,2109,2110,2112,2114,2116,2118,2120,2122],{"class":68,"line":69},[66,2111,250],{"class":80},[66,2113,103],{"class":76},[66,2115,73],{"class":72},[66,2117,404],{"class":76},[66,2119,407],{"class":80},[66,2121,410],{"class":72},[66,2123,266],{"class":76},[66,2125,2126,2128,2130,2132,2134,2136,2138,2140,2142,2144],{"class":68,"line":97},[66,2127,188],{"class":72},[66,2129,120],{"class":76},[66,2131,193],{"class":84},[66,2133,126],{"class":76},[66,2135,198],{"class":72},[66,2137,201],{"class":76},[66,2139,204],{"class":179},[66,2141,201],{"class":76},[66,2143,433],{"class":179},[66,2145,132],{"class":76},[66,2147,2148],{"class":68,"line":114},[66,2149,292],{"class":80},[66,2151,2152],{"class":68,"line":135},[66,2153,222],{"class":76},[10,2155,446],{},[38,2157,449],{"id":449},[10,2159,452],{},[454,2161,2162,2166,2168],{},[457,2163,459,2164,463],{},[14,2165,462],{},[457,2167,466],{},[457,2169,469],{},[10,2171,472],{},[38,2173,475],{"id":475},[10,2175,478],{},[10,2177,481,2178,485],{},[14,2179,484],{},[57,2181,2182],{"className":59,"code":488,"language":61,"meta":62,"style":62},[14,2183,2184,2194,2206,2218,2222,2228,2238,2248,2268,2278,2288,2304,2314,2326,2336,2348,2362,2370,2380,2384,2388],{"__ignoreMap":62},[66,2185,2186,2188,2190,2192],{"class":68,"line":69},[66,2187,495],{"class":80},[66,2189,498],{"class":72},[66,2191,501],{"class":80},[66,2193,504],{"class":72},[66,2195,2196,2198,2200,2202,2204],{"class":68,"line":97},[66,2197,495],{"class":80},[66,2199,511],{"class":72},[66,2201,501],{"class":80},[66,2203,516],{"class":72},[66,2205,519],{"class":347},[66,2207,2208,2210,2212,2214,2216],{"class":68,"line":114},[66,2209,495],{"class":80},[66,2211,526],{"class":72},[66,2213,501],{"class":80},[66,2215,531],{"class":72},[66,2217,534],{"class":347},[66,2219,2220],{"class":68,"line":135},[66,2221,540],{"emptyLinePlaceholder":539},[66,2223,2224,2226],{"class":68,"line":141},[66,2225,545],{"class":84},[66,2227,548],{"class":76},[66,2229,2230,2232,2234,2236],{"class":68,"line":299},[66,2231,188],{"class":72},[66,2233,120],{"class":76},[66,2235,557],{"class":84},[66,2237,287],{"class":76},[66,2239,2240,2242,2244,2246],{"class":68,"line":334},[66,2241,188],{"class":72},[66,2243,404],{"class":76},[66,2245,81],{"class":80},[66,2247,570],{"class":72},[66,2249,2250,2252,2254,2256,2258,2260,2262,2264,2266],{"class":68,"line":351},[66,2251,100],{"class":80},[66,2253,577],{"class":76},[66,2255,580],{"class":80},[66,2257,583],{"class":179},[66,2259,586],{"class":76},[66,2261,73],{"class":72},[66,2263,257],{"class":76},[66,2265,81],{"class":80},[66,2267,570],{"class":72},[66,2269,2270,2272,2274,2276],{"class":68,"line":363},[66,2271,188],{"class":72},[66,2273,601],{"class":76},[66,2275,81],{"class":80},[66,2277,606],{"class":179},[66,2279,2280,2282,2284,2286],{"class":68,"line":609},[66,2281,188],{"class":72},[66,2283,614],{"class":76},[66,2285,81],{"class":80},[66,2287,619],{"class":72},[66,2289,2290,2292,2294,2296,2298,2300,2302],{"class":68,"line":622},[66,2291,188],{"class":72},[66,2293,77],{"class":76},[66,2295,81],{"class":80},[66,2297,85],{"class":84},[66,2299,88],{"class":76},[66,2301,91],{"class":80},[66,2303,94],{"class":76},[66,2305,2306,2308,2310,2312],{"class":68,"line":639},[66,2307,117],{"class":72},[66,2309,77],{"class":76},[66,2311,81],{"class":80},[66,2313,648],{"class":72},[66,2315,2316,2318,2320,2322,2324],{"class":68,"line":651},[66,2317,654],{"class":80},[66,2319,103],{"class":76},[66,2321,73],{"class":72},[66,2323,661],{"class":76},[66,2325,366],{"class":80},[66,2327,2328,2330,2332,2334],{"class":68,"line":666},[66,2329,117],{"class":72},[66,2331,601],{"class":76},[66,2333,81],{"class":80},[66,2335,675],{"class":179},[66,2337,2338,2340,2342,2344,2346],{"class":68,"line":678},[66,2339,654],{"class":80},[66,2341,577],{"class":76},[66,2343,580],{"class":80},[66,2345,583],{"class":179},[66,2347,266],{"class":76},[66,2349,2350,2352,2354,2356,2358,2360],{"class":68,"line":691},[66,2351,694],{"class":72},[66,2353,120],{"class":76},[66,2355,123],{"class":84},[66,2357,126],{"class":76},[66,2359,129],{"class":72},[66,2361,132],{"class":76},[66,2363,2364,2366,2368],{"class":68,"line":707},[66,2365,710],{"class":76},[66,2367,713],{"class":80},[66,2369,94],{"class":76},[66,2371,2372,2374,2376,2378],{"class":68,"line":718},[66,2373,694],{"class":72},[66,2375,120],{"class":76},[66,2377,725],{"class":84},[66,2379,287],{"class":76},[66,2381,2382],{"class":68,"line":730},[66,2383,733],{"class":76},[66,2385,2386],{"class":68,"line":736},[66,2387,739],{"class":76},[66,2389,2390],{"class":68,"line":742},[66,2391,222],{"class":76},[10,2393,747,2394,750,2396,754],{},[14,2395,54],{},[14,2397,753],{},[10,2399,757],{},[10,2401,2402,763,2404,767],{},[33,2403,762],{},[14,2405,766],{},[10,2407,2408,773,2410,777,2412,780],{},[33,2409,772],{},[14,2411,776],{},[14,2413,462],{},[10,2415,2416,786,2418,790],{},[33,2417,785],{},[14,2419,789],{},[10,2421,793],{},[38,2423,796],{"id":796},[10,2425,799,2426,803],{},[14,2427,802],{},[454,2429,2430,2436,2444,2446,2452,2458,2462],{},[457,2431,808,2432,811,2434],{},[14,2433,725],{},[14,2435,123],{},[457,2437,816,2438,820,2440,823,2442],{},[14,2439,819],{},[14,2441,174],{},[14,2443,725],{},[457,2445,828],{},[457,2447,831,2448,834,2450],{},[14,2449,776],{},[14,2451,837],{},[457,2453,831,2454,842,2456],{},[14,2455,789],{},[14,2457,725],{},[457,2459,2460,849],{},[14,2461,462],{},[457,2463,2464,854],{},[14,2465,123],{},[10,2467,857],{},[38,2469,860],{"id":860},[10,2471,863],{},[38,2473,866],{"id":866},[10,2475,869],{},[57,2477,2478],{"className":59,"code":872,"language":61,"meta":62,"style":62},[14,2479,2480,2488,2492,2496,2500,2504,2508],{"__ignoreMap":62},[66,2481,2482,2484,2486],{"class":68,"line":69},[66,2483,879],{"class":76},[66,2485,882],{"class":84},[66,2487,287],{"class":76},[66,2489,2490],{"class":68,"line":97},[66,2491,889],{"class":347},[66,2493,2494],{"class":68,"line":114},[66,2495,894],{"class":347},[66,2497,2498],{"class":68,"line":135},[66,2499,899],{"class":347},[66,2501,2502],{"class":68,"line":141},[66,2503,904],{"class":347},[66,2505,2506],{"class":68,"line":299},[66,2507,909],{"class":347},[66,2509,2510],{"class":68,"line":334},[66,2511,914],{"class":347},[10,2513,2514,920,2516,924,2518,920,2520,931],{},[14,2515,919],{},[14,2517,923],{},[14,2519,927],{},[14,2521,930],{},[10,2523,934],{},[936,2525,938],{},{"title":62,"searchDepth":97,"depth":97,"links":2527},[2528,2533,2534,2535,2536,2537],{"id":40,"depth":97,"text":40,"children":2529},[2530,2531,2532],{"id":47,"depth":114,"text":48},{"id":236,"depth":114,"text":237},{"id":380,"depth":114,"text":381},{"id":449,"depth":97,"text":449},{"id":475,"depth":97,"text":475},{"id":796,"depth":97,"text":796},{"id":860,"depth":97,"text":860},{"id":866,"depth":97,"text":866},{},{"title":5,"description":953},[963,964,965,966,967],1785406912227]