[{"data":1,"prerenderedAt":2176},["ShallowReactive",2],{"\u002F2026-05-11-fa-008-stale-minisig":3,"\u002F2026-05-11-fa-008-stale-minisig-rel":369},{"id":4,"title":5,"body":6,"column":351,"date":352,"description":353,"extension":354,"hero_image":355,"meta":356,"navigation":276,"path":357,"seo":358,"series_id":359,"severity":355,"stem":360,"summary":361,"tags":362,"__hash__":368},"posts\u002F2026-05-11-FA-008-stale-minisig验签失败.md","签名文件比这次更新，早了三小时",{"type":7,"value":8,"toc":344},"minimark",[9,18,21,72,77,86,97,174,177,180,191,194,212,225,228,231,250,257,307,318,321,324,334,337,340],[10,11,12,13,17],"p",{},"状态目录里有个文件时间戳不对劲。",[14,15,16],"code",{},"download.zip.minisig"," 的修改时间是 20:19:35，而本次 OTA 启动于 23:25，相差三小时。验签应该是在 OTA 最后一步才做的事，为什么这个文件早了三小时？",[10,19,20],{},"这是 Windows build host 上 1.0.1 升级到 1.0.2 的 OTA 流程。实测现场的完整时序如下：",[22,23,24,32,47,50,69],"ul",{},[25,26,27,28,31],"li",{},"下载阶段正常完成：",[14,29,30],{},"download.zip"," 体积 24.4 MB，hash 校验通过",[25,33,34,35,38,39,42,43,46],{},"状态文件卡死在 ",[14,36,37],{},"state.phase = \"verifying\"","，且 ",[14,40,41],{},"state.json"," 记录 ",[14,44,45],{},"download.bytes_downloaded = 0","（worker 进入 verify 后被杀，没有机会更新这个字段）",[25,48,49],{},"后台 worker 进程已退出，没有活进程推进 OTA 链路",[25,51,52,53,56,57,59,60,64,65,68],{},"状态目录（",[14,54,55],{},"Update\\\u003Cserial-hash>\\","）里存放签名文件 ",[14,58,16],{},"，修改时间戳是 ",[61,62,63],"strong",{},"20:19:35","，而本次 OTA 启动于 ",[61,66,67],{},"23:25","，相差三小时",[25,70,71],{},"launcher 和面板端陷入重试循环：launcher 看到有效状态标记，调用 chain 重新运行 → worker verify 失败 → worker 进程被杀 → launcher 重启 → 再次重新运行同一条 chain",[73,74,76],"h2",{"id":75},"根因旧签名文件的复用","根因：旧签名文件的复用",[10,78,79,80,82,83,85],{},"状态目录里的 ",[14,81,16],{}," 来自上一轮失败的 OTA。某个时刻的升级（比如从 1.0.0）在 verify 之前或之中中断，签名文件就留在了磁盘上。当前 OTA（1.0.2）下完包后进入验签，代码发现磁盘上已存在 ",[14,84,16],{},"，就直接读取，而不下载新版本对应的签名。",[10,87,88,89,92,93,96],{},"根本原因在 ",[14,90,91],{},"ota_worker.rs"," 和 ",[14,94,95],{},"ota_main_chain.rs"," 两处的同一个逻辑块：",[98,99,104],"pre",{"className":100,"code":101,"language":102,"meta":103,"style":103},"language-rust shiki shiki-themes github-light github-dark","\u002F\u002F ota_worker.rs:240\nlet minisig_path = state_base.join(\"download.zip.minisig\");\nlet sig_text = if minisig_path.exists() {\n    std::fs::read_to_string(&minisig_path)?  \u002F\u002F 文件存在 → 直接读磁盘\n} else {\n    let sig_url = &manifest.download.signature.url;\n    let text = reqwest::blocking::get(sig_url)?.text()?;\n    std::fs::write(&minisig_path, &text)?;\n    text\n};\nverify_zip::verify_zip(&zip_path, &sig_text, ...)?;\n","rust","",[14,105,106,114,120,126,132,138,144,150,156,162,168],{"__ignoreMap":103},[107,108,111],"span",{"class":109,"line":110},"line",1,[107,112,113],{},"\u002F\u002F ota_worker.rs:240\n",[107,115,117],{"class":109,"line":116},2,[107,118,119],{},"let minisig_path = state_base.join(\"download.zip.minisig\");\n",[107,121,123],{"class":109,"line":122},3,[107,124,125],{},"let sig_text = if minisig_path.exists() {\n",[107,127,129],{"class":109,"line":128},4,[107,130,131],{},"    std::fs::read_to_string(&minisig_path)?  \u002F\u002F 文件存在 → 直接读磁盘\n",[107,133,135],{"class":109,"line":134},5,[107,136,137],{},"} else {\n",[107,139,141],{"class":109,"line":140},6,[107,142,143],{},"    let sig_url = &manifest.download.signature.url;\n",[107,145,147],{"class":109,"line":146},7,[107,148,149],{},"    let text = reqwest::blocking::get(sig_url)?.text()?;\n",[107,151,153],{"class":109,"line":152},8,[107,154,155],{},"    std::fs::write(&minisig_path, &text)?;\n",[107,157,159],{"class":109,"line":158},9,[107,160,161],{},"    text\n",[107,163,165],{"class":109,"line":164},10,[107,166,167],{},"};\n",[107,169,171],{"class":109,"line":170},11,[107,172,173],{},"verify_zip::verify_zip(&zip_path, &sig_text, ...)?;\n",[10,175,176],{},"代码本意是支持 resume（同一轮 OTA 中途退出后不重复下载），但\"存在就复用\"太脆弱。",[10,178,179],{},"触发链路简述：前一轮 OTA（1.0.0）中断 → minisig 残留 → 当前 OTA（1.0.2）看到文件存在 → 读取老签名 → 用 1.0.0 签名验 1.0.2 zip → SHA256 不匹配 → verify 失败 → worker 进程异常退出 → launcher 误认为可恢复 → 触发 RunChain → 同一条路径再次失败 → 死循环。",[10,181,182,183,186,187,190],{},"这次故障与之前的 FA-005~FA-007 形成递进关系。前面三篇处理的是 OTA ",[61,184,185],{},"流程级残留","（marker 文件格式、state.phase 卡死、worker 僵尸进程）；这一篇处理 ",[61,188,189],{},"artifact 级残留","——中间产物的生命周期跨越了多轮 OTA。",[73,192,193],{"id":193},"排查过程",[10,195,196,197,200,201,203,204,207,208,211],{},"发现 stale minisig 的关键线索是",[61,198,199],{},"文件时间戳对比","。当 worker 反复失败时，查看 ",[14,202,41],{}," 的 phase 和 ",[14,205,206],{},"ota-events.jsonl"," 的最后一条事件：phase 卡在 ",[14,209,210],{},"verifying","，但 events 里没有 verify 失败的日志，说明 worker 在 verify 时崩溃了。",[10,213,214,215,217,218,220,221,224],{},"接下来检查状态目录：",[14,216,30],{}," 的时间戳是当前 OTA 启动后的（体积、hash 都对），但 ",[14,219,16],{}," 的时间戳远早三小时。minisig 在 verify 前才下载，时间相隔三小时就是上一轮 OTA 残留的文件。用磁盘上的旧 minisig 对当前 zip 进行 ",[14,222,223],{},"minisign -Vm"," 验证会看到 SHA256 mismatch，印证了根因。",[73,226,227],{"id":227},"修复方案",[10,229,230],{},"修复分为两处，逻辑完全相同：",[232,233,234,242],"ol",{},[25,235,236],{},[61,237,238,241],{},[14,239,240],{},"ota_worker.rs::phase1_download_and_verify"," 第 240 行",[25,243,244],{},[61,245,246,249],{},[14,247,248],{},"ota_main_chain.rs::run_full_apply_chain"," Stage B verify 段第 302 行",[10,251,252,253,256],{},"改动是",[61,254,255],{},"移除文件存在性判断，永远 fresh fetch","：",[98,258,260],{"className":100,"code":259,"language":102,"meta":103,"style":103},"\u002F\u002F 改前：存在就读，不存在才下载\n\u002F\u002F let sig_text = if minisig_path.exists() { ... } else { ... };\n\n\u002F\u002F 改后：永远删除后重新下载\nlet minisig_path = state_base.join(\"download.zip.minisig\");\nlet _ = std::fs::remove_file(&minisig_path);  \u002F\u002F 强制删除旧文件\nlet sig_url = &manifest.download.signature.url;\nlet sig_text = reqwest::blocking::get(sig_url)?.text()?;\nstd::fs::write(&minisig_path, &sig_text)?;\n",[14,261,262,267,272,278,283,287,292,297,302],{"__ignoreMap":103},[107,263,264],{"class":109,"line":110},[107,265,266],{},"\u002F\u002F 改前：存在就读，不存在才下载\n",[107,268,269],{"class":109,"line":116},[107,270,271],{},"\u002F\u002F let sig_text = if minisig_path.exists() { ... } else { ... };\n",[107,273,274],{"class":109,"line":122},[107,275,277],{"emptyLinePlaceholder":276},true,"\n",[107,279,280],{"class":109,"line":128},[107,281,282],{},"\u002F\u002F 改后：永远删除后重新下载\n",[107,284,285],{"class":109,"line":134},[107,286,119],{},[107,288,289],{"class":109,"line":140},[107,290,291],{},"let _ = std::fs::remove_file(&minisig_path);  \u002F\u002F 强制删除旧文件\n",[107,293,294],{"class":109,"line":146},[107,295,296],{},"let sig_url = &manifest.download.signature.url;\n",[107,298,299],{"class":109,"line":152},[107,300,301],{},"let sig_text = reqwest::blocking::get(sig_url)?.text()?;\n",[107,303,304],{"class":109,"line":158},[107,305,306],{},"std::fs::write(&minisig_path, &sig_text)?;\n",[10,308,309,310,313,314,317],{},"代价极小：minisig 约 400 字节，网络耗时几十毫秒，在整个 OTA 流程中可忽略。这个改动体现了核心原则——",[61,311,312],{},"中间文件的生命周期必须绑定到任务\u002F版本，而不是绑定到文件是否存在","。未来可以让文件名包含版本号（如 ",[14,315,316],{},"download-1.0.2.zip.minisig","），但这次为了最小变动只删除了文件存在性缓存。",[73,319,320],{"id":320},"验证",[10,322,323],{},"实测在状态目录放置老版本 minisig，触发 1.0.2 OTA，worker 自动覆盖后 verify 通过，整条 chain 完成。修复后实测 1.0.2 OTA 全链路 22 秒完成（download 8s + verify 5s + ready→applied 2s + USB sync 5s + done），不再出现 retry loop。",[10,325,326,329,330,333],{},[61,327,328],{},"防回归","：这是设计问题而非代码 bug。核心原则是",[61,331,332],{},"任何中间文件的有效性不能单靠存在判断，必须包含版本号或 hash 标识","。同理，chain 入口读取 minisig 和 download.zip 时都应该先校验与 manifest 的一致性，不只依赖后续的 verify_zip。",[73,335,336],{"id":336},"系列位置",[10,338,339],{},"这是 OTA 死循环系列（FA-005~FA-008）的最后一环。前三篇处理流程级残留（marker、state.phase、worker 进程）；这一篇处理 artifact 级残留（中间文件生命周期）。症状相同（verify 失败 → 重试循环），但根因逐层深入。每一层修复都必要——只解决流程层而不解决 artifact 层，下次跨版本 OTA 中断还会踩坑。",[341,342,343],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":103,"searchDepth":116,"depth":116,"links":345},[346,347,348,349,350],{"id":75,"depth":116,"text":76},{"id":193,"depth":116,"text":193},{"id":227,"depth":116,"text":227},{"id":320,"depth":116,"text":320},{"id":336,"depth":116,"text":336},"故障档案","2026-05-11","状态目录里有个文件时间戳不对劲。download.zip.minisig 的修改时间是 20:19:35，而本次 OTA 启动于 23:25，相差三小时。验签应该是在 OTA 最后一步才做的事，为什么这个文件早了三小时？","md",null,{},"\u002F2026-05-11-fa-008-stale-minisig",{"title":5,"description":353},"FA-008","2026-05-11-FA-008-stale-minisig验签失败","跨版本 OTA 时复用了上一轮失败留下的旧签名文件，导致所有后续升级验签必然失败。问题根源在文件存在性判断替代了版本校验。",[363,364,365,366,367],"OTA","文件缓存","Windows","签名验证","状态管理","taq7T9Q_otx3dgdBoDl8gs7c3YFVd-tuHTjQdKJp5Mk",[370,785,1245],{"id":371,"title":372,"body":373,"column":351,"date":770,"description":377,"extension":354,"hero_image":355,"meta":771,"navigation":276,"path":772,"seo":773,"series_id":774,"severity":355,"stem":775,"summary":776,"tags":777,"__hash__":784},"posts\u002F2026-07-23-FA-018-共享存储的subPath陷阱.md","一个 PVC 装所有用户，当时看不出有什么问题",{"type":7,"value":374,"toc":761},[375,378,381,385,388,391,398,446,449,459,463,469,472,483,486,490,493,496,499,517,520,527,530,545,549,552,555,566,577,580,583,586,654,657,664,667,670,696,699,710,713,716,755,758],[10,376,377],{},"平台从 Docker Swarm 迁移到 Kubernetes，存储架构做了一个看似理想的调整：用共享 PVC + subPath 替代原来的\"每用户独占 PVC\"。方案优势显而易见——管理简单（1 个 PVC 对 574 个）、自动扩展、成本低。当时看不出有什么缺点。",[10,379,380],{},"但实施中遇到了四个隐蔽的问题。",[73,382,384],{"id":383},"权限错配bind-型数据的属主陷阱","权限错配：bind 型数据的属主陷阱",[10,386,387],{},"旧系统中，用户数据通过两种方式持久化：bind mount 和 Docker volume。迁移时需要从这两个来源完整导出数据。",[10,389,390],{},"对于 bind mount 型数据，文件属主通常是 10001（或其他固定 UID）。当我登上旧集群的宿主机，以普通用户身份尝试读取这些文件时，只能看到 13 个文件。权限拒绝。同一目录里其实有 4778 个文件，但大多数因为属主权限不匹配而不可见。",[10,392,393,394,397],{},"只有用 sudo 才能完整读到。所以迁移脚本必须用 ",[14,395,396],{},"sudo tar"," 来打包：",[98,399,403],{"className":400,"code":401,"language":402,"meta":103,"style":103},"language-bash shiki shiki-themes github-light github-dark","sudo tar -czf \u002Ftmp\u002Fmig\u002F${uid}\u002Fdata.tgz \\\n  -C \u002Fmnt\u002Fyun-claw\u002Fusers\u002F${uid} .\n","bash",[14,404,405,432],{"__ignoreMap":103},[107,406,407,411,415,419,422,426,429],{"class":109,"line":110},[107,408,410],{"class":409},"sScJk","sudo",[107,412,414],{"class":413},"sZZnC"," tar",[107,416,418],{"class":417},"sj4cs"," -czf",[107,420,421],{"class":413}," \u002Ftmp\u002Fmig\u002F",[107,423,425],{"class":424},"sVt8B","${uid}",[107,427,428],{"class":413},"\u002Fdata.tgz",[107,430,431],{"class":417}," \\\n",[107,433,434,437,440,443],{"class":109,"line":116},[107,435,436],{"class":417},"  -C",[107,438,439],{"class":413}," \u002Fmnt\u002Fyun-claw\u002Fusers\u002F",[107,441,442],{"class":424},"${uid} ",[107,444,445],{"class":413},".\n",[10,447,448],{},"这不是脚本设计的问题，而是 Kubernetes 环境下容器权限与宿主权限映射的一个陷阱。容器内运行的网关进程可能以容器用户身份运行，但宿主上的文件属主是不同的 UID。当两个权限体系接不上，访问就会失败。",[10,450,451,452,455,456,458],{},"新方案中，用户数据通过 subPath 挂到容器的 ",[14,453,454],{},"\u002Fdata"," 目录。如果 subPath 指向的目录是由 Kubernetes 在宿主机上创建的，属主可能是 root 或其他用户，容器内进程（以容器指定的 UID 运行）仍然会遭遇权限问题。这个风险需要在 Dockerfile 和容器启动时明确处理：容器内应该预先创建 ",[14,457,454],{}," 并设置正确的属主，或者使用 securityContext 的 fsGroup 或 runAsUser 来强制权限。",[73,460,462],{"id":461},"subpath-目录创建的时机与属主处理","subPath 目录创建的时机与属主处理",[10,464,465,466],{},"Kubernetes 对 subPath 的处理有个微妙之处：如果挂载的 subPath 目录在宿主机上不存在，kubelet 会自动创建它。但 ",[107,467,468],{},"待补：这个自动创建的目录的属主是什么？是 root 还是其他用户？容器的 securityContext 能否保证挂载后的权限符合预期？",[10,470,471],{},"设计文档中没有明确说明这一点。在实施前，需要验证：",[22,473,474,477,480],{},[25,475,476],{},"首次 subPath 目录不存在时，kubelet 创建它并将其属主设置为谁",[25,478,479],{},"容器的 securityContext（fsGroup、runAsUser）是否能在挂载时生效",[25,481,482],{},"是否应该预先在共享 PVC 上创建所有 subPath 目录并设置好属主，而不是依赖 Kubernetes 的隐式创建",[10,484,485],{},"没有明确的答案，就容易踩坑。一个保险做法是在 destroy 用户账户时清空 subPath 目录，同时在 provision 时让一个 init-container 验证或修复目录属主，确保容器进程有写权限。",[73,487,489],{"id":488},"单-pvc-中的节点级故障域","单 PVC 中的节点级故障域",[10,491,492],{},"这是最严重的问题，也是与 FA-015（节点假 Ready）的关联点。",[10,494,495],{},"当一个节点的 NFS 客户端或到 SFS 的网络链路发生 hang 时，所有依赖共享 PVC 的 Pod 都会受影响。这不是共享 PVC 本身的问题，而是故障隔离的问题。",[10,497,498],{},"具体现象：",[22,500,501,508,511,514],{},[25,502,503,504,507],{},"如果 NFS 挂载使用了硬 mount（",[14,505,506],{},"hard"," 参数是 NFS 的默认值），而网络故障或存储服务中断，NFS 客户端会无限重试",[25,509,510],{},"重试期间，所有试图访问 NFS 的进程都会被阻塞在 I\u002FO 上（D 状态，不可中断）",[25,512,513],{},"如果 kubelet 或 containerd 的某个操作（如创建容器的卷挂载步骤）陷入 I\u002FO 阻塞，整个节点就会冻死",[25,515,516],{},"该节点上的所有实例（无论是否在访问存储）都会因为无法创建或删除 Pod 而故障",[10,518,519],{},"这与独立 PVC 的效果截然不同。旧架构中，每个用户有独立 PVC，意味着如果某个 PVC 对应的存储有问题，只会影响这一个用户。其他用户的 PVC 可能分散在不同的存储卷甚至不同的节点上，相对独立。",[10,521,522,523,526],{},"新架构下，单个共享 PVC 承载所有用户，一旦这个 PVC 对应的网络链路或挂载出问题，",[61,524,525],{},"同节点上所有实例都会被殃及","。如果该节点碰巧堆积了 50 个实例，一次节点级的存储 hang 就会导致 50 个实例集体故障，且外表看起来是\"节点 Ready，但实例卡住\"——kubelet 的心跳正常，Pod 状态却卡在 ContainerCreating 或 Terminating。",[10,528,529],{},"这正是 FA-015 在 2026-06-05 观察到的现象：节点假 Ready，但大量实例网关无响应。防护措施包括：",[22,531,532,539,542],{},[25,533,534,535,538],{},"NFS 挂载参数改为软 mount（",[14,536,537],{},"soft,timeo=100,retrans=3","），让 I\u002FO 超时而不是无限等待",[25,540,541],{},"节点加健康检测，检查 ContainerCreating 堆积数和网关探测失败率，及早发现节点冻死",[25,543,544],{},"实例分散部署，用 topologySpreadConstraints 避免单节点堆积",[73,546,548],{"id":547},"无-per-subpath-硬配额","无 per-subPath 硬配额",[10,550,551],{},"共享存储的架构决定了无法在存储层面按 subPath 限制用户配额。SFS（及 NFS 一般）没有 per-directory 的硬配额功能，只能在整个卷级别限制。",[10,553,554],{},"这意味着：",[22,556,557,560,563],{},[25,558,559],{},"无法阻止某个用户的数据膨胀而挤占其他用户的空间",[25,561,562],{},"无法在存储层面实现\"超额用户的写入失败\"",[25,564,565],{},"只能在应用层进行监控、告警和逻辑控制",[10,567,568,569,572,573,576],{},"新系统的方案是应用层监控：定期 ",[14,570,571],{},"du -sb \u002Fdata"," 扫描每个用户的目录大小，落库到 Instance 表，",[14,574,575],{},"\u002Fstatus"," 接口返回超额标志位。前端据此提示用户\"存储已满\"。这是纯监控，不阻断。如果用户无视提示继续写入，直到共享卷真的满了，才会出现\"所有用户集体写入失败\"的惨淡局面。",[10,578,579],{},"这个限制是方案选择的代价。",[73,581,582],{"id":582},"为什么仍然选了这个方案",[10,584,585],{},"尽管有这些问题，平台仍然采用了共享 PVC + subPath 的方案。原因是对比了替代方案：",[587,588,589,608],"table",{},[590,591,592],"thead",{},[593,594,595,599,602,605],"tr",{},[596,597,598],"th",{},"方案",[596,600,601],{},"优点",[596,603,604],{},"缺点",[596,606,607],{},"故障隔离",[609,610,611,626,640],"tbody",{},[593,612,613,617,620,623],{},[614,615,616],"td",{},"独立 PVC（旧架构）",[614,618,619],{},"天然的每用户隔离；故障域清晰",[614,621,622],{},"管理复杂（574 个 PVC）；扩展性差；成本高",[614,624,625],{},"✓ 最好",[593,627,628,631,634,637],{},[614,629,630],{},"共享 PVC + subPath（新架构）",[614,632,633],{},"管理简单（1 个 PVC）；自动扩展；成本低",[614,635,636],{},"故障隔离性差；无硬配额；权限管理复杂",[614,638,639],{},"✗ 较差",[593,641,642,645,648,651],{},[614,643,644],{},"对象存储（S3\u002FOSS）",[614,646,647],{},"真正的多租户隔离；天然分布",[614,649,650],{},"延迟高；成本更高；应用改造大",[614,652,653],{},"✓ 最好，但代价大",[10,655,656],{},"独立 PVC 的管理开销是关键问题。574 个用户意味着 574 个 PVC 对象、574 条 PV 绑定、574 个存储卷。每次实例创建、删除或迁移都要涉及 PVC 的生命周期管理。扩展到 5000 用户时，这种开销会成为瓶颈。对象存储虽然隔离性最好，但需要应用层改造（兼容 S3 API、处理延迟、调整备份策略），而且成本更高。",[10,658,659,660,663],{},"共享 PVC 方案的核心优势是",[61,661,662],{},"运维简洁","：增删用户只需改 subPath（一行 YAML），不涉及存储层操作。代价是故障隔离性从\"用户级\"降到\"节点级\"。",[73,665,666],{"id":666},"适用边界",[10,668,669],{},"这个方案适合以下场景：",[22,671,672,678,684,690],{},[25,673,674,677],{},[61,675,676],{},"用户数量有上限","（几百到几千）。用户过多时，共享卷的单点压力会成问题。",[25,679,680,683],{},[61,681,682],{},"用户数据量可控","（单用户通常 GB 级）。如果单用户数据量达 TB，一次 du 扫描会拖累整体。",[25,685,686,689],{},[61,687,688],{},"可以接受节点级故障隔离","。只要网络和存储配置足够稳定（硬化 NFS 参数、冗余链路），节点 hang 的概率不会很高。",[25,691,692,695],{},[61,693,694],{},"能够实施应用层监控和告警","。无硬配额，就必须有实时监控。",[10,697,698],{},"不适合的场景包括：",[22,700,701,704,707],{},[25,702,703],{},"超大规模用户（万级以上）",[25,705,706],{},"用户数据特别不均衡（少数用户占大头）的场景",[25,708,709],{},"对故障隔离要求极高的系统（比如金融交易）",[73,711,712],{"id":712},"防护与调整",[10,714,715],{},"基于这四个问题，实施中做了以下调整：",[232,717,718,727,737,743,749],{},[25,719,720,723,724,726],{},[61,721,722],{},"权限处理","：容器 Dockerfile 中预先创建 ",[14,725,454],{}," 并设置正确的属主；启动脚本检查并修复权限。",[25,728,729,732,733,736],{},[61,730,731],{},"NFS 参数硬化","：StorageClass 的挂载选项改为 ",[14,734,735],{},"soft,timeo=100,retrans=3,intr","，避免硬 mount 导致节点冻。",[25,738,739,742],{},[61,740,741],{},"节点健康检测","：cron 每 60 秒检查各节点的 ContainerCreating 堆积和网关探测失败率，及早发现故障。",[25,744,745,748],{},[61,746,747],{},"实例分散","：StatefulSet 加 topologySpreadConstraints，避免单节点堆积 50+ 个实例。",[25,750,751,754],{},[61,752,753],{},"应用层配额","：\u002Fstatus 接口返回 diskUsedMi 和 overQuota 标志位，前端告警用户。",[10,756,757],{},"这些措施不能消除风险，但可以大幅降低故障概率和影响范围。",[341,759,760],{},"html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":103,"searchDepth":116,"depth":116,"links":762},[763,764,765,766,767,768,769],{"id":383,"depth":116,"text":384},{"id":461,"depth":116,"text":462},{"id":488,"depth":116,"text":489},{"id":547,"depth":116,"text":548},{"id":582,"depth":116,"text":582},{"id":666,"depth":116,"text":666},{"id":712,"depth":116,"text":712},"2026-07-23",{},"\u002F2026-07-23-fa-018-subpath",{"title":372,"description":377},"FA-018","2026-07-23-FA-018-共享存储的subPath陷阱","K8s 共享 PVC + subPath 方案在实施中暴露的四个陷阱：权限不匹配、目录创建时机、节点级故障域、无 per-subPath 配额。",[778,779,780,781,782,783,607],"Kubernetes","存储","subPath","PVC","NFS","权限","4pJJyzbII4OOc9jJS74HUpUfyVc9CfxdE5ovso8ylSE",{"id":786,"title":787,"body":788,"column":351,"date":1229,"description":1230,"extension":354,"hero_image":355,"meta":1231,"navigation":276,"path":1232,"seo":1233,"series_id":1234,"severity":355,"stem":1235,"summary":1236,"tags":1237,"__hash__":1244},"posts\u002F2026-07-11-FA-017-长任务异步化.md","三个超时机制，一起把长任务掐死了",{"type":7,"value":789,"toc":1221},[790,801,804,807,813,819,825,832,838,842,845,848,856,859,870,876,880,883,890,896,899,914,918,921,932,981,996,1003,1007,1010,1013,1031,1034,1042,1045,1060,1063,1066,1212,1215,1218],[10,791,792,793,796,797,800],{},"数字人口播的「分析」功能从视频提取口播文稿、分镜脚本、结构卖点，需要下载视频、压缩、调 Gemini 视觉 API，耗时从几十秒到几分钟。最初同步实现：前端调 ",[14,794,795],{},"POST \u002Fapi\u002Fworkflow\u002Fdub\u002Fanalyze-parsed","，后端直接跑完再返回。这个方案暴露的问题不是单点，而是",[61,798,799],{},"三个独立的缺陷叠加","。",[10,802,803],{},"网关超时、CDN 空闲切断、客户端重试——这三个没有一个是\"bug\"，每个都是合理的系统设计。但组合在一起就是连锁灾难：客户端看不到结果（超时或连接断），认为失败了重试，后端却已经扣过费了；或者分析其实已跑完，但因为响应发不出去，重试又跑了一遍。退款时无法幂等判断，某些用户最终被多扣了好几倍。",[10,805,806],{},"具体来说，三个问题各自是什么：",[10,808,809,812],{},[61,810,811],{},"第一个问题是网关超时","。云平台的入口网关（API Gateway）有一个固定的读超时，通常设为几十秒。当后端分析耗时超过这个阈值，网关主动断连接，客户端收到 504 或连接重置，但后端的分析进程并不知道客户端已经走了，继续跑完整个流程。如果分析结果的退款逻辑放在 try-catch 的 finally 里，此时 HTTP 响应已发不出去，客户就看到\"分析失败\"，但款已经扣了。",[10,814,815,818],{},[61,816,817],{},"第二个问题是 CDN 空闲切断","。不少 CDN 和负载均衡层有这样的策略：如果一条 HTTP 连接在某个时间段内没有数据往返，就认为它已死而主动关闭。这与 FA-013 那次踩到的问题一样——长连接因为没有心跳数据而被 CDN 中间件切断，导致请求被突然中断。这个机制在分析请求上也会造成麻烦：如果分析用时较长，连接就可能被切，响应发不出去。",[10,820,821,824],{},[61,822,823],{},"第三个问题是客户端重试导致重复扣费","。当客户端没有收到响应（网关超时或 CDN 切断），一般会重试这个请求。问题在于这次重试是一条完全独立的请求，后端会把它当作新任务处理，再次扣费。而且如果第一次的分析实际上已经跑完了，就会出现\"分析跑了两遍、扣费扣了两遍、用户只看到了一个结果\"的情况。如果第一次分析中途被 CDN 切了，第二次重试重新开始，那扣费可能是三倍。而退款的时候，因为业务流程上没有幂等设计，很难追溯哪个 operationId 应该被退，哪个不应该。",[10,826,827,828,831],{},"这三个问题是",[61,829,830],{},"叠加的","，不是独立的。它们共同指向一个根本的架构问题。",[10,833,834,835,800],{},"根本原因一个：",[61,836,837],{},"不该让耗时任务挂在 HTTP 连接上",[73,839,841],{"id":840},"异步任务是必需的不是可选的","异步任务是必需的，不是可选的",[10,843,844],{},"解决方案很简单：把分析改成异步任务模式。",[10,846,847],{},"前端提交分析请求直接返回任务 ID（HTTP 耗时极短：校验参数、建数据库记录、预扣费）。后端后台 worker 跑分析重逻辑，完成后持久化结果。前端轮询查询状态，得到结果后停止。",[98,849,854],{"className":850,"code":852,"language":853},[851],"language-text","客户端              后端\n  |                 |\n  +--提交---------->|\n  |             创建任务，返回 ID\n  |\u003C--返回 ID-----+\n  |                 |（后台运行分析）\n  |                 |\n  +--轮询--------->|\n  |             查询状态\n  |\u003C--返回状态---+\n  |                 |\n  |（重复轮询）\n  +--轮询--------->|\n  |             查询状态\n  |\u003C--返回结果---+\n  |                 |\n","text",[14,855,852],{"__ignoreMap":103},[10,857,858],{},"好处显而易见：",[22,860,861,864,867],{},[25,862,863],{},"单次 HTTP 请求的耗时从分钟级降到秒级，不会触发网关或 CDN 的超时。",[25,865,866],{},"后端分析的运行生命周期与 HTTP 连接完全解耦。即使连接在中途被切，已经启动的分析会继续在后台跑，结果最终还是会被保存。",[25,868,869],{},"客户端重试不会产生新的分析任务。只要实现幂等判重，同一个请求在短时间内重复提交也只会产生一个任务。",[10,871,872,873,800],{},"但异步化不是万灵药，它把问题转移到了三个新的维度上：",[61,874,875],{},"幂等、持久化、失败语义",[73,877,879],{"id":878},"幂等同一请求只产生一个任务","幂等：同一请求只产生一个任务",[10,881,882],{},"假设客户端因为网络抖动，在短时间内连续发了两次\"开始分析\"请求，后端收到两条独立的 HTTP 请求。如果没有幂等设计，就会创建两个任务、扣两次费。",[10,884,885,886,889],{},"幂等的实现最直接的办法是在",[61,887,888],{},"提交阶段就判重","：为这个用户、这个资源设置一个\"只有一个进行中的分析任务\"的约束。素材中的设计是这样做的：",[98,891,894],{"className":892,"code":893,"language":853},[851],"if 该用户已有 status=running && kind=analyze_parsed 的任务:\n    return 409 Conflict（分析已在进行中）\n",[14,895,893],{"__ignoreMap":103},[10,897,898],{},"这样双击提交同一个视频，第一次返回 201 + taskId，第二次返回 409，客户端看到 409 就知道不要再建新任务，拿之前返回的 ID 去轮询。",[10,900,901,902,905,906,909,910,913],{},"另一个幂等层次是",[61,903,904],{},"计费操作的幂等","。预扣费的时候用一个全局唯一的 ",[14,907,908],{},"operationId","（比如 ",[14,911,912],{},"dub-analyze:{uuid}","），这个 ID 关联了这次预扣。如果扣费 API 被重复调用，因为 operationId 重复，系统只会扣一次。退款时也用同一个 operationId，保证退款与预扣能正确配对。",[73,915,917],{"id":916},"持久化进程重启后任务不丢","持久化：进程重启后任务不丢",[10,919,920],{},"后台分析 worker 可能因为发版、机器重启、容器被杀等原因在中途退出。如果任务的状态只保存在内存里，进程一死任务就丢了。",[10,922,923,924,927,928,931],{},"持久化的关键是",[61,925,926],{},"任务状态表","。素材中复用了既有的 ",[14,929,930],{},"SkyhumanTask"," 表，包含字段：",[22,933,934,940,946,951,957,963,969,975],{},[25,935,936,939],{},[14,937,938],{},"id",": 任务 ID",[25,941,942,945],{},[14,943,944],{},"status",": 进度（running \u002F completed \u002F failed）",[25,947,948,950],{},[14,949,908],{},": 幂等 ID",[25,952,953,956],{},[14,954,955],{},"chargedPoints",": 预扣费用",[25,958,959,962],{},[14,960,961],{},"resultPayload",": 分析结果的 JSON",[25,964,965,968],{},[14,966,967],{},"error",": 失败原因",[25,970,971,974],{},[14,972,973],{},"updatedAt",": 最后更新时间戳",[25,976,977,980],{},[14,978,979],{},"sourceObjectKey",": 源视频对象路径（用于幂等判重和重试）",[10,982,983,984,987,988,991,992,995],{},"任务创建时插一条 ",[14,985,986],{},"status=running"," 的记录。后台 worker 定期通过 heartbeat（每 30 秒做一次空 update，仅触碰 updatedAt）来证明自己还活着。分析完成时更新为 ",[14,989,990],{},"status=completed, resultPayload=...","；失败时更新为 ",[14,993,994],{},"status=failed, error=..."," 并触发退款。",[10,997,998,999,1002],{},"这样即使 worker 进程突然死了，任务记录还在数据库里。新启起的 worker 可以扫描表里的 ",[14,1000,1001],{},"status=running && updatedAt 过期"," 的任务，判断它们已经没有 worker 在跑了，执行清理逻辑。",[73,1004,1006],{"id":1005},"失败语义区分任务失败和查询失败","失败语义：区分\"任务失败\"和\"查询失败\"",[10,1008,1009],{},"异步设计引入了一个新的错误维度：客户端查询任务状态时得到的 404，可能是\"这个任务 ID 不存在\"（用户输错了），也可能是\"任务 ID 之前存在但已被清理\"（进程清理了过期数据）。这两种情况的含义很不一样。",[10,1011,1012],{},"更重要的是，前端轮询收到的错误需要区分是否应该重试：",[22,1014,1015,1025],{},[25,1016,1017,1020,1021,1024],{},[61,1018,1019],{},"任务本身失败","（",[14,1022,1023],{},"status=failed, error=\"Gemini API 返回错误\"","）：这种情况不该重试，因为重试也会失败。应该直接向用户展示失败信息，并根据 operationId 触发退款。",[25,1026,1027,1030],{},[61,1028,1029],{},"查询接口失败","（500、网络超时）：这种情况应该重试查询，因为任务本身可能还在继续跑。",[10,1032,1033],{},"素材中的设计把这两类区分得很清楚：",[22,1035,1036,1039],{},[25,1037,1038],{},"任务最终的状态（completed 或 failed）持久化到数据库，客户端查询会拿到明确的业务语义。",[25,1040,1041],{},"查询接口的 HTTP 错误（5xx）是技术层故障，客户端应该重试。",[10,1043,1044],{},"同时，后端的 reaper 机制（定期扫表清理超期任务）也遵循这个语义：",[22,1046,1047,1053],{},[25,1048,1049,1050,1052],{},"如果一个 ",[14,1051,986],{}," 的任务超过 90 秒没有 heartbeat 更新，说明 worker 已死，reaper 会强制置为 failed 并退款。",[25,1054,1055,1056,1059],{},"但这个强制置为 failed 不应该抛错，因为此时可能有其他进程也想更新这个任务。更新语句应该带条件：",[14,1057,1058],{},"UPDATE SkyhumanTask SET status=failed WHERE id=? AND status=running","，这样如果 reaper 和其他 worker 同时操作，只有一方会成功。",[73,1061,1062],{"id":1062},"具体实现模式",[10,1064,1065],{},"数字人口播的分析异步化采用了这样的模式：",[232,1067,1068,1113,1159,1182],{},[25,1069,1070,1073,1074],{},[61,1071,1072],{},"提交阶段","（同步）：",[22,1075,1076,1079,1082,1088,1094,1101,1107],{},[25,1077,1078],{},"校验用户和资源",[25,1080,1081],{},"判重：是否已有进行中的任务（409）",[25,1083,1084,1087],{},[14,1085,1086],{},"getObject"," 拉视频，检测时长（若≤0 返 400 不扣费）",[25,1089,1090,1093],{},[14,1091,1092],{},"chargeResource"," 预扣费用，获得 operationId",[25,1095,1096,1097,1100],{},"创建 ",[14,1098,1099],{},"SkyhumanTask{ status:running, operationId, sourceObjectKey }"," 记录",[25,1102,1103,1106],{},[14,1104,1105],{},"scheduleTask"," 丢到后台队列",[25,1108,1109,1110],{},"返回 ",[14,1111,1112],{},"{ taskId }",[25,1114,1115,256,1118],{},[61,1116,1117],{},"后台运行阶段",[22,1119,1120,1123,1126,1133,1139,1149,1156],{},[25,1121,1122],{},"Worker 取出任务记录",[25,1124,1125],{},"开启 heartbeat 定时器（每 30s update 一次 updatedAt）",[25,1127,1128,1129,1132],{},"执行 ",[14,1130,1131],{},"analyzeDubVisionOnly","（下载→压缩→调用 Gemini→解析）",[25,1134,1135,1136],{},"成功：",[14,1137,1138],{},"update( status:completed, resultPayload )",[25,1140,1141,1142,1145,1146],{},"失败：",[14,1143,1144],{},"update( status:failed, error )"," + ",[14,1147,1148],{},"refundResource(operationId)",[25,1150,1151,1152,1155],{},"所有 update 都带条件 ",[14,1153,1154],{},"WHERE id=? AND status=running","（防被 reaper 覆盖）",[25,1157,1158],{},"Finally 清理 heartbeat 定时器",[25,1160,1161,1164,1165],{},[61,1162,1163],{},"查询阶段","（前端轮询）：",[22,1166,1167,1176,1179],{},[25,1168,1169,1172,1173],{},[14,1170,1171],{},"GET \u002Fapi\u002Fworkflow\u002Fdub\u002Ftasks\u002F:id"," 返回 ",[14,1174,1175],{},"{ status, resultPayload, error }",[25,1177,1178],{},"若 status=completed 或 failed，停止轮询",[25,1180,1181],{},"若查询接口返回 5xx，则后续重试",[25,1183,1184,1187,1188],{},[61,1185,1186],{},"清理阶段","（reaper）：",[22,1189,1190,1193,1200,1203,1209],{},[25,1191,1192],{},"后台定期扫表",[25,1194,1195,1196,1199],{},"对于 ",[14,1197,1198],{},"status=running && updatedAt > 90s"," 的任务，判定 worker 已死",[25,1201,1202],{},"若 providerTaskId（这里没有）存在，调用上游的 finalize API",[25,1204,1205,1206,1208],{},"若不存在，直接 ",[14,1207,1148],{}," + 置 failed",[25,1210,1211],{},"本设计无 providerTaskId，所以 reaper 自动走退款分支，无需改动 reaper 代码",[10,1213,1214],{},"这个模式的关键是三道防线：提交时判重（409）、后台心跳（定期 touch）、reaper 兜底（自动退款）。任何一个环节出问题，都有后续环节来补救。",[73,1216,1217],{"id":1217},"防线缺一不可",[10,1219,1220],{},"长耗时任务不能挂在 HTTP 连接上不只是超时问题，而是连接脆弱性与重复处理的共谋。异步化表面是\"返回 ID、后台跑\"这一步，真正的难点在三个维度的同时保证：幂等（不重复扣费）、持久化（不丢数据）、失败语义（不破坏重试逻辑）。缺一就会在某个场景暴露。",{"title":103,"searchDepth":116,"depth":116,"links":1222},[1223,1224,1225,1226,1227,1228],{"id":840,"depth":116,"text":841},{"id":878,"depth":116,"text":879},{"id":916,"depth":116,"text":917},{"id":1005,"depth":116,"text":1006},{"id":1062,"depth":116,"text":1062},{"id":1217,"depth":116,"text":1217},"2026-07-11","数字人口播的「分析」功能从视频提取口播文稿、分镜脚本、结构卖点，需要下载视频、压缩、调 Gemini 视觉 API，耗时从几十秒到几分钟。最初同步实现：前端调 POST \u002Fapi\u002Fworkflow\u002Fdub\u002Fanalyze-parsed，后端直接跑完再返回。这个方案暴露的问题不是单点，而是三个独立的缺陷叠加。",{},"\u002F2026-07-11-fa-017",{"title":787,"description":1230},"FA-017","2026-07-11-FA-017-长任务异步化","长耗时分析请求（几十秒到分钟级）用同步 HTTP 导致网关超时、CDN切断、客户端重试重复扣费。改异步任务需同时解决幂等、持久化、失败语义。",[1238,1239,1240,1241,1242,1243],"异步任务","HTTP超时","CDN","幂等","任务队列","微服务","ARFDvBMGX3tw-E3Rj_gw1RYEjrykplO6u1dxtdS7Uto",{"id":1246,"title":1247,"body":1248,"column":351,"date":2161,"description":2162,"extension":354,"hero_image":355,"meta":2163,"navigation":276,"path":2164,"seo":2165,"series_id":2166,"severity":355,"stem":2167,"summary":2168,"tags":2169,"__hash__":2175},"posts\u002F2026-06-17-FA-016-WS永久断开三条路径.md","日志里没有重连记录——它根本没在重连",{"type":7,"value":1249,"toc":2149},[1250,1261,1268,1275,1278,1281,1286,1293,1370,1376,1446,1457,1461,1464,1586,1597,1601,1607,1662,1665,1668,1671,1686,1689,1691,1694,1701,1956,1966,1969,1979,1992,2002,2005,2008,2015,2066,2069,2072,2075,2078,2081,2126,2143,2146],[10,1251,1252,1253,1256,1257,1260],{},"从 2026-05-11 开始，多台便携包客户反馈 WebSocket 断线后面板卡死。网关进程（",[14,1254,1255],{},"openclaw_gateway.exe","）明确在运行，",[14,1258,1259],{},"\u002Fhealth"," 返回 200，但面板显示\"已停止重连，请手动刷新\"——用户必须手动重启才能恢复。",[10,1262,1263,1264,1267],{},"关键线索在控制台：看不到任何 ",[14,1265,1266],{},"[ws] 计划重连"," 的日志。",[10,1269,1270,1271,1274],{},"通常断线后客户端会频繁尝试重连，日志里应该是满屏的重连记录。没有日志意味着问题不在重连失败，而在",[61,1272,1273],{},"压根没启动重连机制","。说明客户端进入了某个永久休眠状态。",[73,1276,1277],{"id":1277},"三条独立的永久断开路径",[10,1279,1280],{},"代码审查发现了三条完全不同的、独立的永久断开路径。任意一条命中，就会停止调度任何重连计时器。",[1282,1283,1285],"h3",{"id":1284},"路径-a凭据刷新异常","路径 A：凭据刷新异常",[10,1287,1288,1289,1292],{},"在 ",[14,1290,1291],{},"_scheduleReconnect"," 方法末尾，每次普通重连都会调用：",[98,1294,1298],{"className":1295,"code":1296,"language":1297,"meta":103,"style":103},"language-javascript shiki shiki-themes github-light github-dark","this._reconnectTimer = setTimeout(() => {\n  if (!this._intentionalClose) {\n    this._refreshCredentialsAndReconnect(0)\n  }\n}, delay)\n","javascript",[14,1299,1300,1324,1340,1360,1365],{"__ignoreMap":103},[107,1301,1302,1305,1308,1312,1315,1318,1321],{"class":109,"line":110},[107,1303,1304],{"class":417},"this",[107,1306,1307],{"class":424},"._reconnectTimer ",[107,1309,1311],{"class":1310},"szBVR","=",[107,1313,1314],{"class":409}," setTimeout",[107,1316,1317],{"class":424},"(() ",[107,1319,1320],{"class":1310},"=>",[107,1322,1323],{"class":424}," {\n",[107,1325,1326,1329,1332,1335,1337],{"class":109,"line":116},[107,1327,1328],{"class":1310},"  if",[107,1330,1331],{"class":424}," (",[107,1333,1334],{"class":1310},"!",[107,1336,1304],{"class":417},[107,1338,1339],{"class":424},"._intentionalClose) {\n",[107,1341,1342,1345,1348,1351,1354,1357],{"class":109,"line":122},[107,1343,1344],{"class":417},"    this",[107,1346,1347],{"class":424},".",[107,1349,1350],{"class":409},"_refreshCredentialsAndReconnect",[107,1352,1353],{"class":424},"(",[107,1355,1356],{"class":417},"0",[107,1358,1359],{"class":424},")\n",[107,1361,1362],{"class":109,"line":128},[107,1363,1364],{"class":424},"  }\n",[107,1366,1367],{"class":109,"line":134},[107,1368,1369],{"class":424},"}, delay)\n",[10,1371,1372,1373,1375],{},"而 ",[14,1374,1350],{}," 是异步方法，catch 分支没有任何后续处理：",[98,1377,1379],{"className":1295,"code":1378,"language":1297,"meta":103,"style":103},"} catch (e) {\n  console.error('[ws] 刷新凭据失败:', e)\n  this._setConnected(false, 'error', `凭据刷新失败: ${e}`)\n}\n",[14,1380,1381,1392,1407,1441],{"__ignoreMap":103},[107,1382,1383,1386,1389],{"class":109,"line":110},[107,1384,1385],{"class":424},"} ",[107,1387,1388],{"class":1310},"catch",[107,1390,1391],{"class":424}," (e) {\n",[107,1393,1394,1397,1399,1401,1404],{"class":109,"line":116},[107,1395,1396],{"class":424},"  console.",[107,1398,967],{"class":409},[107,1400,1353],{"class":424},[107,1402,1403],{"class":413},"'[ws] 刷新凭据失败:'",[107,1405,1406],{"class":424},", e)\n",[107,1408,1409,1412,1414,1417,1419,1422,1425,1428,1430,1433,1436,1439],{"class":109,"line":122},[107,1410,1411],{"class":417},"  this",[107,1413,1347],{"class":424},[107,1415,1416],{"class":409},"_setConnected",[107,1418,1353],{"class":424},[107,1420,1421],{"class":417},"false",[107,1423,1424],{"class":424},", ",[107,1426,1427],{"class":413},"'error'",[107,1429,1424],{"class":424},[107,1431,1432],{"class":413},"`凭据刷新失败: ${",[107,1434,1435],{"class":424},"e",[107,1437,1438],{"class":413},"}`",[107,1440,1359],{"class":424},[107,1442,1443],{"class":109,"line":128},[107,1444,1445],{"class":424},"}\n",[10,1447,1448,1449,1452,1453,1456],{},"如果 ",[14,1450,1451],{},"api.readOpenclawConfig()"," 或 ",[14,1454,1455],{},"api.autoPairDevice()"," 抛错——磁盘 IO 抖、Rust 端处理器忙、Tauri IPC 队列阻塞——这次重连就永久终止。便携磁盘的 IO 抖动 + 配置文件读写争抢时最容易发生。",[1282,1458,1460],{"id":1459},"路径-b认证失败后的强制关闭","路径 B：认证失败后的强制关闭",[10,1462,1463],{},"当 WebSocket 收到 1008 unauthorized 响应时的处理逻辑：",[98,1465,1467],{"className":1295,"code":1466,"language":1297,"meta":103,"style":103},"if (this._authRetryCount \u003C 2) {\n  this._authRetryCount++\n  this._refreshCredentialsAndReconnect()\n  return\n}\nthis._setConnected(false, 'auth_failed', `认证失败: ${e.reason}。请检查 Gateway Token 配置。`)\nthis._intentionalClose = true   \u002F\u002F ← 永久关闭标记\nthis._flushPending()\nreturn\n",[14,1468,1469,1490,1500,1511,1516,1520,1554,1570,1581],{"__ignoreMap":103},[107,1470,1471,1474,1476,1478,1481,1484,1487],{"class":109,"line":110},[107,1472,1473],{"class":1310},"if",[107,1475,1331],{"class":424},[107,1477,1304],{"class":417},[107,1479,1480],{"class":424},"._authRetryCount ",[107,1482,1483],{"class":1310},"\u003C",[107,1485,1486],{"class":417}," 2",[107,1488,1489],{"class":424},") {\n",[107,1491,1492,1494,1497],{"class":109,"line":116},[107,1493,1411],{"class":417},[107,1495,1496],{"class":424},"._authRetryCount",[107,1498,1499],{"class":1310},"++\n",[107,1501,1502,1504,1506,1508],{"class":109,"line":122},[107,1503,1411],{"class":417},[107,1505,1347],{"class":424},[107,1507,1350],{"class":409},[107,1509,1510],{"class":424},"()\n",[107,1512,1513],{"class":109,"line":128},[107,1514,1515],{"class":1310},"  return\n",[107,1517,1518],{"class":109,"line":134},[107,1519,1445],{"class":424},[107,1521,1522,1524,1526,1528,1530,1532,1534,1537,1539,1542,1544,1546,1549,1552],{"class":109,"line":140},[107,1523,1304],{"class":417},[107,1525,1347],{"class":424},[107,1527,1416],{"class":409},[107,1529,1353],{"class":424},[107,1531,1421],{"class":417},[107,1533,1424],{"class":424},[107,1535,1536],{"class":413},"'auth_failed'",[107,1538,1424],{"class":424},[107,1540,1541],{"class":413},"`认证失败: ${",[107,1543,1435],{"class":424},[107,1545,1347],{"class":413},[107,1547,1548],{"class":424},"reason",[107,1550,1551],{"class":413},"}。请检查 Gateway Token 配置。`",[107,1553,1359],{"class":424},[107,1555,1556,1558,1561,1563,1566],{"class":109,"line":146},[107,1557,1304],{"class":417},[107,1559,1560],{"class":424},"._intentionalClose ",[107,1562,1311],{"class":1310},[107,1564,1565],{"class":417}," true",[107,1567,1569],{"class":1568},"sJ8bj","   \u002F\u002F ← 永久关闭标记\n",[107,1571,1572,1574,1576,1579],{"class":109,"line":152},[107,1573,1304],{"class":417},[107,1575,1347],{"class":424},[107,1577,1578],{"class":409},"_flushPending",[107,1580,1510],{"class":424},[107,1582,1583],{"class":109,"line":158},[107,1584,1585],{"class":1310},"return\n",[10,1587,1588,1589,1592,1593,1596],{},"设置 ",[14,1590,1591],{},"_intentionalClose = true"," 之后，所有重连逻辑都被 ",[14,1594,1595],{},"if (!this._intentionalClose)"," 短路。即使只是 Gateway 重启窗口碰好出现 token 短暂不匹配，也会一次性把客户端打死，必须刷新页面才能恢复。",[1282,1598,1600],{"id":1599},"路径-c快速重连配额耗尽","路径 C：快速重连配额耗尽",[10,1602,1603,1604,256],{},"定义了常数 ",[14,1605,1606],{},"MAX_RECONNECT_ATTEMPTS = 60",[98,1608,1610],{"className":1295,"code":1609,"language":1297,"meta":103,"style":103},"if (this._reconnectAttempts >= MAX_RECONNECT_ATTEMPTS) {\n  this._setConnected(false, 'error', `连接失败，已停止重连。请手动刷新页面重试。`)\n  return\n}\n",[14,1611,1612,1631,1654,1658],{"__ignoreMap":103},[107,1613,1614,1616,1618,1620,1623,1626,1629],{"class":109,"line":110},[107,1615,1473],{"class":1310},[107,1617,1331],{"class":424},[107,1619,1304],{"class":417},[107,1621,1622],{"class":424},"._reconnectAttempts ",[107,1624,1625],{"class":1310},">=",[107,1627,1628],{"class":417}," MAX_RECONNECT_ATTEMPTS",[107,1630,1489],{"class":424},[107,1632,1633,1635,1637,1639,1641,1643,1645,1647,1649,1652],{"class":109,"line":116},[107,1634,1411],{"class":417},[107,1636,1347],{"class":424},[107,1638,1416],{"class":409},[107,1640,1353],{"class":424},[107,1642,1421],{"class":417},[107,1644,1424],{"class":424},[107,1646,1427],{"class":413},[107,1648,1424],{"class":424},[107,1650,1651],{"class":413},"`连接失败，已停止重连。请手动刷新页面重试。`",[107,1653,1359],{"class":424},[107,1655,1656],{"class":109,"line":122},[107,1657,1515],{"class":1310},[107,1659,1660],{"class":109,"line":128},[107,1661,1445],{"class":424},[10,1663,1664],{},"客户机晚上挂机，Gateway 因为便携磁盘 GC 或 Windows 休眠争抢资源短暂掉线，客户端尝试 60 次仍未成功重连，就永久停摆。早上用户回来面板已成死链。",[73,1666,1667],{"id":1667},"为什么三条缺陷同时存在",[10,1669,1670],{},"这是逐步累加的历史债：",[22,1672,1673,1680,1683],{},[25,1674,1675,1676,1679],{},"路径 B 是在修复\"避免无限自动配对循环\"时添加的保护，但用错了对象——",[14,1677,1678],{},"_intentionalClose=true"," 本来是用户主动断开的语义，不该用在被动失败上",[25,1681,1682],{},"路径 C 是早期\"避免无穷重试\"的防护，但 60 次后完全放弃而不留任何复活路径是绝对错误",[25,1684,1685],{},"路径 A 是在\"凭据 reload\"改动时把所有重连都改走 refreshCredentials，没留意 catch 分支已经成了终态",[10,1687,1688],{},"三条各自独立、都能单独打死客户端，组合在一起命中率非常高。",[73,1690,227],{"id":227},[10,1692,1693],{},"核心思想是永不彻底放弃。任何\"快速重连配额耗尽\"的分支都转入慢轮询，留出窗口让用户改配置或等待 Gateway 恢复后能自动复连。",[10,1695,1696,1697,1700],{},"新增辅助方法 ",[14,1698,1699],{},"_schedulePoll(delayMs, kind)"," 作为慢轮询的触发器：",[98,1702,1704],{"className":1295,"code":1703,"language":1297,"meta":103,"style":103},"const AUTH_RETRY_LIMIT = 2\nconst SLOW_POLL_DELAY_AUTH = 60_000      \u002F\u002F 认证持续失败：1 分钟探一次\nconst SLOW_POLL_DELAY_GENERAL = 300_000  \u002F\u002F 一般持续失败：5 分钟探一次\n\n_schedulePoll(delayMs, kind) {\n  this._clearReconnectTimer()\n  this._reconnectAttempts = 0\n  if (kind === 'auth') this._authRetryCount = 0\n  this._reconnectState = 'scheduled'\n  this._pendingReconnect = true\n  this._reconnectTimer = setTimeout(() => {\n    this._reconnectTimer = null\n    if (this._intentionalClose) return\n    this._reconnectState = 'attempting'\n    if (kind === 'auth') {\n      this._refreshCredentialsAndReconnect(0)\n    } else {\n      this._doConnect()\n    }\n  }, delayMs)\n}\n",[14,1705,1706,1720,1735,1750,1754,1762,1773,1784,1808,1820,1832,1848,1860,1875,1887,1900,1916,1927,1939,1945,1951],{"__ignoreMap":103},[107,1707,1708,1711,1714,1717],{"class":109,"line":110},[107,1709,1710],{"class":1310},"const",[107,1712,1713],{"class":417}," AUTH_RETRY_LIMIT",[107,1715,1716],{"class":1310}," =",[107,1718,1719],{"class":417}," 2\n",[107,1721,1722,1724,1727,1729,1732],{"class":109,"line":116},[107,1723,1710],{"class":1310},[107,1725,1726],{"class":417}," SLOW_POLL_DELAY_AUTH",[107,1728,1716],{"class":1310},[107,1730,1731],{"class":417}," 60_000",[107,1733,1734],{"class":1568},"      \u002F\u002F 认证持续失败：1 分钟探一次\n",[107,1736,1737,1739,1742,1744,1747],{"class":109,"line":122},[107,1738,1710],{"class":1310},[107,1740,1741],{"class":417}," SLOW_POLL_DELAY_GENERAL",[107,1743,1716],{"class":1310},[107,1745,1746],{"class":417}," 300_000",[107,1748,1749],{"class":1568},"  \u002F\u002F 一般持续失败：5 分钟探一次\n",[107,1751,1752],{"class":109,"line":128},[107,1753,277],{"emptyLinePlaceholder":276},[107,1755,1756,1759],{"class":109,"line":134},[107,1757,1758],{"class":409},"_schedulePoll",[107,1760,1761],{"class":424},"(delayMs, kind) {\n",[107,1763,1764,1766,1768,1771],{"class":109,"line":140},[107,1765,1411],{"class":417},[107,1767,1347],{"class":424},[107,1769,1770],{"class":409},"_clearReconnectTimer",[107,1772,1510],{"class":424},[107,1774,1775,1777,1779,1781],{"class":109,"line":146},[107,1776,1411],{"class":417},[107,1778,1622],{"class":424},[107,1780,1311],{"class":1310},[107,1782,1783],{"class":417}," 0\n",[107,1785,1786,1788,1791,1794,1797,1800,1802,1804,1806],{"class":109,"line":152},[107,1787,1328],{"class":1310},[107,1789,1790],{"class":424}," (kind ",[107,1792,1793],{"class":1310},"===",[107,1795,1796],{"class":413}," 'auth'",[107,1798,1799],{"class":424},") ",[107,1801,1304],{"class":417},[107,1803,1480],{"class":424},[107,1805,1311],{"class":1310},[107,1807,1783],{"class":417},[107,1809,1810,1812,1815,1817],{"class":109,"line":158},[107,1811,1411],{"class":417},[107,1813,1814],{"class":424},"._reconnectState ",[107,1816,1311],{"class":1310},[107,1818,1819],{"class":413}," 'scheduled'\n",[107,1821,1822,1824,1827,1829],{"class":109,"line":164},[107,1823,1411],{"class":417},[107,1825,1826],{"class":424},"._pendingReconnect ",[107,1828,1311],{"class":1310},[107,1830,1831],{"class":417}," true\n",[107,1833,1834,1836,1838,1840,1842,1844,1846],{"class":109,"line":170},[107,1835,1411],{"class":417},[107,1837,1307],{"class":424},[107,1839,1311],{"class":1310},[107,1841,1314],{"class":409},[107,1843,1317],{"class":424},[107,1845,1320],{"class":1310},[107,1847,1323],{"class":424},[107,1849,1851,1853,1855,1857],{"class":109,"line":1850},12,[107,1852,1344],{"class":417},[107,1854,1307],{"class":424},[107,1856,1311],{"class":1310},[107,1858,1859],{"class":417}," null\n",[107,1861,1863,1866,1868,1870,1873],{"class":109,"line":1862},13,[107,1864,1865],{"class":1310},"    if",[107,1867,1331],{"class":424},[107,1869,1304],{"class":417},[107,1871,1872],{"class":424},"._intentionalClose) ",[107,1874,1585],{"class":1310},[107,1876,1878,1880,1882,1884],{"class":109,"line":1877},14,[107,1879,1344],{"class":417},[107,1881,1814],{"class":424},[107,1883,1311],{"class":1310},[107,1885,1886],{"class":413}," 'attempting'\n",[107,1888,1890,1892,1894,1896,1898],{"class":109,"line":1889},15,[107,1891,1865],{"class":1310},[107,1893,1790],{"class":424},[107,1895,1793],{"class":1310},[107,1897,1796],{"class":413},[107,1899,1489],{"class":424},[107,1901,1903,1906,1908,1910,1912,1914],{"class":109,"line":1902},16,[107,1904,1905],{"class":417},"      this",[107,1907,1347],{"class":424},[107,1909,1350],{"class":409},[107,1911,1353],{"class":424},[107,1913,1356],{"class":417},[107,1915,1359],{"class":424},[107,1917,1919,1922,1925],{"class":109,"line":1918},17,[107,1920,1921],{"class":424},"    } ",[107,1923,1924],{"class":1310},"else",[107,1926,1323],{"class":424},[107,1928,1930,1932,1934,1937],{"class":109,"line":1929},18,[107,1931,1905],{"class":417},[107,1933,1347],{"class":424},[107,1935,1936],{"class":409},"_doConnect",[107,1938,1510],{"class":424},[107,1940,1942],{"class":109,"line":1941},19,[107,1943,1944],{"class":424},"    }\n",[107,1946,1948],{"class":109,"line":1947},20,[107,1949,1950],{"class":424},"  }, delayMs)\n",[107,1952,1954],{"class":109,"line":1953},21,[107,1955,1445],{"class":424},[10,1957,1958,1959,1961,1962,1965],{},"慢轮询命中后失败会重新进入 ",[14,1960,1291],{}," 快速重连周期（因为 ",[14,1963,1964],{},"_reconnectAttempts"," 重置为 0），相当于\"快速 60 次 → 慢一次 → 快速 60 次 → 慢一次\"的循环，永不放弃。",[10,1967,1968],{},"三条修复并行：",[10,1970,1971,1974,1975,1978],{},[61,1972,1973],{},"Fix A","：普通重连直接走 ",[14,1976,1977],{},"_doConnect()","，凭据刷新隔离到认证失败分支，避免配置读取错误牵连普通重连。",[10,1980,1981,1984,1985,1988,1989,1991],{},[61,1982,1983],{},"Fix B","：认证失败耗尽后转入 ",[14,1986,1987],{},"_schedulePoll('auth')","，不再设置 ",[14,1990,1678],{},"，给认证恢复留出 60 秒的探测窗口。",[10,1993,1994,1997,1998,2001],{},[61,1995,1996],{},"Fix C","：重连次数超过 MAX_RECONNECT_ATTEMPTS 后转入 ",[14,1999,2000],{},"_schedulePoll('general')","，设置 UI 状态为\"连接持续失败，300 秒后重试\"而不是终态。",[10,2003,2004],{},"慢轮询失败后重新进入快速重连周期，形成\"快速 60 次 → 慢一次 → 快速 60 次\"的循环，永不放弃。",[73,2006,2007],{"id":2007},"防回归验证",[10,2009,2010,2011,2014],{},"新增 ",[14,2012,2013],{},"wakou-full\u002Fsrc\u002Flib\u002Fws-client.slow-poll.test.js","，覆盖 7 个 case：",[22,2016,2017,2025,2037,2040,2049,2056,2061],{},[25,2018,2019,2020,2022,2023],{},"Fix A：普通重连命中 ",[14,2021,1936],{},"，不调用 ",[14,2024,1350],{},[25,2026,2027,2028,2031,2032,2034,2035],{},"Fix C：MAX_RECONNECT_ATTEMPTS 后状态是 ",[14,2029,2030],{},"reconnecting","（非终态 ",[14,2033,967],{},"），5 分钟后触发 ",[14,2036,1936],{},[25,2038,2039],{},"Fix C：第二轮慢轮询失败后还能再调度快速重连",[25,2041,2042,2043,2045,2046],{},"Fix B：",[14,2044,1987],{}," 等待 60 秒调用 ",[14,2047,2048],{},"_refreshCredentialsAndReconnect(0)",[25,2050,2042,2051,2053,2054],{},[14,2052,2000],{}," 等待后调用 ",[14,2055,1936],{},[25,2057,2058,2060],{},[14,2059,1678],{}," 时慢轮询应跳过",[25,2062,2063,2065],{},[14,2064,1350],{}," 抛错路径调度慢轮询",[10,2067,2068],{},"测试全部通过（7\u002F7）。",[73,2070,2071],{"id":2071},"外部因素的叠加",[10,2073,2074],{},"这个时期同步发生了另一个外部问题：CDN 对 WebSocket 空闲连接会在 4-9 分钟后强制切断。客户端收到连接中断后根据指数退避策略重连，如果恰好命中上述三条路径之一，就进入永久断开状态。这解释了为什么故障特别在长时间挂机后高发——空闲足够长，CDN 必定切断，而后续重连很容易落入某条缺陷路径。两个问题各自独立，但组合效果是\"挂机过夜必死\"。",[73,2076,2077],{"id":2077},"排查验证",[10,2079,2080],{},"故障修复后，可以用以下方式验证重连状态：",[98,2082,2084],{"className":1295,"code":2083,"language":1297,"meta":103,"style":103},"__clawpanelWsClient.getConnectionInfo()\n\u002F\u002F {\n\u002F\u002F   connected: false,\n\u002F\u002F   reconnectState: 'scheduled' | 'attempting',\n\u002F\u002F   reconnectAttempts: 0~60,\n\u002F\u002F   ...\n\u002F\u002F }\n",[14,2085,2086,2096,2101,2106,2111,2116,2121],{"__ignoreMap":103},[107,2087,2088,2091,2094],{"class":109,"line":110},[107,2089,2090],{"class":424},"__clawpanelWsClient.",[107,2092,2093],{"class":409},"getConnectionInfo",[107,2095,1510],{"class":424},[107,2097,2098],{"class":109,"line":116},[107,2099,2100],{"class":1568},"\u002F\u002F {\n",[107,2102,2103],{"class":109,"line":122},[107,2104,2105],{"class":1568},"\u002F\u002F   connected: false,\n",[107,2107,2108],{"class":109,"line":128},[107,2109,2110],{"class":1568},"\u002F\u002F   reconnectState: 'scheduled' | 'attempting',\n",[107,2112,2113],{"class":109,"line":134},[107,2114,2115],{"class":1568},"\u002F\u002F   reconnectAttempts: 0~60,\n",[107,2117,2118],{"class":109,"line":140},[107,2119,2120],{"class":1568},"\u002F\u002F   ...\n",[107,2122,2123],{"class":109,"line":146},[107,2124,2125],{"class":1568},"\u002F\u002F }\n",[10,2127,2128,2131,2132,2135,2136,2131,2139,2142],{},[14,2129,2130],{},"reconnectState='scheduled'"," 且 ",[14,2133,2134],{},"reconnectAttempts=0"," 表示在慢轮询窗口正常工作；",[14,2137,2138],{},"reconnectState='idle'",[14,2140,2141],{},"connected=false"," 是老的永久断开路径。",[10,2144,2145],{},"重连机制里避免静默终态是最基本的要求。任何异常路径都必须有可见的状态提示和后续操作入口，否则用户端看到的就是死机。这次故障的根本教训是，不能让客户端在任何情况下进入\"不可恢复\"的状态而没有任何提示。慢轮询的引入给了所有失败情景一个\"最后的机会\"，即使前面的快速重连机制彻底耗尽了，用户等待足够长的时间后系统仍有自动恢复的可能。",[341,2147,2148],{},"html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .szBVR, html code.shiki .szBVR{--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sJ8bj, html code.shiki .sJ8bj{--shiki-default:#6A737D;--shiki-dark:#6A737D}",{"title":103,"searchDepth":116,"depth":116,"links":2150},[2151,2156,2157,2158,2159,2160],{"id":1277,"depth":116,"text":1277,"children":2152},[2153,2154,2155],{"id":1284,"depth":122,"text":1285},{"id":1459,"depth":122,"text":1460},{"id":1599,"depth":122,"text":1600},{"id":1667,"depth":116,"text":1667},{"id":227,"depth":116,"text":227},{"id":2007,"depth":116,"text":2007},{"id":2071,"depth":116,"text":2071},{"id":2077,"depth":116,"text":2077},"2026-06-17","从 2026-05-11 开始，多台便携包客户反馈 WebSocket 断线后面板卡死。网关进程（openclaw_gateway.exe）明确在运行，\u002Fhealth 返回 200，但面板显示\"已停止重连，请手动刷新\"——用户必须手动重启才能恢复。",{},"\u002F2026-06-17-fa-016-ws",{"title":1247,"description":2162},"FA-016","2026-06-17-FA-016-WS永久断开三条路径","客户端重连逻辑中三处独立的\"静默终止\"缺陷导致 WS 永久断开，任何一条路径命中就不再调度重连计时器。",[2170,2171,2172,2173,2174],"WebSocket","Node.js","客户端重连机制","异常处理","便携包","qaISKpAwjYnhiSdxEMYYnJtJ-A8QaOq-djOEJ0W9-bM",1785406912384]