[{"data":1,"prerenderedAt":2128},["ShallowReactive",2],{"\u002F2026-06-05-fa-015-ready":3,"\u002F2026-06-05-fa-015-ready-rel":310},{"id":4,"title":5,"body":6,"column":292,"date":293,"description":12,"extension":294,"hero_image":295,"meta":296,"navigation":297,"path":298,"seo":299,"series_id":300,"severity":295,"stem":301,"summary":302,"tags":303,"__hash__":309},"posts\u002F2026-06-05-FA-015-节点Ready但运行时卡死.md","监控全绿。52 个实例已经死了几小时。",{"type":7,"value":8,"toc":281},"minimark",[9,13,16,33,36,39,43,46,49,65,72,76,79,82,85,96,99,102,105,108,112,115,118,121,131,137,163,170,173,180,184,187,208,211,214,217,220,238,241,244,247,252,255,258,278],[10,11,12],"p",{},"凌晨 01:30 左右，某节点（default-9d23b）的监控曲线呈现完整的断崖。CPU 从 70% 跌至 0%，内存从 37% 跌至 3%，该节点上 52 个用户实例的容器集体消失。",[10,14,15],{},"但——",[10,17,18,19,23,24,28,29,32],{},"K8s 控制面全绿。零条告警。kubelet 仍在向 API server 汇报该节点 ",[20,21,22],"strong",{},"Ready","。容器卡在 ",[25,26,27],"code",{},"ContainerCreating"," 或 ",[25,30,31],{},"Terminating","，既创建不出来也删不掉。",[10,34,35],{},"52 个网关同时失效，系统无声无息。直到数小时后收到用户反馈\"连不上\"，才发现已经故障了好几个小时。",[10,37,38],{},"矛盾。监控看起来节点宕了，但 K8s 说它 Ready。宿主机断崖，用户实例无法启动，但没有告警。这才是真正的问题——故障本身可恢复，但看不见的故障无法自愈。",[40,41,42],"h2",{"id":42},"监控数据在说什么",[10,44,45],{},"查共享存储（SFS Turbo）的监控。01:30 时刻写 IOPS 出现尖峰：2800 次\u002F秒，写吞吐 70MB\u002Fs。直观看起来是存储压力导致了节点卡死——节点在做大量 I\u002FO 操作，触发了限流或延迟，进程全部卡住。",[10,47,48],{},"但关键数据打了脸。SFS Turbo 的规格是 30K IOPS \u002F 2GB·s⁻¹。实际使用率：",[50,51,52,59],"ul",{},[53,54,55,56],"li",{},"IOPS：2800 ÷ 30000 = ",[20,57,58],{},"9%",[53,60,61,62],{},"吞吐：70MB\u002Fs ÷ 2GB\u002Fs = ",[20,63,64],{},"3.4%",[10,66,67,68,71],{},"服务端远未饱和。容量用量也只有 14%。这不可能是存储容量不足或聚合性能饱和。所以问题不在存储服务端。在哪呢？",[20,69,70],{},"节点侧","。具体说，该节点的 NFS 客户端（内核网络栈的 NFS 模块）或它到 SFS 的网络链路在 01:30 时 hang 住了。",[40,73,75],{"id":74},"nfs-hard-挂载下的连锁反应","NFS hard 挂载下的连锁反应",[10,77,78],{},"当 NFS 客户端因网络问题或 SFS 暂时无响应而卡死时，所有试图访问这个挂载点的进程会被阻塞在 I\u002FO 操作上，进入 D 状态（disk sleep，不可中断睡眠）。D 状态的进程无法被信号杀死，也无法被超时打断——内核会一直等待 I\u002FO 完成。这个行为在网络稳定的数据中心是安全的，但在任何抖动的环境里都是致命的。",[10,80,81],{},"D 状态的进程很特殊——无法被信号杀死，无法被超时打断，内核会无限期等待 I\u002FO 完成。即便网络彻底断了，进程仍在死等。",[10,83,84],{},"这时宿主机上发生了什么：",[50,86,87,90,93],{},[53,88,89],{},"容器的启动脚本卡在读配置文件",[53,91,92],{},"containerd 的生命周期管理调用卡在存储操作",[53,94,95],{},"kubelet 的 pod 创建\u002F删除操作卡在与 containerd 的 gRPC 通信",[10,97,98],{},"但有个奇特的例外：kubelet 的心跳协程——那个定期向 API server 汇报节点状态的 goroutine——不依赖文件系统操作。它继续活着。每 10 秒一次，向 API server 汇报该节点 Ready。",[10,100,101],{},"从 K8s 的角度看，这个节点一切正常。有心跳，有回应，不存在问题。",[10,103,104],{},"而实际上，任何与 containerd 交互的操作都已无响应。容器创建失败。现有容器无法优雅终止。网关进程要么无法启动（配置文件在 NFS 上读不到），要么启动后立即因存储操作超时而崩溃。",[10,106,107],{},"52 个实例，52 个网关，集体失效。外界完全看不见。",[40,109,111],{"id":110},"为什么-nfs-hang-导致全节点瘫痪","为什么 NFS hang 导致全节点瘫痪",[10,113,114],{},"该节点上的 NFS 挂载点是共享存储卷的唯一入口。一旦这条路被 hang 住，整个节点上所有容器的存储操作（读配置、写日志、挂载 subPath）都会卡死，进而拖累 containerd 的生命周期管理。最终的结果就是：整个节点上的所有容器无法启动、无法优雅停止、无法查询状态，只能处于永久的\"创建中\"或\"删除中\"状态，直到人工介入。",[10,116,117],{},"这是一个级联故障的典型案例：底层的存储客户端出问题 → 容器运行时无法工作 → 整个节点失效 → 上层的 Kubernetes 控制面却仍然说一切正常（因为 kubelet 心跳进程不依赖存储）。",[40,119,120],{"id":120},"根因与修复思路",[10,122,123,124,127,128,130],{},"NFS 挂载使用了默认参数 ",[25,125,126],{},"hard","。",[25,129,126],{}," 的含义：当 NFS 连接中断时，客户端无限重试 I\u002FO 操作，永不超时。这在网络完全可靠的数据中心是安全的。在任何有抖动的网络里，都是定时炸弹。",[10,132,133,134,127],{},"修复：改为 ",[25,135,136],{},"soft,timeo=100,retrans=3,intr",[50,138,139,145,151,157],{},[53,140,141,144],{},[25,142,143],{},"soft","：超时后放弃重试，返回 I\u002FO 错误给应用",[53,146,147,150],{},[25,148,149],{},"timeo=100","：I\u002FO 操作超时 100ms（原默认 600ms，太长）",[53,152,153,156],{},[25,154,155],{},"retrans=3","：放弃前最多重试 3 次（原默认无限）",[53,158,159,162],{},[25,160,161],{},"intr","：允许信号中断 I\u002FO 操作",[10,164,165,166,169],{},"改了以后，当网络或 NFS 客户端出问题时，I\u002FO 操作在 100ms 后返回错误。容器报错退出。关键是：",[20,167,168],{},"进程不进入 D 状态，宿主机不冻死","。ReplicaSet 自动重启容器，或调度器把实例分配到健康节点。故障隔离在单容器，不演变成整个节点瘫痪。",[10,171,172],{},"但这需要滚动重启所有使用该 NFS 卷的实例。一次性代价，可以接受。",[10,174,175,176,179],{},"kubelet 的 Ready 状态检测只验证两件事：kubelet 进程活着 + API server 能收到心跳。它",[20,177,178],{},"不","验证容器运行时是否真的可用、节点的存储链路是否畅通、能否实际创建和销毁容器。当 NFS 客户端 hang 住时，这三个全部失败。但 Ready 信号一点都不知道。这是监控最大的盲点。",[40,181,183],{"id":182},"防回归健康守卫机制","防回归：健康守卫机制",[10,185,186],{},"应该补充一个独立的节点健康守卫机制，定期（每 60 秒）验证：",[188,189,190,198,205],"ol",{},[53,191,192,193,28,195,197],{},"该节点是否有异常堆积的 ",[25,194,27],{},[25,196,31],{}," pod（超过 5 分钟仍未完成）",[53,199,200,201,204],{},"从该节点的各 pod 内部进行网关探测（HTTP 请求实例网关的 ",[25,202,203],{},"\u002Fpanel\u002F"," 端点），检查网关是否可达",[53,206,207],{},"对于发现无响应的节点，进行重试确认，避免误判",[10,209,210],{},"一旦发现节点虽 Ready 但存在大量卡住的 pod 或网关大范围失效，立即触发告警。告警应该同时推送到监控系统和人工值班渠道（钉钉、邮件或短信）。",[10,212,213],{},"后续可以进一步自动化：确认节点真的故障后，自动隔离该节点（cordon），然后驱散它上面的所有实例到健康节点。但这是较高风险的自愈动作，应该先在告警阶段验证机制的准确性，再考虑自动化。",[10,215,216],{},"另外，应该在监控告警规则里补上对宿主机级别异常的检测：CPU \u002F 内存 \u002F 网络指标的断崖式下跌通常表示故障，应该立即告警，而不是等待用户反馈。",[40,218,219],{"id":219},"现场处理",[188,221,222,225,232,235],{},[53,223,224],{},"人工 cordon 该节点，禁止新 pod 调度",[53,226,227,228,231],{},"强制删除所有卡住的 pod（",[25,229,230],{},"kubectl delete pod --grace-period=0 --force","）",[53,233,234],{},"这些 pod 被 ReplicaSet 自动重新调度到健康节点",[53,236,237],{},"原节点标记待重启",[10,239,240],{},"52 个实例逐渐恢复在线。重启时间取决于宿主机厂商的故障排查流程，不受我控制。",[40,242,243],{"id":243},"最后的教训",[10,245,246],{},"故障本身不致命。52 个实例暂时离线是可接受的代价（虽然用户感受很差），可以人工隔离、迁移、恢复。",[10,248,249,127],{},[20,250,251],{},"致命的是不可见加无自愈",[10,253,254],{},"K8s 控制面的 Ready 只代表 kubelet 心跳协程还活着，不代表容器运行时真的可用。当节点侧 NFS 客户端 hang 住时，这个事实在数小时内完全对外界不可见。系统既没自动隔离故障节点，也没自动驱散实例。直到人类察觉到用户反馈，才开始行动。",[10,256,257],{},"后续系统设计缺三不可：",[188,259,260,266,272],{},[53,261,262,265],{},[20,263,264],{},"检测","：定期验证节点运行时的真实可用性，不只依赖心跳信号",[53,267,268,271],{},[20,269,270],{},"告警","：一旦检测到节点虽 Ready 但运行时卡死，立即告警",[53,273,274,277],{},[20,275,276],{},"自愈","：故障确认后，自动隔离节点、驱散实例、释放资源",[10,279,280],{},"没有检测，问题永远看不见。没有告警，人无法及时知道。没有自愈，即使知道了也要等待人工干预，浪费宝贵的恢复时间。这次故障的代价，最终是整个系统的可见性和自愈能力。",{"title":282,"searchDepth":283,"depth":283,"links":284},"",2,[285,286,287,288,289,290,291],{"id":42,"depth":283,"text":42},{"id":74,"depth":283,"text":75},{"id":110,"depth":283,"text":111},{"id":120,"depth":283,"text":120},{"id":182,"depth":283,"text":183},{"id":219,"depth":283,"text":219},{"id":243,"depth":283,"text":243},"故障档案","2026-06-05","md",null,{},true,"\u002F2026-06-05-fa-015-ready",{"title":5,"description":12},"FA-015","2026-06-05-FA-015-节点Ready但运行时卡死","节点 kubelet 虚假 Ready，但容器运行时 hang 住导致 52 个用户实例网关集体失效，数小时无告警。",[304,305,306,307,308],"Kubernetes","NFS","容器运行时","网络故障","节点健康","xmzKVQ7IcVJBJqojkHYyhtx1bDMgLc-5qZ7qnwZOyE8",[311,727,1187],{"id":312,"title":313,"body":314,"column":292,"date":714,"description":318,"extension":294,"hero_image":295,"meta":715,"navigation":297,"path":716,"seo":717,"series_id":718,"severity":295,"stem":719,"summary":720,"tags":721,"__hash__":726},"posts\u002F2026-07-23-FA-018-共享存储的subPath陷阱.md","一个 PVC 装所有用户，当时看不出有什么问题",{"type":7,"value":315,"toc":705},[316,319,322,326,329,332,339,391,394,404,408,414,417,428,431,435,438,441,444,461,464,471,474,489,493,496,499,510,521,524,527,530,598,601,608,611,614,640,643,654,657,660,698,701],[10,317,318],{},"平台从 Docker Swarm 迁移到 Kubernetes，存储架构做了一个看似理想的调整：用共享 PVC + subPath 替代原来的\"每用户独占 PVC\"。方案优势显而易见——管理简单（1 个 PVC 对 574 个）、自动扩展、成本低。当时看不出有什么缺点。",[10,320,321],{},"但实施中遇到了四个隐蔽的问题。",[40,323,325],{"id":324},"权限错配bind-型数据的属主陷阱","权限错配：bind 型数据的属主陷阱",[10,327,328],{},"旧系统中，用户数据通过两种方式持久化：bind mount 和 Docker volume。迁移时需要从这两个来源完整导出数据。",[10,330,331],{},"对于 bind mount 型数据，文件属主通常是 10001（或其他固定 UID）。当我登上旧集群的宿主机，以普通用户身份尝试读取这些文件时，只能看到 13 个文件。权限拒绝。同一目录里其实有 4778 个文件，但大多数因为属主权限不匹配而不可见。",[10,333,334,335,338],{},"只有用 sudo 才能完整读到。所以迁移脚本必须用 ",[25,336,337],{},"sudo tar"," 来打包：",[340,341,345],"pre",{"className":342,"code":343,"language":344,"meta":282,"style":282},"language-bash shiki shiki-themes github-light github-dark","sudo tar -czf \u002Ftmp\u002Fmig\u002F${uid}\u002Fdata.tgz \\\n  -C \u002Fmnt\u002Fyun-claw\u002Fusers\u002F${uid} .\n","bash",[25,346,347,377],{"__ignoreMap":282},[348,349,352,356,360,364,367,371,374],"span",{"class":350,"line":351},"line",1,[348,353,355],{"class":354},"sScJk","sudo",[348,357,359],{"class":358},"sZZnC"," tar",[348,361,363],{"class":362},"sj4cs"," -czf",[348,365,366],{"class":358}," \u002Ftmp\u002Fmig\u002F",[348,368,370],{"class":369},"sVt8B","${uid}",[348,372,373],{"class":358},"\u002Fdata.tgz",[348,375,376],{"class":362}," \\\n",[348,378,379,382,385,388],{"class":350,"line":283},[348,380,381],{"class":362},"  -C",[348,383,384],{"class":358}," \u002Fmnt\u002Fyun-claw\u002Fusers\u002F",[348,386,387],{"class":369},"${uid} ",[348,389,390],{"class":358},".\n",[10,392,393],{},"这不是脚本设计的问题，而是 Kubernetes 环境下容器权限与宿主权限映射的一个陷阱。容器内运行的网关进程可能以容器用户身份运行，但宿主上的文件属主是不同的 UID。当两个权限体系接不上，访问就会失败。",[10,395,396,397,400,401,403],{},"新方案中，用户数据通过 subPath 挂到容器的 ",[25,398,399],{},"\u002Fdata"," 目录。如果 subPath 指向的目录是由 Kubernetes 在宿主机上创建的，属主可能是 root 或其他用户，容器内进程（以容器指定的 UID 运行）仍然会遭遇权限问题。这个风险需要在 Dockerfile 和容器启动时明确处理：容器内应该预先创建 ",[25,402,399],{}," 并设置正确的属主，或者使用 securityContext 的 fsGroup 或 runAsUser 来强制权限。",[40,405,407],{"id":406},"subpath-目录创建的时机与属主处理","subPath 目录创建的时机与属主处理",[10,409,410,411],{},"Kubernetes 对 subPath 的处理有个微妙之处：如果挂载的 subPath 目录在宿主机上不存在，kubelet 会自动创建它。但 ",[348,412,413],{},"待补：这个自动创建的目录的属主是什么？是 root 还是其他用户？容器的 securityContext 能否保证挂载后的权限符合预期？",[10,415,416],{},"设计文档中没有明确说明这一点。在实施前，需要验证：",[50,418,419,422,425],{},[53,420,421],{},"首次 subPath 目录不存在时，kubelet 创建它并将其属主设置为谁",[53,423,424],{},"容器的 securityContext（fsGroup、runAsUser）是否能在挂载时生效",[53,426,427],{},"是否应该预先在共享 PVC 上创建所有 subPath 目录并设置好属主，而不是依赖 Kubernetes 的隐式创建",[10,429,430],{},"没有明确的答案，就容易踩坑。一个保险做法是在 destroy 用户账户时清空 subPath 目录，同时在 provision 时让一个 init-container 验证或修复目录属主，确保容器进程有写权限。",[40,432,434],{"id":433},"单-pvc-中的节点级故障域","单 PVC 中的节点级故障域",[10,436,437],{},"这是最严重的问题，也是与 FA-015（节点假 Ready）的关联点。",[10,439,440],{},"当一个节点的 NFS 客户端或到 SFS 的网络链路发生 hang 时，所有依赖共享 PVC 的 Pod 都会受影响。这不是共享 PVC 本身的问题，而是故障隔离的问题。",[10,442,443],{},"具体现象：",[50,445,446,452,455,458],{},[53,447,448,449,451],{},"如果 NFS 挂载使用了硬 mount（",[25,450,126],{}," 参数是 NFS 的默认值），而网络故障或存储服务中断，NFS 客户端会无限重试",[53,453,454],{},"重试期间，所有试图访问 NFS 的进程都会被阻塞在 I\u002FO 上（D 状态，不可中断）",[53,456,457],{},"如果 kubelet 或 containerd 的某个操作（如创建容器的卷挂载步骤）陷入 I\u002FO 阻塞，整个节点就会冻死",[53,459,460],{},"该节点上的所有实例（无论是否在访问存储）都会因为无法创建或删除 Pod 而故障",[10,462,463],{},"这与独立 PVC 的效果截然不同。旧架构中，每个用户有独立 PVC，意味着如果某个 PVC 对应的存储有问题，只会影响这一个用户。其他用户的 PVC 可能分散在不同的存储卷甚至不同的节点上，相对独立。",[10,465,466,467,470],{},"新架构下，单个共享 PVC 承载所有用户，一旦这个 PVC 对应的网络链路或挂载出问题，",[20,468,469],{},"同节点上所有实例都会被殃及","。如果该节点碰巧堆积了 50 个实例，一次节点级的存储 hang 就会导致 50 个实例集体故障，且外表看起来是\"节点 Ready，但实例卡住\"——kubelet 的心跳正常，Pod 状态却卡在 ContainerCreating 或 Terminating。",[10,472,473],{},"这正是 FA-015 在 2026-06-05 观察到的现象：节点假 Ready，但大量实例网关无响应。防护措施包括：",[50,475,476,483,486],{},[53,477,478,479,482],{},"NFS 挂载参数改为软 mount（",[25,480,481],{},"soft,timeo=100,retrans=3","），让 I\u002FO 超时而不是无限等待",[53,484,485],{},"节点加健康检测，检查 ContainerCreating 堆积数和网关探测失败率，及早发现节点冻死",[53,487,488],{},"实例分散部署，用 topologySpreadConstraints 避免单节点堆积",[40,490,492],{"id":491},"无-per-subpath-硬配额","无 per-subPath 硬配额",[10,494,495],{},"共享存储的架构决定了无法在存储层面按 subPath 限制用户配额。SFS（及 NFS 一般）没有 per-directory 的硬配额功能，只能在整个卷级别限制。",[10,497,498],{},"这意味着：",[50,500,501,504,507],{},[53,502,503],{},"无法阻止某个用户的数据膨胀而挤占其他用户的空间",[53,505,506],{},"无法在存储层面实现\"超额用户的写入失败\"",[53,508,509],{},"只能在应用层进行监控、告警和逻辑控制",[10,511,512,513,516,517,520],{},"新系统的方案是应用层监控：定期 ",[25,514,515],{},"du -sb \u002Fdata"," 扫描每个用户的目录大小，落库到 Instance 表，",[25,518,519],{},"\u002Fstatus"," 接口返回超额标志位。前端据此提示用户\"存储已满\"。这是纯监控，不阻断。如果用户无视提示继续写入，直到共享卷真的满了，才会出现\"所有用户集体写入失败\"的惨淡局面。",[10,522,523],{},"这个限制是方案选择的代价。",[40,525,526],{"id":526},"为什么仍然选了这个方案",[10,528,529],{},"尽管有这些问题，平台仍然采用了共享 PVC + subPath 的方案。原因是对比了替代方案：",[531,532,533,552],"table",{},[534,535,536],"thead",{},[537,538,539,543,546,549],"tr",{},[540,541,542],"th",{},"方案",[540,544,545],{},"优点",[540,547,548],{},"缺点",[540,550,551],{},"故障隔离",[553,554,555,570,584],"tbody",{},[537,556,557,561,564,567],{},[558,559,560],"td",{},"独立 PVC（旧架构）",[558,562,563],{},"天然的每用户隔离；故障域清晰",[558,565,566],{},"管理复杂（574 个 PVC）；扩展性差；成本高",[558,568,569],{},"✓ 最好",[537,571,572,575,578,581],{},[558,573,574],{},"共享 PVC + subPath（新架构）",[558,576,577],{},"管理简单（1 个 PVC）；自动扩展；成本低",[558,579,580],{},"故障隔离性差；无硬配额；权限管理复杂",[558,582,583],{},"✗ 较差",[537,585,586,589,592,595],{},[558,587,588],{},"对象存储（S3\u002FOSS）",[558,590,591],{},"真正的多租户隔离；天然分布",[558,593,594],{},"延迟高；成本更高；应用改造大",[558,596,597],{},"✓ 最好，但代价大",[10,599,600],{},"独立 PVC 的管理开销是关键问题。574 个用户意味着 574 个 PVC 对象、574 条 PV 绑定、574 个存储卷。每次实例创建、删除或迁移都要涉及 PVC 的生命周期管理。扩展到 5000 用户时，这种开销会成为瓶颈。对象存储虽然隔离性最好，但需要应用层改造（兼容 S3 API、处理延迟、调整备份策略），而且成本更高。",[10,602,603,604,607],{},"共享 PVC 方案的核心优势是",[20,605,606],{},"运维简洁","：增删用户只需改 subPath（一行 YAML），不涉及存储层操作。代价是故障隔离性从\"用户级\"降到\"节点级\"。",[40,609,610],{"id":610},"适用边界",[10,612,613],{},"这个方案适合以下场景：",[50,615,616,622,628,634],{},[53,617,618,621],{},[20,619,620],{},"用户数量有上限","（几百到几千）。用户过多时，共享卷的单点压力会成问题。",[53,623,624,627],{},[20,625,626],{},"用户数据量可控","（单用户通常 GB 级）。如果单用户数据量达 TB，一次 du 扫描会拖累整体。",[53,629,630,633],{},[20,631,632],{},"可以接受节点级故障隔离","。只要网络和存储配置足够稳定（硬化 NFS 参数、冗余链路），节点 hang 的概率不会很高。",[53,635,636,639],{},[20,637,638],{},"能够实施应用层监控和告警","。无硬配额，就必须有实时监控。",[10,641,642],{},"不适合的场景包括：",[50,644,645,648,651],{},[53,646,647],{},"超大规模用户（万级以上）",[53,649,650],{},"用户数据特别不均衡（少数用户占大头）的场景",[53,652,653],{},"对故障隔离要求极高的系统（比如金融交易）",[40,655,656],{"id":656},"防护与调整",[10,658,659],{},"基于这四个问题，实施中做了以下调整：",[188,661,662,671,680,686,692],{},[53,663,664,667,668,670],{},[20,665,666],{},"权限处理","：容器 Dockerfile 中预先创建 ",[25,669,399],{}," 并设置正确的属主；启动脚本检查并修复权限。",[53,672,673,676,677,679],{},[20,674,675],{},"NFS 参数硬化","：StorageClass 的挂载选项改为 ",[25,678,136],{},"，避免硬 mount 导致节点冻。",[53,681,682,685],{},[20,683,684],{},"节点健康检测","：cron 每 60 秒检查各节点的 ContainerCreating 堆积和网关探测失败率，及早发现故障。",[53,687,688,691],{},[20,689,690],{},"实例分散","：StatefulSet 加 topologySpreadConstraints，避免单节点堆积 50+ 个实例。",[53,693,694,697],{},[20,695,696],{},"应用层配额","：\u002Fstatus 接口返回 diskUsedMi 和 overQuota 标志位，前端告警用户。",[10,699,700],{},"这些措施不能消除风险，但可以大幅降低故障概率和影响范围。",[702,703,704],"style",{},"html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":282,"searchDepth":283,"depth":283,"links":706},[707,708,709,710,711,712,713],{"id":324,"depth":283,"text":325},{"id":406,"depth":283,"text":407},{"id":433,"depth":283,"text":434},{"id":491,"depth":283,"text":492},{"id":526,"depth":283,"text":526},{"id":610,"depth":283,"text":610},{"id":656,"depth":283,"text":656},"2026-07-23",{},"\u002F2026-07-23-fa-018-subpath",{"title":313,"description":318},"FA-018","2026-07-23-FA-018-共享存储的subPath陷阱","K8s 共享 PVC + subPath 方案在实施中暴露的四个陷阱：权限不匹配、目录创建时机、节点级故障域、无 per-subPath 配额。",[304,722,723,724,305,725,551],"存储","subPath","PVC","权限","4pJJyzbII4OOc9jJS74HUpUfyVc9CfxdE5ovso8ylSE",{"id":728,"title":729,"body":730,"column":292,"date":1171,"description":1172,"extension":294,"hero_image":295,"meta":1173,"navigation":297,"path":1174,"seo":1175,"series_id":1176,"severity":295,"stem":1177,"summary":1178,"tags":1179,"__hash__":1186},"posts\u002F2026-07-11-FA-017-长任务异步化.md","三个超时机制，一起把长任务掐死了",{"type":7,"value":731,"toc":1163},[732,742,745,748,754,760,766,773,779,783,786,789,797,800,811,817,821,824,831,837,840,855,859,862,873,922,937,944,948,951,954,972,975,983,986,1001,1004,1007,1154,1157,1160],[10,733,734,735,738,739,127],{},"数字人口播的「分析」功能从视频提取口播文稿、分镜脚本、结构卖点，需要下载视频、压缩、调 Gemini 视觉 API，耗时从几十秒到几分钟。最初同步实现：前端调 ",[25,736,737],{},"POST \u002Fapi\u002Fworkflow\u002Fdub\u002Fanalyze-parsed","，后端直接跑完再返回。这个方案暴露的问题不是单点，而是",[20,740,741],{},"三个独立的缺陷叠加",[10,743,744],{},"网关超时、CDN 空闲切断、客户端重试——这三个没有一个是\"bug\"，每个都是合理的系统设计。但组合在一起就是连锁灾难：客户端看不到结果（超时或连接断），认为失败了重试，后端却已经扣过费了；或者分析其实已跑完，但因为响应发不出去，重试又跑了一遍。退款时无法幂等判断，某些用户最终被多扣了好几倍。",[10,746,747],{},"具体来说，三个问题各自是什么：",[10,749,750,753],{},[20,751,752],{},"第一个问题是网关超时","。云平台的入口网关（API Gateway）有一个固定的读超时，通常设为几十秒。当后端分析耗时超过这个阈值，网关主动断连接，客户端收到 504 或连接重置，但后端的分析进程并不知道客户端已经走了，继续跑完整个流程。如果分析结果的退款逻辑放在 try-catch 的 finally 里，此时 HTTP 响应已发不出去，客户就看到\"分析失败\"，但款已经扣了。",[10,755,756,759],{},[20,757,758],{},"第二个问题是 CDN 空闲切断","。不少 CDN 和负载均衡层有这样的策略：如果一条 HTTP 连接在某个时间段内没有数据往返，就认为它已死而主动关闭。这与 FA-013 那次踩到的问题一样——长连接因为没有心跳数据而被 CDN 中间件切断，导致请求被突然中断。这个机制在分析请求上也会造成麻烦：如果分析用时较长，连接就可能被切，响应发不出去。",[10,761,762,765],{},[20,763,764],{},"第三个问题是客户端重试导致重复扣费","。当客户端没有收到响应（网关超时或 CDN 切断），一般会重试这个请求。问题在于这次重试是一条完全独立的请求，后端会把它当作新任务处理，再次扣费。而且如果第一次的分析实际上已经跑完了，就会出现\"分析跑了两遍、扣费扣了两遍、用户只看到了一个结果\"的情况。如果第一次分析中途被 CDN 切了，第二次重试重新开始，那扣费可能是三倍。而退款的时候，因为业务流程上没有幂等设计，很难追溯哪个 operationId 应该被退，哪个不应该。",[10,767,768,769,772],{},"这三个问题是",[20,770,771],{},"叠加的","，不是独立的。它们共同指向一个根本的架构问题。",[10,774,775,776,127],{},"根本原因一个：",[20,777,778],{},"不该让耗时任务挂在 HTTP 连接上",[40,780,782],{"id":781},"异步任务是必需的不是可选的","异步任务是必需的，不是可选的",[10,784,785],{},"解决方案很简单：把分析改成异步任务模式。",[10,787,788],{},"前端提交分析请求直接返回任务 ID（HTTP 耗时极短：校验参数、建数据库记录、预扣费）。后端后台 worker 跑分析重逻辑，完成后持久化结果。前端轮询查询状态，得到结果后停止。",[340,790,795],{"className":791,"code":793,"language":794},[792],"language-text","客户端              后端\n  |                 |\n  +--提交---------->|\n  |             创建任务，返回 ID\n  |\u003C--返回 ID-----+\n  |                 |（后台运行分析）\n  |                 |\n  +--轮询--------->|\n  |             查询状态\n  |\u003C--返回状态---+\n  |                 |\n  |（重复轮询）\n  +--轮询--------->|\n  |             查询状态\n  |\u003C--返回结果---+\n  |                 |\n","text",[25,796,793],{"__ignoreMap":282},[10,798,799],{},"好处显而易见：",[50,801,802,805,808],{},[53,803,804],{},"单次 HTTP 请求的耗时从分钟级降到秒级，不会触发网关或 CDN 的超时。",[53,806,807],{},"后端分析的运行生命周期与 HTTP 连接完全解耦。即使连接在中途被切，已经启动的分析会继续在后台跑，结果最终还是会被保存。",[53,809,810],{},"客户端重试不会产生新的分析任务。只要实现幂等判重，同一个请求在短时间内重复提交也只会产生一个任务。",[10,812,813,814,127],{},"但异步化不是万灵药，它把问题转移到了三个新的维度上：",[20,815,816],{},"幂等、持久化、失败语义",[40,818,820],{"id":819},"幂等同一请求只产生一个任务","幂等：同一请求只产生一个任务",[10,822,823],{},"假设客户端因为网络抖动，在短时间内连续发了两次\"开始分析\"请求，后端收到两条独立的 HTTP 请求。如果没有幂等设计，就会创建两个任务、扣两次费。",[10,825,826,827,830],{},"幂等的实现最直接的办法是在",[20,828,829],{},"提交阶段就判重","：为这个用户、这个资源设置一个\"只有一个进行中的分析任务\"的约束。素材中的设计是这样做的：",[340,832,835],{"className":833,"code":834,"language":794},[792],"if 该用户已有 status=running && kind=analyze_parsed 的任务:\n    return 409 Conflict（分析已在进行中）\n",[25,836,834],{"__ignoreMap":282},[10,838,839],{},"这样双击提交同一个视频，第一次返回 201 + taskId，第二次返回 409，客户端看到 409 就知道不要再建新任务，拿之前返回的 ID 去轮询。",[10,841,842,843,846,847,850,851,854],{},"另一个幂等层次是",[20,844,845],{},"计费操作的幂等","。预扣费的时候用一个全局唯一的 ",[25,848,849],{},"operationId","（比如 ",[25,852,853],{},"dub-analyze:{uuid}","），这个 ID 关联了这次预扣。如果扣费 API 被重复调用，因为 operationId 重复，系统只会扣一次。退款时也用同一个 operationId，保证退款与预扣能正确配对。",[40,856,858],{"id":857},"持久化进程重启后任务不丢","持久化：进程重启后任务不丢",[10,860,861],{},"后台分析 worker 可能因为发版、机器重启、容器被杀等原因在中途退出。如果任务的状态只保存在内存里，进程一死任务就丢了。",[10,863,864,865,868,869,872],{},"持久化的关键是",[20,866,867],{},"任务状态表","。素材中复用了既有的 ",[25,870,871],{},"SkyhumanTask"," 表，包含字段：",[50,874,875,881,887,892,898,904,910,916],{},[53,876,877,880],{},[25,878,879],{},"id",": 任务 ID",[53,882,883,886],{},[25,884,885],{},"status",": 进度（running \u002F completed \u002F failed）",[53,888,889,891],{},[25,890,849],{},": 幂等 ID",[53,893,894,897],{},[25,895,896],{},"chargedPoints",": 预扣费用",[53,899,900,903],{},[25,901,902],{},"resultPayload",": 分析结果的 JSON",[53,905,906,909],{},[25,907,908],{},"error",": 失败原因",[53,911,912,915],{},[25,913,914],{},"updatedAt",": 最后更新时间戳",[53,917,918,921],{},[25,919,920],{},"sourceObjectKey",": 源视频对象路径（用于幂等判重和重试）",[10,923,924,925,928,929,932,933,936],{},"任务创建时插一条 ",[25,926,927],{},"status=running"," 的记录。后台 worker 定期通过 heartbeat（每 30 秒做一次空 update，仅触碰 updatedAt）来证明自己还活着。分析完成时更新为 ",[25,930,931],{},"status=completed, resultPayload=...","；失败时更新为 ",[25,934,935],{},"status=failed, error=..."," 并触发退款。",[10,938,939,940,943],{},"这样即使 worker 进程突然死了，任务记录还在数据库里。新启起的 worker 可以扫描表里的 ",[25,941,942],{},"status=running && updatedAt 过期"," 的任务，判断它们已经没有 worker 在跑了，执行清理逻辑。",[40,945,947],{"id":946},"失败语义区分任务失败和查询失败","失败语义：区分\"任务失败\"和\"查询失败\"",[10,949,950],{},"异步设计引入了一个新的错误维度：客户端查询任务状态时得到的 404，可能是\"这个任务 ID 不存在\"（用户输错了），也可能是\"任务 ID 之前存在但已被清理\"（进程清理了过期数据）。这两种情况的含义很不一样。",[10,952,953],{},"更重要的是，前端轮询收到的错误需要区分是否应该重试：",[50,955,956,966],{},[53,957,958,961,962,965],{},[20,959,960],{},"任务本身失败","（",[25,963,964],{},"status=failed, error=\"Gemini API 返回错误\"","）：这种情况不该重试，因为重试也会失败。应该直接向用户展示失败信息，并根据 operationId 触发退款。",[53,967,968,971],{},[20,969,970],{},"查询接口失败","（500、网络超时）：这种情况应该重试查询，因为任务本身可能还在继续跑。",[10,973,974],{},"素材中的设计把这两类区分得很清楚：",[50,976,977,980],{},[53,978,979],{},"任务最终的状态（completed 或 failed）持久化到数据库，客户端查询会拿到明确的业务语义。",[53,981,982],{},"查询接口的 HTTP 错误（5xx）是技术层故障，客户端应该重试。",[10,984,985],{},"同时，后端的 reaper 机制（定期扫表清理超期任务）也遵循这个语义：",[50,987,988,994],{},[53,989,990,991,993],{},"如果一个 ",[25,992,927],{}," 的任务超过 90 秒没有 heartbeat 更新，说明 worker 已死，reaper 会强制置为 failed 并退款。",[53,995,996,997,1000],{},"但这个强制置为 failed 不应该抛错，因为此时可能有其他进程也想更新这个任务。更新语句应该带条件：",[25,998,999],{},"UPDATE SkyhumanTask SET status=failed WHERE id=? AND status=running","，这样如果 reaper 和其他 worker 同时操作，只有一方会成功。",[40,1002,1003],{"id":1003},"具体实现模式",[10,1005,1006],{},"数字人口播的分析异步化采用了这样的模式：",[188,1008,1009,1054,1101,1124],{},[53,1010,1011,1014,1015],{},[20,1012,1013],{},"提交阶段","（同步）：",[50,1016,1017,1020,1023,1029,1035,1042,1048],{},[53,1018,1019],{},"校验用户和资源",[53,1021,1022],{},"判重：是否已有进行中的任务（409）",[53,1024,1025,1028],{},[25,1026,1027],{},"getObject"," 拉视频，检测时长（若≤0 返 400 不扣费）",[53,1030,1031,1034],{},[25,1032,1033],{},"chargeResource"," 预扣费用，获得 operationId",[53,1036,1037,1038,1041],{},"创建 ",[25,1039,1040],{},"SkyhumanTask{ status:running, operationId, sourceObjectKey }"," 记录",[53,1043,1044,1047],{},[25,1045,1046],{},"scheduleTask"," 丢到后台队列",[53,1049,1050,1051],{},"返回 ",[25,1052,1053],{},"{ taskId }",[53,1055,1056,1059,1060],{},[20,1057,1058],{},"后台运行阶段","：",[50,1061,1062,1065,1068,1075,1081,1091,1098],{},[53,1063,1064],{},"Worker 取出任务记录",[53,1066,1067],{},"开启 heartbeat 定时器（每 30s update 一次 updatedAt）",[53,1069,1070,1071,1074],{},"执行 ",[25,1072,1073],{},"analyzeDubVisionOnly","（下载→压缩→调用 Gemini→解析）",[53,1076,1077,1078],{},"成功：",[25,1079,1080],{},"update( status:completed, resultPayload )",[53,1082,1083,1084,1087,1088],{},"失败：",[25,1085,1086],{},"update( status:failed, error )"," + ",[25,1089,1090],{},"refundResource(operationId)",[53,1092,1093,1094,1097],{},"所有 update 都带条件 ",[25,1095,1096],{},"WHERE id=? AND status=running","（防被 reaper 覆盖）",[53,1099,1100],{},"Finally 清理 heartbeat 定时器",[53,1102,1103,1106,1107],{},[20,1104,1105],{},"查询阶段","（前端轮询）：",[50,1108,1109,1118,1121],{},[53,1110,1111,1114,1115],{},[25,1112,1113],{},"GET \u002Fapi\u002Fworkflow\u002Fdub\u002Ftasks\u002F:id"," 返回 ",[25,1116,1117],{},"{ status, resultPayload, error }",[53,1119,1120],{},"若 status=completed 或 failed，停止轮询",[53,1122,1123],{},"若查询接口返回 5xx，则后续重试",[53,1125,1126,1129,1130],{},[20,1127,1128],{},"清理阶段","（reaper）：",[50,1131,1132,1135,1142,1145,1151],{},[53,1133,1134],{},"后台定期扫表",[53,1136,1137,1138,1141],{},"对于 ",[25,1139,1140],{},"status=running && updatedAt > 90s"," 的任务，判定 worker 已死",[53,1143,1144],{},"若 providerTaskId（这里没有）存在，调用上游的 finalize API",[53,1146,1147,1148,1150],{},"若不存在，直接 ",[25,1149,1090],{}," + 置 failed",[53,1152,1153],{},"本设计无 providerTaskId，所以 reaper 自动走退款分支，无需改动 reaper 代码",[10,1155,1156],{},"这个模式的关键是三道防线：提交时判重（409）、后台心跳（定期 touch）、reaper 兜底（自动退款）。任何一个环节出问题，都有后续环节来补救。",[40,1158,1159],{"id":1159},"防线缺一不可",[10,1161,1162],{},"长耗时任务不能挂在 HTTP 连接上不只是超时问题，而是连接脆弱性与重复处理的共谋。异步化表面是\"返回 ID、后台跑\"这一步，真正的难点在三个维度的同时保证：幂等（不重复扣费）、持久化（不丢数据）、失败语义（不破坏重试逻辑）。缺一就会在某个场景暴露。",{"title":282,"searchDepth":283,"depth":283,"links":1164},[1165,1166,1167,1168,1169,1170],{"id":781,"depth":283,"text":782},{"id":819,"depth":283,"text":820},{"id":857,"depth":283,"text":858},{"id":946,"depth":283,"text":947},{"id":1003,"depth":283,"text":1003},{"id":1159,"depth":283,"text":1159},"2026-07-11","数字人口播的「分析」功能从视频提取口播文稿、分镜脚本、结构卖点，需要下载视频、压缩、调 Gemini 视觉 API，耗时从几十秒到几分钟。最初同步实现：前端调 POST \u002Fapi\u002Fworkflow\u002Fdub\u002Fanalyze-parsed，后端直接跑完再返回。这个方案暴露的问题不是单点，而是三个独立的缺陷叠加。",{},"\u002F2026-07-11-fa-017",{"title":729,"description":1172},"FA-017","2026-07-11-FA-017-长任务异步化","长耗时分析请求（几十秒到分钟级）用同步 HTTP 导致网关超时、CDN切断、客户端重试重复扣费。改异步任务需同时解决幂等、持久化、失败语义。",[1180,1181,1182,1183,1184,1185],"异步任务","HTTP超时","CDN","幂等","任务队列","微服务","ARFDvBMGX3tw-E3Rj_gw1RYEjrykplO6u1dxtdS7Uto",{"id":1188,"title":1189,"body":1190,"column":292,"date":2113,"description":2114,"extension":294,"hero_image":295,"meta":2115,"navigation":297,"path":2116,"seo":2117,"series_id":2118,"severity":295,"stem":2119,"summary":2120,"tags":2121,"__hash__":2127},"posts\u002F2026-06-17-FA-016-WS永久断开三条路径.md","日志里没有重连记录——它根本没在重连",{"type":7,"value":1191,"toc":2101},[1192,1203,1210,1217,1220,1223,1228,1235,1315,1321,1391,1401,1405,1408,1534,1545,1549,1555,1610,1613,1616,1619,1634,1637,1640,1643,1650,1908,1918,1921,1931,1944,1954,1957,1960,1967,2018,2021,2024,2027,2030,2033,2078,2095,2098],[10,1193,1194,1195,1198,1199,1202],{},"从 2026-05-11 开始，多台便携包客户反馈 WebSocket 断线后面板卡死。网关进程（",[25,1196,1197],{},"openclaw_gateway.exe","）明确在运行，",[25,1200,1201],{},"\u002Fhealth"," 返回 200，但面板显示\"已停止重连，请手动刷新\"——用户必须手动重启才能恢复。",[10,1204,1205,1206,1209],{},"关键线索在控制台：看不到任何 ",[25,1207,1208],{},"[ws] 计划重连"," 的日志。",[10,1211,1212,1213,1216],{},"通常断线后客户端会频繁尝试重连，日志里应该是满屏的重连记录。没有日志意味着问题不在重连失败，而在",[20,1214,1215],{},"压根没启动重连机制","。说明客户端进入了某个永久休眠状态。",[40,1218,1219],{"id":1219},"三条独立的永久断开路径",[10,1221,1222],{},"代码审查发现了三条完全不同的、独立的永久断开路径。任意一条命中，就会停止调度任何重连计时器。",[1224,1225,1227],"h3",{"id":1226},"路径-a凭据刷新异常","路径 A：凭据刷新异常",[10,1229,1230,1231,1234],{},"在 ",[25,1232,1233],{},"_scheduleReconnect"," 方法末尾，每次普通重连都会调用：",[340,1236,1240],{"className":1237,"code":1238,"language":1239,"meta":282,"style":282},"language-javascript shiki shiki-themes github-light github-dark","this._reconnectTimer = setTimeout(() => {\n  if (!this._intentionalClose) {\n    this._refreshCredentialsAndReconnect(0)\n  }\n}, delay)\n","javascript",[25,1241,1242,1266,1282,1303,1309],{"__ignoreMap":282},[348,1243,1244,1247,1250,1254,1257,1260,1263],{"class":350,"line":351},[348,1245,1246],{"class":362},"this",[348,1248,1249],{"class":369},"._reconnectTimer ",[348,1251,1253],{"class":1252},"szBVR","=",[348,1255,1256],{"class":354}," setTimeout",[348,1258,1259],{"class":369},"(() ",[348,1261,1262],{"class":1252},"=>",[348,1264,1265],{"class":369}," {\n",[348,1267,1268,1271,1274,1277,1279],{"class":350,"line":283},[348,1269,1270],{"class":1252},"  if",[348,1272,1273],{"class":369}," (",[348,1275,1276],{"class":1252},"!",[348,1278,1246],{"class":362},[348,1280,1281],{"class":369},"._intentionalClose) {\n",[348,1283,1285,1288,1291,1294,1297,1300],{"class":350,"line":1284},3,[348,1286,1287],{"class":362},"    this",[348,1289,1290],{"class":369},".",[348,1292,1293],{"class":354},"_refreshCredentialsAndReconnect",[348,1295,1296],{"class":369},"(",[348,1298,1299],{"class":362},"0",[348,1301,1302],{"class":369},")\n",[348,1304,1306],{"class":350,"line":1305},4,[348,1307,1308],{"class":369},"  }\n",[348,1310,1312],{"class":350,"line":1311},5,[348,1313,1314],{"class":369},"}, delay)\n",[10,1316,1317,1318,1320],{},"而 ",[25,1319,1293],{}," 是异步方法，catch 分支没有任何后续处理：",[340,1322,1324],{"className":1237,"code":1323,"language":1239,"meta":282,"style":282},"} catch (e) {\n  console.error('[ws] 刷新凭据失败:', e)\n  this._setConnected(false, 'error', `凭据刷新失败: ${e}`)\n}\n",[25,1325,1326,1337,1352,1386],{"__ignoreMap":282},[348,1327,1328,1331,1334],{"class":350,"line":351},[348,1329,1330],{"class":369},"} ",[348,1332,1333],{"class":1252},"catch",[348,1335,1336],{"class":369}," (e) {\n",[348,1338,1339,1342,1344,1346,1349],{"class":350,"line":283},[348,1340,1341],{"class":369},"  console.",[348,1343,908],{"class":354},[348,1345,1296],{"class":369},[348,1347,1348],{"class":358},"'[ws] 刷新凭据失败:'",[348,1350,1351],{"class":369},", e)\n",[348,1353,1354,1357,1359,1362,1364,1367,1370,1373,1375,1378,1381,1384],{"class":350,"line":1284},[348,1355,1356],{"class":362},"  this",[348,1358,1290],{"class":369},[348,1360,1361],{"class":354},"_setConnected",[348,1363,1296],{"class":369},[348,1365,1366],{"class":362},"false",[348,1368,1369],{"class":369},", ",[348,1371,1372],{"class":358},"'error'",[348,1374,1369],{"class":369},[348,1376,1377],{"class":358},"`凭据刷新失败: ${",[348,1379,1380],{"class":369},"e",[348,1382,1383],{"class":358},"}`",[348,1385,1302],{"class":369},[348,1387,1388],{"class":350,"line":1305},[348,1389,1390],{"class":369},"}\n",[10,1392,1393,1394,28,1397,1400],{},"如果 ",[25,1395,1396],{},"api.readOpenclawConfig()",[25,1398,1399],{},"api.autoPairDevice()"," 抛错——磁盘 IO 抖、Rust 端处理器忙、Tauri IPC 队列阻塞——这次重连就永久终止。便携磁盘的 IO 抖动 + 配置文件读写争抢时最容易发生。",[1224,1402,1404],{"id":1403},"路径-b认证失败后的强制关闭","路径 B：认证失败后的强制关闭",[10,1406,1407],{},"当 WebSocket 收到 1008 unauthorized 响应时的处理逻辑：",[340,1409,1411],{"className":1237,"code":1410,"language":1239,"meta":282,"style":282},"if (this._authRetryCount \u003C 2) {\n  this._authRetryCount++\n  this._refreshCredentialsAndReconnect()\n  return\n}\nthis._setConnected(false, 'auth_failed', `认证失败: ${e.reason}。请检查 Gateway Token 配置。`)\nthis._intentionalClose = true   \u002F\u002F ← 永久关闭标记\nthis._flushPending()\nreturn\n",[25,1412,1413,1434,1444,1455,1460,1464,1499,1516,1528],{"__ignoreMap":282},[348,1414,1415,1418,1420,1422,1425,1428,1431],{"class":350,"line":351},[348,1416,1417],{"class":1252},"if",[348,1419,1273],{"class":369},[348,1421,1246],{"class":362},[348,1423,1424],{"class":369},"._authRetryCount ",[348,1426,1427],{"class":1252},"\u003C",[348,1429,1430],{"class":362}," 2",[348,1432,1433],{"class":369},") {\n",[348,1435,1436,1438,1441],{"class":350,"line":283},[348,1437,1356],{"class":362},[348,1439,1440],{"class":369},"._authRetryCount",[348,1442,1443],{"class":1252},"++\n",[348,1445,1446,1448,1450,1452],{"class":350,"line":1284},[348,1447,1356],{"class":362},[348,1449,1290],{"class":369},[348,1451,1293],{"class":354},[348,1453,1454],{"class":369},"()\n",[348,1456,1457],{"class":350,"line":1305},[348,1458,1459],{"class":1252},"  return\n",[348,1461,1462],{"class":350,"line":1311},[348,1463,1390],{"class":369},[348,1465,1467,1469,1471,1473,1475,1477,1479,1482,1484,1487,1489,1491,1494,1497],{"class":350,"line":1466},6,[348,1468,1246],{"class":362},[348,1470,1290],{"class":369},[348,1472,1361],{"class":354},[348,1474,1296],{"class":369},[348,1476,1366],{"class":362},[348,1478,1369],{"class":369},[348,1480,1481],{"class":358},"'auth_failed'",[348,1483,1369],{"class":369},[348,1485,1486],{"class":358},"`认证失败: ${",[348,1488,1380],{"class":369},[348,1490,1290],{"class":358},[348,1492,1493],{"class":369},"reason",[348,1495,1496],{"class":358},"}。请检查 Gateway Token 配置。`",[348,1498,1302],{"class":369},[348,1500,1502,1504,1507,1509,1512],{"class":350,"line":1501},7,[348,1503,1246],{"class":362},[348,1505,1506],{"class":369},"._intentionalClose ",[348,1508,1253],{"class":1252},[348,1510,1511],{"class":362}," true",[348,1513,1515],{"class":1514},"sJ8bj","   \u002F\u002F ← 永久关闭标记\n",[348,1517,1519,1521,1523,1526],{"class":350,"line":1518},8,[348,1520,1246],{"class":362},[348,1522,1290],{"class":369},[348,1524,1525],{"class":354},"_flushPending",[348,1527,1454],{"class":369},[348,1529,1531],{"class":350,"line":1530},9,[348,1532,1533],{"class":1252},"return\n",[10,1535,1536,1537,1540,1541,1544],{},"设置 ",[25,1538,1539],{},"_intentionalClose = true"," 之后，所有重连逻辑都被 ",[25,1542,1543],{},"if (!this._intentionalClose)"," 短路。即使只是 Gateway 重启窗口碰好出现 token 短暂不匹配，也会一次性把客户端打死，必须刷新页面才能恢复。",[1224,1546,1548],{"id":1547},"路径-c快速重连配额耗尽","路径 C：快速重连配额耗尽",[10,1550,1551,1552,1059],{},"定义了常数 ",[25,1553,1554],{},"MAX_RECONNECT_ATTEMPTS = 60",[340,1556,1558],{"className":1237,"code":1557,"language":1239,"meta":282,"style":282},"if (this._reconnectAttempts >= MAX_RECONNECT_ATTEMPTS) {\n  this._setConnected(false, 'error', `连接失败，已停止重连。请手动刷新页面重试。`)\n  return\n}\n",[25,1559,1560,1579,1602,1606],{"__ignoreMap":282},[348,1561,1562,1564,1566,1568,1571,1574,1577],{"class":350,"line":351},[348,1563,1417],{"class":1252},[348,1565,1273],{"class":369},[348,1567,1246],{"class":362},[348,1569,1570],{"class":369},"._reconnectAttempts ",[348,1572,1573],{"class":1252},">=",[348,1575,1576],{"class":362}," MAX_RECONNECT_ATTEMPTS",[348,1578,1433],{"class":369},[348,1580,1581,1583,1585,1587,1589,1591,1593,1595,1597,1600],{"class":350,"line":283},[348,1582,1356],{"class":362},[348,1584,1290],{"class":369},[348,1586,1361],{"class":354},[348,1588,1296],{"class":369},[348,1590,1366],{"class":362},[348,1592,1369],{"class":369},[348,1594,1372],{"class":358},[348,1596,1369],{"class":369},[348,1598,1599],{"class":358},"`连接失败，已停止重连。请手动刷新页面重试。`",[348,1601,1302],{"class":369},[348,1603,1604],{"class":350,"line":1284},[348,1605,1459],{"class":1252},[348,1607,1608],{"class":350,"line":1305},[348,1609,1390],{"class":369},[10,1611,1612],{},"客户机晚上挂机，Gateway 因为便携磁盘 GC 或 Windows 休眠争抢资源短暂掉线，客户端尝试 60 次仍未成功重连，就永久停摆。早上用户回来面板已成死链。",[40,1614,1615],{"id":1615},"为什么三条缺陷同时存在",[10,1617,1618],{},"这是逐步累加的历史债：",[50,1620,1621,1628,1631],{},[53,1622,1623,1624,1627],{},"路径 B 是在修复\"避免无限自动配对循环\"时添加的保护，但用错了对象——",[25,1625,1626],{},"_intentionalClose=true"," 本来是用户主动断开的语义，不该用在被动失败上",[53,1629,1630],{},"路径 C 是早期\"避免无穷重试\"的防护，但 60 次后完全放弃而不留任何复活路径是绝对错误",[53,1632,1633],{},"路径 A 是在\"凭据 reload\"改动时把所有重连都改走 refreshCredentials，没留意 catch 分支已经成了终态",[10,1635,1636],{},"三条各自独立、都能单独打死客户端，组合在一起命中率非常高。",[40,1638,1639],{"id":1639},"修复方案",[10,1641,1642],{},"核心思想是永不彻底放弃。任何\"快速重连配额耗尽\"的分支都转入慢轮询，留出窗口让用户改配置或等待 Gateway 恢复后能自动复连。",[10,1644,1645,1646,1649],{},"新增辅助方法 ",[25,1647,1648],{},"_schedulePoll(delayMs, kind)"," 作为慢轮询的触发器：",[340,1651,1653],{"className":1237,"code":1652,"language":1239,"meta":282,"style":282},"const AUTH_RETRY_LIMIT = 2\nconst SLOW_POLL_DELAY_AUTH = 60_000      \u002F\u002F 认证持续失败：1 分钟探一次\nconst SLOW_POLL_DELAY_GENERAL = 300_000  \u002F\u002F 一般持续失败：5 分钟探一次\n\n_schedulePoll(delayMs, kind) {\n  this._clearReconnectTimer()\n  this._reconnectAttempts = 0\n  if (kind === 'auth') this._authRetryCount = 0\n  this._reconnectState = 'scheduled'\n  this._pendingReconnect = true\n  this._reconnectTimer = setTimeout(() => {\n    this._reconnectTimer = null\n    if (this._intentionalClose) return\n    this._reconnectState = 'attempting'\n    if (kind === 'auth') {\n      this._refreshCredentialsAndReconnect(0)\n    } else {\n      this._doConnect()\n    }\n  }, delayMs)\n}\n",[25,1654,1655,1669,1684,1699,1704,1712,1723,1734,1758,1770,1783,1800,1812,1827,1839,1852,1868,1879,1891,1897,1903],{"__ignoreMap":282},[348,1656,1657,1660,1663,1666],{"class":350,"line":351},[348,1658,1659],{"class":1252},"const",[348,1661,1662],{"class":362}," AUTH_RETRY_LIMIT",[348,1664,1665],{"class":1252}," =",[348,1667,1668],{"class":362}," 2\n",[348,1670,1671,1673,1676,1678,1681],{"class":350,"line":283},[348,1672,1659],{"class":1252},[348,1674,1675],{"class":362}," SLOW_POLL_DELAY_AUTH",[348,1677,1665],{"class":1252},[348,1679,1680],{"class":362}," 60_000",[348,1682,1683],{"class":1514},"      \u002F\u002F 认证持续失败：1 分钟探一次\n",[348,1685,1686,1688,1691,1693,1696],{"class":350,"line":1284},[348,1687,1659],{"class":1252},[348,1689,1690],{"class":362}," SLOW_POLL_DELAY_GENERAL",[348,1692,1665],{"class":1252},[348,1694,1695],{"class":362}," 300_000",[348,1697,1698],{"class":1514},"  \u002F\u002F 一般持续失败：5 分钟探一次\n",[348,1700,1701],{"class":350,"line":1305},[348,1702,1703],{"emptyLinePlaceholder":297},"\n",[348,1705,1706,1709],{"class":350,"line":1311},[348,1707,1708],{"class":354},"_schedulePoll",[348,1710,1711],{"class":369},"(delayMs, kind) {\n",[348,1713,1714,1716,1718,1721],{"class":350,"line":1466},[348,1715,1356],{"class":362},[348,1717,1290],{"class":369},[348,1719,1720],{"class":354},"_clearReconnectTimer",[348,1722,1454],{"class":369},[348,1724,1725,1727,1729,1731],{"class":350,"line":1501},[348,1726,1356],{"class":362},[348,1728,1570],{"class":369},[348,1730,1253],{"class":1252},[348,1732,1733],{"class":362}," 0\n",[348,1735,1736,1738,1741,1744,1747,1750,1752,1754,1756],{"class":350,"line":1518},[348,1737,1270],{"class":1252},[348,1739,1740],{"class":369}," (kind ",[348,1742,1743],{"class":1252},"===",[348,1745,1746],{"class":358}," 'auth'",[348,1748,1749],{"class":369},") ",[348,1751,1246],{"class":362},[348,1753,1424],{"class":369},[348,1755,1253],{"class":1252},[348,1757,1733],{"class":362},[348,1759,1760,1762,1765,1767],{"class":350,"line":1530},[348,1761,1356],{"class":362},[348,1763,1764],{"class":369},"._reconnectState ",[348,1766,1253],{"class":1252},[348,1768,1769],{"class":358}," 'scheduled'\n",[348,1771,1773,1775,1778,1780],{"class":350,"line":1772},10,[348,1774,1356],{"class":362},[348,1776,1777],{"class":369},"._pendingReconnect ",[348,1779,1253],{"class":1252},[348,1781,1782],{"class":362}," true\n",[348,1784,1786,1788,1790,1792,1794,1796,1798],{"class":350,"line":1785},11,[348,1787,1356],{"class":362},[348,1789,1249],{"class":369},[348,1791,1253],{"class":1252},[348,1793,1256],{"class":354},[348,1795,1259],{"class":369},[348,1797,1262],{"class":1252},[348,1799,1265],{"class":369},[348,1801,1803,1805,1807,1809],{"class":350,"line":1802},12,[348,1804,1287],{"class":362},[348,1806,1249],{"class":369},[348,1808,1253],{"class":1252},[348,1810,1811],{"class":362}," null\n",[348,1813,1815,1818,1820,1822,1825],{"class":350,"line":1814},13,[348,1816,1817],{"class":1252},"    if",[348,1819,1273],{"class":369},[348,1821,1246],{"class":362},[348,1823,1824],{"class":369},"._intentionalClose) ",[348,1826,1533],{"class":1252},[348,1828,1830,1832,1834,1836],{"class":350,"line":1829},14,[348,1831,1287],{"class":362},[348,1833,1764],{"class":369},[348,1835,1253],{"class":1252},[348,1837,1838],{"class":358}," 'attempting'\n",[348,1840,1842,1844,1846,1848,1850],{"class":350,"line":1841},15,[348,1843,1817],{"class":1252},[348,1845,1740],{"class":369},[348,1847,1743],{"class":1252},[348,1849,1746],{"class":358},[348,1851,1433],{"class":369},[348,1853,1855,1858,1860,1862,1864,1866],{"class":350,"line":1854},16,[348,1856,1857],{"class":362},"      this",[348,1859,1290],{"class":369},[348,1861,1293],{"class":354},[348,1863,1296],{"class":369},[348,1865,1299],{"class":362},[348,1867,1302],{"class":369},[348,1869,1871,1874,1877],{"class":350,"line":1870},17,[348,1872,1873],{"class":369},"    } ",[348,1875,1876],{"class":1252},"else",[348,1878,1265],{"class":369},[348,1880,1882,1884,1886,1889],{"class":350,"line":1881},18,[348,1883,1857],{"class":362},[348,1885,1290],{"class":369},[348,1887,1888],{"class":354},"_doConnect",[348,1890,1454],{"class":369},[348,1892,1894],{"class":350,"line":1893},19,[348,1895,1896],{"class":369},"    }\n",[348,1898,1900],{"class":350,"line":1899},20,[348,1901,1902],{"class":369},"  }, delayMs)\n",[348,1904,1906],{"class":350,"line":1905},21,[348,1907,1390],{"class":369},[10,1909,1910,1911,1913,1914,1917],{},"慢轮询命中后失败会重新进入 ",[25,1912,1233],{}," 快速重连周期（因为 ",[25,1915,1916],{},"_reconnectAttempts"," 重置为 0），相当于\"快速 60 次 → 慢一次 → 快速 60 次 → 慢一次\"的循环，永不放弃。",[10,1919,1920],{},"三条修复并行：",[10,1922,1923,1926,1927,1930],{},[20,1924,1925],{},"Fix A","：普通重连直接走 ",[25,1928,1929],{},"_doConnect()","，凭据刷新隔离到认证失败分支，避免配置读取错误牵连普通重连。",[10,1932,1933,1936,1937,1940,1941,1943],{},[20,1934,1935],{},"Fix B","：认证失败耗尽后转入 ",[25,1938,1939],{},"_schedulePoll('auth')","，不再设置 ",[25,1942,1626],{},"，给认证恢复留出 60 秒的探测窗口。",[10,1945,1946,1949,1950,1953],{},[20,1947,1948],{},"Fix C","：重连次数超过 MAX_RECONNECT_ATTEMPTS 后转入 ",[25,1951,1952],{},"_schedulePoll('general')","，设置 UI 状态为\"连接持续失败，300 秒后重试\"而不是终态。",[10,1955,1956],{},"慢轮询失败后重新进入快速重连周期，形成\"快速 60 次 → 慢一次 → 快速 60 次\"的循环，永不放弃。",[40,1958,1959],{"id":1959},"防回归验证",[10,1961,1962,1963,1966],{},"新增 ",[25,1964,1965],{},"wakou-full\u002Fsrc\u002Flib\u002Fws-client.slow-poll.test.js","，覆盖 7 个 case：",[50,1968,1969,1977,1989,1992,2001,2008,2013],{},[53,1970,1971,1972,1974,1975],{},"Fix A：普通重连命中 ",[25,1973,1888],{},"，不调用 ",[25,1976,1293],{},[53,1978,1979,1980,1983,1984,1986,1987],{},"Fix C：MAX_RECONNECT_ATTEMPTS 后状态是 ",[25,1981,1982],{},"reconnecting","（非终态 ",[25,1985,908],{},"），5 分钟后触发 ",[25,1988,1888],{},[53,1990,1991],{},"Fix C：第二轮慢轮询失败后还能再调度快速重连",[53,1993,1994,1995,1997,1998],{},"Fix B：",[25,1996,1939],{}," 等待 60 秒调用 ",[25,1999,2000],{},"_refreshCredentialsAndReconnect(0)",[53,2002,1994,2003,2005,2006],{},[25,2004,1952],{}," 等待后调用 ",[25,2007,1888],{},[53,2009,2010,2012],{},[25,2011,1626],{}," 时慢轮询应跳过",[53,2014,2015,2017],{},[25,2016,1293],{}," 抛错路径调度慢轮询",[10,2019,2020],{},"测试全部通过（7\u002F7）。",[40,2022,2023],{"id":2023},"外部因素的叠加",[10,2025,2026],{},"这个时期同步发生了另一个外部问题：CDN 对 WebSocket 空闲连接会在 4-9 分钟后强制切断。客户端收到连接中断后根据指数退避策略重连，如果恰好命中上述三条路径之一，就进入永久断开状态。这解释了为什么故障特别在长时间挂机后高发——空闲足够长，CDN 必定切断，而后续重连很容易落入某条缺陷路径。两个问题各自独立，但组合效果是\"挂机过夜必死\"。",[40,2028,2029],{"id":2029},"排查验证",[10,2031,2032],{},"故障修复后，可以用以下方式验证重连状态：",[340,2034,2036],{"className":1237,"code":2035,"language":1239,"meta":282,"style":282},"__clawpanelWsClient.getConnectionInfo()\n\u002F\u002F {\n\u002F\u002F   connected: false,\n\u002F\u002F   reconnectState: 'scheduled' | 'attempting',\n\u002F\u002F   reconnectAttempts: 0~60,\n\u002F\u002F   ...\n\u002F\u002F }\n",[25,2037,2038,2048,2053,2058,2063,2068,2073],{"__ignoreMap":282},[348,2039,2040,2043,2046],{"class":350,"line":351},[348,2041,2042],{"class":369},"__clawpanelWsClient.",[348,2044,2045],{"class":354},"getConnectionInfo",[348,2047,1454],{"class":369},[348,2049,2050],{"class":350,"line":283},[348,2051,2052],{"class":1514},"\u002F\u002F {\n",[348,2054,2055],{"class":350,"line":1284},[348,2056,2057],{"class":1514},"\u002F\u002F   connected: false,\n",[348,2059,2060],{"class":350,"line":1305},[348,2061,2062],{"class":1514},"\u002F\u002F   reconnectState: 'scheduled' | 'attempting',\n",[348,2064,2065],{"class":350,"line":1311},[348,2066,2067],{"class":1514},"\u002F\u002F   reconnectAttempts: 0~60,\n",[348,2069,2070],{"class":350,"line":1466},[348,2071,2072],{"class":1514},"\u002F\u002F   ...\n",[348,2074,2075],{"class":350,"line":1501},[348,2076,2077],{"class":1514},"\u002F\u002F }\n",[10,2079,2080,2083,2084,2087,2088,2083,2091,2094],{},[25,2081,2082],{},"reconnectState='scheduled'"," 且 ",[25,2085,2086],{},"reconnectAttempts=0"," 表示在慢轮询窗口正常工作；",[25,2089,2090],{},"reconnectState='idle'",[25,2092,2093],{},"connected=false"," 是老的永久断开路径。",[10,2096,2097],{},"重连机制里避免静默终态是最基本的要求。任何异常路径都必须有可见的状态提示和后续操作入口，否则用户端看到的就是死机。这次故障的根本教训是，不能让客户端在任何情况下进入\"不可恢复\"的状态而没有任何提示。慢轮询的引入给了所有失败情景一个\"最后的机会\"，即使前面的快速重连机制彻底耗尽了，用户等待足够长的时间后系统仍有自动恢复的可能。",[702,2099,2100],{},"html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .szBVR, html code.shiki .szBVR{--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sJ8bj, html code.shiki .sJ8bj{--shiki-default:#6A737D;--shiki-dark:#6A737D}",{"title":282,"searchDepth":283,"depth":283,"links":2102},[2103,2108,2109,2110,2111,2112],{"id":1219,"depth":283,"text":1219,"children":2104},[2105,2106,2107],{"id":1226,"depth":1284,"text":1227},{"id":1403,"depth":1284,"text":1404},{"id":1547,"depth":1284,"text":1548},{"id":1615,"depth":283,"text":1615},{"id":1639,"depth":283,"text":1639},{"id":1959,"depth":283,"text":1959},{"id":2023,"depth":283,"text":2023},{"id":2029,"depth":283,"text":2029},"2026-06-17","从 2026-05-11 开始，多台便携包客户反馈 WebSocket 断线后面板卡死。网关进程（openclaw_gateway.exe）明确在运行，\u002Fhealth 返回 200，但面板显示\"已停止重连，请手动刷新\"——用户必须手动重启才能恢复。",{},"\u002F2026-06-17-fa-016-ws",{"title":1189,"description":2114},"FA-016","2026-06-17-FA-016-WS永久断开三条路径","客户端重连逻辑中三处独立的\"静默终止\"缺陷导致 WS 永久断开，任何一条路径命中就不再调度重连计时器。",[2122,2123,2124,2125,2126],"WebSocket","Node.js","客户端重连机制","异常处理","便携包","qaISKpAwjYnhiSdxEMYYnJtJ-A8QaOq-djOEJ0W9-bM",1785406912281]