[{"data":1,"prerenderedAt":2056},["ShallowReactive",2],{"\u002F2026-05-01-fa-002-22":3,"\u002F2026-05-01-fa-002-22-rel":239},{"id":4,"title":5,"body":6,"column":221,"date":222,"description":12,"extension":223,"hero_image":224,"meta":225,"navigation":226,"path":227,"seo":228,"series_id":229,"severity":224,"stem":230,"summary":231,"tags":232,"__hash__":238},"posts\u002F2026-05-01-FA-002-事件循环阻塞22秒.md","一次 sessions.list，23 秒",{"type":7,"value":8,"toc":212},"minimark",[9,13,17,20,31,34,40,43,49,52,55,58,64,67,70,73,92,95,98,105,120,123,126,132,138,143,154,159,162,175,178,197,200,203,206,209],[10,11,12],"p",{},"sessions.list 23293 毫秒。单次 tick 延迟 21944.6 毫秒。eventLoopUtilization 是 1。这三个数字同时出现，说明的不是网络问题，是主线程被完全占满。",[14,15,16],"h2",{"id":16},"现象",[10,18,19],{},"liveness 告警显示 event loop 延迟峰值达到 22 秒，单核占用率 0.93+，event loop 实际利用率接近 1（无空闲时间）：",[21,22,27],"pre",{"className":23,"code":25,"language":26},[24],"language-text","[diagnostic] liveness warning: reasons=event_loop_delay,event_loop_utilization,cpu interval=34s\n  eventLoopDelayP99Ms=14822.7 eventLoopDelayMaxMs=14822.7 eventLoopUtilization=0.994 cpuCoreRatio=0.977\n  active=1 waiting=0 queued=1\n[diagnostic] liveness warning: reasons=event_loop_delay,event_loop_utilization,cpu interval=38s\n  eventLoopDelayP99Ms=21944.6 eventLoopDelayMaxMs=21944.6 eventLoopUtilization=1 cpuCoreRatio=0.969\n  active=2 waiting=0 queued=5\n[diagnostic] liveness warning: reasons=event_loop_delay,cpu interval=36s\n  eventLoopDelayP99Ms=12977.2 eventLoopDelayMaxMs=19394.5 eventLoopUtilization=0.943 cpuCoreRatio=0.929\n","text",[28,29,25],"code",{"__ignoreMap":30},"",[10,32,33],{},"并发处理能力完全坍塌：",[21,35,38],{"className":36,"code":37,"language":26},[24],"[ws] ←res ↓sessions.list 23293ms\n[ws] ←res ↓sessions.list 13868ms\n",[28,39,37],{"__ignoreMap":30},[10,41,42],{},"正常情况 sessions.list 响应在几十毫秒，这里到了 23 秒。同一时间窗口里 embedded agent 首次启动的阶段拆解显示：",[21,44,47],{"className":45,"code":46,"language":26},[24],"[agent\u002Fembedded] [trace:embedded-run] startup stages:\n  runId=...\n  phase=attempt-dispatch totalMs=32433\n  stages=workspace:1ms@1ms,\n         runtime-plugins:5ms@6ms,\n         hooks:0ms@6ms,\n         model-resolution:2974ms@2980ms,\n         auth:14045ms@17025ms,\n         context-engine:1ms@17026ms,\n         attempt-dispatch:15407ms@32433ms\n",[28,48,46],{"__ignoreMap":30},[10,50,51],{},"auth 阶段独占 14 秒，attempt-dispatch 又花 15 秒，鉴权和派发两步合计 29 秒。用户的核心感受就是\"打开面板就卡，发消息也卡\"。",[14,53,54],{"id":54},"排查过程",[10,56,57],{},"从 liveness 告警时间戳倒推启动序列。Gateway 起来后立刻开始 plugin staging——这个阶段在 err.log 里有清晰的 trace：",[21,59,62],{"className":60,"code":61,"language":26},[24],"[plugins] file-transfer staging bundled runtime deps\n[plugins] runway staging bundled runtime deps\n",[28,63,61],{"__ignoreMap":30},[10,65,66],{},"plugin staging 的工作是从 npm 仓库抽取 plugin 的 runtime dependencies，解包、提取，是典型的 CPU 密集 + IO 密集操作。同时前端 ws 连接可以立刻建立（端口已监听），一旦有调用（例如 sessions.list）就会被 event loop delay 阻塞。",[10,68,69],{},"embedded agent 首次启动时 auth 阶段 14 秒是突出的瓶颈。这一段按 stage 拆分来看，从 model-resolution（2974ms）到 auth 开始还要再花 45ms，auth 本身占了 14045ms。auth 阶段的常见操作是 token 校验或 OAuth 刷新，如果这一步走了同步 HTTP 路径（而非异步），就会把整个 event loop 卡住。",[10,71,72],{},"时序完全吻合：",[74,75,76,80,83,86,89],"ul",{},[77,78,79],"li",{},"重启 Gateway → launcher 拉起 Gateway 进程",[77,81,82],{},"Gateway 进入 plugin staging（npm 包解压，CPU 拉满）",[77,84,85],{},"同时前端 ws 连接进来，调 sessions.list（被 event loop delay 阻塞 → 耗时 23 秒）",[77,87,88],{},"用户在面板上 send 消息 → embedded agent 触发 auth（卡 14 秒）+ dispatch（卡 15 秒）",[77,90,91],{},"用户感觉\"系统完全冻住了\"",[10,93,94],{},"第二次启动明显变快的原因是 plugin staging 的结果有缓存（不需要重新拉 npm 包），token 也有缓存（不需要重新走 OAuth 流程）。所以这个问题严格限定在\"首次启动 \u002F 升级后第一次启动\"这个窗口。",[14,96,97],{"id":97},"根因",[10,99,100,104],{},[101,102,103],"strong",{},"未完全定位","，但根据阶段耗时和 CPU 占用率，两个怀疑点都指向同一时间窗口：",[106,107,108,114],"ol",{},[77,109,110,113],{},[101,111,112],{},"plugin staging 跑在主线程","：npm 包的抽取是 CPU + IO 密集操作，如果不 fork 子进程而是在主线程处理，会直接阻塞 event loop。从 cpuCoreRatio 0.93+ 看，至少有一个核心被打得很满。",[77,115,116,119],{},[101,117,118],{},"embedded agent auth 阶段是同步阻塞","：14 秒的 auth 很可能是某个云端 token 校验或 OAuth 刷新走了同步 HTTP 路径，socket 读写未超时的情况下会永远等下去，此间 event loop 毫无机会让出来。",[10,121,122],{},"两个问题叠加的后果是客户感受到的 22 秒冻结：plugin staging 把 CPU 占满，auth 同步等待把 event loop 卡死，任何其他操作都排队等待。",[14,124,125],{"id":125},"修复方案",[10,127,128,131],{},[101,129,130],{},"应急处理","：客户首次打开面板时等待 2 分钟再使用，或重启 Gateway 后手动刷新两次。问题会逐渐缓解，因为 plugin 缓存和 token 都会落地。这不是长期方案，但能缓解首次体验。",[10,133,134,137],{},[101,135,136],{},"工程改进","（三个方向）：",[10,139,140],{},[101,141,142],{},"1. plugin staging 从主线程移出",[10,144,145,146,149,150,153],{},"改造 staging 过程使用 ",[28,147,148],{},"worker_threads"," 或 ",[28,151,152],{},"child_process"," 运行，主线程只等结果信号。参考 npm\u002Fyarn 的做法：安装依赖时 fork 出一个子进程跑 install，主进程保持响应。对应修改点是 plugin 管理器的 staging 入口，把同步的文件操作改成非阻塞的 spawn\u002Ffork 调用。",[10,155,156],{},[101,157,158],{},"2. embedded agent auth 加超时并预热",[10,160,161],{},"OAuth 刷新或 token 校验加超时，超时走降级路径。更激进的做法是启动时就发起 auth 预热，不等用户第一次 send 才触发，这样用户真正交互时 token 已经热了。",[10,163,164],{},[101,165,166,167,170,171,174],{},"3. 分离 ",[28,168,169],{},"\u002Fhealth"," 和 ",[28,172,173],{},"\u002Fready"," 端点",[10,176,177],{},"现在 Gateway 只要端口能监听就被认为\"健康\"，但实际工作状态是\"正在初始化\"。建议：",[74,179,180,185,190],{},[77,181,182,184],{},[28,183,169],{}," 保持轻量级，返回存活状态",[77,186,187,189],{},[28,188,173],{}," 检查真实就绪条件（plugin staging 完成、token 预热完成）",[77,191,192,193,196],{},"ws 连接在收到 401 或 503 时附带 ",[28,194,195],{},"Retry-After"," header 让前端等待",[10,198,199],{},"这比让用户等一秒感受到的卡顿好得多。\"假就绪 + 真卡顿\"的体验等于欺骗，\"真就绪 + 明确告诉你还要等\"的体验是可控的。",[14,201,202],{"id":202},"防回归",[10,204,205],{},"构建期应增加代码静态检查，确保 plugin staging 的实现确实 fork 了子进程而不是在主线程同步运行。liveness 告警的精细度也要提升，eventLoopDelayMaxMs 达到一定阈值时向前端报警，而非仅作后台日志。冷启动时的性能验证目前还没有明确目标，需要补充。",[14,207,208],{"id":208},"结论",[10,210,211],{},"单一的性能问题往往由多个原因叠加触发。这里是 plugin staging 的 CPU 密集操作 + auth 同步等待 + 缺乏就绪状态隔离，三层一起压向用户。解决任何一层都能改善，但完整修复需要同时改三个地方：把 CPU 工作从主线程移走，把网络等待加超时，把\"我能接请求\"和\"我真的准备好\"分开。",{"title":30,"searchDepth":213,"depth":213,"links":214},2,[215,216,217,218,219,220],{"id":16,"depth":213,"text":16},{"id":54,"depth":213,"text":54},{"id":97,"depth":213,"text":97},{"id":125,"depth":213,"text":125},{"id":202,"depth":213,"text":202},{"id":208,"depth":213,"text":208},"故障档案","2026-05-01","md",null,{},true,"\u002F2026-05-01-fa-002-22",{"title":5,"description":12},"FA-002","2026-05-01-FA-002-事件循环阻塞22秒","Gateway 启动首阶段 event loop 长期阻塞，sessions.list 慢到 23 秒；根因是 plugin staging 跑主线程，结合 auth 同步等待被卡 22 秒。",[233,234,235,236,237],"Node.js","event loop","plugin staging","concurrent","gateway","3WDjfTt61Op_m8jF-QXsl-HMQqxBzcGc0ss811YakMk",[240,659,1118],{"id":241,"title":242,"body":243,"column":221,"date":644,"description":247,"extension":223,"hero_image":224,"meta":645,"navigation":226,"path":646,"seo":647,"series_id":648,"severity":224,"stem":649,"summary":650,"tags":651,"__hash__":658},"posts\u002F2026-07-23-FA-018-共享存储的subPath陷阱.md","一个 PVC 装所有用户，当时看不出有什么问题",{"type":7,"value":244,"toc":635},[245,248,251,255,258,261,268,319,322,332,336,342,345,356,359,363,366,369,372,390,393,400,403,418,422,425,428,439,450,453,456,459,527,530,537,540,543,569,572,583,586,589,628,631],[10,246,247],{},"平台从 Docker Swarm 迁移到 Kubernetes，存储架构做了一个看似理想的调整：用共享 PVC + subPath 替代原来的\"每用户独占 PVC\"。方案优势显而易见——管理简单（1 个 PVC 对 574 个）、自动扩展、成本低。当时看不出有什么缺点。",[10,249,250],{},"但实施中遇到了四个隐蔽的问题。",[14,252,254],{"id":253},"权限错配bind-型数据的属主陷阱","权限错配：bind 型数据的属主陷阱",[10,256,257],{},"旧系统中，用户数据通过两种方式持久化：bind mount 和 Docker volume。迁移时需要从这两个来源完整导出数据。",[10,259,260],{},"对于 bind mount 型数据，文件属主通常是 10001（或其他固定 UID）。当我登上旧集群的宿主机，以普通用户身份尝试读取这些文件时，只能看到 13 个文件。权限拒绝。同一目录里其实有 4778 个文件，但大多数因为属主权限不匹配而不可见。",[10,262,263,264,267],{},"只有用 sudo 才能完整读到。所以迁移脚本必须用 ",[28,265,266],{},"sudo tar"," 来打包：",[21,269,273],{"className":270,"code":271,"language":272,"meta":30,"style":30},"language-bash shiki shiki-themes github-light github-dark","sudo tar -czf \u002Ftmp\u002Fmig\u002F${uid}\u002Fdata.tgz \\\n  -C \u002Fmnt\u002Fyun-claw\u002Fusers\u002F${uid} .\n","bash",[28,274,275,305],{"__ignoreMap":30},[276,277,280,284,288,292,295,299,302],"span",{"class":278,"line":279},"line",1,[276,281,283],{"class":282},"sScJk","sudo",[276,285,287],{"class":286},"sZZnC"," tar",[276,289,291],{"class":290},"sj4cs"," -czf",[276,293,294],{"class":286}," \u002Ftmp\u002Fmig\u002F",[276,296,298],{"class":297},"sVt8B","${uid}",[276,300,301],{"class":286},"\u002Fdata.tgz",[276,303,304],{"class":290}," \\\n",[276,306,307,310,313,316],{"class":278,"line":213},[276,308,309],{"class":290},"  -C",[276,311,312],{"class":286}," \u002Fmnt\u002Fyun-claw\u002Fusers\u002F",[276,314,315],{"class":297},"${uid} ",[276,317,318],{"class":286},".\n",[10,320,321],{},"这不是脚本设计的问题，而是 Kubernetes 环境下容器权限与宿主权限映射的一个陷阱。容器内运行的网关进程可能以容器用户身份运行，但宿主上的文件属主是不同的 UID。当两个权限体系接不上，访问就会失败。",[10,323,324,325,328,329,331],{},"新方案中，用户数据通过 subPath 挂到容器的 ",[28,326,327],{},"\u002Fdata"," 目录。如果 subPath 指向的目录是由 Kubernetes 在宿主机上创建的，属主可能是 root 或其他用户，容器内进程（以容器指定的 UID 运行）仍然会遭遇权限问题。这个风险需要在 Dockerfile 和容器启动时明确处理：容器内应该预先创建 ",[28,330,327],{}," 并设置正确的属主，或者使用 securityContext 的 fsGroup 或 runAsUser 来强制权限。",[14,333,335],{"id":334},"subpath-目录创建的时机与属主处理","subPath 目录创建的时机与属主处理",[10,337,338,339],{},"Kubernetes 对 subPath 的处理有个微妙之处：如果挂载的 subPath 目录在宿主机上不存在，kubelet 会自动创建它。但 ",[276,340,341],{},"待补：这个自动创建的目录的属主是什么？是 root 还是其他用户？容器的 securityContext 能否保证挂载后的权限符合预期？",[10,343,344],{},"设计文档中没有明确说明这一点。在实施前，需要验证：",[74,346,347,350,353],{},[77,348,349],{},"首次 subPath 目录不存在时，kubelet 创建它并将其属主设置为谁",[77,351,352],{},"容器的 securityContext（fsGroup、runAsUser）是否能在挂载时生效",[77,354,355],{},"是否应该预先在共享 PVC 上创建所有 subPath 目录并设置好属主，而不是依赖 Kubernetes 的隐式创建",[10,357,358],{},"没有明确的答案，就容易踩坑。一个保险做法是在 destroy 用户账户时清空 subPath 目录，同时在 provision 时让一个 init-container 验证或修复目录属主，确保容器进程有写权限。",[14,360,362],{"id":361},"单-pvc-中的节点级故障域","单 PVC 中的节点级故障域",[10,364,365],{},"这是最严重的问题，也是与 FA-015（节点假 Ready）的关联点。",[10,367,368],{},"当一个节点的 NFS 客户端或到 SFS 的网络链路发生 hang 时，所有依赖共享 PVC 的 Pod 都会受影响。这不是共享 PVC 本身的问题，而是故障隔离的问题。",[10,370,371],{},"具体现象：",[74,373,374,381,384,387],{},[77,375,376,377,380],{},"如果 NFS 挂载使用了硬 mount（",[28,378,379],{},"hard"," 参数是 NFS 的默认值），而网络故障或存储服务中断，NFS 客户端会无限重试",[77,382,383],{},"重试期间，所有试图访问 NFS 的进程都会被阻塞在 I\u002FO 上（D 状态，不可中断）",[77,385,386],{},"如果 kubelet 或 containerd 的某个操作（如创建容器的卷挂载步骤）陷入 I\u002FO 阻塞，整个节点就会冻死",[77,388,389],{},"该节点上的所有实例（无论是否在访问存储）都会因为无法创建或删除 Pod 而故障",[10,391,392],{},"这与独立 PVC 的效果截然不同。旧架构中，每个用户有独立 PVC，意味着如果某个 PVC 对应的存储有问题，只会影响这一个用户。其他用户的 PVC 可能分散在不同的存储卷甚至不同的节点上，相对独立。",[10,394,395,396,399],{},"新架构下，单个共享 PVC 承载所有用户，一旦这个 PVC 对应的网络链路或挂载出问题，",[101,397,398],{},"同节点上所有实例都会被殃及","。如果该节点碰巧堆积了 50 个实例，一次节点级的存储 hang 就会导致 50 个实例集体故障，且外表看起来是\"节点 Ready，但实例卡住\"——kubelet 的心跳正常，Pod 状态却卡在 ContainerCreating 或 Terminating。",[10,401,402],{},"这正是 FA-015 在 2026-06-05 观察到的现象：节点假 Ready，但大量实例网关无响应。防护措施包括：",[74,404,405,412,415],{},[77,406,407,408,411],{},"NFS 挂载参数改为软 mount（",[28,409,410],{},"soft,timeo=100,retrans=3","），让 I\u002FO 超时而不是无限等待",[77,413,414],{},"节点加健康检测，检查 ContainerCreating 堆积数和网关探测失败率，及早发现节点冻死",[77,416,417],{},"实例分散部署，用 topologySpreadConstraints 避免单节点堆积",[14,419,421],{"id":420},"无-per-subpath-硬配额","无 per-subPath 硬配额",[10,423,424],{},"共享存储的架构决定了无法在存储层面按 subPath 限制用户配额。SFS（及 NFS 一般）没有 per-directory 的硬配额功能，只能在整个卷级别限制。",[10,426,427],{},"这意味着：",[74,429,430,433,436],{},[77,431,432],{},"无法阻止某个用户的数据膨胀而挤占其他用户的空间",[77,434,435],{},"无法在存储层面实现\"超额用户的写入失败\"",[77,437,438],{},"只能在应用层进行监控、告警和逻辑控制",[10,440,441,442,445,446,449],{},"新系统的方案是应用层监控：定期 ",[28,443,444],{},"du -sb \u002Fdata"," 扫描每个用户的目录大小，落库到 Instance 表，",[28,447,448],{},"\u002Fstatus"," 接口返回超额标志位。前端据此提示用户\"存储已满\"。这是纯监控，不阻断。如果用户无视提示继续写入，直到共享卷真的满了，才会出现\"所有用户集体写入失败\"的惨淡局面。",[10,451,452],{},"这个限制是方案选择的代价。",[14,454,455],{"id":455},"为什么仍然选了这个方案",[10,457,458],{},"尽管有这些问题，平台仍然采用了共享 PVC + subPath 的方案。原因是对比了替代方案：",[460,461,462,481],"table",{},[463,464,465],"thead",{},[466,467,468,472,475,478],"tr",{},[469,470,471],"th",{},"方案",[469,473,474],{},"优点",[469,476,477],{},"缺点",[469,479,480],{},"故障隔离",[482,483,484,499,513],"tbody",{},[466,485,486,490,493,496],{},[487,488,489],"td",{},"独立 PVC（旧架构）",[487,491,492],{},"天然的每用户隔离；故障域清晰",[487,494,495],{},"管理复杂（574 个 PVC）；扩展性差；成本高",[487,497,498],{},"✓ 最好",[466,500,501,504,507,510],{},[487,502,503],{},"共享 PVC + subPath（新架构）",[487,505,506],{},"管理简单（1 个 PVC）；自动扩展；成本低",[487,508,509],{},"故障隔离性差；无硬配额；权限管理复杂",[487,511,512],{},"✗ 较差",[466,514,515,518,521,524],{},[487,516,517],{},"对象存储（S3\u002FOSS）",[487,519,520],{},"真正的多租户隔离；天然分布",[487,522,523],{},"延迟高；成本更高；应用改造大",[487,525,526],{},"✓ 最好，但代价大",[10,528,529],{},"独立 PVC 的管理开销是关键问题。574 个用户意味着 574 个 PVC 对象、574 条 PV 绑定、574 个存储卷。每次实例创建、删除或迁移都要涉及 PVC 的生命周期管理。扩展到 5000 用户时，这种开销会成为瓶颈。对象存储虽然隔离性最好，但需要应用层改造（兼容 S3 API、处理延迟、调整备份策略），而且成本更高。",[10,531,532,533,536],{},"共享 PVC 方案的核心优势是",[101,534,535],{},"运维简洁","：增删用户只需改 subPath（一行 YAML），不涉及存储层操作。代价是故障隔离性从\"用户级\"降到\"节点级\"。",[14,538,539],{"id":539},"适用边界",[10,541,542],{},"这个方案适合以下场景：",[74,544,545,551,557,563],{},[77,546,547,550],{},[101,548,549],{},"用户数量有上限","（几百到几千）。用户过多时，共享卷的单点压力会成问题。",[77,552,553,556],{},[101,554,555],{},"用户数据量可控","（单用户通常 GB 级）。如果单用户数据量达 TB，一次 du 扫描会拖累整体。",[77,558,559,562],{},[101,560,561],{},"可以接受节点级故障隔离","。只要网络和存储配置足够稳定（硬化 NFS 参数、冗余链路），节点 hang 的概率不会很高。",[77,564,565,568],{},[101,566,567],{},"能够实施应用层监控和告警","。无硬配额，就必须有实时监控。",[10,570,571],{},"不适合的场景包括：",[74,573,574,577,580],{},[77,575,576],{},"超大规模用户（万级以上）",[77,578,579],{},"用户数据特别不均衡（少数用户占大头）的场景",[77,581,582],{},"对故障隔离要求极高的系统（比如金融交易）",[14,584,585],{"id":585},"防护与调整",[10,587,588],{},"基于这四个问题，实施中做了以下调整：",[106,590,591,600,610,616,622],{},[77,592,593,596,597,599],{},[101,594,595],{},"权限处理","：容器 Dockerfile 中预先创建 ",[28,598,327],{}," 并设置正确的属主；启动脚本检查并修复权限。",[77,601,602,605,606,609],{},[101,603,604],{},"NFS 参数硬化","：StorageClass 的挂载选项改为 ",[28,607,608],{},"soft,timeo=100,retrans=3,intr","，避免硬 mount 导致节点冻。",[77,611,612,615],{},[101,613,614],{},"节点健康检测","：cron 每 60 秒检查各节点的 ContainerCreating 堆积和网关探测失败率，及早发现故障。",[77,617,618,621],{},[101,619,620],{},"实例分散","：StatefulSet 加 topologySpreadConstraints，避免单节点堆积 50+ 个实例。",[77,623,624,627],{},[101,625,626],{},"应用层配额","：\u002Fstatus 接口返回 diskUsedMi 和 overQuota 标志位，前端告警用户。",[10,629,630],{},"这些措施不能消除风险，但可以大幅降低故障概率和影响范围。",[632,633,634],"style",{},"html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":30,"searchDepth":213,"depth":213,"links":636},[637,638,639,640,641,642,643],{"id":253,"depth":213,"text":254},{"id":334,"depth":213,"text":335},{"id":361,"depth":213,"text":362},{"id":420,"depth":213,"text":421},{"id":455,"depth":213,"text":455},{"id":539,"depth":213,"text":539},{"id":585,"depth":213,"text":585},"2026-07-23",{},"\u002F2026-07-23-fa-018-subpath",{"title":242,"description":247},"FA-018","2026-07-23-FA-018-共享存储的subPath陷阱","K8s 共享 PVC + subPath 方案在实施中暴露的四个陷阱：权限不匹配、目录创建时机、节点级故障域、无 per-subPath 配额。",[652,653,654,655,656,657,480],"Kubernetes","存储","subPath","PVC","NFS","权限","4pJJyzbII4OOc9jJS74HUpUfyVc9CfxdE5ovso8ylSE",{"id":660,"title":661,"body":662,"column":221,"date":1102,"description":1103,"extension":223,"hero_image":224,"meta":1104,"navigation":226,"path":1105,"seo":1106,"series_id":1107,"severity":224,"stem":1108,"summary":1109,"tags":1110,"__hash__":1117},"posts\u002F2026-07-11-FA-017-长任务异步化.md","三个超时机制，一起把长任务掐死了",{"type":7,"value":663,"toc":1094},[664,675,678,681,687,693,699,706,712,716,719,722,728,731,742,748,752,755,762,768,771,786,790,793,804,853,868,875,879,882,885,903,906,914,917,932,935,938,1085,1088,1091],[10,665,666,667,670,671,674],{},"数字人口播的「分析」功能从视频提取口播文稿、分镜脚本、结构卖点，需要下载视频、压缩、调 Gemini 视觉 API，耗时从几十秒到几分钟。最初同步实现：前端调 ",[28,668,669],{},"POST \u002Fapi\u002Fworkflow\u002Fdub\u002Fanalyze-parsed","，后端直接跑完再返回。这个方案暴露的问题不是单点，而是",[101,672,673],{},"三个独立的缺陷叠加","。",[10,676,677],{},"网关超时、CDN 空闲切断、客户端重试——这三个没有一个是\"bug\"，每个都是合理的系统设计。但组合在一起就是连锁灾难：客户端看不到结果（超时或连接断），认为失败了重试，后端却已经扣过费了；或者分析其实已跑完，但因为响应发不出去，重试又跑了一遍。退款时无法幂等判断，某些用户最终被多扣了好几倍。",[10,679,680],{},"具体来说，三个问题各自是什么：",[10,682,683,686],{},[101,684,685],{},"第一个问题是网关超时","。云平台的入口网关（API Gateway）有一个固定的读超时，通常设为几十秒。当后端分析耗时超过这个阈值，网关主动断连接，客户端收到 504 或连接重置，但后端的分析进程并不知道客户端已经走了，继续跑完整个流程。如果分析结果的退款逻辑放在 try-catch 的 finally 里，此时 HTTP 响应已发不出去，客户就看到\"分析失败\"，但款已经扣了。",[10,688,689,692],{},[101,690,691],{},"第二个问题是 CDN 空闲切断","。不少 CDN 和负载均衡层有这样的策略：如果一条 HTTP 连接在某个时间段内没有数据往返，就认为它已死而主动关闭。这与 FA-013 那次踩到的问题一样——长连接因为没有心跳数据而被 CDN 中间件切断，导致请求被突然中断。这个机制在分析请求上也会造成麻烦：如果分析用时较长，连接就可能被切，响应发不出去。",[10,694,695,698],{},[101,696,697],{},"第三个问题是客户端重试导致重复扣费","。当客户端没有收到响应（网关超时或 CDN 切断），一般会重试这个请求。问题在于这次重试是一条完全独立的请求，后端会把它当作新任务处理，再次扣费。而且如果第一次的分析实际上已经跑完了，就会出现\"分析跑了两遍、扣费扣了两遍、用户只看到了一个结果\"的情况。如果第一次分析中途被 CDN 切了，第二次重试重新开始，那扣费可能是三倍。而退款的时候，因为业务流程上没有幂等设计，很难追溯哪个 operationId 应该被退，哪个不应该。",[10,700,701,702,705],{},"这三个问题是",[101,703,704],{},"叠加的","，不是独立的。它们共同指向一个根本的架构问题。",[10,707,708,709,674],{},"根本原因一个：",[101,710,711],{},"不该让耗时任务挂在 HTTP 连接上",[14,713,715],{"id":714},"异步任务是必需的不是可选的","异步任务是必需的，不是可选的",[10,717,718],{},"解决方案很简单：把分析改成异步任务模式。",[10,720,721],{},"前端提交分析请求直接返回任务 ID（HTTP 耗时极短：校验参数、建数据库记录、预扣费）。后端后台 worker 跑分析重逻辑，完成后持久化结果。前端轮询查询状态，得到结果后停止。",[21,723,726],{"className":724,"code":725,"language":26},[24],"客户端              后端\n  |                 |\n  +--提交---------->|\n  |             创建任务，返回 ID\n  |\u003C--返回 ID-----+\n  |                 |（后台运行分析）\n  |                 |\n  +--轮询--------->|\n  |             查询状态\n  |\u003C--返回状态---+\n  |                 |\n  |（重复轮询）\n  +--轮询--------->|\n  |             查询状态\n  |\u003C--返回结果---+\n  |                 |\n",[28,727,725],{"__ignoreMap":30},[10,729,730],{},"好处显而易见：",[74,732,733,736,739],{},[77,734,735],{},"单次 HTTP 请求的耗时从分钟级降到秒级，不会触发网关或 CDN 的超时。",[77,737,738],{},"后端分析的运行生命周期与 HTTP 连接完全解耦。即使连接在中途被切，已经启动的分析会继续在后台跑，结果最终还是会被保存。",[77,740,741],{},"客户端重试不会产生新的分析任务。只要实现幂等判重，同一个请求在短时间内重复提交也只会产生一个任务。",[10,743,744,745,674],{},"但异步化不是万灵药，它把问题转移到了三个新的维度上：",[101,746,747],{},"幂等、持久化、失败语义",[14,749,751],{"id":750},"幂等同一请求只产生一个任务","幂等：同一请求只产生一个任务",[10,753,754],{},"假设客户端因为网络抖动，在短时间内连续发了两次\"开始分析\"请求，后端收到两条独立的 HTTP 请求。如果没有幂等设计，就会创建两个任务、扣两次费。",[10,756,757,758,761],{},"幂等的实现最直接的办法是在",[101,759,760],{},"提交阶段就判重","：为这个用户、这个资源设置一个\"只有一个进行中的分析任务\"的约束。素材中的设计是这样做的：",[21,763,766],{"className":764,"code":765,"language":26},[24],"if 该用户已有 status=running && kind=analyze_parsed 的任务:\n    return 409 Conflict（分析已在进行中）\n",[28,767,765],{"__ignoreMap":30},[10,769,770],{},"这样双击提交同一个视频，第一次返回 201 + taskId，第二次返回 409，客户端看到 409 就知道不要再建新任务，拿之前返回的 ID 去轮询。",[10,772,773,774,777,778,781,782,785],{},"另一个幂等层次是",[101,775,776],{},"计费操作的幂等","。预扣费的时候用一个全局唯一的 ",[28,779,780],{},"operationId","（比如 ",[28,783,784],{},"dub-analyze:{uuid}","），这个 ID 关联了这次预扣。如果扣费 API 被重复调用，因为 operationId 重复，系统只会扣一次。退款时也用同一个 operationId，保证退款与预扣能正确配对。",[14,787,789],{"id":788},"持久化进程重启后任务不丢","持久化：进程重启后任务不丢",[10,791,792],{},"后台分析 worker 可能因为发版、机器重启、容器被杀等原因在中途退出。如果任务的状态只保存在内存里，进程一死任务就丢了。",[10,794,795,796,799,800,803],{},"持久化的关键是",[101,797,798],{},"任务状态表","。素材中复用了既有的 ",[28,801,802],{},"SkyhumanTask"," 表，包含字段：",[74,805,806,812,818,823,829,835,841,847],{},[77,807,808,811],{},[28,809,810],{},"id",": 任务 ID",[77,813,814,817],{},[28,815,816],{},"status",": 进度（running \u002F completed \u002F failed）",[77,819,820,822],{},[28,821,780],{},": 幂等 ID",[77,824,825,828],{},[28,826,827],{},"chargedPoints",": 预扣费用",[77,830,831,834],{},[28,832,833],{},"resultPayload",": 分析结果的 JSON",[77,836,837,840],{},[28,838,839],{},"error",": 失败原因",[77,842,843,846],{},[28,844,845],{},"updatedAt",": 最后更新时间戳",[77,848,849,852],{},[28,850,851],{},"sourceObjectKey",": 源视频对象路径（用于幂等判重和重试）",[10,854,855,856,859,860,863,864,867],{},"任务创建时插一条 ",[28,857,858],{},"status=running"," 的记录。后台 worker 定期通过 heartbeat（每 30 秒做一次空 update，仅触碰 updatedAt）来证明自己还活着。分析完成时更新为 ",[28,861,862],{},"status=completed, resultPayload=...","；失败时更新为 ",[28,865,866],{},"status=failed, error=..."," 并触发退款。",[10,869,870,871,874],{},"这样即使 worker 进程突然死了，任务记录还在数据库里。新启起的 worker 可以扫描表里的 ",[28,872,873],{},"status=running && updatedAt 过期"," 的任务，判断它们已经没有 worker 在跑了，执行清理逻辑。",[14,876,878],{"id":877},"失败语义区分任务失败和查询失败","失败语义：区分\"任务失败\"和\"查询失败\"",[10,880,881],{},"异步设计引入了一个新的错误维度：客户端查询任务状态时得到的 404，可能是\"这个任务 ID 不存在\"（用户输错了），也可能是\"任务 ID 之前存在但已被清理\"（进程清理了过期数据）。这两种情况的含义很不一样。",[10,883,884],{},"更重要的是，前端轮询收到的错误需要区分是否应该重试：",[74,886,887,897],{},[77,888,889,892,893,896],{},[101,890,891],{},"任务本身失败","（",[28,894,895],{},"status=failed, error=\"Gemini API 返回错误\"","）：这种情况不该重试，因为重试也会失败。应该直接向用户展示失败信息，并根据 operationId 触发退款。",[77,898,899,902],{},[101,900,901],{},"查询接口失败","（500、网络超时）：这种情况应该重试查询，因为任务本身可能还在继续跑。",[10,904,905],{},"素材中的设计把这两类区分得很清楚：",[74,907,908,911],{},[77,909,910],{},"任务最终的状态（completed 或 failed）持久化到数据库，客户端查询会拿到明确的业务语义。",[77,912,913],{},"查询接口的 HTTP 错误（5xx）是技术层故障，客户端应该重试。",[10,915,916],{},"同时，后端的 reaper 机制（定期扫表清理超期任务）也遵循这个语义：",[74,918,919,925],{},[77,920,921,922,924],{},"如果一个 ",[28,923,858],{}," 的任务超过 90 秒没有 heartbeat 更新，说明 worker 已死，reaper 会强制置为 failed 并退款。",[77,926,927,928,931],{},"但这个强制置为 failed 不应该抛错，因为此时可能有其他进程也想更新这个任务。更新语句应该带条件：",[28,929,930],{},"UPDATE SkyhumanTask SET status=failed WHERE id=? AND status=running","，这样如果 reaper 和其他 worker 同时操作，只有一方会成功。",[14,933,934],{"id":934},"具体实现模式",[10,936,937],{},"数字人口播的分析异步化采用了这样的模式：",[106,939,940,985,1032,1055],{},[77,941,942,945,946],{},[101,943,944],{},"提交阶段","（同步）：",[74,947,948,951,954,960,966,973,979],{},[77,949,950],{},"校验用户和资源",[77,952,953],{},"判重：是否已有进行中的任务（409）",[77,955,956,959],{},[28,957,958],{},"getObject"," 拉视频，检测时长（若≤0 返 400 不扣费）",[77,961,962,965],{},[28,963,964],{},"chargeResource"," 预扣费用，获得 operationId",[77,967,968,969,972],{},"创建 ",[28,970,971],{},"SkyhumanTask{ status:running, operationId, sourceObjectKey }"," 记录",[77,974,975,978],{},[28,976,977],{},"scheduleTask"," 丢到后台队列",[77,980,981,982],{},"返回 ",[28,983,984],{},"{ taskId }",[77,986,987,990,991],{},[101,988,989],{},"后台运行阶段","：",[74,992,993,996,999,1006,1012,1022,1029],{},[77,994,995],{},"Worker 取出任务记录",[77,997,998],{},"开启 heartbeat 定时器（每 30s update 一次 updatedAt）",[77,1000,1001,1002,1005],{},"执行 ",[28,1003,1004],{},"analyzeDubVisionOnly","（下载→压缩→调用 Gemini→解析）",[77,1007,1008,1009],{},"成功：",[28,1010,1011],{},"update( status:completed, resultPayload )",[77,1013,1014,1015,1018,1019],{},"失败：",[28,1016,1017],{},"update( status:failed, error )"," + ",[28,1020,1021],{},"refundResource(operationId)",[77,1023,1024,1025,1028],{},"所有 update 都带条件 ",[28,1026,1027],{},"WHERE id=? AND status=running","（防被 reaper 覆盖）",[77,1030,1031],{},"Finally 清理 heartbeat 定时器",[77,1033,1034,1037,1038],{},[101,1035,1036],{},"查询阶段","（前端轮询）：",[74,1039,1040,1049,1052],{},[77,1041,1042,1045,1046],{},[28,1043,1044],{},"GET \u002Fapi\u002Fworkflow\u002Fdub\u002Ftasks\u002F:id"," 返回 ",[28,1047,1048],{},"{ status, resultPayload, error }",[77,1050,1051],{},"若 status=completed 或 failed，停止轮询",[77,1053,1054],{},"若查询接口返回 5xx，则后续重试",[77,1056,1057,1060,1061],{},[101,1058,1059],{},"清理阶段","（reaper）：",[74,1062,1063,1066,1073,1076,1082],{},[77,1064,1065],{},"后台定期扫表",[77,1067,1068,1069,1072],{},"对于 ",[28,1070,1071],{},"status=running && updatedAt > 90s"," 的任务，判定 worker 已死",[77,1074,1075],{},"若 providerTaskId（这里没有）存在，调用上游的 finalize API",[77,1077,1078,1079,1081],{},"若不存在，直接 ",[28,1080,1021],{}," + 置 failed",[77,1083,1084],{},"本设计无 providerTaskId，所以 reaper 自动走退款分支，无需改动 reaper 代码",[10,1086,1087],{},"这个模式的关键是三道防线：提交时判重（409）、后台心跳（定期 touch）、reaper 兜底（自动退款）。任何一个环节出问题，都有后续环节来补救。",[14,1089,1090],{"id":1090},"防线缺一不可",[10,1092,1093],{},"长耗时任务不能挂在 HTTP 连接上不只是超时问题，而是连接脆弱性与重复处理的共谋。异步化表面是\"返回 ID、后台跑\"这一步，真正的难点在三个维度的同时保证：幂等（不重复扣费）、持久化（不丢数据）、失败语义（不破坏重试逻辑）。缺一就会在某个场景暴露。",{"title":30,"searchDepth":213,"depth":213,"links":1095},[1096,1097,1098,1099,1100,1101],{"id":714,"depth":213,"text":715},{"id":750,"depth":213,"text":751},{"id":788,"depth":213,"text":789},{"id":877,"depth":213,"text":878},{"id":934,"depth":213,"text":934},{"id":1090,"depth":213,"text":1090},"2026-07-11","数字人口播的「分析」功能从视频提取口播文稿、分镜脚本、结构卖点，需要下载视频、压缩、调 Gemini 视觉 API，耗时从几十秒到几分钟。最初同步实现：前端调 POST \u002Fapi\u002Fworkflow\u002Fdub\u002Fanalyze-parsed，后端直接跑完再返回。这个方案暴露的问题不是单点，而是三个独立的缺陷叠加。",{},"\u002F2026-07-11-fa-017",{"title":661,"description":1103},"FA-017","2026-07-11-FA-017-长任务异步化","长耗时分析请求（几十秒到分钟级）用同步 HTTP 导致网关超时、CDN切断、客户端重试重复扣费。改异步任务需同时解决幂等、持久化、失败语义。",[1111,1112,1113,1114,1115,1116],"异步任务","HTTP超时","CDN","幂等","任务队列","微服务","ARFDvBMGX3tw-E3Rj_gw1RYEjrykplO6u1dxtdS7Uto",{"id":1119,"title":1120,"body":1121,"column":221,"date":2042,"description":2043,"extension":223,"hero_image":224,"meta":2044,"navigation":226,"path":2045,"seo":2046,"series_id":2047,"severity":224,"stem":2048,"summary":2049,"tags":2050,"__hash__":2055},"posts\u002F2026-06-17-FA-016-WS永久断开三条路径.md","日志里没有重连记录——它根本没在重连",{"type":7,"value":1122,"toc":2030},[1123,1133,1140,1147,1150,1153,1158,1165,1245,1251,1321,1331,1335,1338,1464,1475,1479,1485,1540,1543,1546,1549,1564,1567,1569,1572,1579,1837,1847,1850,1860,1873,1883,1886,1889,1896,1947,1950,1953,1956,1959,1962,2007,2024,2027],[10,1124,1125,1126,1129,1130,1132],{},"从 2026-05-11 开始，多台便携包客户反馈 WebSocket 断线后面板卡死。网关进程（",[28,1127,1128],{},"openclaw_gateway.exe","）明确在运行，",[28,1131,169],{}," 返回 200，但面板显示\"已停止重连，请手动刷新\"——用户必须手动重启才能恢复。",[10,1134,1135,1136,1139],{},"关键线索在控制台：看不到任何 ",[28,1137,1138],{},"[ws] 计划重连"," 的日志。",[10,1141,1142,1143,1146],{},"通常断线后客户端会频繁尝试重连，日志里应该是满屏的重连记录。没有日志意味着问题不在重连失败，而在",[101,1144,1145],{},"压根没启动重连机制","。说明客户端进入了某个永久休眠状态。",[14,1148,1149],{"id":1149},"三条独立的永久断开路径",[10,1151,1152],{},"代码审查发现了三条完全不同的、独立的永久断开路径。任意一条命中，就会停止调度任何重连计时器。",[1154,1155,1157],"h3",{"id":1156},"路径-a凭据刷新异常","路径 A：凭据刷新异常",[10,1159,1160,1161,1164],{},"在 ",[28,1162,1163],{},"_scheduleReconnect"," 方法末尾，每次普通重连都会调用：",[21,1166,1170],{"className":1167,"code":1168,"language":1169,"meta":30,"style":30},"language-javascript shiki shiki-themes github-light github-dark","this._reconnectTimer = setTimeout(() => {\n  if (!this._intentionalClose) {\n    this._refreshCredentialsAndReconnect(0)\n  }\n}, delay)\n","javascript",[28,1171,1172,1196,1212,1233,1239],{"__ignoreMap":30},[276,1173,1174,1177,1180,1184,1187,1190,1193],{"class":278,"line":279},[276,1175,1176],{"class":290},"this",[276,1178,1179],{"class":297},"._reconnectTimer ",[276,1181,1183],{"class":1182},"szBVR","=",[276,1185,1186],{"class":282}," setTimeout",[276,1188,1189],{"class":297},"(() ",[276,1191,1192],{"class":1182},"=>",[276,1194,1195],{"class":297}," {\n",[276,1197,1198,1201,1204,1207,1209],{"class":278,"line":213},[276,1199,1200],{"class":1182},"  if",[276,1202,1203],{"class":297}," (",[276,1205,1206],{"class":1182},"!",[276,1208,1176],{"class":290},[276,1210,1211],{"class":297},"._intentionalClose) {\n",[276,1213,1215,1218,1221,1224,1227,1230],{"class":278,"line":1214},3,[276,1216,1217],{"class":290},"    this",[276,1219,1220],{"class":297},".",[276,1222,1223],{"class":282},"_refreshCredentialsAndReconnect",[276,1225,1226],{"class":297},"(",[276,1228,1229],{"class":290},"0",[276,1231,1232],{"class":297},")\n",[276,1234,1236],{"class":278,"line":1235},4,[276,1237,1238],{"class":297},"  }\n",[276,1240,1242],{"class":278,"line":1241},5,[276,1243,1244],{"class":297},"}, delay)\n",[10,1246,1247,1248,1250],{},"而 ",[28,1249,1223],{}," 是异步方法，catch 分支没有任何后续处理：",[21,1252,1254],{"className":1167,"code":1253,"language":1169,"meta":30,"style":30},"} catch (e) {\n  console.error('[ws] 刷新凭据失败:', e)\n  this._setConnected(false, 'error', `凭据刷新失败: ${e}`)\n}\n",[28,1255,1256,1267,1282,1316],{"__ignoreMap":30},[276,1257,1258,1261,1264],{"class":278,"line":279},[276,1259,1260],{"class":297},"} ",[276,1262,1263],{"class":1182},"catch",[276,1265,1266],{"class":297}," (e) {\n",[276,1268,1269,1272,1274,1276,1279],{"class":278,"line":213},[276,1270,1271],{"class":297},"  console.",[276,1273,839],{"class":282},[276,1275,1226],{"class":297},[276,1277,1278],{"class":286},"'[ws] 刷新凭据失败:'",[276,1280,1281],{"class":297},", e)\n",[276,1283,1284,1287,1289,1292,1294,1297,1300,1303,1305,1308,1311,1314],{"class":278,"line":1214},[276,1285,1286],{"class":290},"  this",[276,1288,1220],{"class":297},[276,1290,1291],{"class":282},"_setConnected",[276,1293,1226],{"class":297},[276,1295,1296],{"class":290},"false",[276,1298,1299],{"class":297},", ",[276,1301,1302],{"class":286},"'error'",[276,1304,1299],{"class":297},[276,1306,1307],{"class":286},"`凭据刷新失败: ${",[276,1309,1310],{"class":297},"e",[276,1312,1313],{"class":286},"}`",[276,1315,1232],{"class":297},[276,1317,1318],{"class":278,"line":1235},[276,1319,1320],{"class":297},"}\n",[10,1322,1323,1324,149,1327,1330],{},"如果 ",[28,1325,1326],{},"api.readOpenclawConfig()",[28,1328,1329],{},"api.autoPairDevice()"," 抛错——磁盘 IO 抖、Rust 端处理器忙、Tauri IPC 队列阻塞——这次重连就永久终止。便携磁盘的 IO 抖动 + 配置文件读写争抢时最容易发生。",[1154,1332,1334],{"id":1333},"路径-b认证失败后的强制关闭","路径 B：认证失败后的强制关闭",[10,1336,1337],{},"当 WebSocket 收到 1008 unauthorized 响应时的处理逻辑：",[21,1339,1341],{"className":1167,"code":1340,"language":1169,"meta":30,"style":30},"if (this._authRetryCount \u003C 2) {\n  this._authRetryCount++\n  this._refreshCredentialsAndReconnect()\n  return\n}\nthis._setConnected(false, 'auth_failed', `认证失败: ${e.reason}。请检查 Gateway Token 配置。`)\nthis._intentionalClose = true   \u002F\u002F ← 永久关闭标记\nthis._flushPending()\nreturn\n",[28,1342,1343,1364,1374,1385,1390,1394,1429,1446,1458],{"__ignoreMap":30},[276,1344,1345,1348,1350,1352,1355,1358,1361],{"class":278,"line":279},[276,1346,1347],{"class":1182},"if",[276,1349,1203],{"class":297},[276,1351,1176],{"class":290},[276,1353,1354],{"class":297},"._authRetryCount ",[276,1356,1357],{"class":1182},"\u003C",[276,1359,1360],{"class":290}," 2",[276,1362,1363],{"class":297},") {\n",[276,1365,1366,1368,1371],{"class":278,"line":213},[276,1367,1286],{"class":290},[276,1369,1370],{"class":297},"._authRetryCount",[276,1372,1373],{"class":1182},"++\n",[276,1375,1376,1378,1380,1382],{"class":278,"line":1214},[276,1377,1286],{"class":290},[276,1379,1220],{"class":297},[276,1381,1223],{"class":282},[276,1383,1384],{"class":297},"()\n",[276,1386,1387],{"class":278,"line":1235},[276,1388,1389],{"class":1182},"  return\n",[276,1391,1392],{"class":278,"line":1241},[276,1393,1320],{"class":297},[276,1395,1397,1399,1401,1403,1405,1407,1409,1412,1414,1417,1419,1421,1424,1427],{"class":278,"line":1396},6,[276,1398,1176],{"class":290},[276,1400,1220],{"class":297},[276,1402,1291],{"class":282},[276,1404,1226],{"class":297},[276,1406,1296],{"class":290},[276,1408,1299],{"class":297},[276,1410,1411],{"class":286},"'auth_failed'",[276,1413,1299],{"class":297},[276,1415,1416],{"class":286},"`认证失败: ${",[276,1418,1310],{"class":297},[276,1420,1220],{"class":286},[276,1422,1423],{"class":297},"reason",[276,1425,1426],{"class":286},"}。请检查 Gateway Token 配置。`",[276,1428,1232],{"class":297},[276,1430,1432,1434,1437,1439,1442],{"class":278,"line":1431},7,[276,1433,1176],{"class":290},[276,1435,1436],{"class":297},"._intentionalClose ",[276,1438,1183],{"class":1182},[276,1440,1441],{"class":290}," true",[276,1443,1445],{"class":1444},"sJ8bj","   \u002F\u002F ← 永久关闭标记\n",[276,1447,1449,1451,1453,1456],{"class":278,"line":1448},8,[276,1450,1176],{"class":290},[276,1452,1220],{"class":297},[276,1454,1455],{"class":282},"_flushPending",[276,1457,1384],{"class":297},[276,1459,1461],{"class":278,"line":1460},9,[276,1462,1463],{"class":1182},"return\n",[10,1465,1466,1467,1470,1471,1474],{},"设置 ",[28,1468,1469],{},"_intentionalClose = true"," 之后，所有重连逻辑都被 ",[28,1472,1473],{},"if (!this._intentionalClose)"," 短路。即使只是 Gateway 重启窗口碰好出现 token 短暂不匹配，也会一次性把客户端打死，必须刷新页面才能恢复。",[1154,1476,1478],{"id":1477},"路径-c快速重连配额耗尽","路径 C：快速重连配额耗尽",[10,1480,1481,1482,990],{},"定义了常数 ",[28,1483,1484],{},"MAX_RECONNECT_ATTEMPTS = 60",[21,1486,1488],{"className":1167,"code":1487,"language":1169,"meta":30,"style":30},"if (this._reconnectAttempts >= MAX_RECONNECT_ATTEMPTS) {\n  this._setConnected(false, 'error', `连接失败，已停止重连。请手动刷新页面重试。`)\n  return\n}\n",[28,1489,1490,1509,1532,1536],{"__ignoreMap":30},[276,1491,1492,1494,1496,1498,1501,1504,1507],{"class":278,"line":279},[276,1493,1347],{"class":1182},[276,1495,1203],{"class":297},[276,1497,1176],{"class":290},[276,1499,1500],{"class":297},"._reconnectAttempts ",[276,1502,1503],{"class":1182},">=",[276,1505,1506],{"class":290}," MAX_RECONNECT_ATTEMPTS",[276,1508,1363],{"class":297},[276,1510,1511,1513,1515,1517,1519,1521,1523,1525,1527,1530],{"class":278,"line":213},[276,1512,1286],{"class":290},[276,1514,1220],{"class":297},[276,1516,1291],{"class":282},[276,1518,1226],{"class":297},[276,1520,1296],{"class":290},[276,1522,1299],{"class":297},[276,1524,1302],{"class":286},[276,1526,1299],{"class":297},[276,1528,1529],{"class":286},"`连接失败，已停止重连。请手动刷新页面重试。`",[276,1531,1232],{"class":297},[276,1533,1534],{"class":278,"line":1214},[276,1535,1389],{"class":1182},[276,1537,1538],{"class":278,"line":1235},[276,1539,1320],{"class":297},[10,1541,1542],{},"客户机晚上挂机，Gateway 因为便携磁盘 GC 或 Windows 休眠争抢资源短暂掉线，客户端尝试 60 次仍未成功重连，就永久停摆。早上用户回来面板已成死链。",[14,1544,1545],{"id":1545},"为什么三条缺陷同时存在",[10,1547,1548],{},"这是逐步累加的历史债：",[74,1550,1551,1558,1561],{},[77,1552,1553,1554,1557],{},"路径 B 是在修复\"避免无限自动配对循环\"时添加的保护，但用错了对象——",[28,1555,1556],{},"_intentionalClose=true"," 本来是用户主动断开的语义，不该用在被动失败上",[77,1559,1560],{},"路径 C 是早期\"避免无穷重试\"的防护，但 60 次后完全放弃而不留任何复活路径是绝对错误",[77,1562,1563],{},"路径 A 是在\"凭据 reload\"改动时把所有重连都改走 refreshCredentials，没留意 catch 分支已经成了终态",[10,1565,1566],{},"三条各自独立、都能单独打死客户端，组合在一起命中率非常高。",[14,1568,125],{"id":125},[10,1570,1571],{},"核心思想是永不彻底放弃。任何\"快速重连配额耗尽\"的分支都转入慢轮询，留出窗口让用户改配置或等待 Gateway 恢复后能自动复连。",[10,1573,1574,1575,1578],{},"新增辅助方法 ",[28,1576,1577],{},"_schedulePoll(delayMs, kind)"," 作为慢轮询的触发器：",[21,1580,1582],{"className":1167,"code":1581,"language":1169,"meta":30,"style":30},"const AUTH_RETRY_LIMIT = 2\nconst SLOW_POLL_DELAY_AUTH = 60_000      \u002F\u002F 认证持续失败：1 分钟探一次\nconst SLOW_POLL_DELAY_GENERAL = 300_000  \u002F\u002F 一般持续失败：5 分钟探一次\n\n_schedulePoll(delayMs, kind) {\n  this._clearReconnectTimer()\n  this._reconnectAttempts = 0\n  if (kind === 'auth') this._authRetryCount = 0\n  this._reconnectState = 'scheduled'\n  this._pendingReconnect = true\n  this._reconnectTimer = setTimeout(() => {\n    this._reconnectTimer = null\n    if (this._intentionalClose) return\n    this._reconnectState = 'attempting'\n    if (kind === 'auth') {\n      this._refreshCredentialsAndReconnect(0)\n    } else {\n      this._doConnect()\n    }\n  }, delayMs)\n}\n",[28,1583,1584,1598,1613,1628,1633,1641,1652,1663,1687,1699,1712,1729,1741,1756,1768,1781,1797,1808,1820,1826,1832],{"__ignoreMap":30},[276,1585,1586,1589,1592,1595],{"class":278,"line":279},[276,1587,1588],{"class":1182},"const",[276,1590,1591],{"class":290}," AUTH_RETRY_LIMIT",[276,1593,1594],{"class":1182}," =",[276,1596,1597],{"class":290}," 2\n",[276,1599,1600,1602,1605,1607,1610],{"class":278,"line":213},[276,1601,1588],{"class":1182},[276,1603,1604],{"class":290}," SLOW_POLL_DELAY_AUTH",[276,1606,1594],{"class":1182},[276,1608,1609],{"class":290}," 60_000",[276,1611,1612],{"class":1444},"      \u002F\u002F 认证持续失败：1 分钟探一次\n",[276,1614,1615,1617,1620,1622,1625],{"class":278,"line":1214},[276,1616,1588],{"class":1182},[276,1618,1619],{"class":290}," SLOW_POLL_DELAY_GENERAL",[276,1621,1594],{"class":1182},[276,1623,1624],{"class":290}," 300_000",[276,1626,1627],{"class":1444},"  \u002F\u002F 一般持续失败：5 分钟探一次\n",[276,1629,1630],{"class":278,"line":1235},[276,1631,1632],{"emptyLinePlaceholder":226},"\n",[276,1634,1635,1638],{"class":278,"line":1241},[276,1636,1637],{"class":282},"_schedulePoll",[276,1639,1640],{"class":297},"(delayMs, kind) {\n",[276,1642,1643,1645,1647,1650],{"class":278,"line":1396},[276,1644,1286],{"class":290},[276,1646,1220],{"class":297},[276,1648,1649],{"class":282},"_clearReconnectTimer",[276,1651,1384],{"class":297},[276,1653,1654,1656,1658,1660],{"class":278,"line":1431},[276,1655,1286],{"class":290},[276,1657,1500],{"class":297},[276,1659,1183],{"class":1182},[276,1661,1662],{"class":290}," 0\n",[276,1664,1665,1667,1670,1673,1676,1679,1681,1683,1685],{"class":278,"line":1448},[276,1666,1200],{"class":1182},[276,1668,1669],{"class":297}," (kind ",[276,1671,1672],{"class":1182},"===",[276,1674,1675],{"class":286}," 'auth'",[276,1677,1678],{"class":297},") ",[276,1680,1176],{"class":290},[276,1682,1354],{"class":297},[276,1684,1183],{"class":1182},[276,1686,1662],{"class":290},[276,1688,1689,1691,1694,1696],{"class":278,"line":1460},[276,1690,1286],{"class":290},[276,1692,1693],{"class":297},"._reconnectState ",[276,1695,1183],{"class":1182},[276,1697,1698],{"class":286}," 'scheduled'\n",[276,1700,1702,1704,1707,1709],{"class":278,"line":1701},10,[276,1703,1286],{"class":290},[276,1705,1706],{"class":297},"._pendingReconnect ",[276,1708,1183],{"class":1182},[276,1710,1711],{"class":290}," true\n",[276,1713,1715,1717,1719,1721,1723,1725,1727],{"class":278,"line":1714},11,[276,1716,1286],{"class":290},[276,1718,1179],{"class":297},[276,1720,1183],{"class":1182},[276,1722,1186],{"class":282},[276,1724,1189],{"class":297},[276,1726,1192],{"class":1182},[276,1728,1195],{"class":297},[276,1730,1732,1734,1736,1738],{"class":278,"line":1731},12,[276,1733,1217],{"class":290},[276,1735,1179],{"class":297},[276,1737,1183],{"class":1182},[276,1739,1740],{"class":290}," null\n",[276,1742,1744,1747,1749,1751,1754],{"class":278,"line":1743},13,[276,1745,1746],{"class":1182},"    if",[276,1748,1203],{"class":297},[276,1750,1176],{"class":290},[276,1752,1753],{"class":297},"._intentionalClose) ",[276,1755,1463],{"class":1182},[276,1757,1759,1761,1763,1765],{"class":278,"line":1758},14,[276,1760,1217],{"class":290},[276,1762,1693],{"class":297},[276,1764,1183],{"class":1182},[276,1766,1767],{"class":286}," 'attempting'\n",[276,1769,1771,1773,1775,1777,1779],{"class":278,"line":1770},15,[276,1772,1746],{"class":1182},[276,1774,1669],{"class":297},[276,1776,1672],{"class":1182},[276,1778,1675],{"class":286},[276,1780,1363],{"class":297},[276,1782,1784,1787,1789,1791,1793,1795],{"class":278,"line":1783},16,[276,1785,1786],{"class":290},"      this",[276,1788,1220],{"class":297},[276,1790,1223],{"class":282},[276,1792,1226],{"class":297},[276,1794,1229],{"class":290},[276,1796,1232],{"class":297},[276,1798,1800,1803,1806],{"class":278,"line":1799},17,[276,1801,1802],{"class":297},"    } ",[276,1804,1805],{"class":1182},"else",[276,1807,1195],{"class":297},[276,1809,1811,1813,1815,1818],{"class":278,"line":1810},18,[276,1812,1786],{"class":290},[276,1814,1220],{"class":297},[276,1816,1817],{"class":282},"_doConnect",[276,1819,1384],{"class":297},[276,1821,1823],{"class":278,"line":1822},19,[276,1824,1825],{"class":297},"    }\n",[276,1827,1829],{"class":278,"line":1828},20,[276,1830,1831],{"class":297},"  }, delayMs)\n",[276,1833,1835],{"class":278,"line":1834},21,[276,1836,1320],{"class":297},[10,1838,1839,1840,1842,1843,1846],{},"慢轮询命中后失败会重新进入 ",[28,1841,1163],{}," 快速重连周期（因为 ",[28,1844,1845],{},"_reconnectAttempts"," 重置为 0），相当于\"快速 60 次 → 慢一次 → 快速 60 次 → 慢一次\"的循环，永不放弃。",[10,1848,1849],{},"三条修复并行：",[10,1851,1852,1855,1856,1859],{},[101,1853,1854],{},"Fix A","：普通重连直接走 ",[28,1857,1858],{},"_doConnect()","，凭据刷新隔离到认证失败分支，避免配置读取错误牵连普通重连。",[10,1861,1862,1865,1866,1869,1870,1872],{},[101,1863,1864],{},"Fix B","：认证失败耗尽后转入 ",[28,1867,1868],{},"_schedulePoll('auth')","，不再设置 ",[28,1871,1556],{},"，给认证恢复留出 60 秒的探测窗口。",[10,1874,1875,1878,1879,1882],{},[101,1876,1877],{},"Fix C","：重连次数超过 MAX_RECONNECT_ATTEMPTS 后转入 ",[28,1880,1881],{},"_schedulePoll('general')","，设置 UI 状态为\"连接持续失败，300 秒后重试\"而不是终态。",[10,1884,1885],{},"慢轮询失败后重新进入快速重连周期，形成\"快速 60 次 → 慢一次 → 快速 60 次\"的循环，永不放弃。",[14,1887,1888],{"id":1888},"防回归验证",[10,1890,1891,1892,1895],{},"新增 ",[28,1893,1894],{},"wakou-full\u002Fsrc\u002Flib\u002Fws-client.slow-poll.test.js","，覆盖 7 个 case：",[74,1897,1898,1906,1918,1921,1930,1937,1942],{},[77,1899,1900,1901,1903,1904],{},"Fix A：普通重连命中 ",[28,1902,1817],{},"，不调用 ",[28,1905,1223],{},[77,1907,1908,1909,1912,1913,1915,1916],{},"Fix C：MAX_RECONNECT_ATTEMPTS 后状态是 ",[28,1910,1911],{},"reconnecting","（非终态 ",[28,1914,839],{},"），5 分钟后触发 ",[28,1917,1817],{},[77,1919,1920],{},"Fix C：第二轮慢轮询失败后还能再调度快速重连",[77,1922,1923,1924,1926,1927],{},"Fix B：",[28,1925,1868],{}," 等待 60 秒调用 ",[28,1928,1929],{},"_refreshCredentialsAndReconnect(0)",[77,1931,1923,1932,1934,1935],{},[28,1933,1881],{}," 等待后调用 ",[28,1936,1817],{},[77,1938,1939,1941],{},[28,1940,1556],{}," 时慢轮询应跳过",[77,1943,1944,1946],{},[28,1945,1223],{}," 抛错路径调度慢轮询",[10,1948,1949],{},"测试全部通过（7\u002F7）。",[14,1951,1952],{"id":1952},"外部因素的叠加",[10,1954,1955],{},"这个时期同步发生了另一个外部问题：CDN 对 WebSocket 空闲连接会在 4-9 分钟后强制切断。客户端收到连接中断后根据指数退避策略重连，如果恰好命中上述三条路径之一，就进入永久断开状态。这解释了为什么故障特别在长时间挂机后高发——空闲足够长，CDN 必定切断，而后续重连很容易落入某条缺陷路径。两个问题各自独立，但组合效果是\"挂机过夜必死\"。",[14,1957,1958],{"id":1958},"排查验证",[10,1960,1961],{},"故障修复后，可以用以下方式验证重连状态：",[21,1963,1965],{"className":1167,"code":1964,"language":1169,"meta":30,"style":30},"__clawpanelWsClient.getConnectionInfo()\n\u002F\u002F {\n\u002F\u002F   connected: false,\n\u002F\u002F   reconnectState: 'scheduled' | 'attempting',\n\u002F\u002F   reconnectAttempts: 0~60,\n\u002F\u002F   ...\n\u002F\u002F }\n",[28,1966,1967,1977,1982,1987,1992,1997,2002],{"__ignoreMap":30},[276,1968,1969,1972,1975],{"class":278,"line":279},[276,1970,1971],{"class":297},"__clawpanelWsClient.",[276,1973,1974],{"class":282},"getConnectionInfo",[276,1976,1384],{"class":297},[276,1978,1979],{"class":278,"line":213},[276,1980,1981],{"class":1444},"\u002F\u002F {\n",[276,1983,1984],{"class":278,"line":1214},[276,1985,1986],{"class":1444},"\u002F\u002F   connected: false,\n",[276,1988,1989],{"class":278,"line":1235},[276,1990,1991],{"class":1444},"\u002F\u002F   reconnectState: 'scheduled' | 'attempting',\n",[276,1993,1994],{"class":278,"line":1241},[276,1995,1996],{"class":1444},"\u002F\u002F   reconnectAttempts: 0~60,\n",[276,1998,1999],{"class":278,"line":1396},[276,2000,2001],{"class":1444},"\u002F\u002F   ...\n",[276,2003,2004],{"class":278,"line":1431},[276,2005,2006],{"class":1444},"\u002F\u002F }\n",[10,2008,2009,2012,2013,2016,2017,2012,2020,2023],{},[28,2010,2011],{},"reconnectState='scheduled'"," 且 ",[28,2014,2015],{},"reconnectAttempts=0"," 表示在慢轮询窗口正常工作；",[28,2018,2019],{},"reconnectState='idle'",[28,2021,2022],{},"connected=false"," 是老的永久断开路径。",[10,2025,2026],{},"重连机制里避免静默终态是最基本的要求。任何异常路径都必须有可见的状态提示和后续操作入口，否则用户端看到的就是死机。这次故障的根本教训是，不能让客户端在任何情况下进入\"不可恢复\"的状态而没有任何提示。慢轮询的引入给了所有失败情景一个\"最后的机会\"，即使前面的快速重连机制彻底耗尽了，用户等待足够长的时间后系统仍有自动恢复的可能。",[632,2028,2029],{},"html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .szBVR, html code.shiki .szBVR{--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html pre.shiki code .sJ8bj, html code.shiki .sJ8bj{--shiki-default:#6A737D;--shiki-dark:#6A737D}",{"title":30,"searchDepth":213,"depth":213,"links":2031},[2032,2037,2038,2039,2040,2041],{"id":1149,"depth":213,"text":1149,"children":2033},[2034,2035,2036],{"id":1156,"depth":1214,"text":1157},{"id":1333,"depth":1214,"text":1334},{"id":1477,"depth":1214,"text":1478},{"id":1545,"depth":213,"text":1545},{"id":125,"depth":213,"text":125},{"id":1888,"depth":213,"text":1888},{"id":1952,"depth":213,"text":1952},{"id":1958,"depth":213,"text":1958},"2026-06-17","从 2026-05-11 开始，多台便携包客户反馈 WebSocket 断线后面板卡死。网关进程（openclaw_gateway.exe）明确在运行，\u002Fhealth 返回 200，但面板显示\"已停止重连，请手动刷新\"——用户必须手动重启才能恢复。",{},"\u002F2026-06-17-fa-016-ws",{"title":1120,"description":2043},"FA-016","2026-06-17-FA-016-WS永久断开三条路径","客户端重连逻辑中三处独立的\"静默终止\"缺陷导致 WS 永久断开，任何一条路径命中就不再调度重连计时器。",[2051,233,2052,2053,2054],"WebSocket","客户端重连机制","异常处理","便携包","qaISKpAwjYnhiSdxEMYYnJtJ-A8QaOq-djOEJ0W9-bM",1785406912417]