{"id":1739,"date":"2026-06-24T14:01:03","date_gmt":"2026-06-24T06:01:03","guid":{"rendered":"https:\/\/www.izhuhn.cn\/?p=1739"},"modified":"2026-06-26T11:05:54","modified_gmt":"2026-06-26T03:05:54","slug":"dit4dit-%e6%98%87%e8%85%benpu%e9%80%82%e9%85%8d%e7%ae%80%e8%ae%b0-3","status":"publish","type":"post","link":"https:\/\/www.izhuhn.cn\/index.php\/2026\/06\/24\/dit4dit-%e6%98%87%e8%85%benpu%e9%80%82%e9%85%8d%e7%ae%80%e8%ae%b0-3\/","title":{"rendered":"DiT4DiT \u6607\u817eNPU\u9002\u914d\u7b80\u8bb0"},"content":{"rendered":"\n<p class=\"wp-block-paragraph\">DiT4DiT \u662f\u6e2f\u79d1\u5e7f\u56e2\u961f\u5728 StarVLA \u57fa\u7840\u4e0a\u63d0\u51fa\u7684\u89c6\u89c9-\u52a8\u4f5c\u6a21\u578b\uff08VAM\uff09\uff0c\u5c06\u89c6\u9891\u751f\u6210 Diffusion Transformer \u4e0e Flow Matching \u52a8\u4f5c\u9884\u6d4b\u7ed3\u5408\uff0c\u652f\u6301\u673a\u68b0\u81c2\u7075\u5de7\u64cd\u4f5c\u4e0e\u4eba\u5f62\u673a\u5668\u4eba\u5168\u8eab\u63a7\u5236\u3002\u539f\u59cb\u4ee3\u7801\u5b8c\u5168\u57fa\u4e8e CUDA \u751f\u6001\uff08cuDNN\u3001NCCL\u3001torch.cuda\uff09\uff0c\u800c\u534e\u4e3a\u6607\u817e Atlas 800T A3 \u4f7f\u7528\u81ea\u7814\u8fbe\u82ac\u5947\u67b6\u6784 NPU\uff0c\u4e24\u8005\u5728\u7b97\u5b50\u652f\u6301\u3001\u5206\u5e03\u5f0f\u901a\u4fe1\u3001\u6df7\u5408\u7cbe\u5ea6\u7b49\u65b9\u9762\u5b58\u5728\u663e\u8457\u5dee\u5f02\u3002\u672c\u6587\u8bb0\u5f55\u4e86\u4ece\u96f6\u5c06 DiT4DiT \u9002\u914d\u5230\u6607\u817e NPU \u7684\u5168\u8fc7\u7a0b\uff0c\u5305\u62ec\u524d\u671f\u8c03\u7814\u3001\u73af\u5883\u914d\u7f6e\u3001\u4ee3\u7801\u79fb\u690d\u3001ZeRO-2\/ZeRO-3 \u4e24\u79cd\u5206\u5e03\u5f0f\u7b56\u7565\u7684\u8c03\u4f18\uff0c\u4ee5\u53ca 12+ \u4e2a\u8e29\u5751\u70b9\u7684\u5b9a\u4f4d\u4e0e\u4fee\u590d\uff0c\u6700\u7ec8\u5728 RoboTwin \u53cc\u81c2\u6570\u636e\u96c6\u4e0a\u6210\u529f\u8dd1\u901a\u8bad\u7ec3\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Git \u4ed3\u5e93\u7248\u672c<\/strong>\uff1acommit <code>1ae6efd<\/code>\uff08<a href=\"https:\/\/github.com\/Mondo-Robotics\/DiT4DiT\" target=\"_blank\"  rel=\"nofollow\" >github.com\/Mondo-Robotics\/DiT4DiT<\/a>\uff09\uff0c\u9002\u914d patch \u5df2\u4e0a\u4f20\u81f3\u670d\u52a1\u5668\uff0c\u6587\u672b\u9644\u4e0b\u8f7d\u94fe\u63a5\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">\u4e00\u3001\u524d\u671f\u8c03\u7814\u4e0e\u9002\u914d\u601d\u8def<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\">1.1 \u4e3a\u4ec0\u4e48\u8981\u9002\u914d DiT4DiT<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u5177\u8eab\u667a\u80fd\u8d5b\u9053\u5f53\u524d\u4e3b\u6d41\u7684 VLA \u6a21\u578b\uff08\u03c00.5\u3001OpenVLA\u3001StarVLA\uff09\u5927\u591a\u57fa\u4e8e\u5355\u4e00\u89c6\u89c9\u7f16\u7801\u5668 + \u52a8\u4f5c\u5934\u7ed3\u6784\u3002DiT4DiT \u7684\u4e0d\u540c\u4e4b\u5904\u5728\u4e8e\u5b83\u5f15\u5165\u4e86<strong>\u89c6\u9891\u6269\u6563\u6a21\u578b<\/strong>\u4f5c\u4e3a\u89c6\u89c9 backbone\u2014\u2014\u8fd9\u5e26\u6765\u4e86\u4e24\u4e2a\u5173\u952e\u4f18\u52bf\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u65f6\u5e8f\u5efa\u6a21\u66f4\u5f3a<\/strong>\uff1aCosmos-Predict2.5-2B \u539f\u751f\u652f\u6301\u89c6\u9891\u5e27\u95f4\u52a8\u6001\u5efa\u6a21\uff0c\u6bd4\u9759\u6001\u56fe\u50cf\u7f16\u7801\u5668\u5bf9\u52a8\u4f5c\u8fde\u7eed\u6027\u7684\u611f\u77e5\u66f4\u654f\u9510<\/li>\n<li><strong>\u89c6\u9891\u8f85\u52a9 loss<\/strong>\uff1a\u8bad\u7ec3\u65f6\u53ef\u4ee5\u540c\u65f6\u4f18\u5316\u52a8\u4f5c\u9884\u6d4b\u548c\u672a\u6765\u5e27\u91cd\u5efa\uff0c\u8fd9\u662f\u7eaf VLA \u505a\u4e0d\u5230\u7684\u591a\u6a21\u6001\u76d1\u7763<\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">\u4f46\u4ee3\u4ef7\u4e5f\u5f88\u660e\u663e\uff1a\u89c6\u9891 backbone \u7684\u53c2\u6570\u91cf\u548c\u8ba1\u7b97\u91cf\u8fdc\u5927\u4e8e\u4f20\u7edf ViT\uff0c\u5bf9\u7b97\u529b\u548c\u663e\u5b58\u7684\u8981\u6c42\u66f4\u9ad8\u3002\u6b63\u597d\u6211\u4eec\u6709 Atlas 800T A3\uff08910B\uff098 \u5361\u73af\u5883\uff0c64GB HBM \u5355\u5361\u663e\u5b58\u5bf9 2B \u7ea7\u6a21\u578b\u662f\u5145\u8db3\u7684\uff0c\u503c\u5f97\u4e00\u8bd5\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">1.2 DiT4DiT \u4e0e StarVLA \u7684\u5173\u7cfb<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u4e24\u4e2a\u5f00\u6e90\u4ed3\u5e93\u7684\u5173\u7cfb\u68b3\u7406\u5982\u4e0b\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-table\"><table>\n<thead><tr><th><\/th><th>StarVLA<\/th><th>DiT4DiT<\/th><\/tr><\/thead>\n<tbody>\n<tr><td>\u5b9a\u4f4d<\/td><td>VLA \u57fa\u7840\u6846\u67b6<\/td><td>\u57fa\u4e8e StarVLA \u7684\u89c6\u9891 VAM \u6269\u5c55<\/td><\/tr>\n<tr><td>\u57fa\u5ea7\u6a21\u578b<\/td><td>Qwen3-VL-4B-Instruct<\/td><td>Cosmos-Predict2.5-2B<\/td><\/tr>\n<tr><td>\u52a8\u4f5c\u9884\u6d4b<\/td><td>\u6807\u51c6 diffusion head\uff08DiT-B\uff09<\/td><td>Flow-matching action head<\/td><\/tr>\n<tr><td>\u89c6\u9891\u6a21\u578b<\/td><td>\u65e0<\/td><td>\u89c6\u9891 diffusion + VAE \u7f16\u89e3\u7801<\/td><\/tr>\n<tr><td>\u8bad\u7ec3\u6846\u67b6<\/td><td>accelerate + DeepSpeed<\/td><td>accelerate + DeepSpeed\uff08\u590d\u7528\uff09<\/td><\/tr>\n<tr><td>\u6570\u636e\u683c\u5f0f<\/td><td>LeRobot<\/td><td>LeRobot\uff08\u590d\u7528\uff09<\/td><\/tr>\n<tr><td>\u534e\u4e3a\u9002\u914d<\/td><td>\u2705 \u5df2\u6709 NPU \u7248\uff08DrivingSDK\uff09<\/td><td>\u274c \u6682\u65e0\uff08\u672c\u6587\u5b8c\u6210\uff09<\/td><\/tr>\n<tr><td>\u4ed3\u5e93\u5730\u5740<\/td><td>gitcode.com\/Ascend\/DrivingSDK<\/td><td>github.com\/Mondo-Robotics\/DiT4DiT<\/td><\/tr>\n<\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\">\u53ef\u4ee5\u770b\u51fa DiT4DiT \u5728\u8bad\u7ec3\u6846\u67b6\u5c42\u9762\u5927\u91cf\u590d\u7528\u4e86 StarVLA \u7684\u4ee3\u7801\uff08accelerate \u542f\u52a8\u5668\u3001DeepSpeed \u914d\u7f6e\u3001LeRobot dataloader\u3001DiT-B \u52a8\u4f5c\u5934\uff09\uff0c\u6838\u5fc3\u5dee\u5f02\u53ea\u5728\u4e8e\u89c6\u89c9 backbone \u4ece Qwen3-VL \u6362\u6210\u4e86 Cosmos-Predict2.5-2B\u3002\u8fd9\u610f\u5473\u7740\u9002\u914d\u5de5\u4f5c\u91cf\u4e0d\u4f1a\u592a\u5927\u2014\u2014\u534e\u4e3a\u5df2\u7ecf\u8dd1\u901a\u4e86 StarVLA \u7684 NPU \u7248\u672c\uff0c\u6211\u4eec\u53ea\u9700\u8981\u5728 DiT4DiT \u4e0a\u590d\u523b\u76f8\u540c\u7684\u79fb\u690d\u64cd\u4f5c\u5373\u53ef\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">1.3 \u9002\u914d\u96be\u5ea6\u8bc4\u4f30<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u5728\u52a8\u624b\u4e4b\u524d\uff0c\u5148\u68b3\u7406 DiT4DiT \u4ee3\u7801\u4e2d\u53ef\u80fd\u6d89\u53ca CUDA \u786c\u7f16\u7801\u7684\u6a21\u5757\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-table\"><table>\n<thead><tr><th>\u6a21\u5757<\/th><th>CUDA \u4f9d\u8d56\u70b9<\/th><th>\u9002\u914d\u96be\u5ea6<\/th><\/tr><\/thead>\n<tbody>\n<tr><td>\u8bad\u7ec3\u5165\u53e3 train.py<\/td><td>torch.autocast(\"cuda\")\u3001AdamW fused<\/td><td>\u2b50 \u4f4e\uff08\u5b57\u7b26\u4e32\u66ff\u6362\uff09<\/td><\/tr>\n<tr><td>\u6a21\u578b\u6846\u67b6 DiT4DiT.py<\/td><td>autocast\u3001inference_mode\u3001bf16\u2192numpy<\/td><td>\u2b50\u2b50 \u4e2d\uff08\u8bed\u4e49\u5dee\u5f02\uff09<\/td><\/tr>\n<tr><td>backbone Cosmos25.py<\/td><td>\u6df7\u5408 dtype \u53c2\u6570\u3001VAE \u51bb\u7ed3\u7b56\u7565<\/td><td>\u2b50\u2b50 \u4e2d\uff08ZeRO \u517c\u5bb9\uff09<\/td><\/tr>\n<tr><td>\u89c6\u9891\u7f16\u89e3\u7801 VAE<\/td><td>Conv3D\u3001GroupNorm \u7b97\u5b50<\/td><td>\u2b50 \u4f4e\uff08torch_npu \u539f\u751f\u652f\u6301\uff09<\/td><\/tr>\n<tr><td>DeepSpeed ZeRO-3<\/td><td>\u53c2\u6570\u5206\u7247 gather\u3001defragment<\/td><td>\u2b50\u2b50\u2b50 \u9ad8\uff08\u591a\u5751\uff09<\/td><\/tr>\n<tr><td>\u5206\u5e03\u5f0f\u901a\u4fe1<\/td><td>NCCL \u2192 HCCL\uff08\u81ea\u52a8\u8f6c\u6362\uff09<\/td><td>\u2b50 \u4f4e\uff08torch_npu \u81ea\u52a8\u9002\u914d\uff09<\/td><\/tr>\n<\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\">\u603b\u4f53\u6765\u770b\uff0cDiT4DiT \u6ca1\u6709\u50cf flash-attention\u3001triton kernel \u8fd9\u7c7b\u6df1\u5ea6\u7ed1\u5b9a CUDA \u7684\u81ea\u5b9a\u4e49\u7b97\u5b50\uff0c\u7edd\u5927\u591a\u6570 CUDA \u8c03\u7528\u90fd\u662f PyTorch \u9ad8\u5c42 API\uff08autocast\u3001DistributedDataParallel\uff09\uff0ctorch_npu \u5df2\u7ecf\u505a\u4e86\u9002\u914d\u3002\u771f\u6b63\u7684\u96be\u70b9\u5728 ZeRO-3 \u7684\u517c\u5bb9\u6027\u548c\u4e00\u4e9b NPU \u7279\u6709\u7684\u6570\u636e\u683c\u5f0f\u7ea6\u675f\uff08\u5982 bf16 \u4e0d\u652f\u6301 .numpy()\uff09\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">1.4 \u9002\u914d\u8def\u7ebf<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u786e\u5b9a\u4e09\u6b65\u8d70\u7b56\u7565\uff1a<\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li><strong>\u57fa\u7840 NPU \u79fb\u690d<\/strong>\uff1a\u53c2\u7167\u534e\u4e3a DrivingSDK \u4e2d StarVLA \u7684 starvla.patch\uff0c\u5bf9 DiT4DiT \u505a\u76f8\u540c\u7684 autocast \u66ff\u6362 + torch_npu \u5bfc\u5165 + mx_driving patcher \u521d\u59cb\u5316\u3002\u8fd9\u4e00\u6b65\u5b8c\u6210\u540e\u4ee3\u7801\u5e94\u80fd\u5728\u5355\u5361\u4e0a\u4e0d\u62a5\u9519\u5730\u8dd1\u901a\u524d\u5411\u3002<\/li>\n<li><strong>\u5206\u5e03\u5f0f\u8dd1\u901a<\/strong>\uff1a\u914d\u7f6e DeepSpeed ZeRO-2\uff0c8 \u5361\u52a0\u901f\u8bad\u7ec3\u3002ZeRO-2 \u5bf9\u4ee3\u7801\u4fb5\u5165\u6700\u5c0f\uff0c\u53c2\u6570\u5b8c\u6574\u4fdd\u7559\u5728\u6bcf\u5f20\u5361\u4e0a\uff0c\u5148\u4fdd\u8bc1\u80fd\u8dd1\u8d77\u6765\u3002<\/li>\n<li><strong>ZeRO-3 \u8c03\u4f18<\/strong>\uff1a\u5207\u6362\u5230 ZeRO-3 \u4ee5\u652f\u6301\u66f4\u5927 batch size\u3002\u8fd9\u4e00\u9636\u6bb5\u4f1a\u66b4\u9732\u6700\u591a\u517c\u5bb9\u6027\u95ee\u9898\uff08\u6df7\u5408 dtype\u3001inference_mode\u3001\u6a21\u578b\u4fdd\u5b58\u7b49\uff09\uff0c\u9700\u8981\u9010\u4e00\u4fee\u590d\u3002<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\">\u4e8c\u3001\u73af\u5883\u51c6\u5907<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\">2.1 \u786c\u4ef6\u4e0e\u9a71\u52a8<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u786c\u4ef6\u4e3a Atlas 800T A3 \u8bad\u7ec3\u670d\u52a1\u5668\uff0c\u5355\u673a 8 \u5361\u6607\u817e 910B\uff0c\u6bcf\u5361 64GB HBM\u3002\u5728\u8fdb\u884c NPU \u9002\u914d\u524d\uff0c\u5148\u786e\u8ba4\u9a71\u52a8\u548c\u56fa\u4ef6\u7248\u672c\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code># \u67e5\u770b NPU \u72b6\u6001\nnpu-smi info\n\n# \u786e\u8ba4\u9a71\u52a8\u7248\u672c\nnpu-smi info -t driver -i 0\n\n# \u67e5\u770b HCCL \u7f51\u7edc\u62d3\u6251\uff088 \u5361\u5e94\u5168\u4e92\u8054\uff09\nnpu-smi info -t topo<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u9884\u671f\u8f93\u51fa\uff1a8 \u5f20\u5361\u5168\u90e8\u663e\u793a \"OK\"\uff0c\u5065\u5eb7\u72b6\u6001\u6b63\u5e38\uff0c\u9a71\u52a8\u7248\u672c\u4e0e CANN \u7248\u672c\u5339\u914d\uff08CANN 9.0.0 \u5bf9\u5e94\u9a71\u52a8 24.1.rc1+\uff09\u3002\u5982\u679c\u9a71\u52a8\u7248\u672c\u8fc7\u65e7\uff0c\u9700\u8054\u7cfb\u96c6\u7fa4\u7ba1\u7406\u5458\u66f4\u65b0\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">2.2 \u8f6f\u4ef6\u7248\u672c<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u4ee5\u4e0b\u7248\u672c\u7ec4\u5408\u5df2\u5728 Atlas 800T A3 \u4e0a\u9a8c\u8bc1\u901a\u8fc7\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-table\"><table>\n<thead><tr><th>\u8f6f\u4ef6<\/th><th>\u7248\u672c<\/th><th>\u8bf4\u660e<\/th><\/tr><\/thead>\n<tbody>\n<tr><td>Python<\/td><td>3.10<\/td><td>DiT4DiT \u5b98\u65b9\u8981\u6c42 \u2265 3.10<\/td><\/tr>\n<tr><td>CANN<\/td><td>9.0.0<\/td><td>\u6607\u817e AI \u8ba1\u7b97\u6846\u67b6\uff0c\u542b\u7b97\u5b50\u5e93\u548c\u7f16\u8bd1\u5668<\/td><\/tr>\n<tr><td>PyTorch<\/td><td>2.7.1<\/td><td>\u9700\u914d\u5957 torch_npu \u7248\u672c<\/td><\/tr>\n<tr><td>torch_npu<\/td><td>2.7.1.post2<\/td><td>\u6607\u817e NPU \u540e\u7aef\uff0c\u63d0\u4f9b \"npu\" device<\/td><\/tr>\n<tr><td>torchvision<\/td><td>0.22.1<\/td><td>\u89c6\u9891\u5e27\u9884\u5904\u7406\u4f9d\u8d56<\/td><\/tr>\n<tr><td>DeepSpeed<\/td><td>0.18.4<\/td><td>\u5206\u5e03\u5f0f\u8bad\u7ec3\u6846\u67b6<\/td><\/tr>\n<tr><td>accelerate<\/td><td>1.12.0<\/td><td>HuggingFace \u8bad\u7ec3\u542f\u52a8\u5668<\/td><\/tr>\n<tr><td>diffusers<\/td><td>0.38.0<\/td><td>Cosmos \u6a21\u578b\u52a0\u8f7d<\/td><\/tr>\n<tr><td>transformers<\/td><td>4.57.0<\/td><td>HuggingFace \u6a21\u578b\u751f\u6001<\/td><\/tr>\n<tr><td>decord<\/td><td>0.6.0<\/td><td>\u89c6\u9891\u89e3\u7801\uff08\u9700\u6e90\u7801\u7f16\u8bd1\uff09<\/td><\/tr>\n<tr><td>ffmpeg<\/td><td>4.4.2<\/td><td>\u89c6\u9891\u7f16\u89e3\u7801\u57fa\u7840\u5e93<\/td><\/tr>\n<tr><td>mx_driving<\/td><td>1.0.20260421<\/td><td>\u534e\u4e3a NPU patcher\uff0c\u81ea\u52a8\u4fee\u590d\u7b97\u5b50\u517c\u5bb9\u6027<\/td><\/tr>\n<\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u7248\u672c\u9009\u62e9\u6ce8\u610f<\/strong>\uff1atorch \u548c torch_npu \u5fc5\u987b\u662f\u5c0f\u7248\u672c\u4e25\u683c\u5339\u914d\u7684\u2014\u2014\u4f60\u88c5 torch==2.7.1\uff0ctorch_npu \u5c31\u5f97\u662f 2.7.1.postX\u3002\u5982\u679c\u88c5\u6210 torch==2.7.0 + torch_npu==2.7.1\uff0c\u8fd0\u884c\u65f6\u4f1a\u62a5 <code>libtorch_npu.so: undefined symbol<\/code>\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">2.3 \u590d\u7528 StarVLA conda \u73af\u5883<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u65e2\u7136 StarVLA \u7684 NPU \u73af\u5883\u5df2\u7ecf\u914d\u7f6e\u597d\u4e86\uff08PyTorch + torch_npu + CANN + \u5404\u79cd\u4f9d\u8d56\uff09\uff0c\u76f4\u63a5\u590d\u7528\u53ef\u4ee5\u7701\u53bb\u5927\u91cf\u91cd\u590d\u5b89\u88c5\u5de5\u4f5c\u3002\u53ea\u9700\u5728\u5176\u57fa\u7840\u4e0a\u8865\u5145 DiT4DiT \u7279\u6709\u7684\u4f9d\u8d56\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code># \u5982\u679c\u5df2\u6709 StarVLA \u7684 conda \u73af\u5883\uff0c\u76f4\u63a5\u514b\u9686\u4e00\u4efd\nconda create -n dit4dit --clone starvla_env_name\nconda activate dit4dit\n\n# \u5982\u679c\u4ece\u5934\u5f00\u59cb\uff1a\nconda create -n dit4dit python=3.10\nconda activate dit4dit<\/code><\/pre>\n\n\n\n<h3 class=\"wp-block-heading\">2.4 \u5b89\u88c5 PyTorch + torch_npu<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u5347\u817e PyTorch \u7684 whl \u5305\u901a\u5e38\u7531\u96c6\u7fa4\u7ba1\u7406\u5458\u9884\u7f6e\u5728\u672c\u5730\u955c\u50cf\u6216 SWR \u4ed3\u5e93\u4e2d\u3002\u5982\u679c\u672c\u5730\u5df2\u6709 pip \u6e90\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>pip install torch==2.7.1\npip install torch_npu==2.7.1.post2\npip install torchvision==0.22.1\n\n# \u9a8c\u8bc1 NPU \u53ef\u7528\npython -c \"import torch; import torch_npu; print(torch.npu.is_available()); print(torch.npu.device_count())\"\n# \u9884\u671f\u8f93\u51fa: True \\n 8<\/code><\/pre>\n\n\n\n<h3 class=\"wp-block-heading\">2.5 \u5b89\u88c5 ffmpeg + decord<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">DiT4DiT \u7684\u89c6\u9891\u8f93\u5165\u4f9d\u8d56 decord \u505a\u5e27\u62bd\u53d6\uff0cdecord \u53c8\u4f9d\u8d56 ffmpeg \u505a\u89e3\u7801\u3002\u6ce8\u610f <strong>decord \u4e0d\u652f\u6301 pip install<\/strong>\uff0c\u5fc5\u987b\u6e90\u7801\u7f16\u8bd1\uff1a<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u63a8\u8350\u65b9\u6848\uff08conda \u5b89\u88c5 ffmpeg\uff09\uff1a<\/strong><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>conda install -c conda-forge ffmpeg=4.4.2<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u5907\u9009\u65b9\u6848\uff08\u6e90\u7801\u5b89\u88c5 ffmpeg\uff09\uff1a<\/strong><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>wget https:\/\/ffmpeg.org\/releases\/ffmpeg-4.4.2.tar.bz2\ntar -xvf ffmpeg-4.4.2.tar.bz2\ncd ffmpeg-4.4.2\n.\/configure --enable-shared --prefix=\/usr\/local\/ffmpeg\nmake -j 64\nmake install\ncd ..\necho 'export PATH=\"\/usr\/local\/ffmpeg\/bin:$PATH\"' >> \/etc\/profile.d\/ffmpeg.sh\necho 'export LD_LIBRARY_PATH=\"\/usr\/local\/ffmpeg\/lib:$LD_LIBRARY_PATH\"' >> \/etc\/profile.d\/ffmpeg.sh\nsource \/etc\/profile<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u7f16\u8bd1 decord\uff1a<\/strong><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>git clone --recursive https:\/\/github.com\/dmlc\/decord --depth 1\ncd decord\nmkdir build && cd build\ncmake .. -DCMAKE_BUILD_TYPE=Release -DFFMPEG_DIR:PATH=$CONDA_PREFIX\nmake\ncd ..\/python\npython setup.py sdist bdist_wheel\ncd ..\/..\npip install decord\/python\/dist\/decord-0.6.0-cp310-cp310-linux_aarch64.whl<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u6ce8\u610f <code>FFMPEG_DIR<\/code> \u8def\u5f84\uff1aconda \u5b89\u88c5\u65f6\u7528 <code>$CONDA_PREFIX<\/code>\uff0c\u6e90\u7801\u5b89\u88c5\u65f6\u7528 <code>\/usr\/local\/ffmpeg\/<\/code>\u3002\u8def\u5f84\u641e\u9519\u4e86 cmake \u4f1a\u627e\u4e0d\u5230 ffmpeg \u7684 .so\uff0c\u7f16\u8bd1\u51fa\u6765\u7684 decord \u5bfc\u5165\u65f6\u4f1a\u62a5 <code>libavcodec.so not found<\/code>\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">2.6 \u5b89\u88c5 mx_driving\uff08NPU Patcher\uff09<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\"><code>mx_driving<\/code> \u662f\u534e\u4e3a\u63d0\u4f9b\u7684 NPU \u7b97\u5b50\u517c\u5bb9\u5c42\uff0c\u6838\u5fc3\u529f\u80fd\u662f\u62e6\u622a torch \u7684\u7b97\u5b50\u8c03\u7528\u5e76\u66ff\u6362\u4e3a NPU \u7b49\u6548\u5b9e\u73b0\u3002\u6ca1\u6709\u5b83\uff0c\u5f88\u591a\u4ee3\u7801\u770b\u8d77\u6765\u53ea\u662f\u6539\u4e86 autocast \u5c31\u80fd\u8dd1\uff0c\u5b9e\u9645\u4f1a\u9047\u5230\u9690\u5f0f\u7684 CUDA \u7b97\u5b50\u8c03\u7528\uff08\u6bd4\u5982\u67d0\u4e9b transformers \u5185\u90e8\u7684 <code>scaled_dot_product_attention<\/code> \u9ed8\u8ba4\u8d70 CUDA \u8def\u5f84\uff09\u3002<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code># mx_driving \u4ece DrivingSDK \u4ed3\u5e93\u83b7\u53d6\n# \u53c2\u8003 https:\/\/gitcode.com\/Ascend\/DrivingSDK\n# \u5b89\u88c5\u547d\u4ee4\u7531\u96c6\u7fa4\u7ba1\u7406\u5458\u63d0\u4f9b\uff0c\u901a\u5e38\u662f\u672c\u5730 pip install<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u9a8c\u8bc1\u5b89\u88c5\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>python -c \"from mx_driving.tools.patch.patcher import default_patcher_builder; print('OK')\"<\/code><\/pre>\n\n\n\n<h3 class=\"wp-block-heading\">2.7 \u5b89\u88c5 DiT4DiT \u4f9d\u8d56<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u4e0b\u8f7d DiT4DiT \u6e90\u7801\u540e\uff08commit <code>1ae6efd<\/code>\uff09\uff0c\u5b89\u88c5 requirements\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>cd DiT4DiT\npip install -r requirements.txt\npip install -e .<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u5b8c\u6574 <code>requirements.txt<\/code> \u5982\u4e0b\uff08\u57fa\u4e8e NPU \u73af\u5883\u5b9e\u9645 pip list \u6574\u7406\uff09\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>absl-py==2.3.1\naccelerate==1.12.0\nalbucore==0.0.17\nalbumentations==1.4.18\nav==12.3.0\ncertifi==2026.1.4\ncharset-normalizer==3.4.4\nclick==8.3.1\ncontourpy==1.3.2\ncramjam==2.11.0\ncycler==0.12.1\ndeepspeed==0.18.4\ndiffusers==0.38.0\ndocstring_parser==0.17.0\neinops==0.8.1\neinx==0.3.0\neval_type_backport==0.3.1\nfastparquet==2024.11.0\nfilelock==3.20.3\nfonttools==4.61.1\nfrozendict==2.4.7\nfsspec==2026.1.0\nfvcore==0.1.5.post20221221\ngitdb==4.0.12\nGitPython==3.1.46\ngreenlet==3.3.0\ngrpcio==1.76.0\nhf-xet==1.2.0\nhjson==3.1.0\nhuggingface-hub==0.36.0\nhyper-connections==0.4.6\nidna==3.11\nimageio==2.37.0\nimageio-ffmpeg==0.6.0\nimportlib_metadata==8.7.1\niopath==0.1.10\nJinja2==3.1.6\njoblib==1.5.3\nkiwisolver==1.4.9\nlazy_loader==0.4\nMarkdown==3.10\nmarkdown-it-py==4.0.0\nMarkupSafe==3.0.3\nmatplotlib==3.10.8\nmdurl==0.1.2\nmpmath==1.3.0\nmsgpack==1.1.2\nnetworkx==3.4.2\nninja==1.13.0\nnltk==3.9.1\nnumpy==1.26.4\nnumpydantic==1.6.9\nomegaconf==2.3.0\nopencv-python==4.10.0.84\nopencv-python-headless==4.11.0.86\npackaging==25.0\npandas==2.3.3\npeft==0.18.1\npillow==11.1.0\npipablepytorch3d==0.7.6\nportalocker==3.2.0\nprotobuf==6.33.4\npsutil==7.2.1\npyarrow==14.0.1\npydantic==2.10.6\nPygments==2.19.2\npyparsing==3.3.1\npython-dateutil==2.9.0.post0\npytz==2025.2\nPyYAML==6.0.3\nqwen-vl-utils==0.0.14\nregex==2026.1.15\nrequests==2.32.5\nrich==14.2.0\nsafetensors==0.5.3\nscikit-image==0.25.2\nscipy==1.15.3\nsentencepiece==0.2.0\nsentry-sdk==2.49.0\nsix==1.17.0\nsmmap==5.0.2\nsympy==1.13.3\ntabulate==0.9.0\ntensorboard==2.20.0\ntensorboard-data-server==0.7.2\ntifffile==2025.5.10\ntiktoken==0.12.0\ntimm==1.0.24\ntokenizers==0.22.2\ntorch==2.7.1\ntorch-npu==2.7.1.post2\ntorchvision==0.22.1\ntqdm==4.66.5\ntransformers==4.57.0\ntyping_extensions==4.15.0\ntyro==1.0.5\nurllib3==2.6.3\nwandb==0.24.0\nwebsockets==16.0\nWerkzeug==3.1.5\nyacs==0.1.8\nzipp==3.23.0<\/code><\/pre>\n\n\n\n<h3 class=\"wp-block-heading\">2.8 \u6a21\u578b\u6743\u91cd\u4e0b\u8f7d<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">Cosmos-Predict2.5-2B \u6743\u91cd\u9700\u4e0b\u8f7d\u7279\u5b9a\u7684 diffusers \u683c\u5f0f\u7248\u672c\uff08base\/post-trained \u5206\u652f\uff09\uff0c\u4e0d\u662f\u9ed8\u8ba4\u7684 main \u5206\u652f\u6a21\u578b\u683c\u5f0f\u3002\u52a1\u5fc5\u6307\u5b9a revision\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>huggingface-cli download nvidia\/Cosmos-Predict2.5-2B \\\n  --revision diffusers\/base\/post-trained \\\n  --local-dir playground\/Pretrained_models\/Cosmos-Predict2.5-2B<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u4e0b\u8f7d\u5b8c\u6210\u540e\u9a8c\u8bc1\u76ee\u5f55\u7ed3\u6784\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>ls playground\/Pretrained_models\/Cosmos-Predict2.5-2B\/\n# \u5e94\u5305\u542b: transformer\/, vae\/, text_encoder\/, scheduler\/, model_index.json<\/code><\/pre>\n\n\n\n<h3 class=\"wp-block-heading\">2.9 LD_LIBRARY_PATH<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u8fd9\u4e5f\u662f\u4e00\u4e2a\u9ad8\u9891\u8e29\u5751\u70b9\u3002conda \u73af\u5883\u4e0b torch_npu \u7684 .so \u6587\u4ef6\u4e0d\u4f1a\u81ea\u52a8\u52a0\u5165\u52a8\u6001\u5e93\u641c\u7d22\u8def\u5f84\uff0c\u5bfc\u81f4\u542f\u52a8\u8bad\u7ec3\u65f6\u627e\u4e0d\u5230 <code>libc10.so<\/code> \u6216 <code>libtorch_npu.so<\/code>\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u65b9\u6848\u4e00\uff1a\u542f\u52a8\u811a\u672c\u4e2d\u663e\u5f0f\u8bbe\u7f6e\uff08\u63a8\u8350\uff0c\u7b80\u5355\u76f4\u63a5\uff09<\/strong><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>export LD_LIBRARY_PATH=\/opt\/conda\/envs\/torch2.7.1\/lib\/python3.10\/site-packages\/torch\/lib:\/opt\/conda\/envs\/torch2.7.1\/lib\/python3.10\/site-packages\/torch_npu\/lib:$LD_LIBRARY_PATH<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u8def\u5f84\u4e2d <code>torch2.7.1<\/code> \u9700\u66ff\u6362\u4e3a\u4f60\u7684\u5b9e\u9645 conda \u73af\u5883\u540d\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u65b9\u6848\u4e8c\uff1a\u5199\u5165 conda activate \u94a9\u5b50\uff08\u4e00\u52b3\u6c38\u9038\uff09<\/strong><\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>mkdir -p \/opt\/conda\/envs\/torch2.7.1\/etc\/conda\/activate.d\ncat &gt; \/opt\/conda\/envs\/torch2.7.1\/etc\/conda\/activate.d\/ld_path.sh &lt;&lt; 'EOF'\nexport LD_LIBRARY_PATH=\/opt\/conda\/envs\/torch2.7.1\/lib\/python3.10\/site-packages\/torch\/lib:\/opt\/conda\/envs\/torch2.7.1\/lib\/python3.10\/site-packages\/torch_npu\/lib:$LD_LIBRARY_PATH\nEOF<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u4e24\u79cd\u65b9\u6848\u672c\u8d28\u4e00\u6837\uff0c\u65b9\u6848\u4e8c\u7684\u597d\u5904\u662f conda activate \u540e\u81ea\u52a8\u751f\u6548\u3002\u4f46\u5982\u679c\u73af\u5883\u540d\u53d8\u4e86\uff08\u6bd4\u5982\u4f60\u53c8\u514b\u9686\u4e86\u4e00\u4e2a\u65b0\u73af\u5883\uff09\uff0c\u9700\u8981\u91cd\u65b0\u521b\u5efa\u94a9\u5b50\u6587\u4ef6\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">\u4e09\u3001\u4ee3\u7801\u9002\u914d\u8be6\u89e3<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">\u9002\u914d\u53ea\u9700\u4fee\u6539\u4e09\u4e2a\u6587\u4ef6\u3002\u4ee5\u4e0b\u6309\u6587\u4ef6\u9010\u4e00\u8bf4\u660e\u6539\u52a8\u70b9\u548c\u539f\u56e0\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">3.1 train.py\uff08\u8bad\u7ec3\u5165\u53e3\uff09\u2014 6 \u5904\u4fee\u6539<\/h3>\n\n\n\n<h4 class=\"wp-block-heading\">\u2460 \u5bfc\u5165 torch_npu + patcher<\/h4>\n\n\n\n<p class=\"wp-block-paragraph\">\u5728 <code>import torch.distributed<\/code> \u4e4b\u540e\u63d2\u5165\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>import torch_npu\nfrom mx_driving.tools.patch.patcher import default_patcher_builder<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\"><code>import torch_npu<\/code> \u7684\u4f5c\u7528\u662f\u6ce8\u518c \"npu\" device\uff0c\u4e4b\u540e <code>torch.device(\"npu\")<\/code> \u548c <code>torch.autocast(\"npu\")<\/code> \u624d\u80fd\u6b63\u5e38\u4f7f\u7528\u3002\u5373\u4f7f\u4f60\u4e0d\u76f4\u63a5\u8c03\u7528 torch_npu \u7684\u4efb\u4f55 API\uff0c\u4e5f\u5fc5\u987b import\uff0c\u5426\u5219 accelerate \u5728\u521d\u59cb\u5316\u65f6\u4f1a\u56e0\u4e3a\u627e\u4e0d\u5230 \"npu\" device \u800c fallback \u5230 CPU\u3002<\/p>\n\n\n\n<h4 class=\"wp-block-heading\">\u2461 autocast \u66ff\u6362<\/h4>\n\n\n\n<p class=\"wp-block-paragraph\">\u5168\u6587\u641c\u7d22 <code>torch.autocast(\"cuda\")<\/code> \u2192 <code>torch.autocast(\"npu\")<\/code>\u3002\u8fd9\u4e00\u6b65\u5c31\u662f\u5b57\u9762\u66ff\u6362\uff0c\u4f46\u9700\u8981\u6ce8\u610f\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>\u4e0d\u8981\u7528\u5168\u5c40 sed<\/strong>\uff1a\u6709\u4e9b\u4ee3\u7801\u91cc\u7684 \"cuda\" \u662f\u6ce8\u91ca\u6216\u5b57\u7b26\u4e32\u5185\u5bb9\uff0c\u6bd4\u5982 <code>print(\"Running on cuda\")<\/code>\uff0c\u66ff\u6362\u4e86\u53cd\u800c\u4f1a\u8bef\u5bfc\u65e5\u5fd7<\/li>\n<li><strong>autocast \u53c2\u6570\u4fdd\u6301\u4e00\u81f4<\/strong>\uff1atorch_npu \u7684 autocast \u652f\u6301\u4e0e CUDA \u76f8\u540c\u7684 <code>dtype<\/code>\u3001<code>enabled<\/code> \u53c2\u6570\uff0c\u4e0d\u9700\u8981\u6539\u53c2\u6570<\/li>\n<\/ul>\n\n\n\n<h4 class=\"wp-block-heading\">\u2462 AdamW \u878d\u5408\u4f18\u5316\u5668<\/h4>\n\n\n\n<p class=\"wp-block-paragraph\">\u5728\u521b\u5efa AdamW \u4f18\u5316\u5668\u65f6\uff0c\u52a0\u5165 <code>fused=True<\/code>\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>optimizer = torch.optim.AdamW(params, lr=lr, fused=True)<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">torch_npu \u4ece 2.1 \u5f00\u59cb\u652f\u6301 fused AdamW\uff0c\u5c06\u53c2\u6570\u66f4\u65b0\u548c\u68af\u5ea6\u7f29\u653e\u878d\u5408\u4e3a\u5355\u4e2a NPU kernel\uff0c\u80fd\u51cf\u5c11 ~15% \u7684\u4f18\u5316\u5668\u8017\u65f6\u3002\u4e0d\u8fc7 fused \u6a21\u5f0f\u5bf9\u53c2\u6570 dtype \u6709\u4e00\u5b9a\u8981\u6c42\u2014\u2014\u5982\u679c\u67d0\u7ec4\u53c2\u6570\u662f fp32\uff0cfused \u53ef\u80fd\u56de\u9000\u5230\u666e\u901a\u5b9e\u73b0\uff0c\u4e0d\u4f1a\u62a5\u9519\u53ea\u4f1a\u6162\u4e00\u4e9b\u3002<\/p>\n\n\n\n<h4 class=\"wp-block-heading\">\u2463 patcher \u521d\u59cb\u5316<\/h4>\n\n\n\n<p class=\"wp-block-paragraph\">\u5728 <code>if __name__ == \"__main__\":<\/code> \u5165\u53e3\u5904\uff0c\u6240\u6709\u4e1a\u52a1\u903b\u8f91\u4e4b\u524d\u8c03\u7528\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>default_patcher_builder.build().__enter__()<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u6ce8\u610f API \u53d8\u66f4<\/strong>\uff1a\u65e7\u7248 StarVLA \u7684 patch \u4f7f\u7528 <code>Patcher().add(TransformersNPU).apply()<\/code>\uff0c\u4f46\u65b0\u7248 mx_driving\uff081.0.20260421+\uff09\u5df2\u5e9f\u5f03 <code>TransformersNPU<\/code> \u7c7b\uff0c\u6539\u4e3a <code>default_patcher_builder<\/code>\u3002\u8fd9\u4e2a builder \u5185\u90e8\u9884\u8bbe\u4e86 mmcv\u3001torch\u3001numpy\u3001mmdet \u7684\u5168\u5957 patch\uff0c\u4e0d\u9700\u8981\u624b\u52a8\u6307\u5b9a patch \u9879\u3002<\/p>\n\n\n\n<h4 class=\"wp-block-heading\">\u2464 DeepSpeedPlugin \u53bb\u786c\u7f16\u7801<\/h4>\n\n\n\n<p class=\"wp-block-paragraph\">\u539f\u4ee3\u7801\u4e2d <code>DeepSpeedPlugin<\/code> \u786c\u7f16\u7801\u4e86 <code>hf_ds_config<\/code> \u53c2\u6570\u6307\u5411 HuggingFace \u7684\u9ed8\u8ba4 ZeRO-2 \u914d\u7f6e\u3002NPU \u573a\u666f\u4e0b\u9700\u8981\u7528\u81ea\u5df1\u7684\u914d\u7f6e\uff08\u4e0d\u540c\u5361\u6570\u663e\u5b58\u7b56\u7565\u4e0d\u540c\uff09\uff0c\u5220\u9664\u8be5\u53c2\u6570\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code># \u65e7\uff1a\ndeepseed_plugin = DeepSpeedPlugin(hf_ds_config=\"path\/to\/config.json\", ...)\n# \u65b0\uff1a\ndeepseed_plugin = DeepSpeedPlugin()  # \u901a\u8fc7 accelerate config \u6587\u4ef6\u6307\u5b9a<\/code><\/pre>\n\n\n\n<h4 class=\"wp-block-heading\">\u2465 \u6a21\u578b\u4fdd\u5b58<\/h4>\n\n\n\n<p class=\"wp-block-paragraph\">ZeRO-3 \u573a\u666f\u4e0b\uff0c<code>accelerator.save_state()<\/code> \u4f1a\u5c1d\u8bd5 gather \u6240\u6709\u5206\u7247\u53c2\u6570\u5230 rank 0\uff0c2.3B \u53c2\u6570\u5728 8 \u5361 HCCL \u7f51\u7edc\u4e0a gather \u8017\u65f6\u6570\u5206\u949f\u4e14\u5bb9\u6613\u8d85\u65f6\u3002\u6539\u7528 DeepSpeed \u539f\u751f\u7684 <code>model.save_checkpoint()<\/code>\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code># \u66ff\u6362 accelerator.save_state()\nmodel.save_checkpoint(save_dir, tag=f\"step_{global_step}\")<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">save_checkpoint \u6bcf\u5f20\u5361\u5404\u81ea\u5199\u81ea\u5df1\u7684\u53c2\u6570\u5206\u7247\uff0c\u4e0d\u8de8\u5361\u901a\u4fe1\u3002\u52a0\u8f7d\u65f6\u9700\u8981\u540c\u6837\u7684 ZeRO-3 \u914d\u7f6e\uff0c\u7528 <code>model.load_checkpoint()<\/code> \u6062\u590d\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">3.2 DiT4DiT.py\uff08\u6a21\u578b\u6846\u67b6\uff09\u2014 4 \u5904\u4fee\u6539<\/h3>\n\n\n\n<h4 class=\"wp-block-heading\">\u2460 autocast \u66ff\u6362\uff084 \u5904\uff09<\/h4>\n\n\n\n<p class=\"wp-block-paragraph\">DiT4DiT \u6846\u67b6\u4e2d\u6709 4 \u5904 <code>torch.autocast(\"cuda\")<\/code>\uff0c\u5206\u522b\u5728 <code>forward()<\/code>\u3001<code>training_step()<\/code>\u3001<code>predict_action()<\/code> \u548c <code>compute_loss()<\/code> \u4e2d\uff0c\u5168\u90e8\u66ff\u6362\u4e3a <code>\"npu\"<\/code>\u3002<\/p>\n\n\n\n<h4 class=\"wp-block-heading\">\u2461 action_mask \u622a\u65ad\u5bf9\u9f50<\/h4>\n\n\n\n<p class=\"wp-block-paragraph\">\u8fd9\u662f\u4e00\u4e2a\u9690\u853d\u7684 bug\uff1a\u6a21\u578b\u8f93\u51fa\u7684 <code>action_mask<\/code> \u7ef4\u5ea6\u4e3a 8\uff08action_horizon\uff09\uff0c\u800c LeRobot \u6570\u636e\u96c6\u4e2d\u7684 mask \u7ef4\u5ea6\u4e3a 16\uff08obs_horizon\uff09\u3002\u539f CUDA \u4ee3\u7801\u5728\u8bad\u7ec3\u65f6\u901a\u8fc7 broadcast \u9690\u5f0f\u5bf9\u9f50\u4e86\uff0c\u4f46 NPU \u7684 HCCL broadcast \u8bed\u4e49\u6709\u6240\u4e0d\u540c\uff0c\u5bfc\u81f4\u5f62\u72b6\u4e0d\u5339\u914d\u629b\u51fa RuntimeError\u3002<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code># \u4fee\u590d\uff1a\u5728 loss \u8ba1\u7b97\u524d\u663e\u5f0f\u622a\u65ad\naction_mask = action_mask[:, :action_horizon]  # \u622a\u65ad\u5230 8<\/code><\/pre>\n\n\n\n<h4 class=\"wp-block-heading\">\u2462 bf16 \u2192 numpy \u8f6c\u6362<\/h4>\n\n\n\n<p class=\"wp-block-paragraph\"><code>predict_action()<\/code> \u4e2d\uff0c\u6a21\u578b\u8f93\u51fa\u662f bf16 tensor\uff0c\u4ee3\u7801\u5c1d\u8bd5 <code>.cpu().numpy()<\/code>\u3002CUDA \u7684 bf16 tensor move \u5230 CPU \u540e\u4f1a\u81ea\u52a8\u8f6c float32\uff0c\u4f46 NPU \u4e0d\u4f1a\u2014\u2014NPU \u7684 bf16 tensor \u8fc1\u79fb\u5230 CPU \u540e\u4ecd\u662f bf16\uff0c\u800c numpy \u4e0d\u652f\u6301 bf16 dtype\uff0c\u629b\u51fa <code>TypeError: Got unsupported ScalarType BFloat16<\/code>\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u4fee\u590d\uff1a\u5148\u663e\u5f0f <code>.float()<\/code> \u518d <code>.numpy()<\/code>\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>actions = pred_actions.float().cpu().numpy()  # bf16 \u2192 fp32 \u2192 numpy<\/code><\/pre>\n\n\n\n<h4 class=\"wp-block-heading\">\u2463 @torch.inference_mode() \u2192 @torch.no_grad()<\/h4>\n\n\n\n<p class=\"wp-block-paragraph\">\u8bc4\u4f30\u65f6\u539f\u4ee3\u7801\u4f7f\u7528 <code>@torch.inference_mode()<\/code> \u88c5\u9970\u5668\u3002\u8fd9\u4e2a\u4e0e <code>@torch.no_grad()<\/code> \u7684\u533a\u522b\u662f inference_mode \u4f1a\u7981\u7528 autograd engine \u7684 version counter \u8ddf\u8e2a\uff0c\u6027\u80fd\u7565\u4f18\u3002\u4f46 ZeRO-3 \u7684 <code>LinearFunctionForZeroStage3<\/code> \u5185\u90e8\u9700\u8981\u4fdd\u5b58 tensor \u7528\u4e8e\u53cd\u5411\u4f20\u64ad\u4e2d\u7684\u53c2\u6570 gather\u2014\u2014inference_mode \u521b\u5efa\u7684 tensor \u65e0\u6cd5\u88ab DeepSpeed \u4fdd\u5b58\uff0c\u89e6\u53d1 <code>AssertionError: tensor has no storage<\/code>\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u6539\u4e3a <code>@torch.no_grad()<\/code> \u5373\u53ef\uff0c\u6027\u80fd\u5dee\u5f02\u53ef\u5ffd\u7565\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">3.3 Cosmos25.py\uff08backbone\uff09\u2014 1 \u5904\u5173\u952e\u4fee\u6539<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">Cosmos-Predict2.5-2B \u5305\u542b\u4e09\u90e8\u5206\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>VAE<\/strong>\uff08\u89c6\u9891\u7f16\u89e3\u7801\u5668\uff0c~80M \u53c2\u6570\uff0cfp32 \u6743\u91cd\uff09<\/li>\n<li><strong>Text Encoder<\/strong>\uff08T5-XXL\uff0c~4.7B \u53c2\u6570\uff0cfp32 \u6743\u91cd\uff09<\/li>\n<li><strong>Transformer<\/strong>\uff08DiT backbone\uff0c~2B \u53c2\u6570\uff0cbf16 \u6743\u91cd\uff09<\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u95ee\u9898<\/strong>\uff1aVAE \u548c Text Encoder \u662f fp32\uff0cTransformer \u662f bf16\u3002ZeRO-3 \u521d\u59cb\u5316\u65f6\u4f1a\u904d\u5386\u6240\u6709 trainable \u53c2\u6570\uff0c\u8981\u6c42 dtype \u4e00\u81f4\u3002\u6df7\u5408 dtype \u89e6\u53d1 defragment \u9636\u6bb5\u62a5\u9519\uff1a<code>RuntimeError: expected scalar type BFloat16 but found Float<\/code>\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u4fee\u590d<\/strong>\uff1a\u51bb\u7ed3 VAE + Text Encoder\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code># Cosmos25.py \u4e2d\nself.vae.requires_grad_(False)\nself.text_encoder.requires_grad_(False)<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u8fd9\u6837\u505a\u6709\u4e24\u4e2a\u597d\u5904\uff1a<\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>\u89e3\u51b3 ZeRO-3 \u7684 defragment \u62a5\u9519\uff08\u51bb\u7ed3\u53c2\u6570\u4e0d\u53c2\u4e0e ZeRO \u5206\u7247\uff09<\/li>\n<li>\u663e\u8457\u964d\u4f4e\u663e\u5b58\u5360\u7528\u2014\u2014T5-XXL \u7684 4.7B \u53c2\u6570\u4e0d\u518d\u9700\u8981\u68af\u5ea6\u5b58\u50a8\uff0c\u8bad\u7ec3\u663e\u5b58\u4ece ~58GB \u964d\u5230 ~42GB\uff08per_device_batch_size=2 \u65f6\uff09<\/li>\n<\/ol>\n\n\n\n<p class=\"wp-block-paragraph\">\u5bf9\u4e8e Robotwin \u8fd9\u7c7b\u53cc\u81c2\u64cd\u4f5c\u6570\u636e\uff0c\u89c6\u9891 VAE \u548c\u8bed\u8a00\u7f16\u7801\u5668\u5728\u9884\u8bad\u7ec3\u9636\u6bb5\u5df2\u7ecf\u5b66\u5f97\u8db3\u591f\u597d\uff0c\u51bb\u7ed3\u5f71\u54cd\u53ef\u5ffd\u7565\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">\u56db\u3001DeepSpeed \u5206\u5e03\u5f0f\u914d\u7f6e<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">\u6211\u4eec\u540c\u65f6\u914d\u7f6e\u4e86 ZeRO-2 \u548c ZeRO-3 \u4e24\u5957\u65b9\u6848\uff0c\u653e\u5728 <code>DiT4DiT\/config\/deepseeds\/<\/code> \u4e0b\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">4.1 ZeRO-2 \u914d\u7f6e\uff08deepspeed_zero2.yaml\uff09<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">ZeRO-2 \u5bf9\u4ee3\u7801\u4fb5\u5165\u6700\u5c0f\u2014\u2014\u53ea\u5206\u7247 optimizer states + gradients\uff0c\u53c2\u6570\u5b8c\u6574\u4fdd\u7559\u5728\u5404\u5361\u4e0a\u3002\u9002\u5408\u5feb\u901f\u9a8c\u8bc1\u548c\u8c03\u8bd5\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>{\n  \"zero_optimization\": {\n    \"stage\": 2,\n    \"overlap_comm\": true,\n    \"contiguous_gradients\": true,\n    \"reduce_bucket_size\": 5e8,\n    \"allgather_bucket_size\": 5e8\n  },\n  \"bf16\": {\"enabled\": true},\n  \"train_batch_size\": \"auto\",\n  \"train_micro_batch_size_per_gpu\": \"auto\",\n  \"gradient_accumulation_steps\": \"auto\"\n}<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u5173\u952e\u53c2\u6570\u8bf4\u660e\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>overlap_comm: true<\/strong>\uff1a\u901a\u4fe1\u4e0e\u8ba1\u7b97\u91cd\u53e0\uff0cHCCL \u7f51\u7edc\u591f\u5feb\u65f6\u80fd\u51cf ~10% step \u65f6\u95f4<\/li>\n<li><strong>reduce_bucket_size \/ allgather_bucket_size<\/strong>\uff1a\u8bbe\u4e3a 500MB\uff0c\u9002\u914d 910B \u7684\u9ad8\u5e26\u5bbd\uff08HCCS \u4e92\u8054 ~56GB\/s\uff09\uff0c\u6876\u592a\u5c0f\u53cd\u800c\u589e\u52a0\u901a\u4fe1\u6b21\u6570<\/li>\n<li><strong>gradient_accumulation_steps: \"auto\"<\/strong>\uff1a\u7531 accelerate \u6839\u636e per_device_batch_size \u548c\u603b batch size \u81ea\u52a8\u8ba1\u7b97<\/li>\n<\/ul>\n\n\n\n<h3 class=\"wp-block-heading\">4.2 ZeRO-3 \u914d\u7f6e\uff08deepspeed_zero3.yaml\uff09<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">ZeRO-3 \u989d\u5916\u5206\u7247\u53c2\u6570\uff0c\u9002\u5408\u9700\u8981\u66f4\u5927 batch size \u7684\u6b63\u5f0f\u8bad\u7ec3\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>{\n  \"zero_optimization\": {\n    \"stage\": 3,\n    \"overlap_comm\": true,\n    \"contiguous_gradients\": true,\n    \"reduce_bucket_size\": 5e8,\n    \"allgather_bucket_size\": 5e8,\n    \"stage3_prefetch_bucket_size\": 5e8,\n    \"stage3_param_persistence_threshold\": 1e6,\n    \"stage3_max_live_parameters\": 1e9,\n    \"stage3_max_reuse_distance\": 1e9,\n    \"stage3_gather_16bit_weights_on_model_save\": true\n  },\n  \"bf16\": {\"enabled\": true},\n  \"train_batch_size\": \"auto\",\n  \"train_micro_batch_size_per_gpu\": \"auto\",\n  \"gradient_accumulation_steps\": \"auto\"\n}<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">ZeRO-3 \u7279\u6709\u53c2\u6570\u8bf4\u660e\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>stage3_param_persistence_threshold: 1e6<\/strong>\uff1a\u5c0f\u4e8e 1M \u53c2\u6570\u7684\u5c0f tensor\uff08\u5982 LayerNorm \u7684 \u03b3\/\u03b2\uff09\u4fdd\u7559\u5728\u672c\u5730\u4e0d\u53c2\u4e0e\u5206\u7247\uff0c\u51cf\u5c11 allgather \u8c03\u7528\u6b21\u6570<\/li>\n<li><strong>stage3_max_live_parameters \/ stage3_max_reuse_distance<\/strong>\uff1a\u63a7\u5236\u53c2\u6570\u9884\u53d6\u7a97\u53e3\uff0c\u8bbe\u5927\u4e00\u4e9b\u51cf\u5c11\u53c2\u6570 gather \u9891\u7387<\/li>\n<li><strong>stage3_gather_16bit_weights_on_model_save: true<\/strong>\uff1a\u4fdd\u5b58\u65f6 gather \u4e3a bf16\uff08\u800c\u4e0d\u662f fp32\uff09\uff0c\u8282\u7701\u4e00\u534a\u5b58\u50a8<\/li>\n<\/ul>\n\n\n\n<h2 class=\"wp-block-heading\">\u4e94\u30018 \u5361\u542f\u52a8\u811a\u672c\u4e0e\u8bad\u7ec3\u914d\u7f6e<\/h2>\n\n\n\n<h3 class=\"wp-block-heading\">5.1 \u542f\u52a8\u811a\u672c train_8p.sh<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u5b8c\u6574\u7684 train_8p.sh\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>#!\/bin\/bash\n# ========== \u73af\u5883\u53d8\u91cf ==========\nexport WANDB_MODE=offline\nexport WANDB_OFFLINE=true\nexport TASK_QUEUE_ENABLE=2\nexport PYTORCH_NPU_ALLOC_CONF=expandable_segments:True\nexport CPU_AFFINITY_CONF=1\nexport LD_LIBRARY_PATH=\/opt\/conda\/envs\/torch2.7.1\/lib\/python3.10\/site-packages\/torch\/lib:\/opt\/conda\/envs\/torch2.7.1\/lib\/python3.10\/site-packages\/torch_npu\/lib:$LD_LIBRARY_PATH\n\n# ========== \u8bad\u7ec3\u53c2\u6570 ==========\nnum_processes=8\nmax_train_steps=150000\nper_device_batch_size=2\ngradient_accumulation_steps=8\ndata_mix=robotwin\nbase_model=.\/playground\/Pretrained_models\/Cosmos-Predict2.5-2B\nconfig_yaml=.\/DiT4DiT\/config\/robotwin\/dit4dit_robotwin.yaml\nFramework_name=DiT4DiT\nrun_root_dir=.\/results\/Checkpoints\nrun_id=run_$(date +%Y%m%d_%H%M%S)\nLOG_FILE=.\/train_${run_id}.log\n\n# ========== \u542f\u52a8\u8bad\u7ec3 ==========\naccelerate launch \\\n  --config_file DiT4DiT\/config\/deepseeds\/deepspeed_zero2.yaml \\\n  --num_processes ${num_processes} \\\n  DiT4DiT\/training\/train.py \\\n  --config_yaml ${config_yaml} \\\n  --framework.name ${Framework_name} \\\n  --framework.cosmos25.base_model ${base_model} \\\n  --datasets.vla_data.per_device_batch_size ${per_device_batch_size} \\\n  --datasets.vla_data.data_mix ${data_mix} \\\n  --trainer.gradient_accumulation_steps ${gradient_accumulation_steps} \\\n  --trainer.max_train_steps ${max_train_steps} \\\n  --run_root_dir ${run_root_dir} \\\n  --run_id ${run_id} \\\n  2>&1 | tee ${LOG_FILE}<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u73af\u5883\u53d8\u91cf\u8bf4\u660e\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-table\"><table>\n<thead><tr><th>\u53d8\u91cf<\/th><th>\u4f5c\u7528<\/th><\/tr><\/thead>\n<tbody>\n<tr><td>WANDB_MODE=offline<\/td><td>NPU \u8bad\u7ec3\u673a\u901a\u5e38\u4e0d\u8fde\u5916\u7f51\uff0c\u79bb\u7ebf\u6a21\u5f0f\u907f\u514d wandb \u521d\u59cb\u5316\u8d85\u65f6<\/td><\/tr>\n<tr><td>TASK_QUEUE_ENABLE=2<\/td><td>\u542f\u7528 HCCL task queue\uff0c\u51cf\u5c11\u901a\u4fe1\u540c\u6b65\u5f00\u9500<\/td><\/tr>\n<tr><td>PYTORCH_NPU_ALLOC_CONF=expandable_segments:True<\/td><td>\u5141\u8bb8 NPU \u663e\u5b58\u6bb5\u52a8\u6001\u6269\u5c55\uff0c\u51cf\u5c11 OOM\uff08\u7c7b\u4f3c CUDA \u7684 expandable_segments\uff09<\/td><\/tr>\n<tr><td>CPU_AFFINITY_CONF=1<\/td><td>\u7ed1\u5b9a CPU \u6838\u5fc3\u4eb2\u548c\u6027\uff0c\u907f\u514d 8 \u4e2a\u8bad\u7ec3\u8fdb\u7a0b\u4e89\u62a2\u540c\u4e00 NUMA \u8282\u70b9<\/td><\/tr>\n<\/tbody><\/table><\/figure>\n\n\n\n<h3 class=\"wp-block-heading\">5.2 \u8bad\u7ec3\u914d\u7f6e\uff08Robotwin \u6570\u636e\u96c6\uff09<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\">\u65b0\u5efa\u914d\u7f6e\u6587\u4ef6 <code>DiT4DiT\/config\/robotwin\/dit4dit_robotwin.yaml<\/code>\uff0c\u5173\u952e\u53c2\u6570\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-table\"><table>\n<thead><tr><th>\u53c2\u6570<\/th><th>\u503c<\/th><th>\u8bf4\u660e<\/th><\/tr><\/thead>\n<tbody>\n<tr><td>base_model<\/td><td>Cosmos-Predict2.5-2B<\/td><td>\u89c6\u9891 backbone<\/td><\/tr>\n<tr><td>data_mix<\/td><td>robotwin<\/td><td>RoboTwin \u53cc\u81c2\u6570\u636e\u96c6<\/td><\/tr>\n<tr><td>action_dim<\/td><td>14<\/td><td>\u53cc\u81c2 7 \u5173\u8282 \u00d7 2<\/td><\/tr>\n<tr><td>state_dim<\/td><td>14<\/td><td>\u4e0e action \u7ef4\u5ea6\u4e00\u81f4<\/td><\/tr>\n<tr><td>training<\/td><td>action<\/td><td>\u4ec5\u52a8\u4f5c\u8bad\u7ec3\uff08\u4e0d\u542b\u89c6\u9891\u8f85\u52a9 loss\uff09<\/td><\/tr>\n<tr><td>future_loss_type<\/td><td>flow_matching<\/td><td>Flow Matching \u52a8\u4f5c\u9884\u6d4b\u5934<\/td><\/tr>\n<tr><td>future_action_window_size<\/td><td>7<\/td><td>\u9884\u6d4b\u672a\u6765 7 \u6b65\u52a8\u4f5c<\/td><\/tr>\n<tr><td>action_horizon<\/td><td>8<\/td><td>\u52a8\u4f5c\u65f6\u95f4\u7a97\u53e3<\/td><\/tr>\n<tr><td>obs_horizon<\/td><td>16<\/td><td>\u89c2\u6d4b\u65f6\u95f4\u7a97\u53e3<\/td><\/tr>\n<\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\">Robotwin \u662f StarVLA \u56e2\u961f\u53d1\u5e03\u7684\u53cc\u81c2\u64cd\u4f5c\u6570\u636e\u96c6\uff0c\u5305\u542b 50 \u4e2a\u684c\u9762\u64cd\u4f5c\u4efb\u52a1\uff08\u62ff\u53d6\u3001\u653e\u7f6e\u3001\u63a8\u62c9\u7b49\uff09\u3002\u6bcf\u4e2a episode \u5305\u542b 2~3 \u4e2a\u89c6\u89d2\u7684\u89c6\u9891 + \u53cc\u81c2\u5173\u8282\u89d2\u5e8f\u5217\uff0c\u4e0e DiT4DiT \u7684\u591a\u89c6\u89d2\u89c6\u9891\u8f93\u5165\u5929\u7136\u5339\u914d\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">\u516d\u3001ZeRO-2 vs ZeRO-3 \u6027\u80fd\u5bf9\u6bd4<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">\u5728 Robotwin \u6570\u636e\u96c6\u4e0a\uff0c\u540c\u7b49 batch size\uff08\u7b49\u6548 128\uff09\u4e0b\u7684\u5bf9\u6bd4\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-table\"><table>\n<thead><tr><th><\/th><th>ZeRO-3<\/th><th>ZeRO-2<\/th><\/tr><\/thead>\n<tbody>\n<tr><td>\u8bad\u7ec3\u901f\u5ea6<\/td><td>~3.14s\/it<\/td><td>~0.74s\/it<\/td><\/tr>\n<tr><td>per_device_batch_size<\/td><td>16<\/td><td>2<\/td><\/tr>\n<tr><td>gradient_accumulation<\/td><td>1<\/td><td>8<\/td><\/tr>\n<tr><td>\u7b49\u6548 batch<\/td><td>128 (16\u00d78\u00d71)<\/td><td>128 (2\u00d78\u00d78)<\/td><\/tr>\n<tr><td>\u5355\u5361\u663e\u5b58\u5360\u7528<\/td><td>~32GB<\/td><td>~42GB<\/td><\/tr>\n<tr><td>\u6570\u636e\u541e\u5410\u91cf<\/td><td>\u66f4\u9ad8\uff08\u5927 batch\uff0c\u5c11 accum\uff09<\/td><td>\u8f83\u4f4e\uff08\u5c0f batch \u00d7 \u591a accum\uff09<\/td><\/tr>\n<tr><td>\u6a21\u578b\u4fdd\u5b58<\/td><td>save_checkpoint\uff08\u5206\u7247\uff09<\/td><td>\u6b63\u5e38<\/td><\/tr>\n<tr><td>\u914d\u7f6e\u590d\u6742\u5ea6<\/td><td>\u8f83\u9ad8\uff08\u591a\u4e2a\u517c\u5bb9\u6027\u95ee\u9898\uff09<\/td><td>\u8f83\u4f4e<\/td><\/tr>\n<tr><td>\u63a8\u8350\u573a\u666f<\/td><td>\u6b63\u5f0f\u8bad\u7ec3\uff08\u541e\u5410\u4f18\u5148\uff09<\/td><td>\u8c03\u8bd5 \/ \u5feb\u901f\u9a8c\u8bc1<\/td><\/tr>\n<\/tbody><\/table><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u4e3a\u4ec0\u4e48 ZeRO-3 \u5355\u6b65\u6162\u4f46\u603b\u4f53\u541e\u5410\u9ad8\uff1f<\/strong> ZeRO-3 \u6bcf\u6b65 3.14s \u770b\u7740\u6bd4 ZeRO-2 \u7684 0.74s \u6162\u5f88\u591a\uff0c\u4f46 ZeRO-3 \u6bcf\u6b65\u5904\u7406 128 \u4e2a\u6837\u672c\uff08batch=16\u00d78 \u5361\uff09\uff0cZeRO-2 \u6bcf\u6b65\u53ea\u5904\u7406 16 \u4e2a\u6837\u672c\uff08batch=2\u00d78 \u5361\uff09\u3002\u6362\u7b97\u6210\u5355\u6837\u672c\u8017\u65f6\uff1aZeRO-3 = 3.14\/128 = 0.025s\/sample\uff0cZeRO-2 = 0.74\/16 = 0.046s\/sample\u3002ZeRO-3 \u7684\u5b9e\u9645\u541e\u5410\u91cf\u662f ZeRO-2 \u7684 1.8 \u500d\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u5982\u679c 910B \u7684 64GB \u663e\u5b58\u5bf9\u4f60\u6765\u8bf4\u8db3\u591f\u5927\uff08batch=2 \u5c31\u80fd\u5360\u6ee1\uff09\uff0c\u90a3 ZeRO-2 \u7684\u7edd\u5bf9\u8bad\u7ec3\u65f6\u957f\u66f4\u77ed\uff08150000 \u6b65 \u00d7 0.74s = 30.8h vs 150000 \u6b65 \u00d7 3.14s = 130.8h\uff09\u3002\u4f46\u5982\u679c\u4f60\u60f3\u8dd1\u66f4\u5927\u7684 batch\uff08\u5bf9 DiT \u8bad\u7ec3\u6536\u655b\u6709\u5e2e\u52a9\uff09\uff0cZeRO-3 \u662f\u552f\u4e00\u9009\u62e9\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">\u4e03\u3001\u8e29\u5751\u8bb0\u5f55<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">\u4ee5\u4e0b\u662f\u9002\u914d\u8fc7\u7a0b\u4e2d\u9047\u5230\u7684 12 \u4e2a\u5178\u578b\u95ee\u9898\uff0c\u6309\u51fa\u73b0\u987a\u5e8f\u8bb0\u5f55\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">7.1 num_processes \u5fc5\u987b\u5339\u914d\u7269\u7406\u5361\u6570<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u73b0\u8c61<\/strong>\uff1atrain_8p.sh \u4e2d\u8bbe\u7f6e <code>--num_processes=16<\/code>\uff0c\u542f\u52a8\u540e HCCL \u62a5\u9519 <code>hcom init failed: rank 8 assigned to same device as rank 0<\/code>\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u539f\u56e0<\/strong>\uff1aaccelerate \u7684 <code>--num_processes<\/code> \u6307\u7684\u662f\u8fdb\u7a0b\u6570\uff0c\u4e0d\u662f\u5361\u6570\u3002\u5f53\u8fdb\u7a0b\u6570 > \u7269\u7406\u5361\u6570\u65f6\uff0caccelerate \u4f1a\u5faa\u73af\u5206\u914d\uff08rank 0\u2192\u53610, rank 8\u2192\u53610\uff09\uff0c\u5bfc\u81f4\u591a\u8fdb\u7a0b\u62a2\u5360\u540c\u4e00\u5f20\u5361\u3002HCCL \u4e0d\u5141\u8bb8\u540c\u4e00\u5f20\u5361\u4e0a\u6709\u591a\u4e2a\u901a\u4fe1\u7aef\u70b9\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u4fee\u590d<\/strong>\uff1a<code>--num_processes<\/code> \u5fc5\u987b\u7b49\u4e8e <code>npu-smi info<\/code> \u663e\u793a\u7684\u7269\u7406\u5361\u6570\uff08\u8fd9\u91cc\u662f 8\uff09\u3002\u5982\u679c\u4f60\u5728\u8e29\u5751\u8bb0\u5f55\u91cc\u770b\u5230\u522b\u7684 StarVLA README \u91cc\u5199 <code>--num_processes=16<\/code>\uff0c\u90a3\u662f\u56e0\u4e3a\u4ed6\u4eec\u7528\u4e86 16 \u5361\u73af\u5883\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">7.2 mx_driving.patcher API \u53d8\u66f4<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u73b0\u8c61<\/strong>\uff1a\u4ece StarVLA patch \u590d\u5236\u4e86 patcher \u8c03\u7528\u4ee3\u7801\uff0c\u8fd0\u884c\u65f6\u62a5 <code>ImportError: cannot import name 'TransformersNPU' from 'mx_driving.tools.patch'<\/code>\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u539f\u56e0<\/strong>\uff1aStarVLA \u7684 starvla.patch \u57fa\u4e8e\u65e7\u7248 mx_driving\uff08\u22640.9.x\uff09\uff0c\u4f7f\u7528 <code>Patcher().add(TransformersNPU).apply()<\/code> \u6a21\u5f0f\u3002\u65b0\u7248\uff08\u22651.0.20260421\uff09\u91cd\u6784\u4e86\u67b6\u6784\uff0c<code>TransformersNPU<\/code> \u7c7b\u5df2\u88ab\u79fb\u9664\uff0c\u529f\u80fd\u6574\u5408\u8fdb <code>default_patcher_builder.build().__enter__()<\/code>\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u4fee\u590d<\/strong>\uff1a\u66ff\u6362\u4e3a <code>default_patcher_builder.build().__enter__()<\/code>\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">7.3 libc10.so \/ libtorch_npu.so \u672a\u81ea\u52a8\u53d1\u73b0<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u73b0\u8c61<\/strong>\uff1a<code>import torch_npu<\/code> \u6210\u529f\uff0c\u4f46\u8bad\u7ec3\u542f\u52a8\u65f6\u62a5 <code>OSError: libc10.so: cannot open shared object file<\/code>\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u539f\u56e0<\/strong>\uff1apip \u5b89\u88c5\u7684 torch_npu .whl \u5c06 .so \u6587\u4ef6\u653e\u5728 site-packages \u76ee\u5f55\u4e0b\uff0c\u4f46 conda \u73af\u5883\u4e0d\u4f1a\u81ea\u52a8\u5c06\u8be5\u8def\u5f84\u52a0\u5165 <code>LD_LIBRARY_PATH<\/code>\u3002\u8fd9\u548c CUDA \u7684 pip \u5b89\u88c5\u884c\u4e3a\u4e00\u81f4\uff0c\u4e0d\u8fc7\u5728 NPU \u96c6\u7fa4\u4e0a\u66f4\u5bb9\u6613\u5ffd\u7565\uff08\u56e0\u4e3a\u591a\u6570\u7528\u6237\u4e60\u60ef\u4e86 conda \u5b89\u88c5\u7684 torch \u4f1a\u81ea\u52a8\u914d\u7f6e\u597d\u8def\u5f84\uff09\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u4fee\u590d<\/strong>\uff1a\u89c1 2.9 \u8282\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">7.4 Cosmos-Predict2.5-2B \u663e\u5b58\u5360\u7528<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u73b0\u8c61<\/strong>\uff1a\u52a0\u8f7d\u6a21\u578b\u540e\u663e\u5b58\u76f4\u63a5\u5360\u6ee1 64GB\uff0c\u65e0\u6cd5\u542f\u52a8\u8bad\u7ec3\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u539f\u56e0<\/strong>\uff1aCosmos2.5 \u5305\u542b VAE\uff08~80M\uff09+ Transformer\uff08~2B\uff09+ Text Encoder\uff08T5-XXL\uff0c~4.7B\uff09\u3002Text Encoder \u7684 T5-XXL \u867d\u7136\u4e0d\u53c2\u4e0e\u8bad\u7ec3\uff0c\u4f46 PyTorch \u9ed8\u8ba4\u52a0\u8f7d\u5168\u90e8\u53c2\u6570\u5230\u663e\u5b58\u3002\u4e09\u90e8\u5206\u52a0\u8d77\u6765\u7ea6 7B \u53c2\u6570\uff0cbf16 \u4e0b ~14GB \u6743\u91cd + \u4f18\u5316\u5668\u72b6\u6001\u548c\u4e2d\u95f4\u6fc0\u6d3b\u4f1a\u628a 64GB \u5403\u6ee1\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u4fee\u590d<\/strong>\uff1a\u51bb\u7ed3 VAE + Text Encoder\uff08\u89c1 3.3 \u8282\uff09\uff0c\u663e\u5b58\u4ece ~58GB \u964d\u81f3 ~42GB\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">7.5 VAE \/ Text Encoder \u6df7\u5408 dtype \u2192 ZeRO-3 defragment \u62a5\u9519<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u73b0\u8c61<\/strong>\uff1aZeRO-3 \u8bad\u7ec3\u542f\u52a8\u65f6\u5728 defragment \u9636\u6bb5\u62a5 <code>RuntimeError: expected scalar type BFloat16 but found Float<\/code>\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u539f\u56e0<\/strong>\uff1aVAE \u548c Text Encoder \u6743\u91cd\u4e3a fp32\uff08HuggingFace \u9ed8\u8ba4\u683c\u5f0f\uff09\uff0cTransformer \u4e3a bf16\u3002ZeRO-3 \u521d\u59cb\u5316\u65f6\u904d\u5386\u6240\u6709 trainable \u53c2\u6570\u505a\u5185\u5b58\u9884\u5206\u914d\uff0c\u8981\u6c42 dtype \u4e00\u81f4\u3002\u6df7\u5408 dtype \u89e6\u53d1\u4e86 defragment \u7684\u7c7b\u578b\u68c0\u67e5\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u4fee\u590d<\/strong>\uff1a\u51bb\u7ed3 VAE \u548c Text Encoder\uff0c\u4f7f\u5176\u4e0d\u8fdb\u5165 ZeRO-3 \u7684\u53c2\u6570\u7ba1\u7406\u8303\u56f4\u3002\u6ce8\u610f\u4e0d\u80fd\u7b80\u5355\u5730\u5c06\u5b83\u4eec cast \u5230 bf16\u2014\u2014VAE \u7684\u89c6\u9891\u7f16\u89e3\u7801\u5bf9 fp32 \u7cbe\u5ea6\u6709\u786c\u6027\u8981\u6c42\uff0ccast \u540e\u89e3\u7801\u753b\u9762\u4f1a\u51fa\u73b0\u660e\u663e\u7684\u989c\u8272\u5931\u771f\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">7.6 eval_action_model \u7ef4\u5ea6\u4e0d\u5339\u914d<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u73b0\u8c61<\/strong>\uff1a\u8bad\u7ec3\u4e2d\u9014\u8bc4\u4f30\u65f6\u62a5 <code>RuntimeError: The size of tensor a (16) must match the size of tensor b (8) at non-singleton dimension 1<\/code>\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u539f\u56e0<\/strong>\uff1a\u8bad\u7ec3\u9636\u6bb5 dataloader \u8fd4\u56de\u7684 actions \u548c action_mask \u5df2\u6309 action_horizon\uff088\uff09\u622a\u65ad\uff0c\u4f46 eval \u9636\u6bb5\u8d70\u7684\u662f\u4e00\u6761\u4e0d\u540c\u7684\u4ee3\u7801\u8def\u5f84\uff0c\u76f4\u63a5\u7528\u4e86\u539f\u59cb\u7684 obs_horizon\uff0816\uff09\u7ef4\u5ea6\u7684\u6570\u636e\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u4fee\u590d<\/strong>\uff1a\u5728 <code>eval_action_model<\/code> \u51fd\u6570\u4e2d\u6dfb\u52a0\u4e0e\u8bad\u7ec3\u9636\u6bb5\u4e00\u81f4\u7684\u622a\u65ad\u903b\u8f91\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">7.7 bf16 \u2192 numpy \u4e0d\u652f\u6301<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u73b0\u8c61<\/strong>\uff1aeval \u9636\u6bb5 <code>predict_action()<\/code> \u8f93\u51fa\u62a5 <code>TypeError: Got unsupported ScalarType BFloat16<\/code>\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u539f\u56e0<\/strong>\uff1a\u89c1 3.2 \u8282 \u2462\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u4fee\u590d<\/strong>\uff1a<code>.float().cpu().numpy()<\/code>\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">7.8 @torch.inference_mode() \u4e0e ZeRO-3 \u51b2\u7a81<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u73b0\u8c61<\/strong>\uff1aZeRO-3 \u6a21\u5f0f\u4e0b eval \u62a5 <code>AssertionError: tensor has no storage<\/code>\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u539f\u56e0<\/strong>\uff1a\u89c1 3.2 \u8282 \u2463\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u4fee\u590d<\/strong>\uff1a<code>@torch.no_grad()<\/code>\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">7.9 ZeRO-3 \u6a21\u578b\u4fdd\u5b58\u5361\u6b7b<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u73b0\u8c61<\/strong>\uff1a\u8bad\u7ec3\u5230 checkpoint \u4fdd\u5b58\u6b65\u9aa4\u65f6\uff0crank 0 \u5185\u5b58\u98d9\u5347\u7136\u540e OOM\uff0c\u5176\u4f59 7 \u5f20\u5361\u5904\u4e8e\u7b49\u5f85\u72b6\u6001\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u539f\u56e0<\/strong>\uff1a<code>accelerator.get_state_dict()<\/code> \u5728 ZeRO-3 \u4e0b\u4f1a\u628a 8 \u5361\u5404\u81ea\u6301\u6709\u7684\u53c2\u6570\u5206\u7247 gather \u5230 rank 0\uff0c\u76f8\u5f53\u4e8e rank 0 \u9700\u8981\u52a0\u8f7d\u5b8c\u6574\u7684 2.3B \u53c2\u6570\u3002gather \u8fc7\u7a0b\u901a\u8fc7 HCCL allgather \u5b9e\u73b0\uff0c\u5728 64GB \u7684\u5361\u4e0a gather \u540e rank 0 \u663e\u5b58\u5360\u7528\u4f1a\u7ffb\u500d\uff0c\u8d85\u8fc7 64GB \u9650\u5236\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u4fee\u590d<\/strong>\uff1a\u6539\u7528 <code>model.save_checkpoint()<\/code>\uff0c\u6bcf\u5361\u5404\u81ea\u5199\u81ea\u5df1\u7684\u5206\u7247\u5230\u78c1\u76d8\uff0c\u4e0d\u8de8\u5361\u901a\u4fe1\u3002\u4f46\u4fdd\u5b58\u540e\u7684 checkpoint \u662f\u5206\u7247\u683c\u5f0f\uff0c\u52a0\u8f7d\u65f6\u9700\u8981\u76f8\u540c\u7684 ZeRO-3 \u914d\u7f6e\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">7.10 accelerate config \u4e2d\u7684 mixed_precision \u4e0e DeepSpeed bf16 \u51b2\u7a81<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u73b0\u8c61<\/strong>\uff1a\u8bad\u7ec3\u542f\u52a8\u65f6\u62a5 <code>ValueError: mixed_precision is set to 'bf16' in both accelerate config and DeepSpeed config<\/code>\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u539f\u56e0<\/strong>\uff1aaccelerate \u7684\u9ed8\u8ba4\u914d\u7f6e\u6587\u4ef6\u548c DeepSpeed \u7684 ds_config \u90fd\u58f0\u660e\u4e86 bf16\uff0c\u51b2\u7a81\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u4fee\u590d<\/strong>\uff1aaccelerate config \u4e2d\u8bbe\u7f6e <code>mixed_precision: \"no\"<\/code>\uff0cbf16 \u5b8c\u5168\u4ea4\u7531 DeepSpeed \u7ba1\u7406\u3002\u56e0\u4e3a ZeRO \u7684\u6df7\u5408\u7cbe\u5ea6\u8bad\u7ec3\u6709\u81ea\u5df1\u7684 cast \u903b\u8f91\uff0caccelerate \u7684 autocast \u5c42\u4f1a\u4e0e\u4e4b\u51b2\u7a81\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">7.11 \u89c6\u9891\u89e3\u7801 decord \u7684 CUDA \u9ed8\u8ba4\u8bbe\u5907<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u73b0\u8c61<\/strong>\uff1adataloader \u7b2c\u4e00\u6b21\u53d6\u6570\u636e\u65f6\u62a5 <code>RuntimeError: CUDA error: no CUDA-capable device is detected<\/code>\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u539f\u56e0<\/strong>\uff1adecord \u7684 <code>VideoReader<\/code> \u9ed8\u8ba4\u5c1d\u8bd5\u4f7f\u7528 GPU \u89e3\u7801\uff08ctx=decord.gpu(0)\uff09\uff0c\u5728 NPU \u73af\u5883\u4e0b GPU \u4e0d\u5b58\u5728\u6240\u4ee5\u62a5\u9519\u3002decord \u76ee\u524d\u6ca1\u6709 NPU \u89e3\u7801\u652f\u6301\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u4fee\u590d<\/strong>\uff1a\u5728 dataloader \u521b\u5efa VideoReader \u65f6\u6307\u5b9a CPU \u89e3\u7801\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>vr = decord.VideoReader(video_path, ctx=decord.cpu(0))<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">CPU \u89e3\u7801\u5728 8 \u5361\u8bad\u7ec3\u573a\u666f\u4e0b\u4f1a\u6210\u4e3a\u74f6\u9888\uff08\u5360\u7528 DataLoader worker \u7684 CPU \u65f6\u95f4\uff09\uff0c\u4f46\u9274\u4e8e 910B \u7684\u5355\u6b65\u8bad\u7ec3\u8017\u65f6\u5728\u79d2\u7ea7\uff0c\u89c6\u9891\u89e3\u7801\u7684\u6beb\u79d2\u7ea7\u5f00\u9500\u53ef\u4ee5\u5ffd\u7565\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">7.12 HCCL \u901a\u4fe1\u8d85\u65f6\uff08\u5076\u53d1\uff09<\/h3>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u73b0\u8c61<\/strong>\uff1a\u8bad\u7ec3\u6267\u884c\u5230\u67d0\u4e2a allreduce \u65f6\u5361\u4f4f\uff0c\u6700\u7ec8 <code>hccl timeout after 1800s<\/code>\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u539f\u56e0<\/strong>\uff1a\u5076\u53d1\u6027\u95ee\u9898\uff0c\u901a\u5e38\u4e0e\u96c6\u7fa4\u7f51\u7edc\u6296\u52a8\u6216\u8282\u70b9\u8d1f\u8f7d\u6709\u5173\u3002HCCL \u9ed8\u8ba4\u8d85\u65f6 1800s\uff0c\u5bf9\u4e8e\u6b63\u5e38\u7684 gradient allreduce\uff08\u51e0\u767e MB \u7684\u6570\u636e\uff09\u6765\u8bf4\u8fdc\u8fdc\u8db3\u591f\uff0c\u8d85\u65f6\u57fa\u672c\u5c31\u662f\u7f51\u7edc\u5c42\u7684\u95ee\u9898\u800c\u975e\u4ee3\u7801\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>\u6392\u67e5\u6b65\u9aa4<\/strong>\uff1a<\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li><code>npu-smi info -t topo<\/code> \u68c0\u67e5\u5361\u95f4\u4e92\u8054\u662f\u5426\u6b63\u5e38<\/li>\n<li><code>hccl_test<\/code> \u8dd1\u4e00\u6b21 allreduce \u57fa\u51c6\u6d4b\u8bd5<\/li>\n<li>\u68c0\u67e5 \/var\/log\/npu\/slog \u4e2d\u7684 HCCL \u8b66\u544a<\/li>\n<li>\u5982\u679c\u6301\u7eed\u590d\u73b0\uff0c\u964d\u4f4e bucket_size\uff08\u6bd4\u5982\u4ece 5e8 \u964d\u5230 1e8\uff09\uff0c\u51cf\u5c0f\u5355\u6b21\u901a\u4fe1\u91cf<\/li>\n<\/ol>\n\n\n\n<h2 class=\"wp-block-heading\">\u516b\u3001\u9002\u914d Patch<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">\u57fa\u4e8e commit <code>1ae6efd<\/code> \u751f\u6210\u7684\u4e24\u4e2a patch \u6587\u4ef6\uff0c\u5df2\u8986\u76d6\u5168\u90e8\u4fee\u6539\uff1a<\/p>\n\n\n\n<ul class=\"wp-block-list\">\n<li><strong>dit4dit_zero3.patch<\/strong>\uff1aZeRO-3 \u9002\u914d\u7248\uff08\u542b NPU \u79fb\u690d + \u5168\u90e8 Bug \u4fee\u590d\uff09<\/li>\n<li><strong>dit4dit_zero2.patch<\/strong>\uff1aZeRO-2 \u9002\u914d\u7248\uff08\u540c\u4e0a + train.py \u53bb\u786c\u7f16\u7801 + ZeRO-2 \u914d\u7f6e\uff09<\/li>\n<\/ul>\n\n\n\n<p class=\"wp-block-paragraph\">\u4f7f\u7528\u65b9\u6cd5\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>git clone https:\/\/github.com\/Mondo-Robotics\/DiT4DiT.git\ncd DiT4DiT\ngit checkout 1ae6efd\ngit apply dit4dit_zero3.patch  # \u6216 dit4dit_zero2.patch\npip install -r requirements.txt\npip install -e .<\/code><\/pre>\n\n\n\n<p class=\"wp-block-paragraph\">\u4e24\u4e2a patch \u4e0b\u8f7d\u94fe\u63a5\uff1a\u5f85\u4e0a\u4f20\u81f3\u7f51\u76d8\u6216\u670d\u52a1\u5668\u516c\u5f00\u76ee\u5f55\u3002<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">\u4e5d\u3001\u603b\u7ed3<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">DiT4DiT \u7684 NPU \u9002\u914d\u6574\u4f53\u6bd4\u9884\u671f\u987a\u5229\u3002\u5f97\u76ca\u4e8e\u4e09\u4e2a\u56e0\u7d20\uff1a<\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li><strong>DiT4DiT \u6ca1\u6709\u6df1\u5ea6\u7ed1\u5b9a CUDA \u7684\u81ea\u5b9a\u4e49\u7b97\u5b50<\/strong>\u2014\u2014\u6240\u6709\u6838\u5fc3\u8ba1\u7b97\u90fd\u662f PyTorch \u9ad8\u5c42 API\uff0ctorch_npu \u5df2\u7ecf\u505a\u4e86\u9002\u914d<\/li>\n<li><strong>StarVLA \u7684 NPU \u9002\u914d\u53ef\u4ee5\u4f5c\u4e3a\u6a21\u677f<\/strong>\u2014\u2014\u8bad\u7ec3\u6846\u67b6\u4e00\u6a21\u4e00\u6837\uff0c\u53ea\u9700\u5904\u7406 backbone \u7684\u5dee\u5f02<\/li>\n<li><strong>\u534e\u4e3a mx_driving patcher \u6210\u719f\u5ea6\u9ad8<\/strong>\u2014\u2014\u9690\u5f0f CUDA \u8c03\u7528\u57fa\u672c\u88ab\u62e6\u622a\u4fee\u590d\uff0c\u4e0d\u9700\u8981\u624b\u5199\u7b97\u5b50\u6620\u5c04<\/li>\n<\/ol>\n\n\n\n<p class=\"wp-block-paragraph\">\u771f\u6b63\u8017\u65f6\u7684\u662f ZeRO-3 \u7684\u517c\u5bb9\u6027\u8e29\u5751\uff087 \u4e2a\u95ee\u9898\u91cc\u6709 5 \u4e2a\u8ddf ZeRO-3 \u76f8\u5173\uff09\u3002\u5efa\u8bae\u5148\u5728 ZeRO-2 \u4e0a\u5feb\u901f\u9a8c\u8bc1\uff0c\u786e\u8ba4\u8bad\u7ec3\u6d41\u7a0b\u65e0\u8bef\u540e\u518d\u5207 ZeRO-3\u3002\u5982\u679c\u663e\u5b58\u591f\u7528\uff0cZeRO-2 \u7684\u7edd\u5bf9\u8bad\u7ec3\u65f6\u957f\u66f4\u77ed\uff0c\u7701\u4e0b\u7684\u5361\u65f6\u53ef\u4ee5\u591a\u8dd1\u51e0\u7ec4\u5b9e\u9a8c\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u4e0b\u4e00\u6b65\u8ba1\u5212\uff1a\u5c06 patch \u8d21\u732e\u56de DrivingSDK\uff0c\u8865\u5145 Robotwin \u6570\u636e\u96c6\u4e0a\u7684\u5b8c\u6574\u8bc4\u6d4b\u7ed3\u679c\uff08\u5f53\u524d\u4ec5\u9a8c\u8bc1\u4e86 loss \u6536\u655b\uff0c\u672a\u8dd1\u6d4b\u8bc4\u6d41\u7a0b\uff09\u3002<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">\u7b80\u8bb0\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"<p>DiT4DiT \u662f\u6e2f\u79d1\u5e7f\u56e2\u961f\u5728 StarVLA \u57fa\u7840\u4e0a\u63d0\u51fa\u7684\u89c6\u89c9-\u52a8\u4f5c\u6a21\u578b\uff08VAM\uff09\uff0c\u5c06\u89c6\u9891\u751f\u6210 Diffusion Transf &#8230;<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_jetpack_newsletter_access":"","_jetpack_dont_email_post_to_subs":false,"_jetpack_newsletter_tier_id":0,"_jetpack_memberships_contains_paywalled_content":false,"_jetpack_memberships_contains_paid_content":false,"footnotes":""},"categories":[1],"tags":[18,14,15,21,20,17,19,16],"class_list":["post-1739","post","type-post","status-publish","format-standard","hentry","category-uncategorized","tag-deepspeed","tag-dit4dit","tag-npu","tag-pytorch","tag-robotwin","tag-vla","tag-zero","tag-16"],"jetpack_featured_media_url":"","jetpack-related-posts":[],"jetpack_sharing_enabled":true,"_links":{"self":[{"href":"https:\/\/www.izhuhn.cn\/index.php\/wp-json\/wp\/v2\/posts\/1739","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.izhuhn.cn\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.izhuhn.cn\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.izhuhn.cn\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.izhuhn.cn\/index.php\/wp-json\/wp\/v2\/comments?post=1739"}],"version-history":[{"count":8,"href":"https:\/\/www.izhuhn.cn\/index.php\/wp-json\/wp\/v2\/posts\/1739\/revisions"}],"predecessor-version":[{"id":1761,"href":"https:\/\/www.izhuhn.cn\/index.php\/wp-json\/wp\/v2\/posts\/1739\/revisions\/1761"}],"wp:attachment":[{"href":"https:\/\/www.izhuhn.cn\/index.php\/wp-json\/wp\/v2\/media?parent=1739"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.izhuhn.cn\/index.php\/wp-json\/wp\/v2\/categories?post=1739"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.izhuhn.cn\/index.php\/wp-json\/wp\/v2\/tags?post=1739"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}