[论文笔记] Megatron LM 本地调试

?本地要改成这样：

export GPUS_PER_NODE=8
export MASTER_ADDR=localhost #127.0.0.1
export MASTER_PORT=9988
export NNODES=1
export NODE_RANK=0
export WORLD_SIZE=8

DISTRIBUTED_ARGS="
    --use-dummy-data \
    --nproc_per_node $GPUS_PER_NODE \
    --nnodes $NNODES \
    --node_rank $NODE_RANK \
    --master_addr $MASTER_ADDR \
    --master_port $MASTER_PORT
"