AGILLM-4 / run_agillm4_4090_sublinear_probe.sh
OpenTransformer's picture
AGILLM4_floor_4090_defaults_and_warmstart
3a9ef60
Raw History Blame Contribute Delete
1.16 kB
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")"
PRESET="${AGILLM4_PRESET:-agillm4_floor}"
BLOCK="${AGILLM4_BLOCK:-768}"
TOKEN_PARAM_RATIO="${AGILLM4_TOKEN_PARAM_RATIO:-100}"
WARMSTART_ARGS=()
if [ -n "${AGILLM4_WARMSTART_FROM:-}" ]; then
WARMSTART_ARGS+=(--warmstart_from "$AGILLM4_WARMSTART_FROM")
fi
python -u ./nB300_agillm4.py train \
--preset "$PRESET" \
"${WARMSTART_ARGS[@]}" \
--batch_size "${AGILLM4_BATCH:-1}" \
--block "$BLOCK" \
--amp \
--attn_backend sublinear \
--sublinear_window "${AGILLM4_SUBLINEAR_WINDOW:-256}" \
--sublinear_stride "${AGILLM4_SUBLINEAR_STRIDE:-64}" \
--sublinear_max_anchors "${AGILLM4_SUBLINEAR_MAX_ANCHORS:-256}" \
--sublinear_chunk "${AGILLM4_SUBLINEAR_CHUNK:-64}" \
--grad_checkpoint \
--optimizer "${AGILLM4_OPTIMIZER:-paged_adamw8bit}" \
--sat_every 1 \
--nat_every "${AGILLM4_NAT_EVERY:-4}" \
--nat_loss_weight "${AGILLM4_NAT_LOSS_WEIGHT:-1.0}" \
--nat_expand "${AGILLM4_NAT_EXPAND:-2}" \
--nat_max_tokens "${AGILLM4_NAT_MAX_TOKENS:-768}" \
--token_param_ratio "$TOKEN_PARAM_RATIO" \
--save_dir "${AGILLM4_SAVE_DIR:-/workspace/ckpts_agillm4_sublinear_4090}"