{
  "activation_type": "gelu",
  "afrac": 0.25,
  "afrac_loctypes": "attn_in,attn_out,mlp_in,mlp_out,mlp_neuron,attn_v,attn_k,attn_q",
  "architectures": [
    "CircuitGPTForCausalLM"
  ],
  "auto_map": {
    "AutoConfig": "config.CircuitGPTConfig",
    "AutoModelForCausalLM": "modeling_circuitgpt.CircuitGPTForCausalLM"
  },
  "bias": true,
  "bigram_table_rank": null,
  "block_size": 1024,
  "bos_token_id": null,
  "d_head": 16,
  "d_mlp": 8192,
  "d_model": 2048,
  "d_pos_emb": 32,
  "dropout": 0.0,
  "dropout_cat_pos_emb": false,
  "enable_bigram_table": true,
  "eos_token_id": 2047,
  "flash": true,
  "is_decoder": true,
  "learnable_bigram_table": true,
  "ln_bias": true,
  "max_position_embeddings": 1024,
  "model_type": "circuitgpt",
  "n_head": 128,
  "n_layer": 8,
  "pad_token_id": null,
  "residual_activation_type": "identity",
  "rms_norm": true,
  "sink": true,
  "sinusoidal_cat_pos_emb": false,
  "tie_word_embeddings": false,
  "tied_unembed": false,
  "torch_dtype": "float32",
  "transformers_version": "4.49.0",
  "unembed_rank": null,
  "use_position_embeddings": true,
  "vocab_size": 2048
}