| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| """ GLM model configuration """ |
|
|
| from typing import Dict |
|
|
| from transformers.configuration_utils import PretrainedConfig |
| from transformers.utils import logging |
|
|
| logger = logging.get_logger(__name__) |
|
|
| GLM_PRETRAINED_CONFIG_ARCHIVE_MAP = { |
| "shunxing1234/GLM": "https://huggingface.co/shunxing1234/GLM/resolve/main/config.json", |
| |
| } |
|
|
|
|
| class GLMConfig(PretrainedConfig): |
| r""" |
| This is the configuration class to store the configuration of a [`~GLMModel`]. |
| It is used to instantiate an GLM model according to the specified arguments, defining the model |
| architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of |
| the GLM [shunxing1234/GLM-base-cased](https://huggingface.co/shunxing1234/GLM-base-cased) architecture. |
| |
| Configuration objects inherit from [`PretrainedConfig`] and can be used |
| to control the model outputs. Read the documentation from [`PretrainedConfig`] |
| for more information. |
| |
| |
| Args: |
| vocab_size (`int`, *optional*, defaults to 30522): |
| Vocabulary size of the GLM model. Defines the number of different tokens that can be represented by the |
| `inputs_ids` passed when calling [`~GLMModel`] or |
| [`~TFGLMModel`]. |
| hidden_size (`int`, *optional*, defaults to 768): |
| Dimension of the encoder layers and the pooler layer. |
| num_hidden_layers (`int`, *optional*, defaults to 12): |
| Number of hidden layers in the Transformer encoder. |
| num_attention_heads (`int`, *optional*, defaults to 12): |
| Number of attention heads for each attention layer in the Transformer encoder. |
| intermediate_size (`int`, *optional*, defaults to 3072): |
| Dimension of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. |
| hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`): |
| The non-linear activation function (function or string) in the encoder and pooler. |
| If string, `"gelu"`, `"relu"`, `"selu"` and `"gelu_new"` are supported. |
| hidden_dropout_prob (`float`, *optional*, defaults to 0.1): |
| The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler. |
| attention_probs_dropout_prob (`float`, *optional*, defaults to 0.1): |
| The dropout ratio for the attention probabilities. |
| max_position_embeddings (`int`, *optional*, defaults to 512): |
| The maximum sequence length that this model might ever be used with. |
| Typically set this to something large just in case (e.g., 512 or 1024 or 2048). |
| type_vocab_size (`int`, *optional*, defaults to 2): |
| The vocabulary size of the `token_type_ids` passed when calling [`~GLMModel`] or |
| [`~TFGLMModel`]. |
| initializer_range (`float`, *optional*, defaults to 0.02): |
| The standard deviation of the truncated_normal_initializer for initializing all weight matrices. |
| layer_norm_eps (`float`, *optional*, defaults to 1e-12): |
| The epsilon used by the layer normalization layers. |
| use_cache (`bool`, *optional*, defaults to `True`): |
| Whether or not the model should return the last key/values attentions (not used by all models). Only |
| relevant if `config.is_decoder=True`. |
| last_logits_l2_alpha ('float', *optional*, defaults to -1.0): |
| Whether use l2 norm for last output logits. |
| If < 0, will not compute last logits l2 norm, |
| elif == 0, will compute l2 norm but not plus in the loss, |
| while > 0, will plus this loss in the total loss. |
| rotary_type (`str` or `function`, *optional*, defaults to `"none"`): |
| The Rotary Embedding type to used in SelfAttention. |
| If string, `"none"`, `"1d"`, `"2d"` are supported. |
| unidirectional ('bool', *optional*, defaults to `False`): |
| Whether or not the model is train with prefix LM or causal LM. |
| Example: |
| |
| ```python |
| >>> from transformers import GLMModel, GLMConfig |
| |
| >>> # Initializing a GLM shunxing1234/GLM-base-cased style configuration |
| >>> configuration = GLMConfig() |
| |
| >>> # Initializing a model from the shunxing1234/GLM-base-cased style configuration |
| >>> model = GLMModel(configuration) |
| |
| >>> # Accessing the model configuration |
| >>> configuration = model.config |
| ``` |
| """ |
|
|
| model_type = "glm" |
| attribute_map = {"num_hidden_layers": "num_layers"} |
|
|
| def __init__( |
| self, |
| num_layers=24, |
| vocab_size=30592, |
| hidden_size=1024, |
| num_experts=1, |
| expert_capacity=None, |
| moe_config: Dict = {}, |
| num_attention_heads=16, |
| num_key_value_heads=0, |
| embedding_dropout_prob=0.1, |
| attention_dropout_prob=0.1, |
| output_dropout_prob=0.1, |
| max_sequence_length=512, |
| checkpoint_activations=False, |
| checkpoint_num_layers=1, |
| parallel_output=True, |
| relative_encoding=False, |
| block_position_encoding=True, |
| output_predict=False, |
| spell_length=None, |
| spell_func="lstm", |
| attention_scale=1.0, |
| initializer_range=0.02, |
| pool_token="cls", |
| max_memory_length=0, |
| bf16=True, |
| intermediate_size=None, |
| last_logits_l2_alpha=-1.0, |
| rotary_type='none', |
| use_rmsnorm=False, |
| use_atorch_rmsnorm=False, |
| use_swiglu=False, |
| rope_scaling=1.0, |
| use_cache=True, |
| focused_attention=False, |
| cache_in_memory=False, |
| attention_grouping=None, |
| output_hidden_states=False, |
| tie_word_embeddings=True, |
| unidirectional=False, |
| use_bias=True, |
| use_qkv_bias=False, |
| mlp_version='v1', |
| norm_softmax=False, |
| norm_head=False, |
| num_decoder_image_token=1024, |
| num_decoder_audio_token=512, |
| **kwargs, |
| ): |
| self.num_layers = num_layers |
| self.vocab_size = vocab_size |
| self.hidden_size = hidden_size |
| self.num_experts = num_experts |
| self.expert_capacity = expert_capacity |
| self.moe_config = moe_config |
| self.num_attention_heads = num_attention_heads |
| self.num_key_value_heads = num_key_value_heads |
| self.embedding_dropout_prob = embedding_dropout_prob |
| self.attention_dropout_prob = attention_dropout_prob |
| self.output_dropout_prob = output_dropout_prob |
| self.max_sequence_length = max_sequence_length |
| self.checkpoint_activations = checkpoint_activations |
| self.checkpoint_num_layers = checkpoint_num_layers |
| self.parallel_output = parallel_output |
| self.relative_encoding = relative_encoding |
| self.block_position_encoding = block_position_encoding |
| self.output_predict = output_predict |
| self.spell_length = spell_length |
| self.spell_func = spell_func |
| self.attention_scale = attention_scale |
| self.initializer_range = initializer_range |
| self.pool_token = pool_token |
| self.max_memory_length = max_memory_length |
| self.bf16 = bf16 |
| self.intermediate_size = intermediate_size |
| self.last_logits_l2_alpha = last_logits_l2_alpha |
| self.rotary_type = rotary_type |
| self.use_rmsnorm = use_rmsnorm |
| self.use_atorch_rmsnorm = use_atorch_rmsnorm |
| self.use_swiglu = use_swiglu |
| self.rope_scaling = rope_scaling |
| self.use_cache = use_cache |
| self.focused_attention = focused_attention |
| self.cache_in_memory = cache_in_memory |
| self.attention_grouping = attention_grouping |
| self.unidirectional = unidirectional |
| self.use_bias = use_bias |
| self.use_qkv_bias = use_qkv_bias |
| self.mlp_version = mlp_version |
| self.norm_softmax = norm_softmax |
| self.norm_head = norm_head |
| self.num_decoder_image_token = num_decoder_image_token |
| self.num_decoder_audio_token = num_decoder_audio_token |
|
|
| super().__init__(output_hidden_states=output_hidden_states, tie_word_embeddings=tie_word_embeddings, **kwargs) |
|
|
|
|
