# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # This file was automatically generated from src/transformers/models/qwen4_exp/modular_qwen4_exp.py. # Do NOT edit this file manually as any edits will be overwritten by the generation of # the file from the modular. If any change should be done, please apply the change to the # modular_qwen4_exp.py file directly. One of our CI enforces this. # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # Copyright 2026 The Qwen Team and The HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from huggingface_hub.dataclasses import strict from ...configuration_utils import PreTrainedConfig from ...modeling_rope_utils import RopeParameters from ...utils import auto_docstring @auto_docstring(checkpoint="Qwen/Qwen4-Exp") @strict class Qwen4ExpTextConfig(PreTrainedConfig): r""" linear_conv_kernel_dim (`int`, *optional*, defaults to 4): Kernel size of the convolution used in linear attention layers. linear_key_head_dim (`int`, *optional*, defaults to 128): Dimension of each key head in linear attention. linear_value_head_dim (`int`, *optional*, defaults to 128): Dimension of each value head in linear attention. linear_num_key_heads (`int`, *optional*, defaults to 16): Number of key heads used in linear attention layers. linear_num_value_heads (`int`, *optional*, defaults to 32): Number of value heads used in linear attention layers. hc_count (`int`, *optional*, defaults to 4): Number of residual streams used by the hyper-connections. hc_lowrank (`int`, *optional*, defaults to 320): Rank of the learned hyper-connection input mixer. ple_layer_ids (`list[int]`, *optional*): One-indexed decoder layer ids that use Per-Layer Embedding (PLE). ple_embed_dim (`int`, *optional*): Total dimension of the embeddings concatenated from all n-gram heads in each PLE module. Defaults to `hidden_size`. ple_conv_kernel_size (`int`, *optional*, defaults to 4): Kernel size of the dilated depthwise convolution in each PLE module. ngram_size (`int`, *optional*, defaults to 3): Largest token n-gram represented by PLE. heads_per_ngram (`int`, *optional*, defaults to 8): Number of independently hashed embedding heads for every n-gram order. ngram_vocab_size_base (`int`, *optional*, defaults to 20000000): Lower bound used to derive a distinct prime vocabulary size for each hashed n-gram head. make_ngram_vocab_size_divisible_by (`int`, *optional*, defaults to 128): Divisor used to pad the combined n-gram embedding vocabulary. seed (`int`, *optional*, defaults to 1234): Seed used to deterministically derive the per-layer n-gram hash multipliers. split_ngram_parts (`int`, *optional*, defaults to 512): Number of checkpoint shards used for each PLE n-gram embedding table. Loading concatenates the shards into a single runtime embedding, while `save_pretrained` restores the configured sharded layout. indexer_n_heads (`int`, *optional*): Number of query heads used by the QSA token indexer. Setting this enables QSA on full-attention layers. indexer_kv_heads (`int`, *optional*): Number of indexer key heads. Qwen4-Exp QSA requires one key head. indexer_head_dim (`int`, *optional*): Dimension of every QSA indexer query and key head. indexer_budget (`int`, *optional*): Maximum number of tokens selected from complete compressed blocks for each query. indexer_compress_ratio (`int`, *optional*): Number of consecutive token keys averaged into one QSA index block. output_gate_type (`str`, *optional*): Activation used by the output gate of linear attention. If unset, `hidden_act` is used. """ model_type = "qwen4_exp_text" keys_to_ignore_at_inference = ["past_key_values"] base_model_tp_plan = { "layers.*.mlp.experts.gate_up_proj": "packed_colwise", "layers.*.mlp.experts.down_proj": "rowwise", "layers.*.mlp.experts": "moe_tp_experts", "layers.*.mlp.shared_expert.gate_proj": "colwise", "layers.*.mlp.shared_expert.up_proj": "colwise", "layers.*.mlp.shared_expert.down_proj": "rowwise", "layers.*.linear_attn.in_proj_qkv": "colwise_gather_output", "layers.*.linear_attn.in_proj_z": "colwise_gather_output", "layers.*.linear_attn.in_proj_b": "colwise_gather_output", "layers.*.linear_attn.in_proj_a": "colwise_gather_output", "layers.*.linear_attn.out_proj": "colwise_gather_output", "layers.*.self_attn.indexer.index_qk_proj": "colwise_gather_output", "layers.*.attn_hyper_connection.input_mix_weight_down": "rowwise_split_input", "layers.*.mlp_hyper_connection.input_mix_weight_down": "rowwise_split_input", "hyper_connection_mixer.input_mix_weight_down": "rowwise_split_input", # It's extremely important to shard this embedding as its size is ~45B == 90 GiB - we shard on dim 1, as the checkpoints # for it are sharded on dim0 (this way Concatenate and tp do not work on the same dim and everything is easy) "layers.*.ple.ple_embedding.ngram_embedding": "colwise_gather_output", } base_model_pp_plan = None base_model_ep_plan = { "layers.*.mlp.gate": "ep_router", "layers.*.mlp.experts.gate_up_proj": "grouped_gemm", "layers.*.mlp.experts.down_proj": "grouped_gemm", "layers.*.mlp.experts": "moe_tp_experts", } vocab_size: int = 248320 hidden_size: int = 2048 num_hidden_layers: int = 40 num_attention_heads: int = 16 num_key_value_heads: int = 2 hidden_act: str = "silu" max_position_embeddings: int = 32768 initializer_range: float = 0.02 rms_norm_eps: float = 1e-6 use_cache: bool = True tie_word_embeddings: bool = False rope_parameters: RopeParameters | dict | None = None attention_bias: bool = False attention_dropout: float | int = 0.0 head_dim: int = 256 linear_conv_kernel_dim: int = 4 linear_key_head_dim: int = 128 linear_value_head_dim: int = 128 linear_num_key_heads: int = 16 linear_num_value_heads: int = 32 moe_intermediate_size: int = 512 shared_expert_intermediate_size: int = 512 num_experts_per_tok: int = 10 num_experts: int = 512 output_router_logits: bool = False router_aux_loss_coef: float = 0.001 layer_types: list[str] | None = None pad_token_id: int | None = None bos_token_id: int | None = None eos_token_id: int | list[int] | None = None base_config_key = "text_config" ignore_keys_at_rope_validation = {"mrope_section", "mrope_interleaved"} base_model_fsdp_plan = { "embed_tokens": "free_full_weight", "layers.*": "free_full_weight", "hyper_connection_mixer": "keep_full_weight", } hc_count: int = 4 hc_lowrank: int = 320 ple_layer_ids: list[int] | None = None ple_embed_dim: int | None = None ple_conv_kernel_size: int = 4 ngram_size: int = 3 heads_per_ngram: int = 8 ngram_vocab_size_base: int = 20_000_000 make_ngram_vocab_size_divisible_by: int = 128 seed: int = 1234 split_ngram_parts: int = 512 indexer_n_heads: int | None = None indexer_kv_heads: int | None = None indexer_head_dim: int | None = None indexer_budget: int | None = None indexer_compress_ratio: int | None = None norm_topk_prob: bool = True output_gate_type: str | None = None def __post_init__(self, **kwargs): self.ple_layer_ids = [] if self.ple_layer_ids is None else sorted(set(self.ple_layer_ids)) self.ple_embed_dim = self.hidden_size if self.ple_embed_dim is None else self.ple_embed_dim # Qwen4-Exp keeps the GatedDeltaNet convolution, PLE convolution and n-gram context in separate cache states. # Without PLE, only the GatedDeltaNet state is needed. self.number_of_conv_states = 3 if self.ple_layer_ids else 1 if self.layer_types is None: interval_pattern = kwargs.pop("full_attention_interval", 4) self.layer_types = [ "linear_attention" if (i + 1) % interval_pattern else "qwen_sparse_attention" for i in range(self.num_hidden_layers) ] # The real checkpoint contains "full_attention" entries for layers that are actually using an indexer elif "full_attention" in self.layer_types: self.layer_types = [ "qwen_sparse_attention" if layer == "full_attention" else layer for layer in self.layer_types ] super().__post_init__(**kwargs) def validate_architecture(self): """Part of `@strict`-powered validation. Validates Qwen4-Exp architecture invariants.""" unsupported_layer_types = sorted(set(self.layer_types) - {"linear_attention", "qwen_sparse_attention"}) if unsupported_layer_types: raise ValueError(f"Unsupported Qwen4-Exp layer types: {unsupported_layer_types}.") output_gate_type = self.output_gate_type or self.hidden_act if output_gate_type not in {"sigmoid", "silu"}: raise ValueError(f"Unsupported Qwen4-Exp output gate activation: {output_gate_type}.") if self.hc_count <= 1: raise ValueError(f"Qwen4-Exp requires hc_count > 1, got {self.hc_count}.") if self.num_experts <= 0: raise ValueError(f"num_experts must be > 0, got {self.num_experts}.") if not 0 < self.num_experts_per_tok <= self.num_experts: raise ValueError( "num_experts_per_tok must be in [1, num_experts], " f"got {self.num_experts_per_tok} and {self.num_experts}." ) if self.moe_intermediate_size <= 0 or self.shared_expert_intermediate_size <= 0: raise ValueError("moe_intermediate_size and shared_expert_intermediate_size must be > 0.") qsa_fields = ( "indexer_n_heads", "indexer_kv_heads", "indexer_head_dim", "indexer_budget", "indexer_compress_ratio", ) qsa_values = {name: getattr(self, name) for name in qsa_fields} if any(value is not None for value in qsa_values.values()): missing = [name for name, value in qsa_values.items() if value is None] if missing: raise ValueError(f"QSA config is missing required fields: {missing}.") if any(value <= 0 for value in qsa_values.values()): raise ValueError(f"QSA config values must be positive: {qsa_values}.") if self.indexer_kv_heads != 1: raise ValueError("Qwen4-Exp QSA requires indexer_kv_heads=1.") if self.indexer_budget % self.indexer_compress_ratio != 0: raise ValueError("indexer_budget must be divisible by indexer_compress_ratio.") partial_rotary_factor = (self.rope_parameters or {}).get("partial_rotary_factor", 1.0) rotary_dim = int(self.head_dim * partial_rotary_factor) if rotary_dim > self.indexer_head_dim: raise ValueError( f"Qwen4-Exp attention RoPE dimensions must fit the QSA index head: rotary_dim={rotary_dim}, " f"indexer_head_dim={self.indexer_head_dim}." ) if self.ple_layer_ids: ngram_heads = (self.ngram_size - 1) * self.heads_per_ngram if ngram_heads <= 0 or self.ple_embed_dim <= 0 or self.ple_embed_dim % ngram_heads != 0: raise ValueError( "ple_embed_dim and the total number of n-gram heads must be positive, and ple_embed_dim must be " f"divisible by the number of heads: {self.ple_embed_dim} % {ngram_heads} != 0." ) invalid_ple_layers = [ layer_id for layer_id in self.ple_layer_ids if layer_id < 1 or layer_id > self.num_hidden_layers ] if invalid_ple_layers: raise ValueError( f"ple_layer_ids must contain one-indexed ids in [1, {self.num_hidden_layers}], " f"got {invalid_ple_layers}." ) non_linear_ple_layers = [ layer_id for layer_id in self.ple_layer_ids if self.layer_types[layer_id - 1] != "linear_attention" ] if non_linear_ple_layers: raise ValueError( "Qwen4-Exp PLE is only supported on linear_attention layers, " f"got PLE on layers {non_linear_ple_layers}." ) if self.eos_token_id is None or isinstance(self.eos_token_id, list) and not self.eos_token_id: raise ValueError("eos_token_id must be set when Qwen4-Exp PLE layers are enabled.") @auto_docstring(checkpoint="Qwen/Qwen4-Exp") @strict class Qwen4ExpVisionConfig(PreTrainedConfig): r""" out_hidden_size (`int`, *optional*, defaults to 3584): The output hidden size of the vision model. num_position_embeddings (`int`, *optional*, defaults to 2304): The maximum sequence length that this model might ever be used with """ model_type = "qwen4_exp_vision" base_config_key = "vision_config" depth: int = 27 hidden_size: int = 1152 hidden_act: str = "gelu_pytorch_tanh" intermediate_size: int = 4304 num_heads: int = 16 in_channels: int = 3 patch_size: int | list[int] | tuple[int, int] = 16 spatial_merge_size: int = 2 temporal_patch_size: int | list[int] | tuple[int, int] = 2 out_hidden_size: int = 3584 num_position_embeddings: int = 2304 initializer_range: float = 0.02 base_model_fsdp_plan = None @auto_docstring(checkpoint="Qwen/Qwen4-Exp") @strict class Qwen4ExpConfig(PreTrainedConfig): r""" Example: ```python >>> from transformers import Qwen4ExpConfig, Qwen4ExpForConditionalGeneration >>> configuration = Qwen4ExpConfig() >>> model = Qwen4ExpForConditionalGeneration(configuration) >>> configuration = model.config ``` """ model_type = "qwen4_exp" sub_configs = {"vision_config": Qwen4ExpVisionConfig, "text_config": Qwen4ExpTextConfig} keys_to_ignore_at_inference = ["past_key_values"] text_config: dict | PreTrainedConfig | None = None vision_config: dict | PreTrainedConfig | None = None image_token_id: int = 248056 video_token_id: int = 248057 vision_start_token_id: int = 248053 vision_end_token_id: int = 248054 tie_word_embeddings: bool = False base_model_fsdp_plan = None def __post_init__(self, **kwargs): if isinstance(self.vision_config, dict): # old ckpt with incorrect model type -> override manually if self.vision_config.get("model_type") == "qwen4_exp": self.vision_config["model_type"] = "qwen4_exp_vision" self.vision_config = self.sub_configs["vision_config"](**self.vision_config) elif self.vision_config is None: self.vision_config = self.sub_configs["vision_config"]() if isinstance(self.text_config, dict): self.text_config = self.sub_configs["text_config"](**self.text_config) elif self.text_config is None: self.text_config = self.sub_configs["text_config"]() super().__post_init__(**kwargs) __all__ = ["Qwen4ExpConfig", "Qwen4ExpTextConfig", "Qwen4ExpVisionConfig"]