# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # This file was automatically generated from src/transformers/models/step3p7/modular_step3p7.py. # Do NOT edit this file manually as any edits will be overwritten by the generation of # the file from the modular. If any change should be done, please apply the change to the # modular_step3p7.py file directly. One of our CI enforces this. # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # Copyright 2026 The StepFun and HuggingFace Inc. team. All rights reserved. # # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from huggingface_hub.dataclasses import strict from ...configuration_utils import PreTrainedConfig from ...modeling_rope_utils import RopeParameters from ...utils import auto_docstring @auto_docstring(checkpoint="stepfun-ai/Step-3.7-Flash") @strict class Step3p7VisionConfig(PreTrainedConfig): r""" Example: ```python >>> from transformers import Step3p7VisionConfig, Step3p7VisionModel >>> # Initializing a Step3p7VisionConfig with google/step3p7-base-patch16-224 style configuration >>> configuration = Step3p7VisionConfig() >>> # Initializing a Step3p7VisionModel (with random weights) from the google/step3p7-base-patch16-224 style configuration >>> model = Step3p7VisionModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config ```""" model_type = "step3p5_vision" base_config_key = "vision_config" # SiGLIP field overrides hidden_size: int = 1536 intermediate_size: int = 3072 num_hidden_layers: int = 47 num_attention_heads: int = 16 num_channels: int = 3 image_size: int = 728 patch_size: int = 14 hidden_act: str = "quick_gelu" layer_norm_eps: float = 1e-5 attention_dropout: float | int = 0.0 # New fields mlp_ratio: float = 8960 / 1536 layer_scale_init_value: float = 0.1 # RoPE config (compatible with Gemma4VisionRotaryEmbedding) rope_parameters: dict | None = None max_position_embeddings: int = 2704 # (image_size // patch_size)^2 = (728//14)^2 def __post_init__(self, **kwargs): self.hidden_size = kwargs.pop("width", self.hidden_size) self.num_hidden_layers = kwargs.pop("layers", self.num_hidden_layers) self.num_attention_heads = kwargs.pop("heads", self.num_attention_heads) self.layer_scale_init_value = kwargs.pop("ls_init_value", self.layer_scale_init_value) super().__post_init__(**kwargs) self.intermediate_size = int(self.hidden_size * self.mlp_ratio) @auto_docstring(checkpoint="stepfun-ai/Step-3.7-Flash") @strict class Step3p7TextConfig(PreTrainedConfig): r""" mlp_layer_types (`list[str]`, *optional*): Per-layer MLP type: `"sparse"` (MoE) or `"dense"`. If not provided, derived from the legacy `moe_layers_enum` hub-config kwarg (comma-separated string or list of MoE layer indices), defaulting to all layers from index 3 onward being MoE. n_routed_experts (`int`, *optional*, defaults to 288): Total number of routed experts. Accessible as `num_local_experts` via `attribute_map`. share_expert_dim (`int`, *optional*, defaults to 1280): Intermediate size of the always-active shared expert. num_sliding_attention_heads (`int`, *optional*): Attention head count for `"sliding_attention"` layers, if different from `num_attention_heads`. Defaults to the legacy `attention_other_setting` hub-config kwarg's `num_attention_heads` entry. Applied via a `per_layer_config` override (see `PreTrainedConfig`), not a per-layer list field. query_pre_attn_scalar (`int` or `float`, *optional*): `Step3p7Attention.__init__` hook point: defaults to `head_dim`, giving standard `head_dim ** -0.5` scaling; overridable per released checkpoint variant. moe_router_scaling_factor (`float`, *optional*, defaults to 1.0): Scaling factor applied to the MoE block's routed-expert output (`routed_scaling_factor` in `Step3p7SparseMoeBlock`). swiglu_limits (`list[float | None]`, *optional*): Per-layer gate/up clamping bound; `None` means no clamping. swiglu_limits_shared (`list[float | int | None]`, *optional*): Per-layer gate/up clamping bound for the always-active shared expert; `None` means no clamping. mtp_layer_types (`list[str]`, *optional*): Per-MTP-layer attention type; split off `layer_types`'s legacy trailing pad instead of being discarded, so `Step3p7Config.get_mtp_config()` can build the MTP layers for `generate(use_mtp=True)`. mtp_mlp_layer_types (`list[str]`, *optional*): Per-MTP-layer MLP type, analogous to `mtp_layer_types` for `mlp_layer_types`. """ model_type = "step3p5" keys_to_ignore_at_inference = ["past_key_values"] # Same as `MiniMaxM3VLTextConfig.base_model_tp_plan` plus `g_proj` (sharded like q/k/v, since it # gates their gathered output). Spelled out in full, not `{**MiniMaxM3VLTextConfig.base_model_tp_plan, ...}`: # generated files have no cross-model imports, so that name wouldn't resolve there. base_model_tp_plan = { "layers.*.self_attn.q_proj": "colwise_gather_output", "layers.*.self_attn.k_proj": "colwise_gather_output", "layers.*.self_attn.v_proj": "colwise_gather_output", "layers.*.self_attn.g_proj": "colwise_gather_output", "layers.*.self_attn.o_proj": "rowwise_split_input", "layers.*.mlp.experts.gate_up_proj": "packed_colwise", "layers.*.mlp.experts.down_proj": "rowwise", "layers.*.mlp.experts": "moe_tp_experts", } base_model_pp_plan = { "embed_tokens": (["input_ids"], ["inputs_embeds"]), "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), "norm": (["hidden_states"], ["hidden_states"]), } base_model_ep_plan = { "layers.*.mlp.gate": "ep_router", "layers.*.mlp.experts.gate_up_proj": "grouped_gemm", "layers.*.mlp.experts.down_proj": "grouped_gemm", "layers.*.mlp.experts": "moe_tp_experts", } attribute_map = { "num_local_experts": "n_routed_experts", "num_attention_groups": "num_key_value_heads", "moe_num_experts": "n_routed_experts", "moe_top_k": "num_experts_per_tok", "share_expert_dims": "share_expert_dim", "num_mtp_layers": "num_nextn_predict_layers", } default_theta = 10000.0 vocab_size: int = 128815 hidden_size: int = 4096 intermediate_size: int = 11264 num_hidden_layers: int = 45 num_attention_heads: int = 64 num_key_value_heads: int = 8 head_dim: int = 128 hidden_act: str = "silu" max_position_embeddings: int = 128000 initializer_range: float = 0.02 rms_norm_eps: float = 1e-5 use_cache: bool = True pad_token_id: int = 1 bos_token_id: int | None = None eos_token_id: int | list[int] | None = None tie_word_embeddings: bool = False attention_dropout: float | int = 0.0 num_experts_per_tok: int = 8 num_local_experts: int = 128 rope_parameters: RopeParameters | dict | None = None base_config_key = "text_config" mlp_layer_types: list[str] | None = None layer_types: list[str] | None = None gating = True use_bidirectional_attention = False moe_intermediate_size: int = 1280 n_routed_experts: int = 288 share_expert_dim: int = 1280 sliding_window: int | None = None num_sliding_attention_heads: int | None = None attention_bias: bool = False query_pre_attn_scalar: int | float | None = None moe_router_scaling_factor: float = 1.0 mlp_bias: bool = False swiglu_limits: list[float | int | None] | None = None swiglu_limits_shared: list[float | int | None] | None = None mtp_layer_types: list[str] | None = None mtp_mlp_layer_types: list[str] | None = None def __post_init__(self, **kwargs): # Legacy hub configs pad these per-layer lists with `num_nextn_predict_layers` trailing MTP # entries. Split each into `mtp_*` (used by `Step3p7Config.get_mtp_config()` for # `generate(use_mtp=True)`) instead of discarding it. Trimming itself is required either way: # `validate_layer_type`'s `@strict` check rejects `num_hidden_layers != len(layer_types)`. num_nextn_predict_layers = kwargs.pop("num_nextn_predict_layers", 0) self.num_nextn_predict_layers = num_nextn_predict_layers n, padded = self.num_hidden_layers, self.num_hidden_layers + num_nextn_predict_layers if num_nextn_predict_layers: for field, mtp_field in ( ("layer_types", "mtp_layer_types"), ("mlp_layer_types", "mtp_mlp_layer_types"), ): value = getattr(self, field) if isinstance(value, list) and len(value) == padded: setattr(self, field, value[:n]) setattr(self, mtp_field, value[n:padded]) for field in ("swiglu_limits", "swiglu_limits_shared"): value = getattr(self, field) if isinstance(value, list) and len(value) == padded: setattr(self, field, value[:n]) for key in ("rope_theta", "partial_rotary_factors"): value = kwargs.get(key) if isinstance(value, list) and len(value) == padded: kwargs[key] = value[:n] if self.layer_types is None: self.layer_types = ["full_attention"] * n if num_nextn_predict_layers: self.mtp_layer_types = ["full_attention"] * num_nextn_predict_layers if self.mlp_layer_types is None: # `moe_layers_enum` is the legacy hub-config alias for `mlp_layer_types`, read here only # Derived over the padded range so the trailing MTP layers get an `mtp_mlp_layer_types` entry. moe_layers_enum = kwargs.pop("moe_layers_enum", None) if moe_layers_enum is not None: items = moe_layers_enum.split(",") if isinstance(moe_layers_enum, str) else moe_layers_enum moe_set = {int(i) for i in items if str(i).strip()} else: moe_set = set(range(3, n)) mlp_layer_types = ["sparse" if i in moe_set else "dense" for i in range(padded)] self.mlp_layer_types = mlp_layer_types[:n] if num_nextn_predict_layers: self.mtp_mlp_layer_types = mlp_layer_types[n:padded] if self.num_sliding_attention_heads is None: # `attention_other_setting` is a legacy hub-config dict overriding num_attention_heads/ # num_key_value_heads/head_dim for "sliding_attention" layers. Keep `num_attention_heads` attention_other_setting = kwargs.pop("attention_other_setting", None) if attention_other_setting: self.num_sliding_attention_heads = attention_other_setting.get( "num_attention_heads", self.num_attention_heads ) else: self.num_sliding_attention_heads = self.num_attention_heads # On reload: `per_layer_config` is already in kwargs from the saved config. kwargs.setdefault( "per_layer_config", { layer_idx: {"num_attention_heads": self.num_sliding_attention_heads} for layer_idx, layer_type in enumerate(self.layer_types) if layer_type == "sliding_attention" }, ) if self.query_pre_attn_scalar is None: self.query_pre_attn_scalar = self.head_dim # `rope_theta`/`partial_rotary_factors` are per-layer (or `rope_theta` a single scalar shared # by all layers). Pop before `super().__post_init__()`: its RoPE handling only supports one # global scalar and would corrupt `self.rope_parameters` with a list. rope_theta = kwargs.pop("rope_theta", self.default_theta) partial_rotary_factors = kwargs.pop("partial_rotary_factors", None) rope_scaling = kwargs.pop("rope_scaling", None) sparse_cfg = kwargs.pop("sparse_attention_config", None) or {} moe_layer_freq = kwargs.pop("moe_layer_freq", None) super().__post_init__(**kwargs) # Checkpoint declares "swigluoai", but the gate is computed inline from swiglu_alpha/limit; hidden_act # is only the pointwise fallback and must be a real ACT2FN key, so normalize it to silu. self.hidden_act = "silu" for flat, legacy in { "index_n_heads": "sparse_num_index_heads", "index_head_dim": "sparse_index_dim", "index_block_size": "sparse_block_size", "index_topk_blocks": "sparse_topk_blocks", "index_local_blocks": "sparse_local_block", }.items(): if legacy in sparse_cfg: setattr(self, flat, sparse_cfg[legacy]) # `layer_types` is the canonical per-layer attention dispatch: it tells # `DynamicCache(config=...)` which layers want the sparse cache and tells # `Step3p7Attention` which layers build a sparse Lightning Indexer. if self.layer_types is None and "sparse_attention_freq" in sparse_cfg: self.layer_types = [ "minimax_m3_sparse" if f else "full_attention" for f in sparse_cfg["sparse_attention_freq"] ] if self.layer_types is None: self.layer_types = ["full_attention"] * self.num_hidden_layers # `mlp_layer_types` is the per-layer MLP dispatch read by `Step3p7DecoderLayer`: if self.mlp_layer_types is None and moe_layer_freq is not None: self.mlp_layer_types = ["sparse" if f else "dense" for f in moe_layer_freq] if self.mlp_layer_types is None: self.mlp_layer_types = ["sparse"] * self.num_hidden_layers # `DSV4Config` pattern: if `rope_parameters` is already resolved per type (reload), keep # only those sub-dicts. Otherwise build fresh, taking each type's value from any one of its layers (they all agree). layer_type_set = set(self.layer_types) rp = self.rope_parameters or {} if all(isinstance(rp.get(layer_type), dict) for layer_type in layer_type_set): self.rope_parameters = {layer_type: rp[layer_type] for layer_type in layer_type_set} else: if not isinstance(rope_theta, list): rope_theta = [rope_theta] * len(self.layer_types) self.rope_parameters = {} for layer_type in layer_type_set: i = self.layer_types.index(layer_type) params = {"rope_type": "default", "rope_theta": rope_theta[i]} if partial_rotary_factors: params["partial_rotary_factor"] = partial_rotary_factors[i] self.rope_parameters[layer_type] = params if rope_scaling and "full_attention" in self.rope_parameters: self.rope_parameters["full_attention"].update(rope_scaling) @auto_docstring(checkpoint="stepfun-ai/Step-3.7-Flash") @strict class Step3p7Config(PreTrainedConfig): model_type = "step3p7" sub_configs = {"vision_config": Step3p7VisionConfig, "text_config": Step3p7TextConfig} vision_config: dict | PreTrainedConfig | None = None text_config: dict | PreTrainedConfig | None = None projector_bias: bool = False image_token_id: int = 151679 def __post_init__(self, **kwargs): if self.vision_config is None: self.vision_config = Step3p7VisionConfig() elif isinstance(self.vision_config, dict): self.vision_config = Step3p7VisionConfig( **{k: v for k, v in self.vision_config.items() if k != "model_type"} ) if self.text_config is None: self.text_config = Step3p7TextConfig() elif isinstance(self.text_config, dict): self.text_config = Step3p7TextConfig(**{k: v for k, v in self.text_config.items() if k != "model_type"}) super().__post_init__(**kwargs) __all__ = ["Step3p7VisionConfig", "Step3p7TextConfig", "Step3p7Config"]