# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # This file was automatically generated from src/transformers/models/step3p7/modular_step3p7.py. # Do NOT edit this file manually as any edits will be overwritten by the generation of # the file from the modular. If any change should be done, please apply the change to the # modular_step3p7.py file directly. One of our CI enforces this. # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # Copyright 2026 The StepFun and HuggingFace Inc. team. All rights reserved. # # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import torch from ...image_processing_backends import TorchvisionBackend from ...image_processing_utils import BatchFeature from ...image_transforms import divide_to_patches, group_images_by_shape, reorder_images from ...image_utils import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD, ImageInput, PILImageResampling, SizeDict from ...processing_utils import ImagesKwargs, Unpack from ...utils import TensorType, auto_docstring class Step3p7ImageProcessorKwargs(ImagesKwargs, total=False): r""" patch_size (`int`, *optional*, defaults to 504): Target size (height = width) for each local patch crop. max_image_size (`int`, *optional*, defaults to 3024): Images larger than this (on their longest side) are scaled down uniformly before patch planning. """ patch_size: int max_image_size: int @auto_docstring class Step3p7ImageProcessor(TorchvisionBackend): """ Image processor for Step-3.7-Flash. Each input image is split into a global down-scaled view plus zero or more local patch crops via a sliding-window strategy, then every sub-image is resized and normalised independently. """ resample = PILImageResampling.BILINEAR size = {"height": 728, "width": 728} patch_size: int = 504 do_rescale = True do_normalize = True image_mean: list[float] = OPENAI_CLIP_MEAN image_std: list[float] = OPENAI_CLIP_STD do_convert_rgb = True valid_kwargs = Step3p7ImageProcessorKwargs model_input_names = ["pixel_values", "pixel_values_local", "num_local_patches"] max_image_size: int = 3024 # ViT patch size (`Step3p7VisionEmbeddings.patch_size`) and the vision tower's total downsampling # stride (two stride-2 convolutions, `downsampler1`/`downsampler2`); `Step3p7Processor.__init__` # derives `num_image_features`/`num_patch_features` from these plus `size`/`patch_size`. vision_patch_size: int = 14 downsampler_stride: int = 4 @staticmethod def _is_extreme_aspect(width: int, height: int) -> bool: """`True` for near-degenerate images (min side < 32px, aspect ratio > 4:1).""" return min(width, height) < 32 and max(width / height, height / width) > 4 def _plan_patches( self, width: int, height: int, image_size: int, patch_size: int ) -> tuple[tuple[int, int], tuple[int, int], int, int, int, bool]: """Compute the sliding-window patch layout for one image. Unlike models with a fixed tile size (Idefics3, LLaVA-NeXT), Step3p7 adapts the window size to the image's aspect ratio, so this cannot be reduced to a simple ``ceil(h / tile) × ceil(w / tile)`` formula. Step 1 = normalise extreme inputs: - extreme-aspect images (min_side < 32, ratio > 4) are squared - images larger than ``max_image_size`` are scaled down uniformly Step 2 — choose window size from the normalised aspect ratio: - fits in global view (long_side ≤ image_size): tile only if elongated (long_side / short_side > 1.5), using short_side as the window - very elongated (ratio > 4): ``min(short_side, patch_size)`` - standard case: ``patch_size`` Step 3 — snap each dimension to the nearest window multiple (snap up when the remainder exceeds 20 % of the window size). Returns: global_width_height: ``(width, height)`` to resize the global view to before squaring crop_width_height: ``(crop_width, crop_height)`` snapped dimensions for patch extraction window_size: tile side length (``0`` → no local patches) num_patches_x: number of patches along the width num_patches_y: number of patches along the height needs_square_pad: whether the raw image must be zero-padded to a square before resizing """ # Step 1 — normalise needs_square_pad = self._is_extreme_aspect(width, height) if needs_square_pad: width = height = max(width, height) if max(height, width) > self.max_image_size: scale = self.max_image_size / max(height, width) width, height = int(width * scale), int(height * scale) short_side, long_side = min(height, width), max(height, width) # Step 2 — choose window size if long_side <= image_size: window_size = short_side if long_side / short_side > 1.5 else 0 elif long_side / short_side > 4: window_size = min(short_side, patch_size) else: window_size = patch_size if window_size == 0: return (width, height), (width, height), 0, 0, 0, needs_square_pad # Step 3 — snap each dimension to the nearest window-size multiple crop_width, crop_height = ( ( window_size * (dim // window_size + (dim % window_size > 0.2 * window_size)) if dim >= window_size else dim ) for dim in (width, height) ) num_patches_x = max(1, crop_width // window_size) num_patches_y = max(1, crop_height // window_size) return (width, height), (crop_width, crop_height), window_size, num_patches_x, num_patches_y, needs_square_pad def get_number_of_image_patches(self, height: int, width: int, images_kwargs=None) -> int: """Return the number of local patches for an image of the given size.""" images_kwargs = images_kwargs or {} size = images_kwargs.get("size", self.size) image_size = size["height"] patch_size = images_kwargs.get("patch_size", self.patch_size) num_patches_x, num_patches_y = self._plan_patches(width, height, image_size, patch_size)[3:5] return num_patches_x * num_patches_y def _get_image_patches( self, img: torch.Tensor, image_size: int, patch_size: int, resample: "PILImageResampling", ) -> tuple[torch.Tensor, list[torch.Tensor], int, int]: """Step3p7-specific cropping: square-pad extreme aspect ratios, resize the global view, and slice out raw (pre-final-resize) local-patch tiles per `_plan_patches`'s layout. Returns the resized global view, the list of raw local-patch tensors (still at `window_size`, not yet resized to `patch_size`), and the patch grid dimensions. """ _, height, width = img.shape ( (global_width, global_height), (crop_width, crop_height), window_size, num_patches_x, num_patches_y, needs_square_pad, ) = self._plan_patches(width, height, image_size, patch_size) # Pad extreme-aspect-ratio images to square (original at top-left, zeros elsewhere) if needs_square_pad: side = max(width, height) img = self.pad([img], pad_size=SizeDict(height=side, width=side))[0] img_batch = img.unsqueeze(0) # Global view: resize to (global_width, global_height) then square to image_size × image_size global_img = self.resize(img_batch, SizeDict(height=global_height, width=global_width), resample=resample) global_img = self.resize(global_img, SizeDict(height=image_size, width=image_size), resample=resample).squeeze( 0 ) if window_size == 0: return global_img, [], num_patches_x, num_patches_y img_for_crop = self.resize( img_batch, SizeDict(height=crop_height, width=crop_width), resample=resample ).squeeze(0) patches = divide_to_patches(img_for_crop, patch_size=window_size) return global_img, patches, num_patches_x, num_patches_y def _preprocess( self, images: list["torch.Tensor"], do_rescale: bool, rescale_factor: float, do_normalize: bool, image_mean: float | list[float] | None, image_std: float | list[float] | None, resample: "PILImageResampling", size: SizeDict, patch_size: int, disable_grouping: bool | None, return_tensors: "str | TensorType | None", **kwargs, ) -> BatchFeature: image_size = size.height global_images, nested_patches, num_local_patches, patch_newline_masks = [], [], [], [] for img in images: global_img, patches, num_patches_x, num_patches_y = self._get_image_patches( img, image_size, patch_size, resample ) global_images.append(global_img) num_local_patches.append(len(patches)) nested_patches.append(patches) # Newline after the last patch in each row except the final row patch_newline_masks.append( [ col == num_patches_x - 1 and row < num_patches_y - 1 for row in range(num_patches_y) for col in range(num_patches_x) ] ) # Global views already share a uniform (image_size × image_size) shape, so batch directly. global_stack = self.rescale_and_normalize( torch.stack(global_images), do_rescale, rescale_factor, do_normalize, image_mean, image_std ) data = { "pixel_values": global_stack, "num_local_patches": num_local_patches, } # Built before the local-patch fields below are assigned: `result[key] = ...` (unlike `data[key] = ...` # pre-construction) bypasses `BatchFeature`'s tensor conversion, which matters for # `patch_newline_masks` — it must stay a plain list of `bool`s, not a tensor. result = BatchFeature(data=data, tensor_type=return_tensors) max_patches = max(num_local_patches, default=0) if max_patches: # Group by shape while keeping each image's patches nested (`is_nested=True`, the same # convention Idefics3/Maskformer use for a variable number of sub-images per sample) # instead of flattening every image's patches into one list and tracking counts by hand. grouped_patches, grouped_index = group_images_by_shape( nested_patches, is_nested=True, disable_grouping=disable_grouping ) for shape, stacked_patches in grouped_patches.items(): resized = self.resize( stacked_patches, SizeDict(height=patch_size, width=patch_size), resample=resample ) grouped_patches[shape] = self.rescale_and_normalize( resized, do_rescale, rescale_factor, do_normalize, image_mean, image_std ) nested_pixel_values_local = reorder_images(grouped_patches, grouped_index, is_nested=True) # Flatten back to (total_patches, C, H, W): `Step3p7Model.get_image_features` slices this # flat tensor per image using `num_local_patches`. result["pixel_values_local"] = torch.stack( [patch for per_image_patches in nested_pixel_values_local for patch in per_image_patches] ) # Pad every image's mask to `max_patches` so the output is a uniform (batch, max_patches) result["patch_newline_masks"] = [ mask + [False] * (max_patches - len(mask)) for mask in patch_newline_masks ] return result @auto_docstring def preprocess(self, images: ImageInput, **kwargs: Unpack[Step3p7ImageProcessorKwargs]) -> BatchFeature: return super().preprocess(images, **kwargs) __all__ = ["Step3p7ImageProcessor"]