# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # This file was automatically generated from src/transformers/models/qianfan_ocr/modular_qianfan_ocr.py. # Do NOT edit this file manually as any edits will be overwritten by the generation of # the file from the modular. If any change should be done, please apply the change to the # modular_qianfan_ocr.py file directly. One of our CI enforces this. # 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 # Copyright 2026 HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import re from ...image_processing_utils import BatchFeature from ...image_utils import ImageInput from ...processing_utils import MultiModalData, ProcessingKwargs, ProcessorMixin, Unpack from ...tokenization_utils_base import PreTokenizedInput, TextInput from ...utils import auto_docstring class QianfanOCRProcessorKwargs(ProcessingKwargs, total=False): _defaults = { "text_kwargs": { "padding_side": "left", "return_mm_token_type_ids": False, }, "images_kwargs": { "crop_to_patches": True, }, "videos_kwargs": { "return_tensors": "pt", }, } @auto_docstring class QianfanOCRProcessor(ProcessorMixin): valid_processor_kwargs = QianfanOCRProcessorKwargs def __init__( self, image_processor=None, tokenizer=None, image_seq_length: int = 256, chat_template=None, image_placeholder_token: str = "", **kwargs, ): r""" image_placeholder_token (`str`, *optional*, defaults to `""`): The token emitted by the chat template to mark image positions. It is replaced by the full ``...`` sequence during processing. """ super().__init__(image_processor, tokenizer, chat_template=chat_template, **kwargs) self.image_seq_length = image_seq_length self.start_image_token = tokenizer.start_image_token self.end_image_token = tokenizer.end_image_token self.start_image_token_id = tokenizer.start_image_token_id self.end_image_token_id = tokenizer.end_image_token_id self.image_token = tokenizer.context_image_token self.image_token_id = tokenizer.context_image_token_id self.image_ids = [self.image_token_id, self.start_image_token_id, self.end_image_token_id] self.image_placeholder_token = image_placeholder_token self.video_token = None self.video_processor = None @property def image_token_ids(self) -> list[int]: return [self.image_token_id, self.start_image_token_id, self.end_image_token_id] @auto_docstring def __call__( self, images: ImageInput | None = None, text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None = None, **kwargs: Unpack[QianfanOCRProcessorKwargs], ) -> BatchFeature: r""" Returns: [`BatchFeature`]: A [`BatchFeature`] with the following fields: - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`. - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not `None`). - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`. """ # remove video from signature as well because the modality isn't supported # some tests pass all modalities from signature, and stumble upon `ValueError` return super().__call__(images=images, text=text, **kwargs) def validate_inputs( self, images: ImageInput | None = None, text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None = None, videos=None, **kwargs: Unpack[QianfanOCRProcessorKwargs], ): super().validate_inputs(images=images, text=text, videos=videos, **kwargs) if text is None: raise ValueError("You have to specify text.") if videos is not None: raise ValueError("QianfanOCR does not support video input.") def replace_image_token(self, image_inputs: dict, image_idx: int, **kwargs) -> str: image_num_patches = image_inputs["num_patches"] return f"{self.start_image_token}{self.image_token * self.image_seq_length * image_num_patches[image_idx]}{self.end_image_token}" def _get_num_multimodal_tokens(self, image_sizes=None, **kwargs): """ Computes the number of placeholder tokens needed for multimodal inputs with the given sizes. Args: image_sizes (`list[list[int]]`, *optional*): The input sizes formatted as (height, width) per each image. Returns: `MultiModalData`: A `MultiModalData` object holding number of tokens per each of the provided input modalities, along with other useful data. """ vision_data = {} if image_sizes is not None: images_kwargs = QianfanOCRProcessorKwargs._defaults.get("images_kwargs", {}) images_kwargs.update(kwargs) num_image_patches = [ self.image_processor.get_number_of_image_patches(*image_size, images_kwargs) for image_size in image_sizes ] # Add 2 for BOI and EOI tokens num_image_tokens = [2 + (self.image_seq_length * num_patches) for num_patches in num_image_patches] vision_data.update({"num_image_tokens": num_image_tokens, "num_image_patches": num_image_patches}) return MultiModalData(**vision_data) @property def model_input_names(self): # Overwritten because QianfanOCR renames video inputs to `pixel_values` before returning tokenizer_input_names = self.tokenizer.model_input_names image_processor_input_names = self.image_processor.model_input_names return tokenizer_input_names + image_processor_input_names def get_text_with_replacements( self, text: list[str], images_replacements: list[str] = [], videos_replacements: list[str] = [], audio_replacements: list[str] = [], ): """ Replace multimodal placeholder tokens in a batch of text strings with their expanded representations, and return the modified texts alongside offset metadata. This method is the core text-side preprocessing step for multimodal inputs. It scans each text in the batch for special tokens (image, video, audio) and replaces them in-order with the pre-computed replacement strings produced by `self.replace_image_token` / `self.replace_video_token` / `self.replace_audio_token`. Replacements are consumed from each modality's list sequentially, so the i-th occurrence of e.g. ``self.image_token`` is replaced by ``images_replacements[i]``. To add a new multimodal processor with placeholder tokens, you need to define a correct `self.image_token` which is the same token that is embedded in input text and also used as placeholder and repeated many times. Then you need to override `self.replace_image_token` to return the correct replacement string for a given image at index `i`. Same goes for all other supported modalities. Args: text (`list[str]`): Batch of raw text strings, each potentially containing multimodal placeholder tokens. Note that it will be modified in-place and returned. images_replacements (`list[str]`, *optional*, defaults to `[]`): Expanded replacement strings for each image, in the order they appear across the batch. Produced by `self._process_images`. videos_replacements (`list[str]`, *optional*, defaults to `[]`): Expanded replacement strings for each video. Produced by `self._process_videos`. audio_replacements (`list[str]`, *optional*, defaults to `[]`): Expanded replacement strings for each audio input. Produced by `self._process_audio`. Returns: `tuple[list[str], list[dict[str, Any]]]`: A tuple of: - The modified `text` batch with all placeholder tokens expanded. - `batch_replacement_offsets`: one entry per batch item, each being a list of dicts with keys: - `"type"` (`str`): modality name — `"image"`, `"video"`, or `"audio"` - `"span"` (`tuple[int, int]`): original `(start, end)` char offsets of the placeholder token - `"new_span"` (`tuple[int, int]`): `(start, end)` offsets of placeholder in the expanded string - `"text"` (`str`): the original placeholder token string that was matched - `"replacement"` (`str`): the string it was replaced with """ # Override: model uses `image_placeholder_token` and `image_token` instead of a single token for both purposes token_groups = [] if len(images_replacements) > 0: token_groups.append(f"(?P{re.escape(self.image_placeholder_token)})") regex_special_mm_tokens = "|".join(token_groups) or r"(?!)" replacements_iters = { "image": iter(images_replacements), } batch_replacement_offsets = [] for batch_idx in range(len(text)): last = 0 offset = 0 replacement_offsets = [] expanded_sample = [] for m in re.finditer(regex_special_mm_tokens, text[batch_idx]): start, end = m.span() expanded_sample.append(text[batch_idx][last:start]) # adjust spans using running offset if one sample has several MM data associated start_with_offset = start + offset mm_type = m.lastgroup replacement_text = next(replacements_iters[mm_type]) replacement_offsets.append( { "type": mm_type, "span": (start, end), "new_span": (start_with_offset, start_with_offset + len(replacement_text)), "text": m.group(), "replacement": replacement_text, } ) expanded_sample.append(replacement_text) # update the offsets and the last position offset += len(replacement_text) - (end - start) last = end expanded_sample.append(text[batch_idx][last:]) text[batch_idx] = "".join(expanded_sample) batch_replacement_offsets.append(replacement_offsets) return text, batch_replacement_offsets @property def unused_input_names(self) -> list[str]: return ["num_patches"] __all__ = ["QianfanOCRProcessor"]