File size: 2,811 Bytes
a04bbbc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | import copy
from transformers import AutoConfig, Qwen3Config
from transformers.configuration_utils import PretrainedConfig
from transformers.utils import logging
from .configuration_dinov3_vit import DINOv3ViTConfig
logger = logging.get_logger(__name__)
class ProjectorConfig(PretrainedConfig):
model_type = "projector"
_auto_class = "AutoConfig"
def __init__(
self,
visual_hidden_size=4096,
llm_hidden_size=4096,
depth=2,
hidden_act="gelu",
bias=True,
**kwargs,
):
self.visual_hidden_size = visual_hidden_size
self.llm_hidden_size = llm_hidden_size
self.depth = depth
self.hidden_act = hidden_act
self.bias = bias
super().__init__(**kwargs)
class VectorLLMConfig(PretrainedConfig):
model_type = 'vectorllm'
processor_class = "VectorLLMProcessor",
is_composition = True
def __init__(
self,
vision_config=None,
llm_config=None,
regression_size=(128, 128),
projector_depth=2,
pixel_idx=0,
**kwargs):
super().__init__(**kwargs)
if vision_config is None:
vision_config = {}
logger.info('vision_config is None. Initializing the DinoV3Config with default values.')
if llm_config is None:
llm_config = {}
logger.info('llm_config is None. Initializing the Qwen3 config with default values.')
self.vision_config = DINOv3ViTConfig(**vision_config)
self.llm_config = Qwen3Config(**llm_config)
self.text_config = self.llm_config
self.hidden_size = self.llm_config.hidden_size
self.vision_hidden_size = self.vision_config.hidden_size
self.projector_config = ProjectorConfig(
visual_hidden_size=self.vision_hidden_size,
llm_hidden_size=self.hidden_size,
depth=projector_depth
)
self.regression_size = regression_size
self.pixel_idx = pixel_idx
self.tie_word_embeddings = False
self.num_cls_register_tokens = 1 + self.vision_config.num_register_tokens
def to_dict(self):
"""
Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
Returns:
`Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
"""
output = copy.deepcopy(self.__dict__)
output['vision_config'] = self.vision_config.to_dict()
output['llm_config'] = self.llm_config.to_dict()
output['text_config'] = output['llm_config']
output['projector_config'] = self.projector_config.to_dict()
output['model_type'] = self.__class__.model_type
return output |