From 85ef6e99ef867fcb78ff76736cd087d9e11524df Mon Sep 17 00:00:00 2001 From: Shuyu Wu Date: Thu, 5 Feb 2026 02:34:41 -0500 Subject: [PATCH 1/2] dinov2 v1 --- configs/models/dino/dinov2.yaml | 10 +++++++ src/main.py | 3 +++ src/models/config.py | 1 + src/models/dino/__init__.py | 9 +++++++ src/models/dino/dino.py | 46 +++++++++++++++++++++++++++++++++ 5 files changed, 69 insertions(+) create mode 100644 configs/models/dino/dinov2.yaml create mode 100644 src/models/dino/__init__.py create mode 100644 src/models/dino/dino.py diff --git a/configs/models/dino/dinov2.yaml b/configs/models/dino/dinov2.yaml new file mode 100644 index 00000000..f2e8a3fb --- /dev/null +++ b/configs/models/dino/dinov2.yaml @@ -0,0 +1,10 @@ +architecture: dino +model_path: facebook/dinov2-base +model: + - torch_dtype: auto +output_db: output/dinov2.db +input_dir: ./data/test-images/ +prompt: "Describe the color in this image in one word." +modules: + - visual_projection + - text_projection diff --git a/src/main.py b/src/main.py index cedb20f4..c32fdb71 100644 --- a/src/main.py +++ b/src/main.py @@ -33,6 +33,9 @@ def get_model( elif model_arch == ModelSelection.COGVLM: from src.models.cogvlm import CogVLMModel return CogVLMModel(config) + elif model_arch == ModelSelection.DINO: + from src.models.dino import DinoModel + return DinoModel(config) elif model_arch == ModelSelection.GLAMM: from src.models.glamm import GlammModel return GlammModel(config) diff --git a/src/models/config.py b/src/models/config.py index cdbbe790..3bb9e8b3 100644 --- a/src/models/config.py +++ b/src/models/config.py @@ -26,6 +26,7 @@ class ModelSelection(str, Enum): BLIP2 = 'blip2' CLIP = 'clip' COGVLM = 'cogvlm' + DINO = 'dino' GLAMM = 'glamm' INTERNLM_XC = 'internlm-xcomposer' INTERNVL = 'internvl' diff --git a/src/models/dino/__init__.py b/src/models/dino/__init__.py new file mode 100644 index 00000000..2777c907 --- /dev/null +++ b/src/models/dino/__init__.py @@ -0,0 +1,9 @@ +"""__init__.py. + +Dinov2 package, includes which of this module to export. +""" +from .dino import DinoModel + +__all__ = [ + 'DinoModel' +] diff --git a/src/models/dino/dino.py b/src/models/dino/dino.py new file mode 100644 index 00000000..bc64b01f --- /dev/null +++ b/src/models/dino/dino.py @@ -0,0 +1,46 @@ +"""Dino.py. + +File for providing the Dino model implementation. +""" + +from transformers import AutoModel + +from src.models.base import ModelBase +from src.models.config import Config + + +class DinoModel(ModelBase): + """Dinov2 model implementation.""" + + def __init__(self, config: Config) -> None: + """Initialization of the Dino model. + + Args: + config (Config): Parsed config + """ + # initialize the parent class + super().__init__(config) + + def _load_specific_model(self) -> None: + """Overridden function to populate self.model.""" + self.model = AutoModel.from_pretrained( + self.model_path, **self.config.model + ) if hasattr(self.config, 'model') else ( + AutoModel.from_pretrained( + self.model_path + ) + ) + + def _generate_prompt(self, prompt: str, add_generation_prompt: bool = True, has_images: bool = False) -> str: + """Generates the dinov2 model prompt which will not use the chat template. + + Args: + prompt (str): The input prompt to be processed. + add_generation_prompt (bool): Whether to add a start token of a bot + response. + has_images (bool): Whether the model has images or not. + + Returns: + str: The prompt to return, set by the config. + """ + return prompt From 3f92a534f85b76275f1bfb8f8e7490c0c94d2e28 Mon Sep 17 00:00:00 2001 From: Shuyu Wu Date: Thu, 5 Feb 2026 18:40:26 -0500 Subject: [PATCH 2/2] dino model list and config --- configs/models/dino/dinov2.yaml | 4 +- logs/facebook/dinov2-base.txt | 224 ++++++++++++++++++++++++++++++++ 2 files changed, 226 insertions(+), 2 deletions(-) create mode 100644 logs/facebook/dinov2-base.txt diff --git a/configs/models/dino/dinov2.yaml b/configs/models/dino/dinov2.yaml index f2e8a3fb..68b900ec 100644 --- a/configs/models/dino/dinov2.yaml +++ b/configs/models/dino/dinov2.yaml @@ -6,5 +6,5 @@ output_db: output/dinov2.db input_dir: ./data/test-images/ prompt: "Describe the color in this image in one word." modules: - - visual_projection - - text_projection + - encoder.layer.5.layer_scale2 + - encoder.layer.10.layer_scale2 diff --git a/logs/facebook/dinov2-base.txt b/logs/facebook/dinov2-base.txt new file mode 100644 index 00000000..6ed67344 --- /dev/null +++ b/logs/facebook/dinov2-base.txt @@ -0,0 +1,224 @@ + +embeddings +embeddings.patch_embeddings +embeddings.patch_embeddings.projection +embeddings.dropout +encoder +encoder.layer +encoder.layer.0 +encoder.layer.0.norm1 +encoder.layer.0.attention +encoder.layer.0.attention.attention +encoder.layer.0.attention.attention.query +encoder.layer.0.attention.attention.key +encoder.layer.0.attention.attention.value +encoder.layer.0.attention.output +encoder.layer.0.attention.output.dense +encoder.layer.0.attention.output.dropout +encoder.layer.0.layer_scale1 +encoder.layer.0.drop_path +encoder.layer.0.norm2 +encoder.layer.0.mlp +encoder.layer.0.mlp.fc1 +encoder.layer.0.mlp.activation +encoder.layer.0.mlp.fc2 +encoder.layer.0.layer_scale2 +encoder.layer.1 +encoder.layer.1.norm1 +encoder.layer.1.attention +encoder.layer.1.attention.attention +encoder.layer.1.attention.attention.query +encoder.layer.1.attention.attention.key +encoder.layer.1.attention.attention.value +encoder.layer.1.attention.output +encoder.layer.1.attention.output.dense +encoder.layer.1.attention.output.dropout +encoder.layer.1.layer_scale1 +encoder.layer.1.drop_path +encoder.layer.1.norm2 +encoder.layer.1.mlp +encoder.layer.1.mlp.fc1 +encoder.layer.1.mlp.activation +encoder.layer.1.mlp.fc2 +encoder.layer.1.layer_scale2 +encoder.layer.2 +encoder.layer.2.norm1 +encoder.layer.2.attention +encoder.layer.2.attention.attention +encoder.layer.2.attention.attention.query +encoder.layer.2.attention.attention.key +encoder.layer.2.attention.attention.value +encoder.layer.2.attention.output +encoder.layer.2.attention.output.dense +encoder.layer.2.attention.output.dropout +encoder.layer.2.layer_scale1 +encoder.layer.2.drop_path +encoder.layer.2.norm2 +encoder.layer.2.mlp +encoder.layer.2.mlp.fc1 +encoder.layer.2.mlp.activation +encoder.layer.2.mlp.fc2 +encoder.layer.2.layer_scale2 +encoder.layer.3 +encoder.layer.3.norm1 +encoder.layer.3.attention +encoder.layer.3.attention.attention +encoder.layer.3.attention.attention.query +encoder.layer.3.attention.attention.key +encoder.layer.3.attention.attention.value +encoder.layer.3.attention.output +encoder.layer.3.attention.output.dense +encoder.layer.3.attention.output.dropout +encoder.layer.3.layer_scale1 +encoder.layer.3.drop_path +encoder.layer.3.norm2 +encoder.layer.3.mlp +encoder.layer.3.mlp.fc1 +encoder.layer.3.mlp.activation +encoder.layer.3.mlp.fc2 +encoder.layer.3.layer_scale2 +encoder.layer.4 +encoder.layer.4.norm1 +encoder.layer.4.attention +encoder.layer.4.attention.attention +encoder.layer.4.attention.attention.query +encoder.layer.4.attention.attention.key +encoder.layer.4.attention.attention.value +encoder.layer.4.attention.output +encoder.layer.4.attention.output.dense +encoder.layer.4.attention.output.dropout +encoder.layer.4.layer_scale1 +encoder.layer.4.drop_path +encoder.layer.4.norm2 +encoder.layer.4.mlp +encoder.layer.4.mlp.fc1 +encoder.layer.4.mlp.activation +encoder.layer.4.mlp.fc2 +encoder.layer.4.layer_scale2 +encoder.layer.5 +encoder.layer.5.norm1 +encoder.layer.5.attention +encoder.layer.5.attention.attention +encoder.layer.5.attention.attention.query +encoder.layer.5.attention.attention.key +encoder.layer.5.attention.attention.value +encoder.layer.5.attention.output +encoder.layer.5.attention.output.dense +encoder.layer.5.attention.output.dropout +encoder.layer.5.layer_scale1 +encoder.layer.5.drop_path +encoder.layer.5.norm2 +encoder.layer.5.mlp +encoder.layer.5.mlp.fc1 +encoder.layer.5.mlp.activation +encoder.layer.5.mlp.fc2 +encoder.layer.5.layer_scale2 +encoder.layer.6 +encoder.layer.6.norm1 +encoder.layer.6.attention +encoder.layer.6.attention.attention +encoder.layer.6.attention.attention.query +encoder.layer.6.attention.attention.key +encoder.layer.6.attention.attention.value +encoder.layer.6.attention.output +encoder.layer.6.attention.output.dense +encoder.layer.6.attention.output.dropout +encoder.layer.6.layer_scale1 +encoder.layer.6.drop_path +encoder.layer.6.norm2 +encoder.layer.6.mlp +encoder.layer.6.mlp.fc1 +encoder.layer.6.mlp.activation +encoder.layer.6.mlp.fc2 +encoder.layer.6.layer_scale2 +encoder.layer.7 +encoder.layer.7.norm1 +encoder.layer.7.attention +encoder.layer.7.attention.attention +encoder.layer.7.attention.attention.query +encoder.layer.7.attention.attention.key +encoder.layer.7.attention.attention.value +encoder.layer.7.attention.output +encoder.layer.7.attention.output.dense +encoder.layer.7.attention.output.dropout +encoder.layer.7.layer_scale1 +encoder.layer.7.drop_path +encoder.layer.7.norm2 +encoder.layer.7.mlp +encoder.layer.7.mlp.fc1 +encoder.layer.7.mlp.activation +encoder.layer.7.mlp.fc2 +encoder.layer.7.layer_scale2 +encoder.layer.8 +encoder.layer.8.norm1 +encoder.layer.8.attention +encoder.layer.8.attention.attention +encoder.layer.8.attention.attention.query +encoder.layer.8.attention.attention.key +encoder.layer.8.attention.attention.value +encoder.layer.8.attention.output +encoder.layer.8.attention.output.dense +encoder.layer.8.attention.output.dropout +encoder.layer.8.layer_scale1 +encoder.layer.8.drop_path +encoder.layer.8.norm2 +encoder.layer.8.mlp +encoder.layer.8.mlp.fc1 +encoder.layer.8.mlp.activation +encoder.layer.8.mlp.fc2 +encoder.layer.8.layer_scale2 +encoder.layer.9 +encoder.layer.9.norm1 +encoder.layer.9.attention +encoder.layer.9.attention.attention +encoder.layer.9.attention.attention.query +encoder.layer.9.attention.attention.key +encoder.layer.9.attention.attention.value +encoder.layer.9.attention.output +encoder.layer.9.attention.output.dense +encoder.layer.9.attention.output.dropout +encoder.layer.9.layer_scale1 +encoder.layer.9.drop_path +encoder.layer.9.norm2 +encoder.layer.9.mlp +encoder.layer.9.mlp.fc1 +encoder.layer.9.mlp.activation +encoder.layer.9.mlp.fc2 +encoder.layer.9.layer_scale2 +encoder.layer.10 +encoder.layer.10.norm1 +encoder.layer.10.attention +encoder.layer.10.attention.attention +encoder.layer.10.attention.attention.query +encoder.layer.10.attention.attention.key +encoder.layer.10.attention.attention.value +encoder.layer.10.attention.output +encoder.layer.10.attention.output.dense +encoder.layer.10.attention.output.dropout +encoder.layer.10.layer_scale1 +encoder.layer.10.drop_path +encoder.layer.10.norm2 +encoder.layer.10.mlp +encoder.layer.10.mlp.fc1 +encoder.layer.10.mlp.activation +encoder.layer.10.mlp.fc2 +encoder.layer.10.layer_scale2 +encoder.layer.11 +encoder.layer.11.norm1 +encoder.layer.11.attention +encoder.layer.11.attention.attention +encoder.layer.11.attention.attention.query +encoder.layer.11.attention.attention.key +encoder.layer.11.attention.attention.value +encoder.layer.11.attention.output +encoder.layer.11.attention.output.dense +encoder.layer.11.attention.output.dropout +encoder.layer.11.layer_scale1 +encoder.layer.11.drop_path +encoder.layer.11.norm2 +encoder.layer.11.mlp +encoder.layer.11.mlp.fc1 +encoder.layer.11.mlp.activation +encoder.layer.11.mlp.fc2 +encoder.layer.11.layer_scale2 +layernorm