[feat] DINOv2 Implementation

configs/models/dino/dinov2.yaml

@@ -0,0 +1,10 @@
+architecture: dino
+model_path: facebook/dinov2-base
+model:
+  - torch_dtype: auto
+output_db: output/dinov2.db
+input_dir: ./data/test-images/
+prompt: "Describe the color in this image in one word."
+modules:
+  - encoder.layer.5.layer_scale2
+  - encoder.layer.10.layer_scale2

logs/facebook/dinov2-base.txt

@@ -0,0 +1,224 @@
+
+embeddings
+embeddings.patch_embeddings
+embeddings.patch_embeddings.projection
+embeddings.dropout
+encoder
+encoder.layer
+encoder.layer.0
+encoder.layer.0.norm1
+encoder.layer.0.attention
+encoder.layer.0.attention.attention
+encoder.layer.0.attention.attention.query
+encoder.layer.0.attention.attention.key
+encoder.layer.0.attention.attention.value
+encoder.layer.0.attention.output
+encoder.layer.0.attention.output.dense
+encoder.layer.0.attention.output.dropout
+encoder.layer.0.layer_scale1
+encoder.layer.0.drop_path
+encoder.layer.0.norm2
+encoder.layer.0.mlp
+encoder.layer.0.mlp.fc1
+encoder.layer.0.mlp.activation
+encoder.layer.0.mlp.fc2
+encoder.layer.0.layer_scale2
+encoder.layer.1
+encoder.layer.1.norm1
+encoder.layer.1.attention
+encoder.layer.1.attention.attention
+encoder.layer.1.attention.attention.query
+encoder.layer.1.attention.attention.key
+encoder.layer.1.attention.attention.value
+encoder.layer.1.attention.output
+encoder.layer.1.attention.output.dense
+encoder.layer.1.attention.output.dropout
+encoder.layer.1.layer_scale1
+encoder.layer.1.drop_path
+encoder.layer.1.norm2
+encoder.layer.1.mlp
+encoder.layer.1.mlp.fc1
+encoder.layer.1.mlp.activation
+encoder.layer.1.mlp.fc2
+encoder.layer.1.layer_scale2
+encoder.layer.2
+encoder.layer.2.norm1
+encoder.layer.2.attention
+encoder.layer.2.attention.attention
+encoder.layer.2.attention.attention.query
+encoder.layer.2.attention.attention.key
+encoder.layer.2.attention.attention.value
+encoder.layer.2.attention.output
+encoder.layer.2.attention.output.dense
+encoder.layer.2.attention.output.dropout
+encoder.layer.2.layer_scale1
+encoder.layer.2.drop_path
+encoder.layer.2.norm2
+encoder.layer.2.mlp
+encoder.layer.2.mlp.fc1
+encoder.layer.2.mlp.activation
+encoder.layer.2.mlp.fc2
+encoder.layer.2.layer_scale2
+encoder.layer.3
+encoder.layer.3.norm1
+encoder.layer.3.attention
+encoder.layer.3.attention.attention
+encoder.layer.3.attention.attention.query
+encoder.layer.3.attention.attention.key
+encoder.layer.3.attention.attention.value
+encoder.layer.3.attention.output
+encoder.layer.3.attention.output.dense
+encoder.layer.3.attention.output.dropout
+encoder.layer.3.layer_scale1
+encoder.layer.3.drop_path
+encoder.layer.3.norm2
+encoder.layer.3.mlp
+encoder.layer.3.mlp.fc1
+encoder.layer.3.mlp.activation
+encoder.layer.3.mlp.fc2
+encoder.layer.3.layer_scale2
+encoder.layer.4
+encoder.layer.4.norm1
+encoder.layer.4.attention
+encoder.layer.4.attention.attention
+encoder.layer.4.attention.attention.query
+encoder.layer.4.attention.attention.key
+encoder.layer.4.attention.attention.value
+encoder.layer.4.attention.output
+encoder.layer.4.attention.output.dense
+encoder.layer.4.attention.output.dropout
+encoder.layer.4.layer_scale1
+encoder.layer.4.drop_path
+encoder.layer.4.norm2
+encoder.layer.4.mlp
+encoder.layer.4.mlp.fc1
+encoder.layer.4.mlp.activation
+encoder.layer.4.mlp.fc2
+encoder.layer.4.layer_scale2
+encoder.layer.5
+encoder.layer.5.norm1
+encoder.layer.5.attention
+encoder.layer.5.attention.attention
+encoder.layer.5.attention.attention.query
+encoder.layer.5.attention.attention.key
+encoder.layer.5.attention.attention.value
+encoder.layer.5.attention.output
+encoder.layer.5.attention.output.dense
+encoder.layer.5.attention.output.dropout
+encoder.layer.5.layer_scale1
+encoder.layer.5.drop_path
+encoder.layer.5.norm2
+encoder.layer.5.mlp
+encoder.layer.5.mlp.fc1
+encoder.layer.5.mlp.activation
+encoder.layer.5.mlp.fc2
+encoder.layer.5.layer_scale2
+encoder.layer.6
+encoder.layer.6.norm1
+encoder.layer.6.attention
+encoder.layer.6.attention.attention
+encoder.layer.6.attention.attention.query
+encoder.layer.6.attention.attention.key
+encoder.layer.6.attention.attention.value
+encoder.layer.6.attention.output
+encoder.layer.6.attention.output.dense
+encoder.layer.6.attention.output.dropout
+encoder.layer.6.layer_scale1
+encoder.layer.6.drop_path
+encoder.layer.6.norm2
+encoder.layer.6.mlp
+encoder.layer.6.mlp.fc1
+encoder.layer.6.mlp.activation
+encoder.layer.6.mlp.fc2
+encoder.layer.6.layer_scale2
+encoder.layer.7
+encoder.layer.7.norm1
+encoder.layer.7.attention
+encoder.layer.7.attention.attention
+encoder.layer.7.attention.attention.query
+encoder.layer.7.attention.attention.key
+encoder.layer.7.attention.attention.value
+encoder.layer.7.attention.output
+encoder.layer.7.attention.output.dense
+encoder.layer.7.attention.output.dropout
+encoder.layer.7.layer_scale1
+encoder.layer.7.drop_path
+encoder.layer.7.norm2
+encoder.layer.7.mlp
+encoder.layer.7.mlp.fc1
+encoder.layer.7.mlp.activation
+encoder.layer.7.mlp.fc2
+encoder.layer.7.layer_scale2
+encoder.layer.8
+encoder.layer.8.norm1
+encoder.layer.8.attention
+encoder.layer.8.attention.attention
+encoder.layer.8.attention.attention.query
+encoder.layer.8.attention.attention.key
+encoder.layer.8.attention.attention.value
+encoder.layer.8.attention.output
+encoder.layer.8.attention.output.dense
+encoder.layer.8.attention.output.dropout
+encoder.layer.8.layer_scale1
+encoder.layer.8.drop_path
+encoder.layer.8.norm2
+encoder.layer.8.mlp
+encoder.layer.8.mlp.fc1
+encoder.layer.8.mlp.activation
+encoder.layer.8.mlp.fc2
+encoder.layer.8.layer_scale2
+encoder.layer.9
+encoder.layer.9.norm1
+encoder.layer.9.attention
+encoder.layer.9.attention.attention
+encoder.layer.9.attention.attention.query
+encoder.layer.9.attention.attention.key
+encoder.layer.9.attention.attention.value
+encoder.layer.9.attention.output
+encoder.layer.9.attention.output.dense
+encoder.layer.9.attention.output.dropout
+encoder.layer.9.layer_scale1
+encoder.layer.9.drop_path
+encoder.layer.9.norm2
+encoder.layer.9.mlp
+encoder.layer.9.mlp.fc1
+encoder.layer.9.mlp.activation
+encoder.layer.9.mlp.fc2
+encoder.layer.9.layer_scale2
+encoder.layer.10
+encoder.layer.10.norm1
+encoder.layer.10.attention
+encoder.layer.10.attention.attention
+encoder.layer.10.attention.attention.query
+encoder.layer.10.attention.attention.key
+encoder.layer.10.attention.attention.value
+encoder.layer.10.attention.output
+encoder.layer.10.attention.output.dense
+encoder.layer.10.attention.output.dropout
+encoder.layer.10.layer_scale1
+encoder.layer.10.drop_path
+encoder.layer.10.norm2
+encoder.layer.10.mlp
+encoder.layer.10.mlp.fc1
+encoder.layer.10.mlp.activation
+encoder.layer.10.mlp.fc2
+encoder.layer.10.layer_scale2
+encoder.layer.11
+encoder.layer.11.norm1
+encoder.layer.11.attention
+encoder.layer.11.attention.attention
+encoder.layer.11.attention.attention.query
+encoder.layer.11.attention.attention.key
+encoder.layer.11.attention.attention.value
+encoder.layer.11.attention.output
+encoder.layer.11.attention.output.dense
+encoder.layer.11.attention.output.dropout
+encoder.layer.11.layer_scale1
+encoder.layer.11.drop_path
+encoder.layer.11.norm2
+encoder.layer.11.mlp
+encoder.layer.11.mlp.fc1
+encoder.layer.11.mlp.activation
+encoder.layer.11.mlp.fc2
+encoder.layer.11.layer_scale2
+layernorm

src/main.py

@@ -33,6 +33,9 @@ def get_model(
     elif model_arch == ModelSelection.COGVLM:
         from src.models.cogvlm import CogVLMModel
         return CogVLMModel(config)
+    elif model_arch == ModelSelection.DINO:
+        from src.models.dino import DinoModel
+        return DinoModel(config)
     elif model_arch == ModelSelection.GLAMM:
         from src.models.glamm import GlammModel
         return GlammModel(config)

src/models/config.py

@@ -26,6 +26,7 @@ class ModelSelection(str, Enum):
     BLIP2 = 'blip2'
     CLIP = 'clip'
     COGVLM = 'cogvlm'
+    DINO = 'dino'
     GLAMM = 'glamm'
     INTERNLM_XC = 'internlm-xcomposer'
     INTERNVL = 'internvl'

src/models/dino/__init__.py

@@ -0,0 +1,9 @@
+"""__init__.py.
+
+Dinov2 package, includes which of this module to export.
+"""
+from .dino import DinoModel
+
+__all__ = [
+    'DinoModel'
+]

src/models/dino/dino.py

@@ -0,0 +1,46 @@
+"""Dino.py.
+
+File for providing the Dino model implementation.
+"""
+
+from transformers import AutoModel
+
+from src.models.base import ModelBase
+from src.models.config import Config
+
+
+class DinoModel(ModelBase):
+    """Dinov2 model implementation."""
+
+    def __init__(self, config: Config) -> None:
+        """Initialization of the Dino model.
+
+        Args:
+            config (Config): Parsed config
+        """
+        # initialize the parent class
+        super().__init__(config)
+
+    def _load_specific_model(self) -> None:
+        """Overridden function to populate self.model."""
+        self.model = AutoModel.from_pretrained(
+            self.model_path, **self.config.model
+        ) if hasattr(self.config, 'model') else (
+            AutoModel.from_pretrained(
+                self.model_path
+            )
+        )
+
+    def _generate_prompt(self, prompt: str, add_generation_prompt: bool = True, has_images: bool = False) -> str:
+        """Generates the dinov2 model prompt which will not use the chat template.
+
+        Args:
+            prompt (str): The input prompt to be processed.
+            add_generation_prompt (bool): Whether to add a start token of a bot
+                response.
+            has_images (bool): Whether the model has images or not.
+
+        Returns:
+            str: The prompt to return, set by the config.
+        """
+        return prompt