Add ORA postprocessor

README.md

@@ -225,7 +225,7 @@ reconstruction: c2e2de,
 classification: fdd7e6,
 distance: f4d5b3 -->
 
-> Post-Hoc Methods (24):
+> Post-Hoc Methods (25):
 > - [x] [![msp](https://img.shields.io/badge/ICLR'17-MSP-fdd7e6?style=for-the-badge)](https://openreview.net/forum?id=Hkg4TI9xl)
 > - [x] [![odin](https://img.shields.io/badge/ICLR'18-ODIN-fdd7e6?style=for-the-badge)](https://openreview.net/forum?id=H1VGkIxRZ)    ![postprocess]
 > - [x] [![mds](https://img.shields.io/badge/NeurIPS'18-MDS-f4d5b3?style=for-the-badge)](https://papers.nips.cc/paper/2018/hash/abdeb6f575ac5c6676b747bca8d09cc2-Abstract.html)    ![postprocess]
@@ -254,6 +254,7 @@ distance: f4d5b3 -->
 > - [x] [![adascale-l](https://img.shields.io/badge/arXiv'25-AdaScale\_L-fdd7e6?style=for-the-badge)](https://github.com/sudarshanregmi/adascale)    ![postprocess]
 > - [x] [![ascood](https://img.shields.io/badge/arXiv'25-iODIN-fdd7e6?style=for-the-badge)](https://github.com/sudarshanregmi/ASCOOD)    ![postprocess]
 > - [x] [![nci](https://img.shields.io/badge/CVPR'25-NCI-fdd7e6?style=for-the-badge)](https://arxiv.org/pdf/2311.01479)    ![postprocess]
+> - [x] [![ora](https://img.shields.io/badge/NeurIPS'25-ORA-f4d5b3?style=for-the-badge)](https://github.com/berkerdemirel/ORA-OOD-Detection-with-Relative-Angles)    ![postprocess]
 
 > Training Methods (14):
 > - [x] [![confbranch](https://img.shields.io/badge/arXiv'18-ConfBranch-fdd7e6?style=for-the-badge)](https://github.com/uoguelph-mlrg/confidence_estimation)    ![preprocess]   ![training]

configs/postprocessors/ora.yml

@@ -0,0 +1,7 @@
+postprocessor:
+  name: ora
+  APS_mode: False
+  postprocessor_args:
+    aggregation: mean
+  postprocessor_sweep:
+    aggregation_list: [mean, max, min]

openood/evaluation_api/postprocessor.py

@@ -16,13 +16,14 @@
     RMDSPostprocessor, SHEPostprocessor, CIDERPostprocessor, NPOSPostprocessor,
     GENPostprocessor, NNGuidePostprocessor, RelationPostprocessor,
     T2FNormPostprocessor, ReweightOODPostprocessor, fDBDPostprocessor,
-    AdaScalePostprocessor, IODINPostprocessor, NCIPostprocessor,CFOODPostprocessor,
-    VRAPostprocessor, GrOODPostprocessor)
+    AdaScalePostprocessor, IODINPostprocessor, NCIPostprocessor,
+    CFOODPostprocessor, VRAPostprocessor, GrOODPostprocessor, ORAPostprocessor)
 from openood.utils.config import Config, merge_configs
 
 postprocessors = {
     'nci': NCIPostprocessor,
     'fdbd': fDBDPostprocessor,
+    'ora': ORAPostprocessor,
     'ash': ASHPostprocessor,
     'cider': CIDERPostprocessor,
     'conf_branch': ConfBranchPostprocessor,

openood/postprocessors/__init__.py

@@ -1,5 +1,6 @@
 from .nci_postprocessor import NCIPostprocessor
 from .fdbd_postprocessor import fDBDPostprocessor
+from .ora_postprocessor import ORAPostprocessor
 from .ash_postprocessor import ASHPostprocessor
 from .base_postprocessor import BasePostprocessor
 from .cider_postprocessor import CIDERPostprocessor
@@ -50,4 +51,3 @@
 from .grood import GrOODPostprocessor
 from .vra_postprocessor import VRAPostprocessor
 from .cfood_postprocessor import CFOODPostprocessor
-

openood/postprocessors/ora_postprocessor.py

@@ -0,0 +1,112 @@
+from typing import Any
+
+from tqdm import tqdm
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from .base_postprocessor import BasePostprocessor
+
+
+class ORAPostprocessor(BasePostprocessor):
+    """ORA: OOD detection with Relative Angles.
+
+    For each test sample, ORA projects the feature onto each class's
+    decision boundary using the linear classifier weights, then measures
+    the angle between (i) the feature vector and (ii) its boundary
+    projection, both centered at the mean of the per-class training
+    feature means. The OOD score aggregates these per-class angles via
+    `aggregation` (mean | max | min). `mean` is the default and is the
+    recommended choice for CE-trained backbones; `max` is recommended
+    for SupCon-trained backbones.
+    """
+    def __init__(self, config):
+        super(ORAPostprocessor, self).__init__(config)
+        self.APS_mode = False
+        self.setup_flag = False
+        self.mean_of_class_means = None
+        self.num_classes = None
+
+        self.args = self.config.postprocessor.postprocessor_args
+        self.aggregation = getattr(self.args, 'aggregation', 'mean')
+        if self.aggregation not in ('mean', 'max', 'min'):
+            raise ValueError(
+                f"aggregation must be one of 'mean', 'max', 'min'; "
+                f'got {self.aggregation!r}')
+        self.args_dict = self.config.postprocessor.postprocessor_sweep
+
+    def setup(self, net: nn.Module, id_loader_dict, ood_loader_dict):
+        if not self.setup_flag:
+            net.eval()
+            num_classes = net.fc.weight.shape[0]
+
+            class_sum = torch.zeros(num_classes, net.fc.weight.shape[1]).cuda()
+            class_count = torch.zeros(num_classes).cuda()
+
+            with torch.no_grad():
+                for batch in tqdm(id_loader_dict['train'],
+                                  desc='Setup: ',
+                                  position=0,
+                                  leave=True):
+                    data = batch['data'].cuda().float()
+                    labels = batch['label'].cuda()
+                    _, feature = net(data, return_feature=True)
+
+                    class_sum.index_add_(0, labels, feature)
+                    class_count.index_add_(
+                        0, labels, torch.ones_like(labels, dtype=torch.float))
+
+            class_means = class_sum / class_count.clamp(min=1).unsqueeze(1)
+            self.mean_of_class_means = class_means.mean(dim=0)
+            self.num_classes = num_classes
+            self.setup_flag = True
+        else:
+            pass
+
+    @torch.no_grad()
+    def postprocess(self, net: nn.Module, data: Any):
+        output, feature = net(data, return_feature=True)
+        preds = output.argmax(dim=1)
+        max_logits = output.max(dim=1).values
+        w = net.fc.weight  # (C, D)
+
+        centered_feats = feature - self.mean_of_class_means
+        norm_centered_feats = F.normalize(centered_feats, p=2, dim=1)
+
+        trajectory = torch.zeros(feature.size(0),
+                                 self.num_classes,
+                                 device=feature.device)
+        for c in range(self.num_classes):
+            logit_diff = max_logits - output[:, c]
+            weight_diff = w[preds] - w[c]
+            weight_diff_norm_sq = (weight_diff * weight_diff).sum(dim=1)
+            scale = (logit_diff / weight_diff_norm_sq).unsqueeze(1)
+            feats_db = feature - scale * weight_diff
+
+            centered_db = feats_db - self.mean_of_class_means
+            norm_centered_db = F.normalize(centered_db, p=2, dim=1)
+            cos_sim = (norm_centered_feats * norm_centered_db).sum(dim=1)
+            trajectory[:,
+                       c] = torch.arccos(cos_sim.clamp(-1.0, 1.0)) / torch.pi
+
+        # Diagonal entries (c == preds) are 0/0 -> NaN. For mean/max we
+        # treat them as 0 (matches the reference); for min we exclude them
+        # since 0 would otherwise dominate.
+        if self.aggregation == 'min':
+            trajectory = torch.where(torch.isnan(trajectory),
+                                     torch.full_like(trajectory, float('inf')),
+                                     trajectory)
+            score = trajectory.min(dim=1).values
+        else:
+            trajectory = torch.nan_to_num(trajectory, nan=0.0)
+            if self.aggregation == 'max':
+                score = trajectory.max(dim=1).values
+            else:
+                score = trajectory.mean(dim=1)
+        return preds, score
+
+    def set_hyperparam(self, hyperparam: list):
+        self.aggregation = hyperparam[0]
+
+    def get_hyperparam(self):
+        return self.aggregation

openood/postprocessors/utils.py

@@ -2,6 +2,7 @@
 
 from .nci_postprocessor import NCIPostprocessor
 from .fdbd_postprocessor import fDBDPostprocessor
+from .ora_postprocessor import ORAPostprocessor
 from .ash_postprocessor import ASHPostprocessor
 from .base_postprocessor import BasePostprocessor
 from .cider_postprocessor import CIDERPostprocessor
@@ -51,6 +52,7 @@ def get_postprocessor(config: Config):
     postprocessors = {
         'nci': NCIPostprocessor,
         'fdbd': fDBDPostprocessor,
+        'ora': ORAPostprocessor,
         'ash': ASHPostprocessor,
         'cider': CIDERPostprocessor,
         'conf_branch': ConfBranchPostprocessor,

scripts/ood/ora/cifar100_test_ood_ora.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# sh scripts/ood/ora/cifar100_test_ood_ora.sh
+
+# GPU=1
+# CPU=1
+# node=73
+# jobname=openood
+
+PYTHONPATH='.':$PYTHONPATH \
+# srun -p dsta --mpi=pmi2 --gres=gpu:${GPU} -n1 \
+# --cpus-per-task=${CPU} --ntasks-per-node=${GPU} \
+# --kill-on-bad-exit=1 --job-name=${jobname} -w SG-IDC1-10-51-2-${node} \
+
+python main.py \
+    --config configs/datasets/cifar100/cifar100.yml \
+    configs/datasets/cifar100/cifar100_ood.yml \
+    configs/networks/resnet18_32x32.yml \
+    configs/pipelines/test/test_ood.yml \
+    configs/preprocessors/base_preprocessor.yml \
+    configs/postprocessors/ora.yml \
+    --network.checkpoint 'results/cifar100_resnet18_32x32_base_e100_lr0.1_default/s0/best.ckpt'
+
+############################################
+# alternatively, we recommend using the
+# new unified, easy-to-use evaluator with
+# the example script scripts/eval_ood.py
+# especially if you want to get results from
+# multiple runs
+python scripts/eval_ood.py \
+    --id-data cifar100 \
+    --root ./results/cifar100_resnet18_32x32_base_e100_lr0.1_default \
+    --postprocessor ora \
+    --save-score --save-csv

scripts/ood/ora/cifar10_test_ood_ora.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# sh scripts/ood/ora/cifar10_test_ood_ora.sh
+
+# GPU=1
+# CPU=1
+# node=73
+# jobname=openood
+
+PYTHONPATH='.':$PYTHONPATH \
+# srun -p dsta --mpi=pmi2 --gres=gpu:${GPU} -n1 \
+# --cpus-per-task=${CPU} --ntasks-per-node=${GPU} \
+# --kill-on-bad-exit=1 --job-name=${jobname} -w SG-IDC1-10-51-2-${node} \
+
+python main.py \
+    --config configs/datasets/cifar10/cifar10.yml \
+    configs/datasets/cifar10/cifar10_ood.yml \
+    configs/networks/resnet18_32x32.yml \
+    configs/pipelines/test/test_ood.yml \
+    configs/preprocessors/base_preprocessor.yml \
+    configs/postprocessors/ora.yml \
+    --num_workers 8 \
+    --network.checkpoint 'results/cifar10_resnet18_32x32_base_e100_lr0.1_default/s0/best.ckpt' \
+    --mark 1
+
+############################################
+# alternatively, we recommend using the
+# new unified, easy-to-use evaluator with
+# the example script scripts/eval_ood.py
+# especially if you want to get results from
+# multiple runs
+python scripts/eval_ood.py \
+    --id-data cifar10 \
+    --root ./results/cifar10_resnet18_32x32_base_e100_lr0.1_default \
+    --postprocessor ora \
+    --save-score --save-csv

scripts/ood/ora/imagenet200_test_ood_ora.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# sh scripts/ood/ora/imagenet200_test_ood_ora.sh
+
+############################################
+# alternatively, we recommend using the
+# new unified, easy-to-use evaluator with
+# the example script scripts/eval_ood.py
+# especially if you want to get results from
+# multiple runs
+
+# ood
+python scripts/eval_ood.py \
+    --id-data imagenet200 \
+    --root ./results/imagenet200_resnet18_224x224_base_e90_lr0.1_default \
+    --postprocessor ora \
+    --save-score --save-csv #--fsood
+
+# full-spectrum ood
+python scripts/eval_ood.py \
+    --id-data imagenet200 \
+    --root ./results/imagenet200_resnet18_224x224_base_e90_lr0.1_default \
+    --postprocessor ora \
+    --save-score --save-csv --fsood

scripts/ood/ora/imagenet_test_ood_ora.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+# sh scripts/ood/ora/imagenet_test_ood_ora.sh
+
+GPU=1
+CPU=1
+node=63
+jobname=openood
+
+PYTHONPATH='.':$PYTHONPATH \
+# srun -p dsta --mpi=pmi2 --gres=gpu:${GPU} -n1 \
+# --cpus-per-task=${CPU} --ntasks-per-node=${GPU} \
+# --kill-on-bad-exit=1 --job-name=${jobname} -w SG-IDC1-10-51-2-${node} \
+# python main.py \
+#     --config configs/datasets/imagenet/imagenet.yml \
+#     configs/datasets/imagenet/imagenet_ood.yml \
+#     configs/networks/resnet50.yml \
+#     configs/pipelines/test/test_ood.yml \
+#     configs/preprocessors/base_preprocessor.yml \
+#     configs/postprocessors/ora.yml \
+#     --num_workers 4 \
+#     --ood_dataset.image_size 256 \
+#     --dataset.test.batch_size 256 \
+#     --dataset.val.batch_size 256 \
+#     --network.pretrained True \
+#     --network.checkpoint 'results/pretrained_weights/resnet50_imagenet1k_v1.pth' \
+#     --merge_option merge
+
+############################################
+# we recommend using the
+# new unified, easy-to-use evaluator with
+# the example script scripts/eval_ood_imagenet.py
+
+# available architectures:
+# resnet50, swin-t, vit-b-16
+# ood
+python scripts/eval_ood_imagenet.py \
+    --tvs-pretrained \
+    --arch resnet50 \
+    --postprocessor ora \
+    --save-score --save-csv #--fsood
+
+# full-spectrum ood
+python scripts/eval_ood_imagenet.py \
+    --tvs-pretrained \
+    --arch resnet50 \
+    --postprocessor ora \
+    --save-score --save-csv --fsood