Legacy Metrics

Starting with 0.23.0, a new metrics module is being introduced to supervision. Metrics here are part of the legacy evaluation API and will be deprecated in the future.

Install the metrics extra before using this page's APIs:

pip install "supervision[metrics]"

ConfusionMatrix

supervision.metrics.detection.ConfusionMatrix dataclass

Confusion matrix for object detection and instance segmentation tasks.

Attributes:

Name	Type	Description
matrix	NDArray[int32]	An 2D np.ndarray of shape (len(classes) + 1, len(classes) + 1) containing the number of TP, FP, FN and TN for each class.
classes	list[str]	Model class names.
conf_threshold	float	Detection confidence threshold between 0 and 1. Detections with lower confidence will be excluded from the matrix.
iou_threshold	float	Detection IoU threshold between 0 and 1. Detections with lower IoU will be classified as FP.
metric_target	MetricTarget	The type of detection data used for IoU computation: MetricTarget.BOXES, MetricTarget.ORIENTED_BOUNDING_BOXES or MetricTarget.MASKS. Informational metadata set by from_detections, from_tensors and benchmark. Excluded from __eq__ comparisons — two ConfusionMatrix instances with identical matrix, classes, conf_threshold, and iou_threshold compare as equal regardless of metric_target.

Source code in src/supervision/metrics/detection.py

@dataclass
class ConfusionMatrix:
    """Confusion matrix for object detection and instance segmentation tasks.

    Attributes:
        matrix: An 2D `np.ndarray` of shape `(len(classes) + 1, len(classes) + 1)`
            containing the number of `TP`, `FP`, `FN` and `TN` for each class.
        classes: Model class names.
        conf_threshold: Detection confidence threshold between `0` and `1`.
            Detections with lower confidence will be excluded from the matrix.
        iou_threshold: Detection IoU threshold between `0` and `1`.
            Detections with lower IoU will be classified as `FP`.
        metric_target: The type of detection data used for IoU computation:
            `MetricTarget.BOXES`, `MetricTarget.ORIENTED_BOUNDING_BOXES` or
            `MetricTarget.MASKS`. Informational metadata set by
            `from_detections`, `from_tensors` and `benchmark`.
            Excluded from `__eq__` comparisons — two `ConfusionMatrix` instances
            with identical `matrix`, `classes`, `conf_threshold`, and
            `iou_threshold` compare as equal regardless of `metric_target`.
    """

    matrix: npt.NDArray[np.int32]
    classes: list[str]
    conf_threshold: float
    iou_threshold: float
    metric_target: MetricTarget = MetricTarget.BOXES

    def __eq__(self, other: object) -> bool:
        if not isinstance(other, ConfusionMatrix):
            return NotImplemented
        return (
            np.array_equal(self.matrix, other.matrix)
            and self.classes == other.classes
            and self.conf_threshold == other.conf_threshold
            and self.iou_threshold == other.iou_threshold
        )

    __hash__ = None  # type: ignore[assignment]

    @classmethod
    def from_detections(
        cls,
        predictions: list[Detections],
        targets: list[Detections],
        classes: list[str],
        conf_threshold: float = 0.3,
        iou_threshold: float = 0.5,
        metric_target: MetricTarget = MetricTarget.BOXES,
    ) -> ConfusionMatrix:
        """Calculate confusion matrix based on predicted and ground-truth detections.

        Args:
            targets: Detections objects from ground-truth.
            predictions: Detections objects predicted by the model.
            classes: Model class names.
            conf_threshold: Detection confidence threshold between `0` and `1`.
                Detections with lower confidence will be excluded.
            iou_threshold: Detection IoU threshold between `0` and `1`.
                Detections with lower IoU will be classified as `FP`.
            metric_target: The type of detection data to use.
                Supports `MetricTarget.BOXES` (default),
                `MetricTarget.ORIENTED_BOUNDING_BOXES` and `MetricTarget.MASKS`.
                When using `MetricTarget.ORIENTED_BOUNDING_BOXES`, each
                `Detections` object must include OBB coordinates in
                `detections.data[ORIENTED_BOX_COORDINATES]` as a float32
                array of shape `(N, 8)` (flat) or `(N, 4, 2)` (as stored by
                `from_ultralytics`); both are normalised to `(N, 8)` internally.
                When using `MetricTarget.MASKS`, every non-empty `Detections`
                object must carry `mask`, either a dense `(N, H, W)` boolean
                array or a `CompactMask`, and predictions and targets of one
                image must share the mask resolution. IoU is then computed on
                the masks, so two instances that share a box but not a shape
                are not matched.

        Returns:
            New instance of ConfusionMatrix.

        Raises:
            ValueError: If `predictions` and `targets` differ in length, if any
                `Detections` lacks `class_id`, if a prediction lacks
                `confidence`, or if `MetricTarget.MASKS` is requested and a
                non-empty `Detections` lacks `mask` or the masks of one image
                differ in resolution.

        Examples:
            ```pycon
            >>> import numpy as np
            >>> import supervision as sv
            >>> targets = [
            ...     sv.Detections(
            ...         xyxy=np.array([[0, 0, 10, 10], [50, 50, 60, 60]]),
            ...         class_id=np.array([0, 0])
            ...     )
            ... ]
            >>> predictions = [
            ...     sv.Detections(
            ...         xyxy=np.array([[0, 0, 10, 10], [100, 100, 110, 110]]),
            ...         class_id=np.array([0, 0]),
            ...         confidence=np.array([0.9, 0.8])
            ...     )
            ... ]
            >>> confusion_matrix = sv.ConfusionMatrix.from_detections(
            ...     predictions=predictions,
            ...     targets=targets,
            ...     classes=['person']
            ... )
            >>> confusion_matrix.matrix
            array([[1, 1],
                   [1, 0]], dtype=int32)

            ```

            Instance segmentation is scored on the masks. Both instances below
            share a box; only the first exceeds the default mask IoU threshold:

            ```pycon
            >>> from supervision.metrics import MetricTarget
            >>> target_masks = np.zeros((2, 20, 20), dtype=bool)
            >>> target_masks[0, 2:8, 2:8] = True
            >>> target_masks[1, 12:18, 12:18] = True
            >>> predicted_masks = np.zeros((2, 20, 20), dtype=bool)
            >>> predicted_masks[0, 2:8, 2:8] = True
            >>> predicted_masks[1, 12:18, 12:14] = True
            >>> targets = [
            ...     sv.Detections(
            ...         xyxy=sv.mask_to_xyxy(target_masks),
            ...         mask=target_masks,
            ...         class_id=np.array([0, 0]),
            ...     )
            ... ]
            >>> predictions = [
            ...     sv.Detections(
            ...         xyxy=sv.mask_to_xyxy(target_masks),
            ...         mask=predicted_masks,
            ...         class_id=np.array([0, 0]),
            ...         confidence=np.array([0.9, 0.8]),
            ...     )
            ... ]
            >>> sv.ConfusionMatrix.from_detections(
            ...     predictions=predictions,
            ...     targets=targets,
            ...     classes=['cell'],
            ...     metric_target=MetricTarget.MASKS,
            ... ).matrix
            array([[1, 1],
                   [1, 0]], dtype=int32)

            ```
        """
        if len(predictions) != len(targets):
            raise ValueError(
                f"Number of predictions ({len(predictions)}) and "
                f"targets ({len(targets)}) must be equal."
            )

        if metric_target == MetricTarget.MASKS:
            num_classes = len(classes)
            matrix: npt.NDArray[np.int32] = np.zeros(
                (num_classes + 1, num_classes + 1), dtype=np.int32
            )
            for prediction, target in zip(predictions, targets):
                matrix += _evaluate_mask_batch(
                    predictions=prediction,
                    targets=target,
                    num_classes=num_classes,
                    conf_threshold=conf_threshold,
                    iou_threshold=iou_threshold,
                )
            return cls(
                matrix=matrix,
                classes=classes,
                conf_threshold=conf_threshold,
                iou_threshold=iou_threshold,
                metric_target=metric_target,
            )

        prediction_tensors = []
        target_tensors = []
        for prediction, target in zip(predictions, targets):
            prediction_tensors.append(
                detections_to_tensor(
                    prediction, with_confidence=True, metric_target=metric_target
                )
            )
            target_tensors.append(
                detections_to_tensor(
                    target, with_confidence=False, metric_target=metric_target
                )
            )
        return cls.from_tensors(
            predictions=prediction_tensors,
            targets=target_tensors,
            classes=classes,
            conf_threshold=conf_threshold,
            iou_threshold=iou_threshold,
            metric_target=metric_target,
        )

    @classmethod
    def from_tensors(
        cls,
        predictions: list[npt.NDArray[np.float32]],
        targets: list[npt.NDArray[np.float32]],
        classes: list[str],
        conf_threshold: float = 0.3,
        iou_threshold: float = 0.5,
        metric_target: MetricTarget = MetricTarget.BOXES,
    ) -> ConfusionMatrix:
        """Calculate confusion matrix based on predicted and ground-truth detections.

        Args:
            predictions: Each element of the list describes a single
                image and has `shape = (M, 6)` or `shape = (M, 10)` depending on
                `metric_target`.
                If `MetricTarget.BOXES`, each row is in
                `(x_min, y_min, x_max, y_max, class, conf)` format.
                If `MetricTarget.ORIENTED_BOUNDING_BOXES`, each row is in
                `(x1, y1, x2, y2, x3, y3, x4, y4, class, conf)` format.
            targets: Each element of the list describes a single
                image and has `shape = (N, 5)` or `shape = (N, 9)` depending on
                `metric_target`.
                If `MetricTarget.BOXES`, each row is in
                `(x_min, y_min, x_max, y_max, class)` format.
                If `MetricTarget.ORIENTED_BOUNDING_BOXES`, each row is in
                `(x1, y1, x2, y2, x3, y3, x4, y4, class)` format.
            classes: Model class names.
            conf_threshold: Detection confidence threshold between `0` and `1`.
                Detections with lower confidence will be excluded.
            iou_threshold: Detection iou threshold between `0` and `1`.
                Detections with lower iou will be classified as `FP`.
            metric_target: The type of detection data to use.
                Determines expected tensor shapes (see Args above for column
                layouts). Masks have no tensor row layout, so
                `MetricTarget.MASKS` is rejected here; use `from_detections`
                for masks.

        Returns:
            New instance of ConfusionMatrix.

        Examples:
            ```pycon
            >>> import supervision as sv
            >>> import numpy as np
            >>> targets = [
            ...     np.array([
            ...         [0.0, 0.0, 3.0, 3.0, 0],
            ...         [2.0, 2.0, 5.0, 5.0, 0],
            ...         [6.0, 1.0, 8.0, 3.0, 1],
            ...     ])
            ... ]
            >>> predictions = [
            ...     np.array([
            ...         [0.0, 0.0, 3.0, 3.0, 0, 0.9],
            ...         [0.1, 0.1, 3.0, 3.0, 0, 0.9],
            ...         [6.0, 1.0, 8.0, 3.0, 1, 0.8],
            ...     ])
            ... ]
            >>> confusion_matrix = sv.ConfusionMatrix.from_tensors(
            ...     predictions=predictions,
            ...     targets=targets,
            ...     classes=['person', 'dog']
            ... )
            >>> confusion_matrix.matrix
            array([[1, 0, 1],
                   [0, 1, 0],
                   [1, 0, 0]], dtype=int32)

            ```
        """
        _assert_tensor_target(metric_target)
        _validate_input_tensors(predictions, targets, metric_target=metric_target)

        num_classes = len(classes)
        matrix: npt.NDArray[np.int32] = np.zeros(
            (num_classes + 1, num_classes + 1), dtype=np.int32
        )
        for true_batch, detection_batch in zip(targets, predictions):
            matrix += cls.evaluate_detection_batch(
                predictions=detection_batch,
                targets=true_batch,
                num_classes=num_classes,
                conf_threshold=conf_threshold,
                iou_threshold=iou_threshold,
                metric_target=metric_target,
            )
        return cls(
            matrix=matrix,
            classes=classes,
            conf_threshold=conf_threshold,
            iou_threshold=iou_threshold,
            metric_target=metric_target,
        )

    @staticmethod
    def evaluate_detection_batch(
        predictions: npt.NDArray[np.float32],
        targets: npt.NDArray[np.float32],
        num_classes: int,
        conf_threshold: float,
        iou_threshold: float,
        metric_target: MetricTarget = MetricTarget.BOXES,
    ) -> npt.NDArray[np.int32]:
        """Calculate confusion matrix for a batch of detections for a single image.

        Args:
            predictions: Batch prediction. Describes a single image and
                has `shape = (M, 6)` or `shape = (M, 10)` depending on
                `metric_target`.
                If `MetricTarget.BOXES`, each row is in
                `(x_min, y_min, x_max, y_max, class, conf)` format.
                If `MetricTarget.ORIENTED_BOUNDING_BOXES`, each row is in
                `(x1, y1, x2, y2, x3, y3, x4, y4, class, conf)` format.
            targets: Batch target labels. Describes a single image and
                has `shape = (N, 5)` or `shape = (N, 9)` depending on
                `metric_target`.
                If `MetricTarget.BOXES`, each row is in
                `(x_min, y_min, x_max, y_max, class)` format.
                If `MetricTarget.ORIENTED_BOUNDING_BOXES`, each row is in
                `(x1, y1, x2, y2, x3, y3, x4, y4, class)` format.
            num_classes: Number of classes.
            conf_threshold: Detection confidence threshold between `0` and `1`.
                Detections with lower confidence will be excluded.
            iou_threshold: Detection iou threshold between `0` and `1`.
                Detections with lower iou will be classified as `FP`.
            metric_target: The type of detection data to use.
                Determines IoU function (`box_iou_batch` vs
                `oriented_box_iou_batch`) and coordinate column count. Masks
                have no tensor row layout, so `MetricTarget.MASKS` is rejected
                here; use `from_detections` for masks.

        Returns:
            Confusion matrix based on a single image.
        """
        _assert_tensor_target(metric_target)

        expected_pred_cols = (
            10 if metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES else 6
        )
        expected_target_cols = (
            9 if metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES else 5
        )
        if predictions.ndim != 2 or predictions.shape[1] != expected_pred_cols:
            raise ValueError(
                f"Predictions must have shape (M, {expected_pred_cols}). "
                f"Got {predictions.shape} instead."
            )
        if targets.ndim != 2 or targets.shape[1] != expected_target_cols:
            raise ValueError(
                f"Targets must have shape (N, {expected_target_cols}). "
                f"Got {targets.shape} instead."
            )

        result_matrix: npt.NDArray[np.int32] = np.zeros(
            (num_classes + 1, num_classes + 1), dtype=np.int32
        )

        # Filter predictions by confidence threshold
        coords_dim = 8 if metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES else 4
        class_id_idx = coords_dim
        conf_idx = coords_dim + 1

        confidence = predictions[:, conf_idx]
        detection_batch_filtered = predictions[confidence >= conf_threshold]

        if len(detection_batch_filtered) == 0:
            true_classes = _validated_class_ids(
                targets[:, class_id_idx], num_classes, "Target"
            )
            for gt_class in true_classes:
                result_matrix[gt_class, num_classes] += 1
            return result_matrix

        if len(targets) == 0:
            detection_classes = _validated_class_ids(
                detection_batch_filtered[:, class_id_idx], num_classes, "Prediction"
            )
            for det_class in detection_classes:
                result_matrix[num_classes, det_class] += 1
            return result_matrix

        true_classes = _validated_class_ids(
            targets[:, class_id_idx], num_classes, "Target"
        )
        detection_classes = _validated_class_ids(
            detection_batch_filtered[:, class_id_idx], num_classes, "Prediction"
        )
        true_boxes = targets[:, :coords_dim]
        detection_boxes = detection_batch_filtered[:, :coords_dim]

        # Calculate IoU matrix
        if metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
            iou_batch = oriented_box_iou_batch(
                boxes_true=true_boxes, boxes_detection=detection_boxes
            )
        else:
            iou_batch = box_iou_batch(
                boxes_true=true_boxes, boxes_detection=detection_boxes
            )

        return _confusion_matrix_from_iou(
            iou_batch=iou_batch,
            true_classes=true_classes,
            detection_classes=detection_classes,
            num_classes=num_classes,
            iou_threshold=iou_threshold,
        )

    @staticmethod
    def _drop_extra_matches(
        matches: npt.NDArray[np.float32],
    ) -> npt.NDArray[np.float32]:
        """Deduplicate matches.

        If there are multiple matches for the same true or predicted box, only the one
        with the highest IoU is kept.
        """
        if matches.shape[0] > 0:
            matches = matches[matches[:, 2].argsort()[::-1]]
            matches = matches[np.unique(matches[:, 1], return_index=True)[1]]
            matches = matches[matches[:, 2].argsort()[::-1]]
            matches = matches[np.unique(matches[:, 0], return_index=True)[1]]
        result: npt.NDArray[np.float32] = matches
        return result

    @classmethod
    def benchmark(
        cls,
        dataset: DetectionDataset,
        callback: Callable[[npt.NDArray[np.uint8]], Detections],
        conf_threshold: float = 0.3,
        iou_threshold: float = 0.5,
        metric_target: MetricTarget = MetricTarget.BOXES,
        *,
        save_directory_path: str | Path | None = None,
    ) -> ConfusionMatrix:
        """Calculate confusion matrix from dataset and callback function.

        Args:
            dataset: Detection or instance segmentation dataset used for
                evaluation.
            callback: Function that takes an image as input and returns a
                Detections object.
            conf_threshold: Detection confidence threshold between `0` and `1`.
                Detections with lower confidence will be excluded.
            iou_threshold: Detection IoU threshold between `0` and `1`.
                Detections with lower IoU will be classified as `FP`.
            save_directory_path: Optional directory where per-image validation
                result grids are saved using the original image filenames. Images
                are written directly to this directory (no subdirectory is added).
                When ``None`` (default), no images are saved.
            metric_target: The type of detection data to use.
                Supports `MetricTarget.BOXES`,
                `MetricTarget.ORIENTED_BOUNDING_BOXES` and `MetricTarget.MASKS`.
                Passed through to `from_detections`. With `MetricTarget.MASKS`
                the dataset annotations and the callback's detections must
                carry `mask`, and the saved validation grids fill each mask.

        Returns:
            New instance of ConfusionMatrix.

        Example:
            ```python
            import supervision as sv
            from rfdetr import RFDETRMedium

            dataset = sv.DetectionDataset.from_yolo(...)

            model = RFDETRMedium()
            def callback(image: np.ndarray) -> sv.Detections:
                return model.predict(image[:, :, ::-1])

            confusion_matrix = sv.ConfusionMatrix.benchmark(
                dataset = dataset,
                callback = callback
            )

            print(confusion_matrix.matrix)
            # np.array([
            #     [0., 0., 0., 0.],
            #     [0., 1., 0., 1.],
            #     [0., 1., 1., 0.],
            #     [1., 1., 0., 0.]
            # ])
            ```
        """
        if save_directory_path is not None:
            save_directory = Path(save_directory_path)
            save_directory.mkdir(parents=True, exist_ok=True)

        predictions, targets = [], []
        for index, (image_name, image, annotation) in enumerate(dataset):
            predictions_batch = callback(image)
            predictions.append(predictions_batch)
            targets.append(annotation)

            if save_directory_path is not None:
                if isinstance(image_name, Path):
                    image_filename = image_name.name
                elif isinstance(image_name, str):
                    image_filename = Path(image_name).name
                else:
                    image_filename = f"image_{index:06d}.jpg"

                if Path(image_filename).suffix == "":
                    image_filename = f"{image_filename}.jpg"

                save_path = save_directory / image_filename
                if save_path.exists():
                    warnings.warn(
                        f"Validation image '{image_filename}' already exists at "
                        f"'{save_path}' and will be overwritten.",
                        UserWarning,
                        stacklevel=2,
                    )
                _save_detection_validation_visualization(
                    scene=image,
                    predictions=predictions_batch,
                    targets=annotation,
                    save_path=save_path,
                    conf_threshold=conf_threshold,
                    iou_threshold=iou_threshold,
                    class_names=dataset.classes,
                    metric_target=metric_target,
                )
        return cls.from_detections(
            predictions=predictions,
            targets=targets,
            classes=dataset.classes,
            conf_threshold=conf_threshold,
            iou_threshold=iou_threshold,
            metric_target=metric_target,
        )

    def plot(
        self,
        save_path: str | None = None,
        title: str | None = None,
        classes: list[str] | None = None,
        normalize: bool = False,
        fig_size: tuple[int, int] = (12, 10),
    ) -> Figure:
        """Create confusion matrix plot and save it at selected location.

        Args:
            save_path: Path to save the plot. If not provided,
                plot will be displayed.
            title: Title of the plot.
            classes: List of classes to be displayed on the plot.
                If not provided, all classes will be displayed.
            normalize: If True, normalize the confusion matrix.
            fig_size: Size of the plot.

        Returns:
            Confusion matrix plot.
        """
        from matplotlib import pyplot as plt

        # Cast to float so that the NaN masking below never hits an integer
        # matrix (assigning NaN into an int array raises ValueError).
        array = self.matrix.astype(np.float64)

        if normalize:
            eps = 1e-8
            array = array / (array.sum(0).reshape(1, -1) + eps)

        array[array < 0.005] = np.nan

        fig, ax = plt.subplots(figsize=fig_size, tight_layout=True, facecolor="white")

        class_names = classes if classes is not None else self.classes
        use_labels_for_ticks = class_names is not None and (0 < len(class_names) < 99)
        if use_labels_for_ticks:
            x_tick_labels = [*class_names, "FN"]
            y_tick_labels = [*class_names, "FP"]
            num_ticks = len(x_tick_labels)
        else:
            x_tick_labels = None
            y_tick_labels = None
            num_ticks = len(array)
        im = ax.imshow(array, cmap="Blues")

        cbar = ax.figure.colorbar(im, ax=ax)
        cbar.mappable.set_clim(vmin=0, vmax=float(np.nanmax(array)))

        if x_tick_labels is None:
            tick_interval = 2
        else:
            tick_interval = 1
        ax.set_xticks(np.arange(0, num_ticks, tick_interval), labels=x_tick_labels)
        ax.set_yticks(np.arange(0, num_ticks, tick_interval), labels=y_tick_labels)

        plt.setp(ax.get_xticklabels(), rotation=90, ha="right", rotation_mode="default")

        labelsize = 10 if num_ticks < 50 else 8
        ax.tick_params(axis="both", which="both", labelsize=labelsize)

        if num_ticks < 30:
            for i in range(array.shape[0]):
                for j in range(array.shape[1]):
                    n_preds = array[i, j]
                    if not np.isnan(n_preds):
                        ax.text(
                            j,
                            i,
                            f"{n_preds:.2f}" if normalize else f"{n_preds:.0f}",
                            ha="center",
                            va="center",
                            color="black"
                            if n_preds < 0.5 * np.nanmax(array)
                            else "white",
                        )

        if title:
            ax.set_title(title, fontsize=20)

        ax.set_xlabel("Predicted")
        ax.set_ylabel("True")
        ax.set_facecolor("white")
        if save_path:
            fig.savefig(
                save_path, dpi=250, facecolor=fig.get_facecolor(), transparent=True
            )
        return fig

Methods:

benchmark(dataset: DetectionDataset, callback: Callable[[npt.NDArray[np.uint8]], Detections], conf_threshold: float = 0.3, iou_threshold: float = 0.5, metric_target: MetricTarget = MetricTarget.BOXES, *, save_directory_path: str | Path | None = None) -> ConfusionMatrix classmethod

Calculate confusion matrix from dataset and callback function.

Parameters:

Name	Type	Description	Default
dataset	DetectionDataset	Detection or instance segmentation dataset used for evaluation.	required
callback	Callable[[NDArray[uint8]], Detections]	Function that takes an image as input and returns a Detections object.	required
conf_threshold	float	Detection confidence threshold between 0 and 1. Detections with lower confidence will be excluded.	0.3
iou_threshold	float	Detection IoU threshold between 0 and 1. Detections with lower IoU will be classified as FP.	0.5
save_directory_path	str | Path | None	Optional directory where per-image validation result grids are saved using the original image filenames. Images are written directly to this directory (no subdirectory is added). When None (default), no images are saved.	None
metric_target	MetricTarget	The type of detection data to use. Supports MetricTarget.BOXES, MetricTarget.ORIENTED_BOUNDING_BOXES and MetricTarget.MASKS. Passed through to from_detections. With MetricTarget.MASKS the dataset annotations and the callback's detections must carry mask, and the saved validation grids fill each mask.	BOXES

Returns:

Type	Description
ConfusionMatrix	New instance of ConfusionMatrix.

Example

import supervision as sv
from rfdetr import RFDETRMedium

dataset = sv.DetectionDataset.from_yolo(...)

model = RFDETRMedium()
def callback(image: np.ndarray) -> sv.Detections:
    return model.predict(image[:, :, ::-1])

confusion_matrix = sv.ConfusionMatrix.benchmark(
    dataset = dataset,
    callback = callback
)

print(confusion_matrix.matrix)
# np.array([
#     [0., 0., 0., 0.],
#     [0., 1., 0., 1.],
#     [0., 1., 1., 0.],
#     [1., 1., 0., 0.]
# ])

Source code in src/supervision/metrics/detection.py

@classmethod
def benchmark(
    cls,
    dataset: DetectionDataset,
    callback: Callable[[npt.NDArray[np.uint8]], Detections],
    conf_threshold: float = 0.3,
    iou_threshold: float = 0.5,
    metric_target: MetricTarget = MetricTarget.BOXES,
    *,
    save_directory_path: str | Path | None = None,
) -> ConfusionMatrix:
    """Calculate confusion matrix from dataset and callback function.

    Args:
        dataset: Detection or instance segmentation dataset used for
            evaluation.
        callback: Function that takes an image as input and returns a
            Detections object.
        conf_threshold: Detection confidence threshold between `0` and `1`.
            Detections with lower confidence will be excluded.
        iou_threshold: Detection IoU threshold between `0` and `1`.
            Detections with lower IoU will be classified as `FP`.
        save_directory_path: Optional directory where per-image validation
            result grids are saved using the original image filenames. Images
            are written directly to this directory (no subdirectory is added).
            When ``None`` (default), no images are saved.
        metric_target: The type of detection data to use.
            Supports `MetricTarget.BOXES`,
            `MetricTarget.ORIENTED_BOUNDING_BOXES` and `MetricTarget.MASKS`.
            Passed through to `from_detections`. With `MetricTarget.MASKS`
            the dataset annotations and the callback's detections must
            carry `mask`, and the saved validation grids fill each mask.

    Returns:
        New instance of ConfusionMatrix.

    Example:
        ```python
        import supervision as sv
        from rfdetr import RFDETRMedium

        dataset = sv.DetectionDataset.from_yolo(...)

        model = RFDETRMedium()
        def callback(image: np.ndarray) -> sv.Detections:
            return model.predict(image[:, :, ::-1])

        confusion_matrix = sv.ConfusionMatrix.benchmark(
            dataset = dataset,
            callback = callback
        )

        print(confusion_matrix.matrix)
        # np.array([
        #     [0., 0., 0., 0.],
        #     [0., 1., 0., 1.],
        #     [0., 1., 1., 0.],
        #     [1., 1., 0., 0.]
        # ])
        ```
    """
    if save_directory_path is not None:
        save_directory = Path(save_directory_path)
        save_directory.mkdir(parents=True, exist_ok=True)

    predictions, targets = [], []
    for index, (image_name, image, annotation) in enumerate(dataset):
        predictions_batch = callback(image)
        predictions.append(predictions_batch)
        targets.append(annotation)

        if save_directory_path is not None:
            if isinstance(image_name, Path):
                image_filename = image_name.name
            elif isinstance(image_name, str):
                image_filename = Path(image_name).name
            else:
                image_filename = f"image_{index:06d}.jpg"

            if Path(image_filename).suffix == "":
                image_filename = f"{image_filename}.jpg"

            save_path = save_directory / image_filename
            if save_path.exists():
                warnings.warn(
                    f"Validation image '{image_filename}' already exists at "
                    f"'{save_path}' and will be overwritten.",
                    UserWarning,
                    stacklevel=2,
                )
            _save_detection_validation_visualization(
                scene=image,
                predictions=predictions_batch,
                targets=annotation,
                save_path=save_path,
                conf_threshold=conf_threshold,
                iou_threshold=iou_threshold,
                class_names=dataset.classes,
                metric_target=metric_target,
            )
    return cls.from_detections(
        predictions=predictions,
        targets=targets,
        classes=dataset.classes,
        conf_threshold=conf_threshold,
        iou_threshold=iou_threshold,
        metric_target=metric_target,
    )

evaluate_detection_batch(predictions: npt.NDArray[np.float32], targets: npt.NDArray[np.float32], num_classes: int, conf_threshold: float, iou_threshold: float, metric_target: MetricTarget = MetricTarget.BOXES) -> npt.NDArray[np.int32] staticmethod

Calculate confusion matrix for a batch of detections for a single image.

Parameters:

Name	Type	Description	Default
predictions	NDArray[float32]	Batch prediction. Describes a single image and has shape = (M, 6) or shape = (M, 10) depending on metric_target. If MetricTarget.BOXES, each row is in (x_min, y_min, x_max, y_max, class, conf) format. If MetricTarget.ORIENTED_BOUNDING_BOXES, each row is in (x1, y1, x2, y2, x3, y3, x4, y4, class, conf) format.	required
targets	NDArray[float32]	Batch target labels. Describes a single image and has shape = (N, 5) or shape = (N, 9) depending on metric_target. If MetricTarget.BOXES, each row is in (x_min, y_min, x_max, y_max, class) format. If MetricTarget.ORIENTED_BOUNDING_BOXES, each row is in (x1, y1, x2, y2, x3, y3, x4, y4, class) format.	required
num_classes	int	Number of classes.	required
conf_threshold	float	Detection confidence threshold between 0 and 1. Detections with lower confidence will be excluded.	required
iou_threshold	float	Detection iou threshold between 0 and 1. Detections with lower iou will be classified as FP.	required
metric_target	MetricTarget	The type of detection data to use. Determines IoU function (box_iou_batch vs oriented_box_iou_batch) and coordinate column count. Masks have no tensor row layout, so MetricTarget.MASKS is rejected here; use from_detections for masks.	BOXES

Returns:

Type	Description
NDArray[int32]	Confusion matrix based on a single image.

Source code in src/supervision/metrics/detection.py

@staticmethod
def evaluate_detection_batch(
    predictions: npt.NDArray[np.float32],
    targets: npt.NDArray[np.float32],
    num_classes: int,
    conf_threshold: float,
    iou_threshold: float,
    metric_target: MetricTarget = MetricTarget.BOXES,
) -> npt.NDArray[np.int32]:
    """Calculate confusion matrix for a batch of detections for a single image.

    Args:
        predictions: Batch prediction. Describes a single image and
            has `shape = (M, 6)` or `shape = (M, 10)` depending on
            `metric_target`.
            If `MetricTarget.BOXES`, each row is in
            `(x_min, y_min, x_max, y_max, class, conf)` format.
            If `MetricTarget.ORIENTED_BOUNDING_BOXES`, each row is in
            `(x1, y1, x2, y2, x3, y3, x4, y4, class, conf)` format.
        targets: Batch target labels. Describes a single image and
            has `shape = (N, 5)` or `shape = (N, 9)` depending on
            `metric_target`.
            If `MetricTarget.BOXES`, each row is in
            `(x_min, y_min, x_max, y_max, class)` format.
            If `MetricTarget.ORIENTED_BOUNDING_BOXES`, each row is in
            `(x1, y1, x2, y2, x3, y3, x4, y4, class)` format.
        num_classes: Number of classes.
        conf_threshold: Detection confidence threshold between `0` and `1`.
            Detections with lower confidence will be excluded.
        iou_threshold: Detection iou threshold between `0` and `1`.
            Detections with lower iou will be classified as `FP`.
        metric_target: The type of detection data to use.
            Determines IoU function (`box_iou_batch` vs
            `oriented_box_iou_batch`) and coordinate column count. Masks
            have no tensor row layout, so `MetricTarget.MASKS` is rejected
            here; use `from_detections` for masks.

    Returns:
        Confusion matrix based on a single image.
    """
    _assert_tensor_target(metric_target)

    expected_pred_cols = (
        10 if metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES else 6
    )
    expected_target_cols = (
        9 if metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES else 5
    )
    if predictions.ndim != 2 or predictions.shape[1] != expected_pred_cols:
        raise ValueError(
            f"Predictions must have shape (M, {expected_pred_cols}). "
            f"Got {predictions.shape} instead."
        )
    if targets.ndim != 2 or targets.shape[1] != expected_target_cols:
        raise ValueError(
            f"Targets must have shape (N, {expected_target_cols}). "
            f"Got {targets.shape} instead."
        )

    result_matrix: npt.NDArray[np.int32] = np.zeros(
        (num_classes + 1, num_classes + 1), dtype=np.int32
    )

    # Filter predictions by confidence threshold
    coords_dim = 8 if metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES else 4
    class_id_idx = coords_dim
    conf_idx = coords_dim + 1

    confidence = predictions[:, conf_idx]
    detection_batch_filtered = predictions[confidence >= conf_threshold]

    if len(detection_batch_filtered) == 0:
        true_classes = _validated_class_ids(
            targets[:, class_id_idx], num_classes, "Target"
        )
        for gt_class in true_classes:
            result_matrix[gt_class, num_classes] += 1
        return result_matrix

    if len(targets) == 0:
        detection_classes = _validated_class_ids(
            detection_batch_filtered[:, class_id_idx], num_classes, "Prediction"
        )
        for det_class in detection_classes:
            result_matrix[num_classes, det_class] += 1
        return result_matrix

    true_classes = _validated_class_ids(
        targets[:, class_id_idx], num_classes, "Target"
    )
    detection_classes = _validated_class_ids(
        detection_batch_filtered[:, class_id_idx], num_classes, "Prediction"
    )
    true_boxes = targets[:, :coords_dim]
    detection_boxes = detection_batch_filtered[:, :coords_dim]

    # Calculate IoU matrix
    if metric_target == MetricTarget.ORIENTED_BOUNDING_BOXES:
        iou_batch = oriented_box_iou_batch(
            boxes_true=true_boxes, boxes_detection=detection_boxes
        )
    else:
        iou_batch = box_iou_batch(
            boxes_true=true_boxes, boxes_detection=detection_boxes
        )

    return _confusion_matrix_from_iou(
        iou_batch=iou_batch,
        true_classes=true_classes,
        detection_classes=detection_classes,
        num_classes=num_classes,
        iou_threshold=iou_threshold,
    )

from_detections(predictions: list[Detections], targets: list[Detections], classes: list[str], conf_threshold: float = 0.3, iou_threshold: float = 0.5, metric_target: MetricTarget = MetricTarget.BOXES) -> ConfusionMatrix classmethod

Calculate confusion matrix based on predicted and ground-truth detections.

Parameters:

Name	Type	Description	Default
targets	list[Detections]	Detections objects from ground-truth.	required
predictions	list[Detections]	Detections objects predicted by the model.	required
classes	list[str]	Model class names.	required
conf_threshold	float	Detection confidence threshold between 0 and 1. Detections with lower confidence will be excluded.	0.3
iou_threshold	float	Detection IoU threshold between 0 and 1. Detections with lower IoU will be classified as FP.	0.5
metric_target	MetricTarget	The type of detection data to use. Supports MetricTarget.BOXES (default), MetricTarget.ORIENTED_BOUNDING_BOXES and MetricTarget.MASKS. When using MetricTarget.ORIENTED_BOUNDING_BOXES, each Detections object must include OBB coordinates in detections.data[ORIENTED_BOX_COORDINATES] as a float32 array of shape (N, 8) (flat) or (N, 4, 2) (as stored by from_ultralytics); both are normalised to (N, 8) internally. When using MetricTarget.MASKS, every non-empty Detections object must carry mask, either a dense (N, H, W) boolean array or a CompactMask, and predictions and targets of one image must share the mask resolution. IoU is then computed on the masks, so two instances that share a box but not a shape are not matched.	BOXES

Returns:

Type	Description
ConfusionMatrix	New instance of ConfusionMatrix.

Raises:

Type	Description
ValueError	If predictions and targets differ in length, if any Detections lacks class_id, if a prediction lacks confidence, or if MetricTarget.MASKS is requested and a non-empty Detections lacks mask or the masks of one image differ in resolution.

Examples:

>>> import numpy as np
>>> import supervision as sv
>>> targets = [
...     sv.Detections(
...         xyxy=np.array([[0, 0, 10, 10], [50, 50, 60, 60]]),
...         class_id=np.array([0, 0])
...     )
... ]
>>> predictions = [
...     sv.Detections(
...         xyxy=np.array([[0, 0, 10, 10], [100, 100, 110, 110]]),
...         class_id=np.array([0, 0]),
...         confidence=np.array([0.9, 0.8])
...     )
... ]
>>> confusion_matrix = sv.ConfusionMatrix.from_detections(
...     predictions=predictions,
...     targets=targets,
...     classes=['person']
... )
>>> confusion_matrix.matrix
array([[1, 1],
       [1, 0]], dtype=int32)

Instance segmentation is scored on the masks. Both instances below share a box; only the first exceeds the default mask IoU threshold:

>>> from supervision.metrics import MetricTarget
>>> target_masks = np.zeros((2, 20, 20), dtype=bool)
>>> target_masks[0, 2:8, 2:8] = True
>>> target_masks[1, 12:18, 12:18] = True
>>> predicted_masks = np.zeros((2, 20, 20), dtype=bool)
>>> predicted_masks[0, 2:8, 2:8] = True
>>> predicted_masks[1, 12:18, 12:14] = True
>>> targets = [
...     sv.Detections(
...         xyxy=sv.mask_to_xyxy(target_masks),
...         mask=target_masks,
...         class_id=np.array([0, 0]),
...     )
... ]
>>> predictions = [
...     sv.Detections(
...         xyxy=sv.mask_to_xyxy(target_masks),
...         mask=predicted_masks,
...         class_id=np.array([0, 0]),
...         confidence=np.array([0.9, 0.8]),
...     )
... ]
>>> sv.ConfusionMatrix.from_detections(
...     predictions=predictions,
...     targets=targets,
...     classes=['cell'],
...     metric_target=MetricTarget.MASKS,
... ).matrix
array([[1, 1],
       [1, 0]], dtype=int32)

Source code in src/supervision/metrics/detection.py

@classmethod
def from_detections(
    cls,
    predictions: list[Detections],
    targets: list[Detections],
    classes: list[str],
    conf_threshold: float = 0.3,
    iou_threshold: float = 0.5,
    metric_target: MetricTarget = MetricTarget.BOXES,
) -> ConfusionMatrix:
    """Calculate confusion matrix based on predicted and ground-truth detections.

    Args:
        targets: Detections objects from ground-truth.
        predictions: Detections objects predicted by the model.
        classes: Model class names.
        conf_threshold: Detection confidence threshold between `0` and `1`.
            Detections with lower confidence will be excluded.
        iou_threshold: Detection IoU threshold between `0` and `1`.
            Detections with lower IoU will be classified as `FP`.
        metric_target: The type of detection data to use.
            Supports `MetricTarget.BOXES` (default),
            `MetricTarget.ORIENTED_BOUNDING_BOXES` and `MetricTarget.MASKS`.
            When using `MetricTarget.ORIENTED_BOUNDING_BOXES`, each
            `Detections` object must include OBB coordinates in
            `detections.data[ORIENTED_BOX_COORDINATES]` as a float32
            array of shape `(N, 8)` (flat) or `(N, 4, 2)` (as stored by
            `from_ultralytics`); both are normalised to `(N, 8)` internally.
            When using `MetricTarget.MASKS`, every non-empty `Detections`
            object must carry `mask`, either a dense `(N, H, W)` boolean
            array or a `CompactMask`, and predictions and targets of one
            image must share the mask resolution. IoU is then computed on
            the masks, so two instances that share a box but not a shape
            are not matched.

    Returns:
        New instance of ConfusionMatrix.

    Raises:
        ValueError: If `predictions` and `targets` differ in length, if any
            `Detections` lacks `class_id`, if a prediction lacks
            `confidence`, or if `MetricTarget.MASKS` is requested and a
            non-empty `Detections` lacks `mask` or the masks of one image
            differ in resolution.

    Examples:
        ```pycon
        >>> import numpy as np
        >>> import supervision as sv
        >>> targets = [
        ...     sv.Detections(
        ...         xyxy=np.array([[0, 0, 10, 10], [50, 50, 60, 60]]),
        ...         class_id=np.array([0, 0])
        ...     )
        ... ]
        >>> predictions = [
        ...     sv.Detections(
        ...         xyxy=np.array([[0, 0, 10, 10], [100, 100, 110, 110]]),
        ...         class_id=np.array([0, 0]),
        ...         confidence=np.array([0.9, 0.8])
        ...     )
        ... ]
        >>> confusion_matrix = sv.ConfusionMatrix.from_detections(
        ...     predictions=predictions,
        ...     targets=targets,
        ...     classes=['person']
        ... )
        >>> confusion_matrix.matrix
        array([[1, 1],
               [1, 0]], dtype=int32)

        ```

        Instance segmentation is scored on the masks. Both instances below
        share a box; only the first exceeds the default mask IoU threshold:

        ```pycon
        >>> from supervision.metrics import MetricTarget
        >>> target_masks = np.zeros((2, 20, 20), dtype=bool)
        >>> target_masks[0, 2:8, 2:8] = True
        >>> target_masks[1, 12:18, 12:18] = True
        >>> predicted_masks = np.zeros((2, 20, 20), dtype=bool)
        >>> predicted_masks[0, 2:8, 2:8] = True
        >>> predicted_masks[1, 12:18, 12:14] = True
        >>> targets = [
        ...     sv.Detections(
        ...         xyxy=sv.mask_to_xyxy(target_masks),
        ...         mask=target_masks,
        ...         class_id=np.array([0, 0]),
        ...     )
        ... ]
        >>> predictions = [
        ...     sv.Detections(
        ...         xyxy=sv.mask_to_xyxy(target_masks),
        ...         mask=predicted_masks,
        ...         class_id=np.array([0, 0]),
        ...         confidence=np.array([0.9, 0.8]),
        ...     )
        ... ]
        >>> sv.ConfusionMatrix.from_detections(
        ...     predictions=predictions,
        ...     targets=targets,
        ...     classes=['cell'],
        ...     metric_target=MetricTarget.MASKS,
        ... ).matrix
        array([[1, 1],
               [1, 0]], dtype=int32)

        ```
    """
    if len(predictions) != len(targets):
        raise ValueError(
            f"Number of predictions ({len(predictions)}) and "
            f"targets ({len(targets)}) must be equal."
        )

    if metric_target == MetricTarget.MASKS:
        num_classes = len(classes)
        matrix: npt.NDArray[np.int32] = np.zeros(
            (num_classes + 1, num_classes + 1), dtype=np.int32
        )
        for prediction, target in zip(predictions, targets):
            matrix += _evaluate_mask_batch(
                predictions=prediction,
                targets=target,
                num_classes=num_classes,
                conf_threshold=conf_threshold,
                iou_threshold=iou_threshold,
            )
        return cls(
            matrix=matrix,
            classes=classes,
            conf_threshold=conf_threshold,
            iou_threshold=iou_threshold,
            metric_target=metric_target,
        )

    prediction_tensors = []
    target_tensors = []
    for prediction, target in zip(predictions, targets):
        prediction_tensors.append(
            detections_to_tensor(
                prediction, with_confidence=True, metric_target=metric_target
            )
        )
        target_tensors.append(
            detections_to_tensor(
                target, with_confidence=False, metric_target=metric_target
            )
        )
    return cls.from_tensors(
        predictions=prediction_tensors,
        targets=target_tensors,
        classes=classes,
        conf_threshold=conf_threshold,
        iou_threshold=iou_threshold,
        metric_target=metric_target,
    )

from_tensors(predictions: list[npt.NDArray[np.float32]], targets: list[npt.NDArray[np.float32]], classes: list[str], conf_threshold: float = 0.3, iou_threshold: float = 0.5, metric_target: MetricTarget = MetricTarget.BOXES) -> ConfusionMatrix classmethod

Calculate confusion matrix based on predicted and ground-truth detections.

Parameters:

Name	Type	Description	Default
predictions	list[NDArray[float32]]	Each element of the list describes a single image and has shape = (M, 6) or shape = (M, 10) depending on metric_target. If MetricTarget.BOXES, each row is in (x_min, y_min, x_max, y_max, class, conf) format. If MetricTarget.ORIENTED_BOUNDING_BOXES, each row is in (x1, y1, x2, y2, x3, y3, x4, y4, class, conf) format.	required
targets	list[NDArray[float32]]	Each element of the list describes a single image and has shape = (N, 5) or shape = (N, 9) depending on metric_target. If MetricTarget.BOXES, each row is in (x_min, y_min, x_max, y_max, class) format. If MetricTarget.ORIENTED_BOUNDING_BOXES, each row is in (x1, y1, x2, y2, x3, y3, x4, y4, class) format.	required
classes	list[str]	Model class names.	required
conf_threshold	float	Detection confidence threshold between 0 and 1. Detections with lower confidence will be excluded.	0.3
iou_threshold	float	Detection iou threshold between 0 and 1. Detections with lower iou will be classified as FP.	0.5
metric_target	MetricTarget	The type of detection data to use. Determines expected tensor shapes (see Args above for column layouts). Masks have no tensor row layout, so MetricTarget.MASKS is rejected here; use from_detections for masks.	BOXES

Returns:

Type	Description
ConfusionMatrix	New instance of ConfusionMatrix.

Examples:

>>> import supervision as sv
>>> import numpy as np
>>> targets = [
...     np.array([
...         [0.0, 0.0, 3.0, 3.0, 0],
...         [2.0, 2.0, 5.0, 5.0, 0],
...         [6.0, 1.0, 8.0, 3.0, 1],
...     ])
... ]
>>> predictions = [
...     np.array([
...         [0.0, 0.0, 3.0, 3.0, 0, 0.9],
...         [0.1, 0.1, 3.0, 3.0, 0, 0.9],
...         [6.0, 1.0, 8.0, 3.0, 1, 0.8],
...     ])
... ]
>>> confusion_matrix = sv.ConfusionMatrix.from_tensors(
...     predictions=predictions,
...     targets=targets,
...     classes=['person', 'dog']
... )
>>> confusion_matrix.matrix
array([[1, 0, 1],
       [0, 1, 0],
       [1, 0, 0]], dtype=int32)

Source code in src/supervision/metrics/detection.py

@classmethod
def from_tensors(
    cls,
    predictions: list[npt.NDArray[np.float32]],
    targets: list[npt.NDArray[np.float32]],
    classes: list[str],
    conf_threshold: float = 0.3,
    iou_threshold: float = 0.5,
    metric_target: MetricTarget = MetricTarget.BOXES,
) -> ConfusionMatrix:
    """Calculate confusion matrix based on predicted and ground-truth detections.

    Args:
        predictions: Each element of the list describes a single
            image and has `shape = (M, 6)` or `shape = (M, 10)` depending on
            `metric_target`.
            If `MetricTarget.BOXES`, each row is in
            `(x_min, y_min, x_max, y_max, class, conf)` format.
            If `MetricTarget.ORIENTED_BOUNDING_BOXES`, each row is in
            `(x1, y1, x2, y2, x3, y3, x4, y4, class, conf)` format.
        targets: Each element of the list describes a single
            image and has `shape = (N, 5)` or `shape = (N, 9)` depending on
            `metric_target`.
            If `MetricTarget.BOXES`, each row is in
            `(x_min, y_min, x_max, y_max, class)` format.
            If `MetricTarget.ORIENTED_BOUNDING_BOXES`, each row is in
            `(x1, y1, x2, y2, x3, y3, x4, y4, class)` format.
        classes: Model class names.
        conf_threshold: Detection confidence threshold between `0` and `1`.
            Detections with lower confidence will be excluded.
        iou_threshold: Detection iou threshold between `0` and `1`.
            Detections with lower iou will be classified as `FP`.
        metric_target: The type of detection data to use.
            Determines expected tensor shapes (see Args above for column
            layouts). Masks have no tensor row layout, so
            `MetricTarget.MASKS` is rejected here; use `from_detections`
            for masks.

    Returns:
        New instance of ConfusionMatrix.

    Examples:
        ```pycon
        >>> import supervision as sv
        >>> import numpy as np
        >>> targets = [
        ...     np.array([
        ...         [0.0, 0.0, 3.0, 3.0, 0],
        ...         [2.0, 2.0, 5.0, 5.0, 0],
        ...         [6.0, 1.0, 8.0, 3.0, 1],
        ...     ])
        ... ]
        >>> predictions = [
        ...     np.array([
        ...         [0.0, 0.0, 3.0, 3.0, 0, 0.9],
        ...         [0.1, 0.1, 3.0, 3.0, 0, 0.9],
        ...         [6.0, 1.0, 8.0, 3.0, 1, 0.8],
        ...     ])
        ... ]
        >>> confusion_matrix = sv.ConfusionMatrix.from_tensors(
        ...     predictions=predictions,
        ...     targets=targets,
        ...     classes=['person', 'dog']
        ... )
        >>> confusion_matrix.matrix
        array([[1, 0, 1],
               [0, 1, 0],
               [1, 0, 0]], dtype=int32)

        ```
    """
    _assert_tensor_target(metric_target)
    _validate_input_tensors(predictions, targets, metric_target=metric_target)

    num_classes = len(classes)
    matrix: npt.NDArray[np.int32] = np.zeros(
        (num_classes + 1, num_classes + 1), dtype=np.int32
    )
    for true_batch, detection_batch in zip(targets, predictions):
        matrix += cls.evaluate_detection_batch(
            predictions=detection_batch,
            targets=true_batch,
            num_classes=num_classes,
            conf_threshold=conf_threshold,
            iou_threshold=iou_threshold,
            metric_target=metric_target,
        )
    return cls(
        matrix=matrix,
        classes=classes,
        conf_threshold=conf_threshold,
        iou_threshold=iou_threshold,
        metric_target=metric_target,
    )

plot(save_path: str | None = None, title: str | None = None, classes: list[str] | None = None, normalize: bool = False, fig_size: tuple[int, int] = (12, 10)) -> Figure

Create confusion matrix plot and save it at selected location.

Parameters:

Name	Type	Description	Default
save_path	str | None	Path to save the plot. If not provided, plot will be displayed.	None
title	str | None	Title of the plot.	None
classes	list[str] | None	List of classes to be displayed on the plot. If not provided, all classes will be displayed.	None
normalize	bool	If True, normalize the confusion matrix.	False
fig_size	tuple[int, int]	Size of the plot.	(12, 10)

Returns:

Type	Description
Figure	Confusion matrix plot.

Source code in src/supervision/metrics/detection.py

def plot(
    self,
    save_path: str | None = None,
    title: str | None = None,
    classes: list[str] | None = None,
    normalize: bool = False,
    fig_size: tuple[int, int] = (12, 10),
) -> Figure:
    """Create confusion matrix plot and save it at selected location.

    Args:
        save_path: Path to save the plot. If not provided,
            plot will be displayed.
        title: Title of the plot.
        classes: List of classes to be displayed on the plot.
            If not provided, all classes will be displayed.
        normalize: If True, normalize the confusion matrix.
        fig_size: Size of the plot.

    Returns:
        Confusion matrix plot.
    """
    from matplotlib import pyplot as plt

    # Cast to float so that the NaN masking below never hits an integer
    # matrix (assigning NaN into an int array raises ValueError).
    array = self.matrix.astype(np.float64)

    if normalize:
        eps = 1e-8
        array = array / (array.sum(0).reshape(1, -1) + eps)

    array[array < 0.005] = np.nan

    fig, ax = plt.subplots(figsize=fig_size, tight_layout=True, facecolor="white")

    class_names = classes if classes is not None else self.classes
    use_labels_for_ticks = class_names is not None and (0 < len(class_names) < 99)
    if use_labels_for_ticks:
        x_tick_labels = [*class_names, "FN"]
        y_tick_labels = [*class_names, "FP"]
        num_ticks = len(x_tick_labels)
    else:
        x_tick_labels = None
        y_tick_labels = None
        num_ticks = len(array)
    im = ax.imshow(array, cmap="Blues")

    cbar = ax.figure.colorbar(im, ax=ax)
    cbar.mappable.set_clim(vmin=0, vmax=float(np.nanmax(array)))

    if x_tick_labels is None:
        tick_interval = 2
    else:
        tick_interval = 1
    ax.set_xticks(np.arange(0, num_ticks, tick_interval), labels=x_tick_labels)
    ax.set_yticks(np.arange(0, num_ticks, tick_interval), labels=y_tick_labels)

    plt.setp(ax.get_xticklabels(), rotation=90, ha="right", rotation_mode="default")

    labelsize = 10 if num_ticks < 50 else 8
    ax.tick_params(axis="both", which="both", labelsize=labelsize)

    if num_ticks < 30:
        for i in range(array.shape[0]):
            for j in range(array.shape[1]):
                n_preds = array[i, j]
                if not np.isnan(n_preds):
                    ax.text(
                        j,
                        i,
                        f"{n_preds:.2f}" if normalize else f"{n_preds:.0f}",
                        ha="center",
                        va="center",
                        color="black"
                        if n_preds < 0.5 * np.nanmax(array)
                        else "white",
                    )

    if title:
        ax.set_title(title, fontsize=20)

    ax.set_xlabel("Predicted")
    ax.set_ylabel("True")
    ax.set_facecolor("white")
    if save_path:
        fig.savefig(
            save_path, dpi=250, facecolor=fig.get_facecolor(), transparent=True
        )
    return fig

Comments