classification

ClassificationDataModule(data_path, dataset=ImageClassificationListDataset, name='classification_datamodule', num_workers=8, batch_size=32, seed=42, val_size=0.2, test_size=0.2, num_data_class=None, exclude_filter=None, include_filter=None, label_map=None, load_aug_images=False, aug_name=None, n_aug_to_take=4, replace_str_from=None, replace_str_to=None, train_transform=None, val_transform=None, test_transform=None, train_split_file=None, test_split_file=None, val_split_file=None, class_to_idx=None)

Bases: BaseDataModule

Base class single folder based classification datamodules. If there is no nested folders, use this class.

Parameters:

• data_path (str) –

  Path to the data main folder.

• name (str) –

  The name for the data module. Defaults to "classification_datamodule".

• num_workers (int) –

  Number of workers for dataloaders. Defaults to 16.

• batch_size (int) –

  Batch size. Defaults to 32.

• seed (int) –

  Random generator seed. Defaults to 42.

• dataset (Type[ImageClassificationListDataset]) –

  Dataset class.

• val_size (Optional[float]) –

  The validation split. Defaults to 0.2.

• test_size (float) –

  The test split. Defaults to 0.2.

• exclude_filter (Optional[List[str]]) –

  The filter for excluding folders. Defaults to None.

• include_filter (Optional[List[str]]) –

  The filter for including folders. Defaults to None.

• label_map (Optional[Dict[str, Any]]) –

  The mapping for labels. Defaults to None.

• num_data_class (Optional[int]) –

  The number of samples per class. Defaults to None.

• train_transform (Optional[albumentations.Compose]) –

  Transformations for train dataset. Defaults to None.

• val_transform (Optional[albumentations.Compose]) –

  Transformations for validation dataset. Defaults to None.

• test_transform (Optional[albumentations.Compose]) –

  Transformations for test dataset. Defaults to None.

• train_split_file (Optional[str]) –

  The file with train split. Defaults to None.

• val_split_file (Optional[str]) –

  The file with validation split. Defaults to None.

• test_split_file (Optional[str]) –

  The file with test split. Defaults to None.

• class_to_idx (Optional[Dict[str, int]]) –

  The mapping from class name to index. Defaults to None.

Source code in quadra/datamodules/classification.py

def __init__(
    self,
    data_path: str,
    dataset: Type[ImageClassificationListDataset] = ImageClassificationListDataset,
    name: str = "classification_datamodule",
    num_workers: int = 8,
    batch_size: int = 32,
    seed: int = 42,
    val_size: Optional[float] = 0.2,
    test_size: float = 0.2,
    num_data_class: Optional[int] = None,
    exclude_filter: Optional[List[str]] = None,
    include_filter: Optional[List[str]] = None,
    label_map: Optional[Dict[str, Any]] = None,
    load_aug_images: bool = False,
    aug_name: Optional[str] = None,
    n_aug_to_take: Optional[int] = 4,
    replace_str_from: Optional[str] = None,
    replace_str_to: Optional[str] = None,
    train_transform: Optional[albumentations.Compose] = None,
    val_transform: Optional[albumentations.Compose] = None,
    test_transform: Optional[albumentations.Compose] = None,
    train_split_file: Optional[str] = None,
    test_split_file: Optional[str] = None,
    val_split_file: Optional[str] = None,
    class_to_idx: Optional[Dict[str, int]] = None,
):
    super().__init__(
        data_path=data_path,
        name=name,
        seed=seed,
        batch_size=batch_size,
        num_workers=num_workers,
        train_transform=train_transform,
        val_transform=val_transform,
        test_transform=test_transform,
        load_aug_images=load_aug_images,
        aug_name=aug_name,
        n_aug_to_take=n_aug_to_take,
        replace_str_from=replace_str_from,
        replace_str_to=replace_str_to,
    )
    self.replace_str = None
    self.exclude_filter = exclude_filter
    self.include_filter = include_filter
    self.val_size = val_size
    self.test_size = test_size
    self.label_map = label_map
    self.num_data_class = num_data_class
    self.dataset = dataset
    self.train_split_file = train_split_file
    self.test_split_file = test_split_file
    self.val_split_file = val_split_file
    self.class_to_idx = class_to_idx

predict_dataloader()

Returns a dataloader used for predictions.

Source code in quadra/datamodules/classification.py

def predict_dataloader(self) -> DataLoader:
    """Returns a dataloader used for predictions."""
    return self.test_dataloader()

setup(stage=None)

Setup data module based on stages of training.

Source code in quadra/datamodules/classification.py

def setup(self, stage: Optional[str] = None) -> None:
    """Setup data module based on stages of training."""
    if stage in ["train", "fit"]:
        self.train_dataset = self.dataset(
            samples=self.data[self.data["split"] == "train"]["samples"].tolist(),
            targets=self.data[self.data["split"] == "train"]["targets"].tolist(),
            transform=self.train_transform,
            class_to_idx=self.class_to_idx,
        )
        self.val_dataset = self.dataset(
            samples=self.data[self.data["split"] == "val"]["samples"].tolist(),
            targets=self.data[self.data["split"] == "val"]["targets"].tolist(),
            transform=self.val_transform,
            class_to_idx=self.class_to_idx,
        )
    if stage in ["test", "predict"]:
        self.test_dataset = self.dataset(
            samples=self.data[self.data["split"] == "test"]["samples"].tolist(),
            targets=self.data[self.data["split"] == "test"]["targets"].tolist(),
            transform=self.test_transform,
            class_to_idx=self.class_to_idx,
        )

test_dataloader()

Returns the test dataloader.

Raises:

• ValueError –

  If test dataset is not initialized.

Returns:

• DataLoader –

  test dataloader.

Source code in quadra/datamodules/classification.py

def test_dataloader(self) -> DataLoader:
    """Returns the test dataloader.

    Raises:
        ValueError: If test dataset is not initialized.


    Returns:
        test dataloader.
    """
    if not self.test_dataset_available:
        raise ValueError("Test dataset is not initialized")

    loader = DataLoader(
        self.test_dataset,
        batch_size=self.batch_size,
        shuffle=False,
        num_workers=self.num_workers,
        drop_last=False,
        pin_memory=True,
        persistent_workers=self.num_workers > 0,
    )
    return loader

train_dataloader()

Returns the train dataloader.

Raises:

• ValueError –

  If train dataset is not initialized.

Returns:

• DataLoader –

  Train dataloader.

Source code in quadra/datamodules/classification.py

def train_dataloader(self) -> DataLoader:
    """Returns the train dataloader.

    Raises:
        ValueError: If train dataset is not initialized.

    Returns:
        Train dataloader.
    """
    if not self.train_dataset_available:
        raise ValueError("Train dataset is not initialized")
    if not isinstance(self.train_dataset, torch.utils.data.Dataset):
        raise ValueError("Train dataset has to be single `torch.utils.data.Dataset` instance.")
    return DataLoader(
        self.train_dataset,
        batch_size=self.batch_size,
        shuffle=True,
        num_workers=self.num_workers,
        drop_last=False,
        pin_memory=True,
        persistent_workers=self.num_workers > 0,
    )

val_dataloader()

Returns the validation dataloader.

Raises:

• ValueError –

  If validation dataset is not initialized.

Returns:

• DataLoader –

  val dataloader.

Source code in quadra/datamodules/classification.py

def val_dataloader(self) -> DataLoader:
    """Returns the validation dataloader.

    Raises:
        ValueError: If validation dataset is not initialized.

    Returns:
        val dataloader.
    """
    if not self.val_dataset_available:
        raise ValueError("Validation dataset is not initialized")
    if not isinstance(self.val_dataset, torch.utils.data.Dataset):
        raise ValueError("Validation dataset has to be single `torch.utils.data.Dataset` instance.")
    return DataLoader(
        self.val_dataset,
        batch_size=self.batch_size,
        shuffle=True,
        num_workers=self.num_workers,
        drop_last=False,
        pin_memory=True,
        persistent_workers=self.num_workers > 0,
    )

MultilabelClassificationDataModule(data_path, images_and_labels_file=None, train_split_file=None, test_split_file=None, val_split_file=None, name='multilabel_datamodule', dataset=MultilabelClassificationDataset, num_classes=None, num_workers=16, batch_size=64, test_batch_size=64, seed=42, val_size=0.2, test_size=0.2, train_transform=None, val_transform=None, test_transform=None, class_to_idx=None, **kwargs)

Bases: BaseDataModule

Base class for all multi-label modules.

Parameters:

• data_path (str) –

  Path to the data main folder.

• images_and_labels_file (Optional[str]) –

  a path to a txt file containing the relative (to data_path) path of images with their relative labels, in a comma-separated way. E.g.:

  • path1,l1,l2,l3
  • path2,l4,l5
  • ...

  One of images_and_label and both train_split_file and test_split_file must be set. Defaults to None.

• name (str) –

  The name for the data module. Defaults to "multilabel_datamodule".

• dataset (Callable) –

  a callable returning a torch.utils.data.Dataset class.

• num_classes (Optional[int]) –

  the number of classes in the dataset. This is used to create one-hot encoded targets. Defaults to None.

• num_workers (int) –

  Number of workers for dataloaders. Defaults to 16.

• batch_size (int) –

  Training batch size. Defaults to 64.

• test_batch_size (int) –

  Testing batch size. Defaults to 64.

• seed (int) –

  Random generator seed. Defaults to SegmentationEvalua2.

• val_size (Optional[float]) –

  The validation split. Defaults to 0.2.

• test_size (Optional[float]) –

  The test split. Defaults to 0.2.

• train_transform (Optional[albumentations.Compose]) –

  Transformations for train dataset. Defaults to None.

• val_transform (Optional[albumentations.Compose]) –

  Transformations for validation dataset. Defaults to None.

• test_transform (Optional[albumentations.Compose]) –

  Transformations for test dataset. Defaults to None.

• train_split_file (Optional[str]) –

  The file with train split. Defaults to None.

• val_split_file (Optional[str]) –

  The file with validation split. Defaults to None.

• test_split_file (Optional[str]) –

  The file with test split. Defaults to None.

• class_to_idx (Optional[Dict[str, int]]) –

  a clss to idx dictionary. Defaults to None.

Source code in quadra/datamodules/classification.py

def __init__(
    self,
    data_path: str,
    images_and_labels_file: Optional[str] = None,
    train_split_file: Optional[str] = None,
    test_split_file: Optional[str] = None,
    val_split_file: Optional[str] = None,
    name: str = "multilabel_datamodule",
    dataset: Callable = MultilabelClassificationDataset,
    num_classes: Optional[int] = None,
    num_workers: int = 16,
    batch_size: int = 64,
    test_batch_size: int = 64,
    seed: int = 42,
    val_size: Optional[float] = 0.2,
    test_size: Optional[float] = 0.2,
    train_transform: Optional[albumentations.Compose] = None,
    val_transform: Optional[albumentations.Compose] = None,
    test_transform: Optional[albumentations.Compose] = None,
    class_to_idx: Optional[Dict[str, int]] = None,
    **kwargs,
):
    super().__init__(
        data_path=data_path,
        name=name,
        num_workers=num_workers,
        batch_size=batch_size,
        seed=seed,
        train_transform=train_transform,
        val_transform=val_transform,
        test_transform=test_transform,
        **kwargs,
    )
    if not (images_and_labels_file is not None or (train_split_file is not None and test_split_file is not None)):
        raise ValueError(
            "Either `images_and_labels_file` or both `train_split_file` and `test_split_file` must be set"
        )
    self.images_and_labels_file = images_and_labels_file
    self.dataset = dataset
    self.num_classes = num_classes
    self.train_batch_size = batch_size
    self.test_batch_size = test_batch_size
    self.val_size = val_size
    self.test_size = test_size
    self.train_split_file = train_split_file
    self.test_split_file = test_split_file
    self.val_split_file = val_split_file
    self.class_to_idx = class_to_idx
    self.train_dataset: MultilabelClassificationDataset
    self.val_dataset: MultilabelClassificationDataset
    self.test_dataset: MultilabelClassificationDataset

predict_dataloader()

Returns a dataloader used for predictions.

Source code in quadra/datamodules/classification.py

def predict_dataloader(self) -> DataLoader:
    """Returns a dataloader used for predictions."""
    return self.test_dataloader()

setup(stage=None)

Setup data module based on stages of training.

Source code in quadra/datamodules/classification.py

def setup(self, stage: Optional[str] = None) -> None:
    """Setup data module based on stages of training."""
    if stage in ["train", "fit"]:
        train_samples = self.data[self.data["split"] == "train"]["samples"].tolist()
        train_targets = self.data[self.data["split"] == "train"]["targets"].tolist()
        val_samples = self.data[self.data["split"] == "val"]["samples"].tolist()
        val_targets = self.data[self.data["split"] == "val"]["targets"].tolist()
        self.train_dataset = self.dataset(
            samples=train_samples,
            targets=train_targets,
            transform=self.train_transform,
            class_to_idx=self.class_to_idx,
        )
        self.val_dataset = self.dataset(
            samples=val_samples,
            targets=val_targets,
            transform=self.val_transform,
            class_to_idx=self.class_to_idx,
        )
    if stage == "test":
        test_samples = self.data[self.data["split"] == "test"]["samples"].tolist()
        test_targets = self.data[self.data["split"] == "test"]["targets"].tolist()
        self.test_dataset = self.dataset(
            samples=test_samples,
            targets=test_targets,
            transform=self.test_transform,
            class_to_idx=self.class_to_idx,
        )

test_dataloader()

Returns the test dataloader.

Raises:

• ValueError –

  If test dataset is not initialized.

Returns:

• DataLoader –

  test dataloader.

Source code in quadra/datamodules/classification.py

def test_dataloader(self) -> DataLoader:
    """Returns the test dataloader.

    Raises:
        ValueError: If test dataset is not initialized.


    Returns:
        test dataloader.
    """
    if not self.test_dataset_available:
        raise ValueError("Test dataset is not initialized")

    loader = DataLoader(
        self.test_dataset,
        batch_size=self.batch_size,
        shuffle=False,
        num_workers=self.num_workers,
        drop_last=False,
        pin_memory=True,
        persistent_workers=self.num_workers > 0,
    )
    return loader

train_dataloader()

Returns the train dataloader.

Raises:

• ValueError –

  If train dataset is not initialized.

Returns:

• DataLoader –

  Train dataloader.

Source code in quadra/datamodules/classification.py

def train_dataloader(self) -> DataLoader:
    """Returns the train dataloader.

    Raises:
        ValueError: If train dataset is not initialized.

    Returns:
        Train dataloader.
    """
    if not self.train_dataset_available:
        raise ValueError("Train dataset is not initialized")
    return DataLoader(
        self.train_dataset,
        batch_size=self.batch_size,
        shuffle=True,
        num_workers=self.num_workers,
        drop_last=False,
        pin_memory=True,
        persistent_workers=self.num_workers > 0,
    )

val_dataloader()

Returns the validation dataloader.

Raises:

• ValueError –

  If validation dataset is not initialized.

Returns:

• DataLoader –

  val dataloader.

Source code in quadra/datamodules/classification.py

def val_dataloader(self) -> DataLoader:
    """Returns the validation dataloader.

    Raises:
        ValueError: If validation dataset is not initialized.

    Returns:
        val dataloader.
    """
    if not self.val_dataset_available:
        raise ValueError("Validation dataset is not initialized")
    return DataLoader(
        self.train_dataset,
        batch_size=self.batch_size,
        shuffle=True,
        num_workers=self.num_workers,
        drop_last=False,
        pin_memory=True,
        persistent_workers=self.num_workers > 0,
    )

SklearnClassificationDataModule(data_path, exclude_filter=None, include_filter=None, val_size=0.2, class_to_idx=None, label_map=None, seed=42, batch_size=32, num_workers=6, train_transform=None, val_transform=None, test_transform=None, roi=None, n_splits=1, phase='train', cache=False, limit_training_data=None, train_split_file=None, test_split_file=None, name='sklearn_classification_datamodule', dataset=ImageClassificationListDataset, **kwargs)

Bases: BaseDataModule

A generic Data Module for classification with frozen torch backbone and sklearn classifier.

It can also handle k-fold cross validation.

Parameters:

• name (str) –

  The name for the data module. Defaults to "sklearn_classification_datamodule".

• data_path (str) –

  Path to images main folder

• exclude_filter (Optional[List[str]]) –

  List of string filter to be used to exclude images. If None no filter will be applied.

• include_filter (Optional[List[str]]) –

  List of string filter to be used to include images. Only images that satisfied at list one of the filter will be included.

• val_size (float) –

  The validation split. Defaults to 0.2.

• class_to_idx (Optional[Dict[str, int]]) –

  Dictionary of conversion btw folder name and index. Only file whose label is in dictionary key list will be considered. If None all files will be considered and a custom conversion is created.

• seed (int) –

  Fixed seed for random operations

• batch_size (int) –

  Dimension of batches for dataloader

• num_workers (int) –

  Number of workers for dataloader

• train_transform (Optional[albumentations.Compose]) –

  Albumentation transformations for training set

• val_transform (Optional[albumentations.Compose]) –

  Albumentation transformations for validation set

• test_transform (Optional[albumentations.Compose]) –

  Albumentation transformations for test set

• roi (Optional[Tuple[int, int, int, int]]) –

  Optional cropping region

• n_splits (int) –

  Number of dataset subdivision (default 1 -> train/test). Use a value >= 2 for cross validation.

• phase (str) –

  Either train or test

• cache (bool) –

  If true disable shuffling in all dataloader to enable feature caching

• limit_training_data (Optional[int]) –

  if defined, each class will be donwsampled to this number. It must be >= 2 to allow splitting

• label_map (Optional[Dict[str, Any]]) –

  Dictionary of conversion btw folder name and label.

• train_split_file (Optional[str]) –

  Optional path to a csv file containing the train split samples.

• test_split_file (Optional[str]) –

  Optional path to a csv file containing the test split samples.

• **kwargs (Any) –

  Additional arguments for BaseDataModule

Source code in quadra/datamodules/classification.py

def __init__(
    self,
    data_path: str,
    exclude_filter: Optional[List[str]] = None,
    include_filter: Optional[List[str]] = None,
    val_size: float = 0.2,
    class_to_idx: Optional[Dict[str, int]] = None,
    label_map: Optional[Dict[str, Any]] = None,
    seed: int = 42,
    batch_size: int = 32,
    num_workers: int = 6,
    train_transform: Optional[albumentations.Compose] = None,
    val_transform: Optional[albumentations.Compose] = None,
    test_transform: Optional[albumentations.Compose] = None,
    roi: Optional[Tuple[int, int, int, int]] = None,
    n_splits: int = 1,
    phase: str = "train",
    cache: bool = False,
    limit_training_data: Optional[int] = None,
    train_split_file: Optional[str] = None,
    test_split_file: Optional[str] = None,
    name: str = "sklearn_classification_datamodule",
    dataset: Type[ImageClassificationListDataset] = ImageClassificationListDataset,
    **kwargs: Any,
):
    super().__init__(
        data_path=data_path,
        name=name,
        seed=seed,
        batch_size=batch_size,
        num_workers=num_workers,
        train_transform=train_transform,
        val_transform=val_transform,
        test_transform=test_transform,
        **kwargs,
    )

    self.class_to_idx = class_to_idx
    self.roi = roi
    self.cache = cache
    self.limit_training_data = limit_training_data

    self.dataset = dataset
    self.phase = phase
    self.n_splits = n_splits
    self.train_split_file = train_split_file
    self.test_split_file = test_split_file
    self.exclude_filter = exclude_filter
    self.include_filter = include_filter
    self.val_size = val_size
    self.label_map = label_map
    self.full_dataset: ImageClassificationListDataset
    self.train_dataset: List[ImageClassificationListDataset]
    self.val_dataset: List[ImageClassificationListDataset]

full_dataloader()

Return a dataloader to perform training on the entire dataset.

Returns:

• DataLoader –

  dataloader to perform training on the entire dataset after evaluation. This is useful

• DataLoader –

  to perform a final training on the entire dataset after the evaluation phase.

Source code in quadra/datamodules/classification.py

def full_dataloader(self) -> DataLoader:
    """Return a dataloader to perform training on the entire dataset.

    Returns:
        dataloader to perform training on the entire dataset after evaluation. This is useful
        to perform a final training on the entire dataset after the evaluation phase.

    """
    if self.full_dataset is None:
        raise ValueError("Full dataset is not initialized")

    return DataLoader(
        self.full_dataset,
        batch_size=self.batch_size,
        shuffle=not self.cache,
        num_workers=self.num_workers,
        drop_last=False,
        pin_memory=True,
    )

predict_dataloader()

Returns a dataloader used for predictions.

Source code in quadra/datamodules/classification.py

def predict_dataloader(self) -> DataLoader:
    """Returns a dataloader used for predictions."""
    return self.test_dataloader()

setup(stage)

Setup data module based on stages of training.

Source code in quadra/datamodules/classification.py

def setup(self, stage: str) -> None:
    """Setup data module based on stages of training."""
    if stage == "fit":
        self.train_dataset = []
        self.val_dataset = []

        for cv_idx in range(self.n_splits):
            cv_df = self.data[self.data["cv"] == cv_idx]
            train_samples = cv_df[cv_df["split"] == "train"]["samples"].tolist()
            train_targets = cv_df[cv_df["split"] == "train"]["targets"].tolist()
            val_samples = cv_df[cv_df["split"] == "val"]["samples"].tolist()
            val_targets = cv_df[cv_df["split"] == "val"]["targets"].tolist()
            self.train_dataset.append(
                self.dataset(
                    class_to_idx=self.class_to_idx,
                    samples=train_samples,
                    targets=train_targets,
                    transform=self.train_transform,
                    roi=self.roi,
                )
            )
            self.val_dataset.append(
                self.dataset(
                    class_to_idx=self.class_to_idx,
                    samples=val_samples,
                    targets=val_targets,
                    transform=self.val_transform,
                    roi=self.roi,
                )
            )
        all_samples = self.data[self.data["cv"] == 0]["samples"].tolist()
        all_targets = self.data[self.data["cv"] == 0]["targets"].tolist()
        self.full_dataset = self.dataset(
            class_to_idx=self.class_to_idx,
            samples=all_samples,
            targets=all_targets,
            transform=self.train_transform,
            roi=self.roi,
        )
    if stage == "test":
        test_samples = self.data[self.data["split"] == "test"]["samples"].tolist()
        test_targets = self.data[self.data["split"] == "test"]["targets"]
        self.test_dataset = self.dataset(
            class_to_idx=self.class_to_idx,
            samples=test_samples,
            targets=test_targets.tolist(),
            transform=self.test_transform,
            roi=self.roi,
            allow_missing_label=True,
        )

test_dataloader()

Returns the test dataloader.

Raises:

• ValueError –

  If test dataset is not initialized.

Returns:

• DataLoader –

  test dataloader.

Source code in quadra/datamodules/classification.py

def test_dataloader(self) -> DataLoader:
    """Returns the test dataloader.

    Raises:
        ValueError: If test dataset is not initialized.


    Returns:
        test dataloader.
    """
    if not self.test_dataset_available:
        raise ValueError("Test dataset is not initialized")

    loader = DataLoader(
        self.test_dataset,
        batch_size=self.batch_size,
        shuffle=False,
        num_workers=self.num_workers,
        drop_last=False,
        pin_memory=True,
        persistent_workers=self.num_workers > 0,
    )
    return loader

train_dataloader()

Returns a list of train dataloader.

Raises:

• ValueError –

  If train dataset is not initialized.

Returns:

• List[DataLoader] –

  list of train dataloader.

Source code in quadra/datamodules/classification.py

def train_dataloader(self) -> List[DataLoader]:
    """Returns a list of train dataloader.

    Raises:
        ValueError: If train dataset is not initialized.

    Returns:
        list of train dataloader.
    """
    if not self.train_dataset_available:
        raise ValueError("Train dataset is not initialized")

    loader = []
    for dataset in self.train_dataset:
        loader.append(
            DataLoader(
                dataset,
                batch_size=self.batch_size,
                shuffle=not self.cache,
                num_workers=self.num_workers,
                drop_last=False,
                pin_memory=True,
            )
        )
    return loader

val_dataloader()

Returns a list of validation dataloader.

Raises:

• ValueError –

  If validation dataset is not initialized.

Returns:

• List[DataLoader] –

  List of validation dataloader.

Source code in quadra/datamodules/classification.py

def val_dataloader(self) -> List[DataLoader]:
    """Returns a list of validation dataloader.

    Raises:
        ValueError: If validation dataset is not initialized.

    Returns:
        List of validation dataloader.
    """
    if not self.val_dataset_available:
        raise ValueError("Validation dataset is not initialized")

    loader = []
    for dataset in self.val_dataset:
        loader.append(
            DataLoader(
                dataset,
                batch_size=self.batch_size,
                shuffle=False,
                num_workers=self.num_workers,
                drop_last=False,
                pin_memory=True,
            )
        )

    return loader