You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
176 lines
7.1 KiB
176 lines
7.1 KiB
import os.path
|
|
from pathlib import Path
|
|
from typing import Any, Callable, cast, Optional, Tuple, Union
|
|
|
|
import numpy as np
|
|
from PIL import Image
|
|
|
|
from .utils import check_integrity, download_and_extract_archive, verify_str_arg
|
|
from .vision import VisionDataset
|
|
|
|
|
|
class STL10(VisionDataset):
|
|
"""`STL10 <https://cs.stanford.edu/~acoates/stl10/>`_ Dataset.
|
|
|
|
Args:
|
|
root (str or ``pathlib.Path``): Root directory of dataset where directory
|
|
``stl10_binary`` exists.
|
|
split (string): One of {'train', 'test', 'unlabeled', 'train+unlabeled'}.
|
|
Accordingly, dataset is selected.
|
|
folds (int, optional): One of {0-9} or None.
|
|
For training, loads one of the 10 pre-defined folds of 1k samples for the
|
|
standard evaluation procedure. If no value is passed, loads the 5k samples.
|
|
transform (callable, optional): A function/transform that takes in a PIL image
|
|
and returns a transformed version. E.g, ``transforms.RandomCrop``
|
|
target_transform (callable, optional): A function/transform that takes in the
|
|
target and transforms it.
|
|
download (bool, optional): If true, downloads the dataset from the internet and
|
|
puts it in root directory. If dataset is already downloaded, it is not
|
|
downloaded again.
|
|
"""
|
|
|
|
base_folder = "stl10_binary"
|
|
url = "http://ai.stanford.edu/~acoates/stl10/stl10_binary.tar.gz"
|
|
filename = "stl10_binary.tar.gz"
|
|
tgz_md5 = "91f7769df0f17e558f3565bffb0c7dfb"
|
|
class_names_file = "class_names.txt"
|
|
folds_list_file = "fold_indices.txt"
|
|
train_list = [
|
|
["train_X.bin", "918c2871b30a85fa023e0c44e0bee87f"],
|
|
["train_y.bin", "5a34089d4802c674881badbb80307741"],
|
|
["unlabeled_X.bin", "5242ba1fed5e4be9e1e742405eb56ca4"],
|
|
]
|
|
|
|
test_list = [["test_X.bin", "7f263ba9f9e0b06b93213547f721ac82"], ["test_y.bin", "36f9794fa4beb8a2c72628de14fa638e"]]
|
|
splits = ("train", "train+unlabeled", "unlabeled", "test")
|
|
|
|
def __init__(
|
|
self,
|
|
root: Union[str, Path],
|
|
split: str = "train",
|
|
folds: Optional[int] = None,
|
|
transform: Optional[Callable] = None,
|
|
target_transform: Optional[Callable] = None,
|
|
download: bool = False,
|
|
) -> None:
|
|
super().__init__(root, transform=transform, target_transform=target_transform)
|
|
self.split = verify_str_arg(split, "split", self.splits)
|
|
self.folds = self._verify_folds(folds)
|
|
|
|
if download:
|
|
self.download()
|
|
elif not self._check_integrity():
|
|
raise RuntimeError("Dataset not found or corrupted. You can use download=True to download it")
|
|
|
|
# now load the picked numpy arrays
|
|
self.labels: Optional[np.ndarray]
|
|
if self.split == "train":
|
|
self.data, self.labels = self.__loadfile(self.train_list[0][0], self.train_list[1][0])
|
|
self.labels = cast(np.ndarray, self.labels)
|
|
self.__load_folds(folds)
|
|
|
|
elif self.split == "train+unlabeled":
|
|
self.data, self.labels = self.__loadfile(self.train_list[0][0], self.train_list[1][0])
|
|
self.labels = cast(np.ndarray, self.labels)
|
|
self.__load_folds(folds)
|
|
unlabeled_data, _ = self.__loadfile(self.train_list[2][0])
|
|
self.data = np.concatenate((self.data, unlabeled_data))
|
|
self.labels = np.concatenate((self.labels, np.asarray([-1] * unlabeled_data.shape[0])))
|
|
|
|
elif self.split == "unlabeled":
|
|
self.data, _ = self.__loadfile(self.train_list[2][0])
|
|
self.labels = np.asarray([-1] * self.data.shape[0])
|
|
else: # self.split == 'test':
|
|
self.data, self.labels = self.__loadfile(self.test_list[0][0], self.test_list[1][0])
|
|
|
|
class_file = os.path.join(self.root, self.base_folder, self.class_names_file)
|
|
if os.path.isfile(class_file):
|
|
with open(class_file) as f:
|
|
self.classes = f.read().splitlines()
|
|
|
|
def _verify_folds(self, folds: Optional[int]) -> Optional[int]:
|
|
if folds is None:
|
|
return folds
|
|
elif isinstance(folds, int):
|
|
if folds in range(10):
|
|
return folds
|
|
msg = "Value for argument folds should be in the range [0, 10), but got {}."
|
|
raise ValueError(msg.format(folds))
|
|
else:
|
|
msg = "Expected type None or int for argument folds, but got type {}."
|
|
raise ValueError(msg.format(type(folds)))
|
|
|
|
def __getitem__(self, index: int) -> Tuple[Any, Any]:
|
|
"""
|
|
Args:
|
|
index (int): Index
|
|
|
|
Returns:
|
|
tuple: (image, target) where target is index of the target class.
|
|
"""
|
|
target: Optional[int]
|
|
if self.labels is not None:
|
|
img, target = self.data[index], int(self.labels[index])
|
|
else:
|
|
img, target = self.data[index], None
|
|
|
|
# doing this so that it is consistent with all other datasets
|
|
# to return a PIL Image
|
|
img = Image.fromarray(np.transpose(img, (1, 2, 0)))
|
|
|
|
if self.transform is not None:
|
|
img = self.transform(img)
|
|
|
|
if self.target_transform is not None:
|
|
target = self.target_transform(target)
|
|
|
|
return img, target
|
|
|
|
def __len__(self) -> int:
|
|
return self.data.shape[0]
|
|
|
|
def __loadfile(self, data_file: str, labels_file: Optional[str] = None) -> Tuple[np.ndarray, Optional[np.ndarray]]:
|
|
labels = None
|
|
if labels_file:
|
|
path_to_labels = os.path.join(self.root, self.base_folder, labels_file)
|
|
with open(path_to_labels, "rb") as f:
|
|
labels = np.fromfile(f, dtype=np.uint8) - 1 # 0-based
|
|
|
|
path_to_data = os.path.join(self.root, self.base_folder, data_file)
|
|
with open(path_to_data, "rb") as f:
|
|
# read whole file in uint8 chunks
|
|
everything = np.fromfile(f, dtype=np.uint8)
|
|
images = np.reshape(everything, (-1, 3, 96, 96))
|
|
images = np.transpose(images, (0, 1, 3, 2))
|
|
|
|
return images, labels
|
|
|
|
def _check_integrity(self) -> bool:
|
|
for filename, md5 in self.train_list + self.test_list:
|
|
fpath = os.path.join(self.root, self.base_folder, filename)
|
|
if not check_integrity(fpath, md5):
|
|
return False
|
|
return True
|
|
|
|
def download(self) -> None:
|
|
if self._check_integrity():
|
|
print("Files already downloaded and verified")
|
|
return
|
|
download_and_extract_archive(self.url, self.root, filename=self.filename, md5=self.tgz_md5)
|
|
self._check_integrity()
|
|
|
|
def extra_repr(self) -> str:
|
|
return "Split: {split}".format(**self.__dict__)
|
|
|
|
def __load_folds(self, folds: Optional[int]) -> None:
|
|
# loads one of the folds if specified
|
|
if folds is None:
|
|
return
|
|
path_to_folds = os.path.join(self.root, self.base_folder, self.folds_list_file)
|
|
with open(path_to_folds) as f:
|
|
str_idx = f.read().splitlines()[folds]
|
|
list_idx = np.fromstring(str_idx, dtype=np.int64, sep=" ")
|
|
self.data = self.data[list_idx, :, :, :]
|
|
if self.labels is not None:
|
|
self.labels = self.labels[list_idx]
|