How do I implement a PyTorch Dataset for use with AWS SageMaker?
I was able to create a PyTorch Dataset
backed by S3 data using boto3
. Here's the snippet if anyone is interested.
class ImageDataset(Dataset):
def __init__(self, path='./images', transform=None):
self.path = path
self.s3 = boto3.resource('s3')
self.bucket = self.s3.Bucket(path)
self.files = [obj.key for obj in self.bucket.objects.all()]
self.transform = transform
if transform is None:
self.transform = transforms.Compose([
transforms.Resize((128, 128)),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
def __len__(self):
return len(files)
def __getitem__(self, idx):
img_name = self.files[idx]
# we may infer the label from the filename
dash_idx = img_name.rfind('-')
dot_idx = img_name.rfind('.')
label = int(img_name[dash_idx + 1:dot_idx])
# we need to download the file from S3 to a temporary file locally
# we need to create the local file name
obj = self.bucket.Object(img_name)
tmp = tempfile.NamedTemporaryFile()
tmp_name = '{}.jpg'.format(tmp.name)
# now we can actually download from S3 to a local place
with open(tmp_name, 'wb') as f:
obj.download_fileobj(f)
f.flush()
f.close()
image = Image.open(tmp_name)
if self.transform:
image = self.transform(image)
return image, label