Processing imagenet2012 datasets

Download the imagenet2012 The dataset is
ILSVRC2012_img_train.tar and ILSVRC2012_img_val.tar These two data
First, unzip the file

mkdir train
mkdir val
tar xvf ILSVRC2012_img_train.tar -C ./train
tar xvf ILSVRC2012_img_val.tar -C ./val

about train The package , After decompression, it is still 1000 individual tar Compressed package （ Corresponding 1000 Categories ）, Need to decompress again

import tarfile
import os
def un_tar(file_name):
    tar = tarfile.open(file_name)
    file_name = file_name.split('.')[0]
    """ decompression tar"""
    print(file_name)
    names = tar.getnames()
    temp_file_path = ''
    if os.path.isdir(file_name):
        print(' file already exist ')
        temp_file_path = os.path.isdir(file_name)
    else:
        temp_file_path = os.mkdir(file_name)
        print(' Create a new file name ')
    # Because there are many files after decompression , Create a directory with the same name in advance 
    for name in names:
        tar.extract(name, file_name)
    tar.close()
    return temp_file_path
filePath = 'train'
for tar in os.listdir(filePath):
    un_tar(filePath + '/' + tar)
    os.remove(filePath + '/' + tar)

Organize test sets （ need ILSVRC2012_devkit_t12.tar.gz）

from scipy import io
import os
import shutil

def move_valimg(val_dir='./val', devkit_dir='./ILSVRC2012_devkit_t12'):
    """ move valimg to correspongding folders. val_id(start from 1) -> ILSVRC_ID(start from 1) -> WIND organize like: /val /n01440764 images /n01443537 images ..... """
    # load synset, val ground truth and val images list
    synset = io.loadmat(os.path.join(devkit_dir, 'data', 'meta.mat'))
    
    ground_truth = open(os.path.join(devkit_dir, 'data', 'ILSVRC2012_validation_ground_truth.txt'))
    lines = ground_truth.readlines()
    labels = [int(line[:-1]) for line in lines]
    
    root, _, filenames = next(os.walk(val_dir))
    for filename in filenames:
        # val image name -> ILSVRC ID -> WIND
        val_id = int(filename.split('.')[0].split('_')[-1])
        ILSVRC_ID = labels[val_id-1]
        WIND = synset['synsets'][ILSVRC_ID-1][0][1][0]
        print("val_id:%d, ILSVRC_ID:%d, WIND:%s" % (val_id, ILSVRC_ID, WIND))

        # move val images
        output_dir = os.path.join(root, WIND)
        if os.path.isdir(output_dir):
            pass
        else:
            os.mkdir(output_dir)
        shutil.move(os.path.join(root, filename), os.path.join(output_dir, filename))

if __name__ == '__main__':
    move_valimg()