grouped conv

.gitignore

@@ -1,9 +1,14 @@
+# Other
+.pytorch-lightning_ipynb_new/
+
+# Log files
+pytorch-lightning_ipynb/mlp/logs/
+pytorch-lightning_ipynb/cnn/logs/
 
 # Datasets
 png-files
 *-ubyte*
-pytorch-lightning_ipynb_new/*/data
-pytorch-lightning_ipynb_new/*/logs
+pytorch-lightning_ipynb/*/data
 pytorch_ipynb/viz/cnns/cats-and-dogs/dogs-vs-cats
 pytorch_ipynb/gan/dogs-vs-cats
 pytorch_ipynb/viz/cnns/cats-and-dogs/dogs-vs-cats

README.md

@@ -49,8 +49,9 @@ A collection of various deep learning architectures, models, and tips for Tensor
 
 #### AlexNet
 
-- AlexNet on CIFAR-10  [![PyTorch Lightning](https://img.shields.io/badge/PyTorch-Lightning-blueviolet)](pytorch-lightning_ipynb/cnn/cnn-alexnet-cifar10.ipynb) [![PyTorch](https://img.shields.io/badge/Py-Torch-red)](pytorch_ipynb/cnn/cnn-alexnet-cifar10.ipynb) 
+- AlexNet Trained on CIFAR-10  [![PyTorch Lightning](https://img.shields.io/badge/PyTorch-Lightning-blueviolet)](pytorch-lightning_ipynb/cnn/cnn-alexnet-cifar10.ipynb) [![PyTorch](https://img.shields.io/badge/Py-Torch-red)](pytorch_ipynb/cnn/cnn-alexnet-cifar10.ipynb) 
 
+- AlexNet with Grouped Convolutions Trained on CIFAR-10  [![PyTorch Lightning](https://img.shields.io/badge/PyTorch-Lightning-blueviolet)](pytorch-lightning_ipynb/cnn/cnn-alexnet-cifar10-grouped.ipynb) [![PyTorch](https://img.shields.io/badge/Py-Torch-red)](pytorch_ipynb/cnn/cnn-alexnet-cifar10-grouped.ipynb) 
 
 #### DenseNet
 

pytorch-lightning_ipynb/mlp/logs/my-mlp/version_4/hparams.yaml

@@ -1,35 +0,0 @@
-model: !!python/object:__main__.PyTorchMLP
-  _backward_hooks: !!python/object/apply:collections.OrderedDict
-  - []
-  _buffers: !!python/object/apply:collections.OrderedDict
-  - []
-  _forward_hooks: !!python/object/apply:collections.OrderedDict
-  - []
-  _forward_pre_hooks: !!python/object/apply:collections.OrderedDict
-  - []
-  _is_full_backward_hook: null
-  _load_state_dict_pre_hooks: !!python/object/apply:collections.OrderedDict
-  - []
-  _modules: !!python/object/apply:collections.OrderedDict
-  - - - linear_1
-      - !!python/object:torch.nn.modules.linear.Linear
-        _backward_hooks: !!python/object/apply:collections.OrderedDict
-        - []
-        _buffers: !!python/object/apply:collections.OrderedDict
-        - []
-        _forward_hooks: !!python/object/apply:collections.OrderedDict
-        - []
-        _forward_pre_hooks: !!python/object/apply:collections.OrderedDict
-        - []
-        _is_full_backward_hook: null
-        _load_state_dict_pre_hooks: !!python/object/apply:collections.OrderedDict
-        - []
-        _modules: !!python/object/apply:collections.OrderedDict
-        - []
-        _non_persistent_buffers_set: !!set {}
-        _parameters: !!python/object/apply:collections.OrderedDict
-        - - - weight
-            - !!python/object/apply:torch._utils._rebuild_parameter
-              - !!python/object/apply:torch._utils._rebuild_tensor_v2
-                - !!python/object/apply:torch.storage._load_from_bytes
-                  -

pytorch-lightning_ipynb_new/code_dataset/dataset_cifar10_check.py

@@ -1,43 +0,0 @@
-from collections import Counter
-from torchvision import datasets
-from torchvision import transforms
-from torch.utils.data import DataLoader
-
-
-train_dataset = datasets.CIFAR10(
-    root="./data", train=True, transform=transforms.ToTensor(), download=True
-)
-
-train_loader = DataLoader(
-    dataset=train_dataset,
-    batch_size=BATCH_SIZE,
-    num_workers=NUM_WORKERS,
-    drop_last=True,
-    shuffle=True,
-)
-
-test_dataset = datasets.CIFAR10(
-    root="./data", train=False, transform=transforms.ToTensor()
-)
-
-test_loader = DataLoader(
-    dataset=test_dataset,
-    batch_size=BATCH_SIZE,
-    num_workers=NUM_WORKERS,
-    drop_last=False,
-    shuffle=False,
-)
-
-train_counter = Counter()
-for images, labels in train_loader:
-    train_counter.update(labels.tolist())
-
-test_counter = Counter()
-for images, labels in test_loader:
-    test_counter.update(labels.tolist())
-
-print("\nTraining label distribution:")
-sorted(train_counter.items())
-
-print("\nTest label distribution:")
-sorted(test_counter.items())

pytorch-lightning_ipynb_new/code_dataset/dataset_mnist_check.py

@@ -1,43 +0,0 @@
-from collections import Counter
-from torchvision import datasets
-from torchvision import transforms
-from torch.utils.data import DataLoader
-
-
-train_dataset = datasets.MNIST(
-    root="./data", train=True, transform=transforms.ToTensor(), download=True
-)
-
-train_loader = DataLoader(
-    dataset=train_dataset,
-    batch_size=BATCH_SIZE,
-    num_workers=NUM_WORKERS,
-    drop_last=True,
-    shuffle=True,
-)
-
-test_dataset = datasets.MNIST(
-    root="./data", train=False, transform=transforms.ToTensor()
-)
-
-test_loader = DataLoader(
-    dataset=test_dataset,
-    batch_size=BATCH_SIZE,
-    num_workers=NUM_WORKERS,
-    drop_last=False,
-    shuffle=False,
-)
-
-train_counter = Counter()
-for images, labels in train_loader:
-    train_counter.update(labels.tolist())
-
-test_counter = Counter()
-for images, labels in test_loader:
-    test_counter.update(labels.tolist())
-
-print("\nTraining label distribution:")
-sorted(train_counter.items())
-
-print("\nTest label distribution:")
-sorted(test_counter.items())

pytorch-lightning_ipynb_new/code_dataset/performance_baseline.py

@@ -1,6 +0,0 @@
-majority_class = test_counter.most_common(1)[0]
-print("Majority class:", majority_class[0])
-
-baseline_acc = majority_class[1] / sum(test_counter.values())
-print("Accuracy when always predicting the majority class:")
-print(f"{baseline_acc:.2f} ({baseline_acc*100:.2f}%)")

pytorch-lightning_ipynb_new/code_dataset/plot_visual-check_basic.py

@@ -1,18 +0,0 @@
-%matplotlib inline
-import matplotlib.pyplot as plt
-import numpy as np
-import torchvision
-
-
-for images, labels in train_loader:  
-    break
-
-plt.figure(figsize=(8, 8))
-plt.axis("off")
-plt.title("Training images")
-plt.imshow(np.transpose(torchvision.utils.make_grid(
-    images[:64], 
-    padding=2,
-    normalize=True),
-    (1, 2, 0)))
-plt.show()

pytorch-lightning_ipynb_new/code_lightningmodule/datamodule_cifar10_basic.py

@@ -1,74 +0,0 @@
-import os
-
-from torch.utils.data.dataset import random_split
-from torch.utils.data import DataLoader
-from torchvision import transforms
-
-
-class DataModule(pl.LightningDataModule):
-    def __init__(self, data_path="./"):
-        super().__init__()
-        self.data_path = data_path
-
-    def prepare_data(self):
-        datasets.CIFAR10(root=self.data_path, download=True)
-
-        self.train_transform = transforms.Compose(
-            [
-                transforms.ToTensor(),
-            ]
-        )
-
-        self.test_transform = transforms.Compose(
-            [
-                transforms.ToTensor(),
-            ]
-        )
-        return
-
-    def setup(self, stage=None):
-        train = datasets.CIFAR10(
-            root=self.data_path,
-            train=True,
-            transform=self.train_transform,
-            download=False,
-        )
-
-        self.test = datasets.CIFAR10(
-            root=self.data_path,
-            train=False,
-            transform=self.test_transform,
-            download=False,
-        )
-
-        self.train, self.valid = random_split(train, lengths=[45000, 5000])
-
-    def train_dataloader(self):
-        train_loader = DataLoader(
-            dataset=self.train,
-            batch_size=BATCH_SIZE,
-            drop_last=True,
-            shuffle=True,
-            num_workers=NUM_WORKERS,
-        )
-        return train_loader
-
-    def val_dataloader(self):
-        valid_loader = DataLoader(
-            dataset=self.valid,
-            batch_size=BATCH_SIZE,
-            drop_last=False,
-            shuffle=False,
-            num_workers=NUM_WORKERS,
-        )
-        return valid_loader
-
-    def test_dataloader(self):
-        test_loader = DataLoader(
-            dataset=self.test,
-            batch_size=BATCH_SIZE,
-            drop_last=False,
-            shuffle=False,
-            num_workers=NUM_WORKERS,
-        )
-        return test_loader

pytorch-lightning_ipynb_new/code_lightningmodule/datamodule_mnist_basic.py

@@ -1,60 +0,0 @@
-from torch.utils.data.dataset import random_split
-
-
-class DataModule(pl.LightningDataModule):
-    def __init__(self, data_path="./"):
-        super().__init__()
-        self.data_path = data_path
-
-    def prepare_data(self):
-        datasets.MNIST(root=self.data_path, download=True)
-        return
-
-    def setup(self, stage=None):
-        # Note transforms.ToTensor() scales input images
-        # to 0-1 range
-        train = datasets.MNIST(
-            root=self.data_path,
-            train=True,
-            transform=transforms.ToTensor(),
-            download=False,
-        )
-
-        self.test = datasets.MNIST(
-            root=self.data_path,
-            train=False,
-            transform=transforms.ToTensor(),
-            download=False,
-        )
-
-        self.train, self.valid = random_split(train, lengths=[55000, 5000])
-
-    def train_dataloader(self):
-        train_loader = DataLoader(
-            dataset=self.train,
-            batch_size=BATCH_SIZE,
-            drop_last=True,
-            shuffle=True,
-            num_workers=NUM_WORKERS,
-        )
-        return train_loader
-
-    def val_dataloader(self):
-        valid_loader = DataLoader(
-            dataset=self.valid,
-            batch_size=BATCH_SIZE,
-            drop_last=False,
-            shuffle=False,
-            num_workers=NUM_WORKERS,
-        )
-        return valid_loader
-
-    def test_dataloader(self):
-        test_loader = DataLoader(
-            dataset=self.test,
-            batch_size=BATCH_SIZE,
-            drop_last=False,
-            shuffle=False,
-            num_workers=NUM_WORKERS,
-        )
-        return test_loader

pytorch-lightning_ipynb_new/code_lightningmodule/datamodule_testloader.py

@@ -1,13 +0,0 @@
-test_dataloader = data_module.test_dataloader()
-acc = torchmetrics.Accuracy()
-
-for batch in test_dataloader:
-    features, true_labels = batch
-
-    with torch.no_grad():
-        logits = lightning_model(features)
-
-    predicted_labels = torch.argmax(logits, dim=1)
-    acc(predicted_labels, true_labels)
-
-predicted_labels[:5]

pytorch-lightning_ipynb_new/code_lightningmodule/lightningmodule_classifier_basic.py

@@ -1,75 +0,0 @@
-import pytorch_lightning as pl
-import torchmetrics
-
-
-# LightningModule that receives a PyTorch model as input
-class LightningModel(pl.LightningModule):
-    def __init__(self, model, learning_rate):
-        super().__init__()
-
-        self.learning_rate = learning_rate
-        # The inherited PyTorch module
-        self.model = model
-        if hasattr(model, "dropout_proba"):
-            self.dropout_proba = model.dropout_proba
-
-        # Save settings and hyperparameters to the log directory
-        # but skip the model parameters
-        self.save_hyperparameters(ignore=["model"])
-
-        # Set up attributes for computing the accuracy
-        self.train_acc = torchmetrics.Accuracy()
-        self.valid_acc = torchmetrics.Accuracy()
-        self.test_acc = torchmetrics.Accuracy()
-
-    # Defining the forward method is only necessary
-    # if you want to use a Trainer's .predict() method (optional)
-    def forward(self, x):
-        return self.model(x)
-
-    # A common forward step to compute the loss and labels
-    # this is used for training, validation, and testing below
-    def _shared_step(self, batch):
-        features, true_labels = batch
-        logits = self(features)
-        loss = torch.nn.functional.cross_entropy(logits, true_labels)
-        predicted_labels = torch.argmax(logits, dim=1)
-
-        return loss, true_labels, predicted_labels
-
-    def training_step(self, batch, batch_idx):
-        loss, true_labels, predicted_labels = self._shared_step(batch)
-        self.log("train_loss", loss)
-
-        # Do another forward pass in .eval() mode to compute accuracy
-        # while accountingfor Dropout, BatchNorm etc. behavior
-        # during evaluation (inference)
-        self.model.eval()
-        with torch.no_grad():
-            _, true_labels, predicted_labels = self._shared_step(batch)
-        self.train_acc(predicted_labels, true_labels)
-        self.log("train_acc", self.train_acc, on_epoch=True, on_step=False)
-        self.model.train()
-
-        return loss  # this is passed to the optimzer for training
-
-    def validation_step(self, batch, batch_idx):
-        loss, true_labels, predicted_labels = self._shared_step(batch)
-        self.log("valid_loss", loss)
-        self.valid_acc(predicted_labels, true_labels)
-        self.log(
-            "valid_acc",
-            self.valid_acc,
-            on_epoch=True,
-            on_step=False,
-            prog_bar=True,
-        )
-
-    def test_step(self, batch, batch_idx):
-        loss, true_labels, predicted_labels = self._shared_step(batch)
-        self.test_acc(predicted_labels, true_labels)
-        self.log("test_acc", self.test_acc, on_epoch=True, on_step=False)
-
-    def configure_optimizers(self):
-        optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)
-        return optimizer

pytorch-lightning_ipynb_new/code_lightningmodule/logger_csv_acc_basic.py

@@ -1,12 +0,0 @@
-from pytorch_lightning.callbacks import ModelCheckpoint
-from pytorch_lightning.loggers import CSVLogger
-
-
-lightning_model = LightningModel(pytorch_model, learning_rate=LEARNING_RATE)
-
-callbacks = [
-    ModelCheckpoint(
-        save_top_k=1, mode="max", monitor="valid_acc"
-    )  # save top 1 model
-]
-logger = CSVLogger(save_dir="logs/", name="my-model")

pytorch-lightning_ipynb_new/code_lightningmodule/logger_csv_plot_basic.py

@@ -1,22 +0,0 @@
-import pandas as pd
-import matplotlib.pyplot as plt
-
-
-metrics = pd.read_csv(f"{trainer.logger.log_dir}/metrics.csv")
-
-aggreg_metrics = []
-agg_col = "epoch"
-for i, dfg in metrics.groupby(agg_col):
-    agg = dict(dfg.mean())
-    agg[agg_col] = i
-    aggreg_metrics.append(agg)
-
-df_metrics = pd.DataFrame(aggreg_metrics)
-df_metrics[["train_loss", "valid_loss"]].plot(
-    grid=True, legend=True, xlabel="Epoch", ylabel="Loss"
-)
-df_metrics[["train_acc", "valid_acc"]].plot(
-    grid=True, legend=True, xlabel="Epoch", ylabel="ACC"
-)
-
-plt.show()

pytorch-lightning_ipynb_new/code_lightningmodule/plot_confusion-matrix_basic.py

@@ -1,25 +0,0 @@
-from torchmetrics import ConfusionMatrix
-import matplotlib
-from mlxtend.plotting import plot_confusion_matrix
-
-
-cmat = ConfusionMatrix(num_classes=len(class_dict))
-
-for x, y in test_dataloader:
-
-    with torch.no_grad():
-        pred = lightning_model(x)
-    cmat(pred, y)
-
-cmat_tensor = cmat.compute()
-cmat = cmat_tensor.numpy()
-
-fig, ax = plot_confusion_matrix(
-    conf_mat=cmat,
-    class_names=class_dict.values(),
-    norm_colormap=matplotlib.colors.LogNorm()  
-    # normed colormaps highlight the off-diagonals 
-    # for high-accuracy models better
-)
-
-plt.show()

pytorch-lightning_ipynb_new/code_lightningmodule/plot_failurecases_basic.py

@@ -1,14 +0,0 @@
-# Append the folder that contains the
-# helper_data.py, helper_plotting.py, and helper_evaluate.py
-# files so we can import from them
-
-import sys
-
-sys.path.append("../pytorch_ipynb")
-
-from helper_plotting import show_examples
-
-
-show_examples(
-    model=lightning_model, data_loader=test_dataloader, class_dict=class_dict
-)

pytorch-lightning_ipynb_new/code_lightningmodule/plot_failurecases_unnormalize-1.py

@@ -1,22 +0,0 @@
-# Append the folder that contains the
-# helper_data.py, helper_plotting.py, and helper_evaluate.py
-# files so we can import from them
-
-import sys
-
-sys.path.append("../pytorch_ipynb")
-
-from helper_data import UnNormalize
-from helper_plotting import show_examples
-
-# We normalized each channel during training; here
-# we are reverting the normalization so that we
-# can plot them as images
-unnormalizer = UnNormalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
-
-show_examples(
-    model=lightning_model,
-    data_loader=test_dataloader,
-    unnormalizer=unnormalizer,
-    class_dict=class_dict,
-)

pytorch-lightning_ipynb_new/code_lightningmodule/trainer_nb_basic.py

@@ -1,19 +0,0 @@
-import time
-
-
-trainer = pl.Trainer(
-    max_epochs=NUM_EPOCHS,
-    callbacks=callbacks,
-    progress_bar_refresh_rate=50,  # recommended for notebooks
-    accelerator="auto",  # Uses GPUs or TPUs if available
-    devices="auto",  # Uses all available GPUs/TPUs if applicable
-    logger=logger,
-    deterministic=True,
-    log_every_n_steps=10,
-)
-
-start_time = time.time()
-trainer.fit(model=lightning_model, datamodule=data_module)
-
-runtime = (time.time() - start_time) / 60
-print(f"Training took {runtime:.2f} min in total.")

pytorch-lightning_ipynb_new/mlp/logs/my-model/version_0/hparams.yaml

@@ -1 +0,0 @@
-learning_rate: 0.005

pytorch-lightning_ipynb_new/mlp/logs/my-model/version_1/hparams.yaml

@@ -1 +0,0 @@
-learning_rate: 0.005

pytorch-lightning_ipynb_new/mlp/logs/my-model/version_2/hparams.yaml

@@ -1 +0,0 @@
-learning_rate: 0.005

pytorch-lightning_ipynb_new/mlp/logs/my-model/version_3/hparams.yaml

@@ -1 +0,0 @@
-learning_rate: 0.005