Inference
Contents
Inference#
This notebooks demonstrates various convolutional neural network (CNN) able to
segment UAV imagery into ground, water, and building classes. This process
is called semantic segmentation.
For the Python code used to train the models in this notebook, see train.ipynb instead.
The purpose of this functionality for the Lakehopper system is twofold:
Provide awareness of the situation under the drone at the moment of flight (âin-situâ). This is as opposed to beforehand (when a certain dataset was captured or when the drone previously collected data). A lake might have dried up or a boat might be present where it wasnât previously.
Provide a wide-area map based on orthographic imagery with features that are relevant to lakehopper. Other maps do not necessarily have the same feature definitions as lakehopper. The
waterclass for Lakehopper refers to a body of surface water where autonomous landing is possible. Swamps or water obscured by bridges or vegetation do not fit this definition. They are however still classified as âwaterâ on most topographic maps.
More information on everything from the model architecture to the motivation for using a CNN can be found in my 2022 masterâs dissertation Autonomous Path Planning and Landing Site Selection for Aquatic UAV.
The dataset is too large to upload via cloud storage (58.6 GiB). Please contact pieter@pfiers.net for a copy.
Google Colaboratory Setup#
Running in Google Colab is useful to inspect the models and preview the dataset. To train the model however, a more powerful runtime is necessary (like a GCE VPS with a TPU).
import sys
IN_COLAB = 'google.colab' in sys.modules
if IN_COLAB:
sys.path.insert(0, "/content/drive/MyDrive/lh-train/lakehopper_semseg")
from google.colab import drive
drive.mount('/content/drive')
Jupyter Notebook Setup#
from env_utils import is_notebook, find_repo
from pathlib import Path
IS_NOTEBOOK = is_notebook()
REPO_ROOT = find_repo(Path.cwd())
if IS_NOTEBOOK:
%load_ext autoreload
%autoreload 2
Imports#
from datetime import datetime
from itertools import islice
import sys, os, csv, json, re
import numpy as np
from IPython.display import clear_output
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import tensorflow as tf
from tensorflow.keras.models import load_model
# Convenience methods
from visualize import show, imshow_rgb, imshow_mask
from tpu import resolve_tpu_strategy, get_tpu_devices
from dataset import load_dataset, single_hot_to_color_mask, split_dataset_paths, read_metadata, has_all_classes, single_hot_to_categorical, categorical_to_single_label_mask_tf, categorical_to_one_hot_label_mask
from masks import colorize_byte_mask
from augment import Augment
from models.create import create_model, compile_model
from models.f1score import F1Score
from preprocess.colormaps import IV_ORTHO_MID_LABEL_COLOR_MAP as COLOR_MAP, IV_ORTHO_BG_COLOR as BG_COLOR
TPU Setup#
USE_TPU = os.environ.get('USE_TPU', False)
if os.environ.get('USE_TPU', False):
PARALLEL_OPS = tf.data.AUTOTUNE
tpu_strategy = resolve_tpu_strategy('us-node')
nbro_tpu_devices = len(get_tpu_devices())
print(f"{nbro_tpu_devices} TPU devices connected!")
else:
PARALLEL_OPS = None
tpu_strategy = None
Dataset Preparation#
CLASSES = ["water", "building"]
IMAGE_SIZE = (224, 224)
if IN_COLAB:
DATASET_FOLDER = "/content/drive/MyDrive/lh-train/tfr"
elif USE_TPU:
DATASET_FOLDER = "gs://semseg-data-aggregated-1/tfr"
elif REPO_ROOT is not None:
DATASET_FOLDER = str(REPO_ROOT / "vision/datasets/aggregated/tfr")
metadata_file_path = DATASET_FOLDER + "/metadata.json"
metadata = read_metadata(metadata_file_path)
TFR_PATTERN = DATASET_FOLDER + "/*.tfr"
if len(CLASSES) == 1:
def should_include_chip(chip_classes):
return all(class_ in chip_classes for class_ in CLASSES)
else:
def should_include_chip(chip_classes):
return any(class_ in chip_classes for class_ in CLASSES)
stem_nbro_label_pixels = metadata["nbroLabelPixels"]
applicable_chips = set()
for stem, nbro_label_pixels in stem_nbro_label_pixels.items():
if should_include_chip(nbro_label_pixels.keys()):
applicable_chips.add(stem)
tfr_paths = list(filter(
lambda p: Path(p).stem in applicable_chips,
tf.io.gfile.glob(TFR_PATTERN)
))
TRAIN_RATIO = 0.75
VALIDATION_RATIO = 0.22 # Remaining 8% is test
train_paths, validate_paths, test_paths = split_dataset_paths(tfr_paths, TRAIN_RATIO, VALIDATION_RATIO)
print(f"Found {len(tfr_paths)} tfrs. Splitting into {len(train_paths)} training, {len(validate_paths)} validation, and {len(test_paths)} test tfrs")
Found 2801 tfrs. Splitting into 2100 training, 616 validation, and 85 test tfrs
Load Dataset#
BATCH_SIZE = 64 # Using TPU v3-8 device => must be divisible by 8 for sharding
BUFFER_SIZE = 1000
def transform_mask(img, mask, weights = None):
if len(CLASSES) == 1:
mask_transformed = categorical_to_single_label_mask_tf(CLASSES.index("water") + 1, mask)
else:
mask_transformed = categorical_to_one_hot_label_mask(len(CLASSES) + 1, mask)
if weights is None:
return img, mask_transformed
else:
return img, categorical_to_one_hot_label_mask(len(CLASSES) + 1, mask), weights
test_dataset = (
load_dataset(test_paths, IMAGE_SIZE, None)
.map(transform_mask)
.prefetch(buffer_size=tf.data.AUTOTUNE)
)
2022-09-25 23:30:25.895008: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:980] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
2022-09-25 23:30:25.895348: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory
2022-09-25 23:30:25.895439: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcublas.so.11'; dlerror: libcublas.so.11: cannot open shared object file: No such file or directory
2022-09-25 23:30:25.895513: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcublasLt.so.11'; dlerror: libcublasLt.so.11: cannot open shared object file: No such file or directory
2022-09-25 23:30:25.895583: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcufft.so.10'; dlerror: libcufft.so.10: cannot open shared object file: No such file or directory
2022-09-25 23:30:25.895651: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcurand.so.10'; dlerror: libcurand.so.10: cannot open shared object file: No such file or directory
2022-09-25 23:30:25.895717: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcusolver.so.11'; dlerror: libcusolver.so.11: cannot open shared object file: No such file or directory
2022-09-25 23:30:25.895785: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcusparse.so.11'; dlerror: libcusparse.so.11: cannot open shared object file: No such file or directory
2022-09-25 23:30:25.895850: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudnn.so.8'; dlerror: libcudnn.so.8: cannot open shared object file: No such file or directory
2022-09-25 23:30:25.895859: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1934] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
2022-09-25 23:30:25.896403: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
Create Model#
INPUT_SHAPE = IMAGE_SIZE + (3,)
if IN_COLAB:
# MODELS_FOLDER = "/content/drive/MyDrive/lh-train/tfr"
assert(False)
elif USE_TPU:
# MODELS_FOLDER = "gs://semseg-data-aggregated-1/tfr"
assert(False)
elif REPO_ROOT is not None:
MODELS_FOLDER = str(REPO_ROOT / "vision/models")
def create_and_compile(encoder, decoder):
model = create_model(CLASSES, INPUT_SHAPE, encoder, decoder)
model = compile_model(model, CLASSES)
return model
def create_compile_and_load_model(encoder, decoder):
if tpu_strategy is not None:
with tpu_strategy.scope():
model = create_and_compile(encoder, decoder)
else:
model = create_and_compile(encoder, decoder)
model.load_weights(str(MODELS_FOLDER + f"/{decoder}_{encoder}_best.h5"))
return model
encoder = os.environ.get('ENCODER', 'EfficientNetB3')
decoder = os.environ.get('DECODER', 'UNet')
model = create_compile_and_load_model(encoder, decoder)
print(f"Model created with encoder: {encoder} and decoder: {decoder}")
if IS_NOTEBOOK:
model.summary()
Model created with encoder: EfficientNetB3 and decoder: UNet
Model: "UNet_EfficientNetB3"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
input_1 (InputLayer) [(None, 224, 224, 3 0 []
)]
rescaling (Rescaling) (None, 224, 224, 3) 0 ['input_1[0][0]']
normalization (Normalization) (None, 224, 224, 3) 7 ['rescaling[0][0]']
rescaling_1 (Rescaling) (None, 224, 224, 3) 0 ['normalization[0][0]']
stem_conv_pad (ZeroPadding2D) (None, 225, 225, 3) 0 ['rescaling_1[0][0]']
stem_conv (Conv2D) (None, 112, 112, 40 1080 ['stem_conv_pad[0][0]']
)
stem_bn (BatchNormalization) (None, 112, 112, 40 160 ['stem_conv[0][0]']
)
stem_activation (Activation) (None, 112, 112, 40 0 ['stem_bn[0][0]']
)
block1a_dwconv (DepthwiseConv2 (None, 112, 112, 40 360 ['stem_activation[0][0]']
D) )
block1a_bn (BatchNormalization (None, 112, 112, 40 160 ['block1a_dwconv[0][0]']
) )
block1a_activation (Activation (None, 112, 112, 40 0 ['block1a_bn[0][0]']
) )
block1a_se_squeeze (GlobalAver (None, 40) 0 ['block1a_activation[0][0]']
agePooling2D)
block1a_se_reshape (Reshape) (None, 1, 1, 40) 0 ['block1a_se_squeeze[0][0]']
block1a_se_reduce (Conv2D) (None, 1, 1, 10) 410 ['block1a_se_reshape[0][0]']
block1a_se_expand (Conv2D) (None, 1, 1, 40) 440 ['block1a_se_reduce[0][0]']
block1a_se_excite (Multiply) (None, 112, 112, 40 0 ['block1a_activation[0][0]',
) 'block1a_se_expand[0][0]']
block1a_project_conv (Conv2D) (None, 112, 112, 24 960 ['block1a_se_excite[0][0]']
)
block1a_project_bn (BatchNorma (None, 112, 112, 24 96 ['block1a_project_conv[0][0]']
lization) )
block1b_dwconv (DepthwiseConv2 (None, 112, 112, 24 216 ['block1a_project_bn[0][0]']
D) )
block1b_bn (BatchNormalization (None, 112, 112, 24 96 ['block1b_dwconv[0][0]']
) )
block1b_activation (Activation (None, 112, 112, 24 0 ['block1b_bn[0][0]']
) )
block1b_se_squeeze (GlobalAver (None, 24) 0 ['block1b_activation[0][0]']
agePooling2D)
block1b_se_reshape (Reshape) (None, 1, 1, 24) 0 ['block1b_se_squeeze[0][0]']
block1b_se_reduce (Conv2D) (None, 1, 1, 6) 150 ['block1b_se_reshape[0][0]']
block1b_se_expand (Conv2D) (None, 1, 1, 24) 168 ['block1b_se_reduce[0][0]']
block1b_se_excite (Multiply) (None, 112, 112, 24 0 ['block1b_activation[0][0]',
) 'block1b_se_expand[0][0]']
block1b_project_conv (Conv2D) (None, 112, 112, 24 576 ['block1b_se_excite[0][0]']
)
block1b_project_bn (BatchNorma (None, 112, 112, 24 96 ['block1b_project_conv[0][0]']
lization) )
block1b_drop (Dropout) (None, 112, 112, 24 0 ['block1b_project_bn[0][0]']
)
block1b_add (Add) (None, 112, 112, 24 0 ['block1b_drop[0][0]',
) 'block1a_project_bn[0][0]']
block2a_expand_conv (Conv2D) (None, 112, 112, 14 3456 ['block1b_add[0][0]']
4)
block2a_expand_bn (BatchNormal (None, 112, 112, 14 576 ['block2a_expand_conv[0][0]']
ization) 4)
block2a_expand_activation (Act (None, 112, 112, 14 0 ['block2a_expand_bn[0][0]']
ivation) 4)
block2a_dwconv_pad (ZeroPaddin (None, 113, 113, 14 0 ['block2a_expand_activation[0][0]
g2D) 4) ']
block2a_dwconv (DepthwiseConv2 (None, 56, 56, 144) 1296 ['block2a_dwconv_pad[0][0]']
D)
block2a_bn (BatchNormalization (None, 56, 56, 144) 576 ['block2a_dwconv[0][0]']
)
block2a_activation (Activation (None, 56, 56, 144) 0 ['block2a_bn[0][0]']
)
block2a_se_squeeze (GlobalAver (None, 144) 0 ['block2a_activation[0][0]']
agePooling2D)
block2a_se_reshape (Reshape) (None, 1, 1, 144) 0 ['block2a_se_squeeze[0][0]']
block2a_se_reduce (Conv2D) (None, 1, 1, 6) 870 ['block2a_se_reshape[0][0]']
block2a_se_expand (Conv2D) (None, 1, 1, 144) 1008 ['block2a_se_reduce[0][0]']
block2a_se_excite (Multiply) (None, 56, 56, 144) 0 ['block2a_activation[0][0]',
'block2a_se_expand[0][0]']
block2a_project_conv (Conv2D) (None, 56, 56, 32) 4608 ['block2a_se_excite[0][0]']
block2a_project_bn (BatchNorma (None, 56, 56, 32) 128 ['block2a_project_conv[0][0]']
lization)
block2b_expand_conv (Conv2D) (None, 56, 56, 192) 6144 ['block2a_project_bn[0][0]']
block2b_expand_bn (BatchNormal (None, 56, 56, 192) 768 ['block2b_expand_conv[0][0]']
ization)
block2b_expand_activation (Act (None, 56, 56, 192) 0 ['block2b_expand_bn[0][0]']
ivation)
block2b_dwconv (DepthwiseConv2 (None, 56, 56, 192) 1728 ['block2b_expand_activation[0][0]
D) ']
block2b_bn (BatchNormalization (None, 56, 56, 192) 768 ['block2b_dwconv[0][0]']
)
block2b_activation (Activation (None, 56, 56, 192) 0 ['block2b_bn[0][0]']
)
block2b_se_squeeze (GlobalAver (None, 192) 0 ['block2b_activation[0][0]']
agePooling2D)
block2b_se_reshape (Reshape) (None, 1, 1, 192) 0 ['block2b_se_squeeze[0][0]']
block2b_se_reduce (Conv2D) (None, 1, 1, 8) 1544 ['block2b_se_reshape[0][0]']
block2b_se_expand (Conv2D) (None, 1, 1, 192) 1728 ['block2b_se_reduce[0][0]']
block2b_se_excite (Multiply) (None, 56, 56, 192) 0 ['block2b_activation[0][0]',
'block2b_se_expand[0][0]']
block2b_project_conv (Conv2D) (None, 56, 56, 32) 6144 ['block2b_se_excite[0][0]']
block2b_project_bn (BatchNorma (None, 56, 56, 32) 128 ['block2b_project_conv[0][0]']
lization)
block2b_drop (Dropout) (None, 56, 56, 32) 0 ['block2b_project_bn[0][0]']
block2b_add (Add) (None, 56, 56, 32) 0 ['block2b_drop[0][0]',
'block2a_project_bn[0][0]']
block2c_expand_conv (Conv2D) (None, 56, 56, 192) 6144 ['block2b_add[0][0]']
block2c_expand_bn (BatchNormal (None, 56, 56, 192) 768 ['block2c_expand_conv[0][0]']
ization)
block2c_expand_activation (Act (None, 56, 56, 192) 0 ['block2c_expand_bn[0][0]']
ivation)
block2c_dwconv (DepthwiseConv2 (None, 56, 56, 192) 1728 ['block2c_expand_activation[0][0]
D) ']
block2c_bn (BatchNormalization (None, 56, 56, 192) 768 ['block2c_dwconv[0][0]']
)
block2c_activation (Activation (None, 56, 56, 192) 0 ['block2c_bn[0][0]']
)
block2c_se_squeeze (GlobalAver (None, 192) 0 ['block2c_activation[0][0]']
agePooling2D)
block2c_se_reshape (Reshape) (None, 1, 1, 192) 0 ['block2c_se_squeeze[0][0]']
block2c_se_reduce (Conv2D) (None, 1, 1, 8) 1544 ['block2c_se_reshape[0][0]']
block2c_se_expand (Conv2D) (None, 1, 1, 192) 1728 ['block2c_se_reduce[0][0]']
block2c_se_excite (Multiply) (None, 56, 56, 192) 0 ['block2c_activation[0][0]',
'block2c_se_expand[0][0]']
block2c_project_conv (Conv2D) (None, 56, 56, 32) 6144 ['block2c_se_excite[0][0]']
block2c_project_bn (BatchNorma (None, 56, 56, 32) 128 ['block2c_project_conv[0][0]']
lization)
block2c_drop (Dropout) (None, 56, 56, 32) 0 ['block2c_project_bn[0][0]']
block2c_add (Add) (None, 56, 56, 32) 0 ['block2c_drop[0][0]',
'block2b_add[0][0]']
block3a_expand_conv (Conv2D) (None, 56, 56, 192) 6144 ['block2c_add[0][0]']
block3a_expand_bn (BatchNormal (None, 56, 56, 192) 768 ['block3a_expand_conv[0][0]']
ization)
block3a_expand_activation (Act (None, 56, 56, 192) 0 ['block3a_expand_bn[0][0]']
ivation)
block3a_dwconv_pad (ZeroPaddin (None, 59, 59, 192) 0 ['block3a_expand_activation[0][0]
g2D) ']
block3a_dwconv (DepthwiseConv2 (None, 28, 28, 192) 4800 ['block3a_dwconv_pad[0][0]']
D)
block3a_bn (BatchNormalization (None, 28, 28, 192) 768 ['block3a_dwconv[0][0]']
)
block3a_activation (Activation (None, 28, 28, 192) 0 ['block3a_bn[0][0]']
)
block3a_se_squeeze (GlobalAver (None, 192) 0 ['block3a_activation[0][0]']
agePooling2D)
block3a_se_reshape (Reshape) (None, 1, 1, 192) 0 ['block3a_se_squeeze[0][0]']
block3a_se_reduce (Conv2D) (None, 1, 1, 8) 1544 ['block3a_se_reshape[0][0]']
block3a_se_expand (Conv2D) (None, 1, 1, 192) 1728 ['block3a_se_reduce[0][0]']
block3a_se_excite (Multiply) (None, 28, 28, 192) 0 ['block3a_activation[0][0]',
'block3a_se_expand[0][0]']
block3a_project_conv (Conv2D) (None, 28, 28, 48) 9216 ['block3a_se_excite[0][0]']
block3a_project_bn (BatchNorma (None, 28, 28, 48) 192 ['block3a_project_conv[0][0]']
lization)
block3b_expand_conv (Conv2D) (None, 28, 28, 288) 13824 ['block3a_project_bn[0][0]']
block3b_expand_bn (BatchNormal (None, 28, 28, 288) 1152 ['block3b_expand_conv[0][0]']
ization)
block3b_expand_activation (Act (None, 28, 28, 288) 0 ['block3b_expand_bn[0][0]']
ivation)
block3b_dwconv (DepthwiseConv2 (None, 28, 28, 288) 7200 ['block3b_expand_activation[0][0]
D) ']
block3b_bn (BatchNormalization (None, 28, 28, 288) 1152 ['block3b_dwconv[0][0]']
)
block3b_activation (Activation (None, 28, 28, 288) 0 ['block3b_bn[0][0]']
)
block3b_se_squeeze (GlobalAver (None, 288) 0 ['block3b_activation[0][0]']
agePooling2D)
block3b_se_reshape (Reshape) (None, 1, 1, 288) 0 ['block3b_se_squeeze[0][0]']
block3b_se_reduce (Conv2D) (None, 1, 1, 12) 3468 ['block3b_se_reshape[0][0]']
block3b_se_expand (Conv2D) (None, 1, 1, 288) 3744 ['block3b_se_reduce[0][0]']
block3b_se_excite (Multiply) (None, 28, 28, 288) 0 ['block3b_activation[0][0]',
'block3b_se_expand[0][0]']
block3b_project_conv (Conv2D) (None, 28, 28, 48) 13824 ['block3b_se_excite[0][0]']
block3b_project_bn (BatchNorma (None, 28, 28, 48) 192 ['block3b_project_conv[0][0]']
lization)
block3b_drop (Dropout) (None, 28, 28, 48) 0 ['block3b_project_bn[0][0]']
block3b_add (Add) (None, 28, 28, 48) 0 ['block3b_drop[0][0]',
'block3a_project_bn[0][0]']
block3c_expand_conv (Conv2D) (None, 28, 28, 288) 13824 ['block3b_add[0][0]']
block3c_expand_bn (BatchNormal (None, 28, 28, 288) 1152 ['block3c_expand_conv[0][0]']
ization)
block3c_expand_activation (Act (None, 28, 28, 288) 0 ['block3c_expand_bn[0][0]']
ivation)
block3c_dwconv (DepthwiseConv2 (None, 28, 28, 288) 7200 ['block3c_expand_activation[0][0]
D) ']
block3c_bn (BatchNormalization (None, 28, 28, 288) 1152 ['block3c_dwconv[0][0]']
)
block3c_activation (Activation (None, 28, 28, 288) 0 ['block3c_bn[0][0]']
)
block3c_se_squeeze (GlobalAver (None, 288) 0 ['block3c_activation[0][0]']
agePooling2D)
block3c_se_reshape (Reshape) (None, 1, 1, 288) 0 ['block3c_se_squeeze[0][0]']
block3c_se_reduce (Conv2D) (None, 1, 1, 12) 3468 ['block3c_se_reshape[0][0]']
block3c_se_expand (Conv2D) (None, 1, 1, 288) 3744 ['block3c_se_reduce[0][0]']
block3c_se_excite (Multiply) (None, 28, 28, 288) 0 ['block3c_activation[0][0]',
'block3c_se_expand[0][0]']
block3c_project_conv (Conv2D) (None, 28, 28, 48) 13824 ['block3c_se_excite[0][0]']
block3c_project_bn (BatchNorma (None, 28, 28, 48) 192 ['block3c_project_conv[0][0]']
lization)
block3c_drop (Dropout) (None, 28, 28, 48) 0 ['block3c_project_bn[0][0]']
block3c_add (Add) (None, 28, 28, 48) 0 ['block3c_drop[0][0]',
'block3b_add[0][0]']
block4a_expand_conv (Conv2D) (None, 28, 28, 288) 13824 ['block3c_add[0][0]']
block4a_expand_bn (BatchNormal (None, 28, 28, 288) 1152 ['block4a_expand_conv[0][0]']
ization)
block4a_expand_activation (Act (None, 28, 28, 288) 0 ['block4a_expand_bn[0][0]']
ivation)
block4a_dwconv_pad (ZeroPaddin (None, 29, 29, 288) 0 ['block4a_expand_activation[0][0]
g2D) ']
block4a_dwconv (DepthwiseConv2 (None, 14, 14, 288) 2592 ['block4a_dwconv_pad[0][0]']
D)
block4a_bn (BatchNormalization (None, 14, 14, 288) 1152 ['block4a_dwconv[0][0]']
)
block4a_activation (Activation (None, 14, 14, 288) 0 ['block4a_bn[0][0]']
)
block4a_se_squeeze (GlobalAver (None, 288) 0 ['block4a_activation[0][0]']
agePooling2D)
block4a_se_reshape (Reshape) (None, 1, 1, 288) 0 ['block4a_se_squeeze[0][0]']
block4a_se_reduce (Conv2D) (None, 1, 1, 12) 3468 ['block4a_se_reshape[0][0]']
block4a_se_expand (Conv2D) (None, 1, 1, 288) 3744 ['block4a_se_reduce[0][0]']
block4a_se_excite (Multiply) (None, 14, 14, 288) 0 ['block4a_activation[0][0]',
'block4a_se_expand[0][0]']
block4a_project_conv (Conv2D) (None, 14, 14, 96) 27648 ['block4a_se_excite[0][0]']
block4a_project_bn (BatchNorma (None, 14, 14, 96) 384 ['block4a_project_conv[0][0]']
lization)
block4b_expand_conv (Conv2D) (None, 14, 14, 576) 55296 ['block4a_project_bn[0][0]']
block4b_expand_bn (BatchNormal (None, 14, 14, 576) 2304 ['block4b_expand_conv[0][0]']
ization)
block4b_expand_activation (Act (None, 14, 14, 576) 0 ['block4b_expand_bn[0][0]']
ivation)
block4b_dwconv (DepthwiseConv2 (None, 14, 14, 576) 5184 ['block4b_expand_activation[0][0]
D) ']
block4b_bn (BatchNormalization (None, 14, 14, 576) 2304 ['block4b_dwconv[0][0]']
)
block4b_activation (Activation (None, 14, 14, 576) 0 ['block4b_bn[0][0]']
)
block4b_se_squeeze (GlobalAver (None, 576) 0 ['block4b_activation[0][0]']
agePooling2D)
block4b_se_reshape (Reshape) (None, 1, 1, 576) 0 ['block4b_se_squeeze[0][0]']
block4b_se_reduce (Conv2D) (None, 1, 1, 24) 13848 ['block4b_se_reshape[0][0]']
block4b_se_expand (Conv2D) (None, 1, 1, 576) 14400 ['block4b_se_reduce[0][0]']
block4b_se_excite (Multiply) (None, 14, 14, 576) 0 ['block4b_activation[0][0]',
'block4b_se_expand[0][0]']
block4b_project_conv (Conv2D) (None, 14, 14, 96) 55296 ['block4b_se_excite[0][0]']
block4b_project_bn (BatchNorma (None, 14, 14, 96) 384 ['block4b_project_conv[0][0]']
lization)
block4b_drop (Dropout) (None, 14, 14, 96) 0 ['block4b_project_bn[0][0]']
block4b_add (Add) (None, 14, 14, 96) 0 ['block4b_drop[0][0]',
'block4a_project_bn[0][0]']
block4c_expand_conv (Conv2D) (None, 14, 14, 576) 55296 ['block4b_add[0][0]']
block4c_expand_bn (BatchNormal (None, 14, 14, 576) 2304 ['block4c_expand_conv[0][0]']
ization)
block4c_expand_activation (Act (None, 14, 14, 576) 0 ['block4c_expand_bn[0][0]']
ivation)
block4c_dwconv (DepthwiseConv2 (None, 14, 14, 576) 5184 ['block4c_expand_activation[0][0]
D) ']
block4c_bn (BatchNormalization (None, 14, 14, 576) 2304 ['block4c_dwconv[0][0]']
)
block4c_activation (Activation (None, 14, 14, 576) 0 ['block4c_bn[0][0]']
)
block4c_se_squeeze (GlobalAver (None, 576) 0 ['block4c_activation[0][0]']
agePooling2D)
block4c_se_reshape (Reshape) (None, 1, 1, 576) 0 ['block4c_se_squeeze[0][0]']
block4c_se_reduce (Conv2D) (None, 1, 1, 24) 13848 ['block4c_se_reshape[0][0]']
block4c_se_expand (Conv2D) (None, 1, 1, 576) 14400 ['block4c_se_reduce[0][0]']
block4c_se_excite (Multiply) (None, 14, 14, 576) 0 ['block4c_activation[0][0]',
'block4c_se_expand[0][0]']
block4c_project_conv (Conv2D) (None, 14, 14, 96) 55296 ['block4c_se_excite[0][0]']
block4c_project_bn (BatchNorma (None, 14, 14, 96) 384 ['block4c_project_conv[0][0]']
lization)
block4c_drop (Dropout) (None, 14, 14, 96) 0 ['block4c_project_bn[0][0]']
block4c_add (Add) (None, 14, 14, 96) 0 ['block4c_drop[0][0]',
'block4b_add[0][0]']
block4d_expand_conv (Conv2D) (None, 14, 14, 576) 55296 ['block4c_add[0][0]']
block4d_expand_bn (BatchNormal (None, 14, 14, 576) 2304 ['block4d_expand_conv[0][0]']
ization)
block4d_expand_activation (Act (None, 14, 14, 576) 0 ['block4d_expand_bn[0][0]']
ivation)
block4d_dwconv (DepthwiseConv2 (None, 14, 14, 576) 5184 ['block4d_expand_activation[0][0]
D) ']
block4d_bn (BatchNormalization (None, 14, 14, 576) 2304 ['block4d_dwconv[0][0]']
)
block4d_activation (Activation (None, 14, 14, 576) 0 ['block4d_bn[0][0]']
)
block4d_se_squeeze (GlobalAver (None, 576) 0 ['block4d_activation[0][0]']
agePooling2D)
block4d_se_reshape (Reshape) (None, 1, 1, 576) 0 ['block4d_se_squeeze[0][0]']
block4d_se_reduce (Conv2D) (None, 1, 1, 24) 13848 ['block4d_se_reshape[0][0]']
block4d_se_expand (Conv2D) (None, 1, 1, 576) 14400 ['block4d_se_reduce[0][0]']
block4d_se_excite (Multiply) (None, 14, 14, 576) 0 ['block4d_activation[0][0]',
'block4d_se_expand[0][0]']
block4d_project_conv (Conv2D) (None, 14, 14, 96) 55296 ['block4d_se_excite[0][0]']
block4d_project_bn (BatchNorma (None, 14, 14, 96) 384 ['block4d_project_conv[0][0]']
lization)
block4d_drop (Dropout) (None, 14, 14, 96) 0 ['block4d_project_bn[0][0]']
block4d_add (Add) (None, 14, 14, 96) 0 ['block4d_drop[0][0]',
'block4c_add[0][0]']
block4e_expand_conv (Conv2D) (None, 14, 14, 576) 55296 ['block4d_add[0][0]']
block4e_expand_bn (BatchNormal (None, 14, 14, 576) 2304 ['block4e_expand_conv[0][0]']
ization)
block4e_expand_activation (Act (None, 14, 14, 576) 0 ['block4e_expand_bn[0][0]']
ivation)
block4e_dwconv (DepthwiseConv2 (None, 14, 14, 576) 5184 ['block4e_expand_activation[0][0]
D) ']
block4e_bn (BatchNormalization (None, 14, 14, 576) 2304 ['block4e_dwconv[0][0]']
)
block4e_activation (Activation (None, 14, 14, 576) 0 ['block4e_bn[0][0]']
)
block4e_se_squeeze (GlobalAver (None, 576) 0 ['block4e_activation[0][0]']
agePooling2D)
block4e_se_reshape (Reshape) (None, 1, 1, 576) 0 ['block4e_se_squeeze[0][0]']
block4e_se_reduce (Conv2D) (None, 1, 1, 24) 13848 ['block4e_se_reshape[0][0]']
block4e_se_expand (Conv2D) (None, 1, 1, 576) 14400 ['block4e_se_reduce[0][0]']
block4e_se_excite (Multiply) (None, 14, 14, 576) 0 ['block4e_activation[0][0]',
'block4e_se_expand[0][0]']
block4e_project_conv (Conv2D) (None, 14, 14, 96) 55296 ['block4e_se_excite[0][0]']
block4e_project_bn (BatchNorma (None, 14, 14, 96) 384 ['block4e_project_conv[0][0]']
lization)
block4e_drop (Dropout) (None, 14, 14, 96) 0 ['block4e_project_bn[0][0]']
block4e_add (Add) (None, 14, 14, 96) 0 ['block4e_drop[0][0]',
'block4d_add[0][0]']
block5a_expand_conv (Conv2D) (None, 14, 14, 576) 55296 ['block4e_add[0][0]']
block5a_expand_bn (BatchNormal (None, 14, 14, 576) 2304 ['block5a_expand_conv[0][0]']
ization)
block5a_expand_activation (Act (None, 14, 14, 576) 0 ['block5a_expand_bn[0][0]']
ivation)
block5a_dwconv (DepthwiseConv2 (None, 14, 14, 576) 14400 ['block5a_expand_activation[0][0]
D) ']
block5a_bn (BatchNormalization (None, 14, 14, 576) 2304 ['block5a_dwconv[0][0]']
)
block5a_activation (Activation (None, 14, 14, 576) 0 ['block5a_bn[0][0]']
)
block5a_se_squeeze (GlobalAver (None, 576) 0 ['block5a_activation[0][0]']
agePooling2D)
block5a_se_reshape (Reshape) (None, 1, 1, 576) 0 ['block5a_se_squeeze[0][0]']
block5a_se_reduce (Conv2D) (None, 1, 1, 24) 13848 ['block5a_se_reshape[0][0]']
block5a_se_expand (Conv2D) (None, 1, 1, 576) 14400 ['block5a_se_reduce[0][0]']
block5a_se_excite (Multiply) (None, 14, 14, 576) 0 ['block5a_activation[0][0]',
'block5a_se_expand[0][0]']
block5a_project_conv (Conv2D) (None, 14, 14, 136) 78336 ['block5a_se_excite[0][0]']
block5a_project_bn (BatchNorma (None, 14, 14, 136) 544 ['block5a_project_conv[0][0]']
lization)
block5b_expand_conv (Conv2D) (None, 14, 14, 816) 110976 ['block5a_project_bn[0][0]']
block5b_expand_bn (BatchNormal (None, 14, 14, 816) 3264 ['block5b_expand_conv[0][0]']
ization)
block5b_expand_activation (Act (None, 14, 14, 816) 0 ['block5b_expand_bn[0][0]']
ivation)
block5b_dwconv (DepthwiseConv2 (None, 14, 14, 816) 20400 ['block5b_expand_activation[0][0]
D) ']
block5b_bn (BatchNormalization (None, 14, 14, 816) 3264 ['block5b_dwconv[0][0]']
)
block5b_activation (Activation (None, 14, 14, 816) 0 ['block5b_bn[0][0]']
)
block5b_se_squeeze (GlobalAver (None, 816) 0 ['block5b_activation[0][0]']
agePooling2D)
block5b_se_reshape (Reshape) (None, 1, 1, 816) 0 ['block5b_se_squeeze[0][0]']
block5b_se_reduce (Conv2D) (None, 1, 1, 34) 27778 ['block5b_se_reshape[0][0]']
block5b_se_expand (Conv2D) (None, 1, 1, 816) 28560 ['block5b_se_reduce[0][0]']
block5b_se_excite (Multiply) (None, 14, 14, 816) 0 ['block5b_activation[0][0]',
'block5b_se_expand[0][0]']
block5b_project_conv (Conv2D) (None, 14, 14, 136) 110976 ['block5b_se_excite[0][0]']
block5b_project_bn (BatchNorma (None, 14, 14, 136) 544 ['block5b_project_conv[0][0]']
lization)
block5b_drop (Dropout) (None, 14, 14, 136) 0 ['block5b_project_bn[0][0]']
block5b_add (Add) (None, 14, 14, 136) 0 ['block5b_drop[0][0]',
'block5a_project_bn[0][0]']
block5c_expand_conv (Conv2D) (None, 14, 14, 816) 110976 ['block5b_add[0][0]']
block5c_expand_bn (BatchNormal (None, 14, 14, 816) 3264 ['block5c_expand_conv[0][0]']
ization)
block5c_expand_activation (Act (None, 14, 14, 816) 0 ['block5c_expand_bn[0][0]']
ivation)
block5c_dwconv (DepthwiseConv2 (None, 14, 14, 816) 20400 ['block5c_expand_activation[0][0]
D) ']
block5c_bn (BatchNormalization (None, 14, 14, 816) 3264 ['block5c_dwconv[0][0]']
)
block5c_activation (Activation (None, 14, 14, 816) 0 ['block5c_bn[0][0]']
)
block5c_se_squeeze (GlobalAver (None, 816) 0 ['block5c_activation[0][0]']
agePooling2D)
block5c_se_reshape (Reshape) (None, 1, 1, 816) 0 ['block5c_se_squeeze[0][0]']
block5c_se_reduce (Conv2D) (None, 1, 1, 34) 27778 ['block5c_se_reshape[0][0]']
block5c_se_expand (Conv2D) (None, 1, 1, 816) 28560 ['block5c_se_reduce[0][0]']
block5c_se_excite (Multiply) (None, 14, 14, 816) 0 ['block5c_activation[0][0]',
'block5c_se_expand[0][0]']
block5c_project_conv (Conv2D) (None, 14, 14, 136) 110976 ['block5c_se_excite[0][0]']
block5c_project_bn (BatchNorma (None, 14, 14, 136) 544 ['block5c_project_conv[0][0]']
lization)
block5c_drop (Dropout) (None, 14, 14, 136) 0 ['block5c_project_bn[0][0]']
block5c_add (Add) (None, 14, 14, 136) 0 ['block5c_drop[0][0]',
'block5b_add[0][0]']
block5d_expand_conv (Conv2D) (None, 14, 14, 816) 110976 ['block5c_add[0][0]']
block5d_expand_bn (BatchNormal (None, 14, 14, 816) 3264 ['block5d_expand_conv[0][0]']
ization)
block5d_expand_activation (Act (None, 14, 14, 816) 0 ['block5d_expand_bn[0][0]']
ivation)
block5d_dwconv (DepthwiseConv2 (None, 14, 14, 816) 20400 ['block5d_expand_activation[0][0]
D) ']
block5d_bn (BatchNormalization (None, 14, 14, 816) 3264 ['block5d_dwconv[0][0]']
)
block5d_activation (Activation (None, 14, 14, 816) 0 ['block5d_bn[0][0]']
)
block5d_se_squeeze (GlobalAver (None, 816) 0 ['block5d_activation[0][0]']
agePooling2D)
block5d_se_reshape (Reshape) (None, 1, 1, 816) 0 ['block5d_se_squeeze[0][0]']
block5d_se_reduce (Conv2D) (None, 1, 1, 34) 27778 ['block5d_se_reshape[0][0]']
block5d_se_expand (Conv2D) (None, 1, 1, 816) 28560 ['block5d_se_reduce[0][0]']
block5d_se_excite (Multiply) (None, 14, 14, 816) 0 ['block5d_activation[0][0]',
'block5d_se_expand[0][0]']
block5d_project_conv (Conv2D) (None, 14, 14, 136) 110976 ['block5d_se_excite[0][0]']
block5d_project_bn (BatchNorma (None, 14, 14, 136) 544 ['block5d_project_conv[0][0]']
lization)
block5d_drop (Dropout) (None, 14, 14, 136) 0 ['block5d_project_bn[0][0]']
block5d_add (Add) (None, 14, 14, 136) 0 ['block5d_drop[0][0]',
'block5c_add[0][0]']
block5e_expand_conv (Conv2D) (None, 14, 14, 816) 110976 ['block5d_add[0][0]']
block5e_expand_bn (BatchNormal (None, 14, 14, 816) 3264 ['block5e_expand_conv[0][0]']
ization)
block5e_expand_activation (Act (None, 14, 14, 816) 0 ['block5e_expand_bn[0][0]']
ivation)
block5e_dwconv (DepthwiseConv2 (None, 14, 14, 816) 20400 ['block5e_expand_activation[0][0]
D) ']
block5e_bn (BatchNormalization (None, 14, 14, 816) 3264 ['block5e_dwconv[0][0]']
)
block5e_activation (Activation (None, 14, 14, 816) 0 ['block5e_bn[0][0]']
)
block5e_se_squeeze (GlobalAver (None, 816) 0 ['block5e_activation[0][0]']
agePooling2D)
block5e_se_reshape (Reshape) (None, 1, 1, 816) 0 ['block5e_se_squeeze[0][0]']
block5e_se_reduce (Conv2D) (None, 1, 1, 34) 27778 ['block5e_se_reshape[0][0]']
block5e_se_expand (Conv2D) (None, 1, 1, 816) 28560 ['block5e_se_reduce[0][0]']
block5e_se_excite (Multiply) (None, 14, 14, 816) 0 ['block5e_activation[0][0]',
'block5e_se_expand[0][0]']
block5e_project_conv (Conv2D) (None, 14, 14, 136) 110976 ['block5e_se_excite[0][0]']
block5e_project_bn (BatchNorma (None, 14, 14, 136) 544 ['block5e_project_conv[0][0]']
lization)
block5e_drop (Dropout) (None, 14, 14, 136) 0 ['block5e_project_bn[0][0]']
block5e_add (Add) (None, 14, 14, 136) 0 ['block5e_drop[0][0]',
'block5d_add[0][0]']
block6a_expand_conv (Conv2D) (None, 14, 14, 816) 110976 ['block5e_add[0][0]']
block6a_expand_bn (BatchNormal (None, 14, 14, 816) 3264 ['block6a_expand_conv[0][0]']
ization)
block6a_expand_activation (Act (None, 14, 14, 816) 0 ['block6a_expand_bn[0][0]']
ivation)
block6a_dwconv_pad (ZeroPaddin (None, 17, 17, 816) 0 ['block6a_expand_activation[0][0]
g2D) ']
block6a_dwconv (DepthwiseConv2 (None, 7, 7, 816) 20400 ['block6a_dwconv_pad[0][0]']
D)
block6a_bn (BatchNormalization (None, 7, 7, 816) 3264 ['block6a_dwconv[0][0]']
)
block6a_activation (Activation (None, 7, 7, 816) 0 ['block6a_bn[0][0]']
)
block6a_se_squeeze (GlobalAver (None, 816) 0 ['block6a_activation[0][0]']
agePooling2D)
block6a_se_reshape (Reshape) (None, 1, 1, 816) 0 ['block6a_se_squeeze[0][0]']
block6a_se_reduce (Conv2D) (None, 1, 1, 34) 27778 ['block6a_se_reshape[0][0]']
block6a_se_expand (Conv2D) (None, 1, 1, 816) 28560 ['block6a_se_reduce[0][0]']
block6a_se_excite (Multiply) (None, 7, 7, 816) 0 ['block6a_activation[0][0]',
'block6a_se_expand[0][0]']
block6a_project_conv (Conv2D) (None, 7, 7, 232) 189312 ['block6a_se_excite[0][0]']
block6a_project_bn (BatchNorma (None, 7, 7, 232) 928 ['block6a_project_conv[0][0]']
lization)
block6b_expand_conv (Conv2D) (None, 7, 7, 1392) 322944 ['block6a_project_bn[0][0]']
block6b_expand_bn (BatchNormal (None, 7, 7, 1392) 5568 ['block6b_expand_conv[0][0]']
ization)
block6b_expand_activation (Act (None, 7, 7, 1392) 0 ['block6b_expand_bn[0][0]']
ivation)
block6b_dwconv (DepthwiseConv2 (None, 7, 7, 1392) 34800 ['block6b_expand_activation[0][0]
D) ']
block6b_bn (BatchNormalization (None, 7, 7, 1392) 5568 ['block6b_dwconv[0][0]']
)
block6b_activation (Activation (None, 7, 7, 1392) 0 ['block6b_bn[0][0]']
)
block6b_se_squeeze (GlobalAver (None, 1392) 0 ['block6b_activation[0][0]']
agePooling2D)
block6b_se_reshape (Reshape) (None, 1, 1, 1392) 0 ['block6b_se_squeeze[0][0]']
block6b_se_reduce (Conv2D) (None, 1, 1, 58) 80794 ['block6b_se_reshape[0][0]']
block6b_se_expand (Conv2D) (None, 1, 1, 1392) 82128 ['block6b_se_reduce[0][0]']
block6b_se_excite (Multiply) (None, 7, 7, 1392) 0 ['block6b_activation[0][0]',
'block6b_se_expand[0][0]']
block6b_project_conv (Conv2D) (None, 7, 7, 232) 322944 ['block6b_se_excite[0][0]']
block6b_project_bn (BatchNorma (None, 7, 7, 232) 928 ['block6b_project_conv[0][0]']
lization)
block6b_drop (Dropout) (None, 7, 7, 232) 0 ['block6b_project_bn[0][0]']
block6b_add (Add) (None, 7, 7, 232) 0 ['block6b_drop[0][0]',
'block6a_project_bn[0][0]']
block6c_expand_conv (Conv2D) (None, 7, 7, 1392) 322944 ['block6b_add[0][0]']
block6c_expand_bn (BatchNormal (None, 7, 7, 1392) 5568 ['block6c_expand_conv[0][0]']
ization)
block6c_expand_activation (Act (None, 7, 7, 1392) 0 ['block6c_expand_bn[0][0]']
ivation)
block6c_dwconv (DepthwiseConv2 (None, 7, 7, 1392) 34800 ['block6c_expand_activation[0][0]
D) ']
block6c_bn (BatchNormalization (None, 7, 7, 1392) 5568 ['block6c_dwconv[0][0]']
)
block6c_activation (Activation (None, 7, 7, 1392) 0 ['block6c_bn[0][0]']
)
block6c_se_squeeze (GlobalAver (None, 1392) 0 ['block6c_activation[0][0]']
agePooling2D)
block6c_se_reshape (Reshape) (None, 1, 1, 1392) 0 ['block6c_se_squeeze[0][0]']
block6c_se_reduce (Conv2D) (None, 1, 1, 58) 80794 ['block6c_se_reshape[0][0]']
block6c_se_expand (Conv2D) (None, 1, 1, 1392) 82128 ['block6c_se_reduce[0][0]']
block6c_se_excite (Multiply) (None, 7, 7, 1392) 0 ['block6c_activation[0][0]',
'block6c_se_expand[0][0]']
block6c_project_conv (Conv2D) (None, 7, 7, 232) 322944 ['block6c_se_excite[0][0]']
block6c_project_bn (BatchNorma (None, 7, 7, 232) 928 ['block6c_project_conv[0][0]']
lization)
block6c_drop (Dropout) (None, 7, 7, 232) 0 ['block6c_project_bn[0][0]']
block6c_add (Add) (None, 7, 7, 232) 0 ['block6c_drop[0][0]',
'block6b_add[0][0]']
block6d_expand_conv (Conv2D) (None, 7, 7, 1392) 322944 ['block6c_add[0][0]']
block6d_expand_bn (BatchNormal (None, 7, 7, 1392) 5568 ['block6d_expand_conv[0][0]']
ization)
block6d_expand_activation (Act (None, 7, 7, 1392) 0 ['block6d_expand_bn[0][0]']
ivation)
block6d_dwconv (DepthwiseConv2 (None, 7, 7, 1392) 34800 ['block6d_expand_activation[0][0]
D) ']
block6d_bn (BatchNormalization (None, 7, 7, 1392) 5568 ['block6d_dwconv[0][0]']
)
block6d_activation (Activation (None, 7, 7, 1392) 0 ['block6d_bn[0][0]']
)
block6d_se_squeeze (GlobalAver (None, 1392) 0 ['block6d_activation[0][0]']
agePooling2D)
block6d_se_reshape (Reshape) (None, 1, 1, 1392) 0 ['block6d_se_squeeze[0][0]']
block6d_se_reduce (Conv2D) (None, 1, 1, 58) 80794 ['block6d_se_reshape[0][0]']
block6d_se_expand (Conv2D) (None, 1, 1, 1392) 82128 ['block6d_se_reduce[0][0]']
block6d_se_excite (Multiply) (None, 7, 7, 1392) 0 ['block6d_activation[0][0]',
'block6d_se_expand[0][0]']
block6d_project_conv (Conv2D) (None, 7, 7, 232) 322944 ['block6d_se_excite[0][0]']
block6d_project_bn (BatchNorma (None, 7, 7, 232) 928 ['block6d_project_conv[0][0]']
lization)
block6d_drop (Dropout) (None, 7, 7, 232) 0 ['block6d_project_bn[0][0]']
block6d_add (Add) (None, 7, 7, 232) 0 ['block6d_drop[0][0]',
'block6c_add[0][0]']
block6e_expand_conv (Conv2D) (None, 7, 7, 1392) 322944 ['block6d_add[0][0]']
block6e_expand_bn (BatchNormal (None, 7, 7, 1392) 5568 ['block6e_expand_conv[0][0]']
ization)
block6e_expand_activation (Act (None, 7, 7, 1392) 0 ['block6e_expand_bn[0][0]']
ivation)
block6e_dwconv (DepthwiseConv2 (None, 7, 7, 1392) 34800 ['block6e_expand_activation[0][0]
D) ']
block6e_bn (BatchNormalization (None, 7, 7, 1392) 5568 ['block6e_dwconv[0][0]']
)
block6e_activation (Activation (None, 7, 7, 1392) 0 ['block6e_bn[0][0]']
)
block6e_se_squeeze (GlobalAver (None, 1392) 0 ['block6e_activation[0][0]']
agePooling2D)
block6e_se_reshape (Reshape) (None, 1, 1, 1392) 0 ['block6e_se_squeeze[0][0]']
block6e_se_reduce (Conv2D) (None, 1, 1, 58) 80794 ['block6e_se_reshape[0][0]']
block6e_se_expand (Conv2D) (None, 1, 1, 1392) 82128 ['block6e_se_reduce[0][0]']
block6e_se_excite (Multiply) (None, 7, 7, 1392) 0 ['block6e_activation[0][0]',
'block6e_se_expand[0][0]']
block6e_project_conv (Conv2D) (None, 7, 7, 232) 322944 ['block6e_se_excite[0][0]']
block6e_project_bn (BatchNorma (None, 7, 7, 232) 928 ['block6e_project_conv[0][0]']
lization)
block6e_drop (Dropout) (None, 7, 7, 232) 0 ['block6e_project_bn[0][0]']
block6e_add (Add) (None, 7, 7, 232) 0 ['block6e_drop[0][0]',
'block6d_add[0][0]']
block6f_expand_conv (Conv2D) (None, 7, 7, 1392) 322944 ['block6e_add[0][0]']
block6f_expand_bn (BatchNormal (None, 7, 7, 1392) 5568 ['block6f_expand_conv[0][0]']
ization)
block6f_expand_activation (Act (None, 7, 7, 1392) 0 ['block6f_expand_bn[0][0]']
ivation)
block6f_dwconv (DepthwiseConv2 (None, 7, 7, 1392) 34800 ['block6f_expand_activation[0][0]
D) ']
block6f_bn (BatchNormalization (None, 7, 7, 1392) 5568 ['block6f_dwconv[0][0]']
)
block6f_activation (Activation (None, 7, 7, 1392) 0 ['block6f_bn[0][0]']
)
block6f_se_squeeze (GlobalAver (None, 1392) 0 ['block6f_activation[0][0]']
agePooling2D)
block6f_se_reshape (Reshape) (None, 1, 1, 1392) 0 ['block6f_se_squeeze[0][0]']
block6f_se_reduce (Conv2D) (None, 1, 1, 58) 80794 ['block6f_se_reshape[0][0]']
block6f_se_expand (Conv2D) (None, 1, 1, 1392) 82128 ['block6f_se_reduce[0][0]']
block6f_se_excite (Multiply) (None, 7, 7, 1392) 0 ['block6f_activation[0][0]',
'block6f_se_expand[0][0]']
block6f_project_conv (Conv2D) (None, 7, 7, 232) 322944 ['block6f_se_excite[0][0]']
block6f_project_bn (BatchNorma (None, 7, 7, 232) 928 ['block6f_project_conv[0][0]']
lization)
block6f_drop (Dropout) (None, 7, 7, 232) 0 ['block6f_project_bn[0][0]']
block6f_add (Add) (None, 7, 7, 232) 0 ['block6f_drop[0][0]',
'block6e_add[0][0]']
block7a_expand_conv (Conv2D) (None, 7, 7, 1392) 322944 ['block6f_add[0][0]']
block7a_expand_bn (BatchNormal (None, 7, 7, 1392) 5568 ['block7a_expand_conv[0][0]']
ization)
block7a_expand_activation (Act (None, 7, 7, 1392) 0 ['block7a_expand_bn[0][0]']
ivation)
block7a_dwconv (DepthwiseConv2 (None, 7, 7, 1392) 12528 ['block7a_expand_activation[0][0]
D) ']
block7a_bn (BatchNormalization (None, 7, 7, 1392) 5568 ['block7a_dwconv[0][0]']
)
block7a_activation (Activation (None, 7, 7, 1392) 0 ['block7a_bn[0][0]']
)
block7a_se_squeeze (GlobalAver (None, 1392) 0 ['block7a_activation[0][0]']
agePooling2D)
block7a_se_reshape (Reshape) (None, 1, 1, 1392) 0 ['block7a_se_squeeze[0][0]']
block7a_se_reduce (Conv2D) (None, 1, 1, 58) 80794 ['block7a_se_reshape[0][0]']
block7a_se_expand (Conv2D) (None, 1, 1, 1392) 82128 ['block7a_se_reduce[0][0]']
block7a_se_excite (Multiply) (None, 7, 7, 1392) 0 ['block7a_activation[0][0]',
'block7a_se_expand[0][0]']
block7a_project_conv (Conv2D) (None, 7, 7, 384) 534528 ['block7a_se_excite[0][0]']
block7a_project_bn (BatchNorma (None, 7, 7, 384) 1536 ['block7a_project_conv[0][0]']
lization)
block7b_expand_conv (Conv2D) (None, 7, 7, 2304) 884736 ['block7a_project_bn[0][0]']
block7b_expand_bn (BatchNormal (None, 7, 7, 2304) 9216 ['block7b_expand_conv[0][0]']
ization)
block7b_expand_activation (Act (None, 7, 7, 2304) 0 ['block7b_expand_bn[0][0]']
ivation)
block7b_dwconv (DepthwiseConv2 (None, 7, 7, 2304) 20736 ['block7b_expand_activation[0][0]
D) ']
block7b_bn (BatchNormalization (None, 7, 7, 2304) 9216 ['block7b_dwconv[0][0]']
)
block7b_activation (Activation (None, 7, 7, 2304) 0 ['block7b_bn[0][0]']
)
block7b_se_squeeze (GlobalAver (None, 2304) 0 ['block7b_activation[0][0]']
agePooling2D)
block7b_se_reshape (Reshape) (None, 1, 1, 2304) 0 ['block7b_se_squeeze[0][0]']
block7b_se_reduce (Conv2D) (None, 1, 1, 96) 221280 ['block7b_se_reshape[0][0]']
block7b_se_expand (Conv2D) (None, 1, 1, 2304) 223488 ['block7b_se_reduce[0][0]']
block7b_se_excite (Multiply) (None, 7, 7, 2304) 0 ['block7b_activation[0][0]',
'block7b_se_expand[0][0]']
block7b_project_conv (Conv2D) (None, 7, 7, 384) 884736 ['block7b_se_excite[0][0]']
block7b_project_bn (BatchNorma (None, 7, 7, 384) 1536 ['block7b_project_conv[0][0]']
lization)
block7b_drop (Dropout) (None, 7, 7, 384) 0 ['block7b_project_bn[0][0]']
block7b_add (Add) (None, 7, 7, 384) 0 ['block7b_drop[0][0]',
'block7a_project_bn[0][0]']
top_conv (Conv2D) (None, 7, 7, 1536) 589824 ['block7b_add[0][0]']
top_bn (BatchNormalization) (None, 7, 7, 1536) 6144 ['top_conv[0][0]']
top_activation (Activation) (None, 7, 7, 1536) 0 ['top_bn[0][0]']
decoder_stage_0_upsampling (Up (None, 14, 14, 1536 0 ['top_activation[0][0]']
Sampling2D) )
decoder_stage_0_concat (Concat (None, 14, 14, 2352 0 ['decoder_stage_0_upsampling[0][0
enate) ) ]',
'block6a_expand_activation[0][0]
']
decoder_stage_0a_conv (Conv2D) (None, 14, 14, 256) 5419008 ['decoder_stage_0_concat[0][0]']
decoder_stage_0a_bn (BatchNorm (None, 14, 14, 256) 1024 ['decoder_stage_0a_conv[0][0]']
alization)
decoder_stage_0a_relu (Activat (None, 14, 14, 256) 0 ['decoder_stage_0a_bn[0][0]']
ion)
decoder_stage_0b_conv (Conv2D) (None, 14, 14, 256) 589824 ['decoder_stage_0a_relu[0][0]']
decoder_stage_0b_bn (BatchNorm (None, 14, 14, 256) 1024 ['decoder_stage_0b_conv[0][0]']
alization)
decoder_stage_0b_relu (Activat (None, 14, 14, 256) 0 ['decoder_stage_0b_bn[0][0]']
ion)
decoder_stage_1_upsampling (Up (None, 28, 28, 256) 0 ['decoder_stage_0b_relu[0][0]']
Sampling2D)
decoder_stage_1_concat (Concat (None, 28, 28, 544) 0 ['decoder_stage_1_upsampling[0][0
enate) ]',
'block4a_expand_activation[0][0]
']
decoder_stage_1a_conv (Conv2D) (None, 28, 28, 128) 626688 ['decoder_stage_1_concat[0][0]']
decoder_stage_1a_bn (BatchNorm (None, 28, 28, 128) 512 ['decoder_stage_1a_conv[0][0]']
alization)
decoder_stage_1a_relu (Activat (None, 28, 28, 128) 0 ['decoder_stage_1a_bn[0][0]']
ion)
decoder_stage_1b_conv (Conv2D) (None, 28, 28, 128) 147456 ['decoder_stage_1a_relu[0][0]']
decoder_stage_1b_bn (BatchNorm (None, 28, 28, 128) 512 ['decoder_stage_1b_conv[0][0]']
alization)
decoder_stage_1b_relu (Activat (None, 28, 28, 128) 0 ['decoder_stage_1b_bn[0][0]']
ion)
decoder_stage_2_upsampling (Up (None, 56, 56, 128) 0 ['decoder_stage_1b_relu[0][0]']
Sampling2D)
decoder_stage_2_concat (Concat (None, 56, 56, 320) 0 ['decoder_stage_2_upsampling[0][0
enate) ]',
'block3a_expand_activation[0][0]
']
decoder_stage_2a_conv (Conv2D) (None, 56, 56, 64) 184320 ['decoder_stage_2_concat[0][0]']
decoder_stage_2a_bn (BatchNorm (None, 56, 56, 64) 256 ['decoder_stage_2a_conv[0][0]']
alization)
decoder_stage_2a_relu (Activat (None, 56, 56, 64) 0 ['decoder_stage_2a_bn[0][0]']
ion)
decoder_stage_2b_conv (Conv2D) (None, 56, 56, 64) 36864 ['decoder_stage_2a_relu[0][0]']
decoder_stage_2b_bn (BatchNorm (None, 56, 56, 64) 256 ['decoder_stage_2b_conv[0][0]']
alization)
decoder_stage_2b_relu (Activat (None, 56, 56, 64) 0 ['decoder_stage_2b_bn[0][0]']
ion)
decoder_stage_3_upsampling (Up (None, 112, 112, 64 0 ['decoder_stage_2b_relu[0][0]']
Sampling2D) )
decoder_stage_3_concat (Concat (None, 112, 112, 20 0 ['decoder_stage_3_upsampling[0][0
enate) 8) ]',
'block2a_expand_activation[0][0]
']
decoder_stage_3a_conv (Conv2D) (None, 112, 112, 32 59904 ['decoder_stage_3_concat[0][0]']
)
decoder_stage_3a_bn (BatchNorm (None, 112, 112, 32 128 ['decoder_stage_3a_conv[0][0]']
alization) )
decoder_stage_3a_relu (Activat (None, 112, 112, 32 0 ['decoder_stage_3a_bn[0][0]']
ion) )
decoder_stage_3b_conv (Conv2D) (None, 112, 112, 32 9216 ['decoder_stage_3a_relu[0][0]']
)
decoder_stage_3b_bn (BatchNorm (None, 112, 112, 32 128 ['decoder_stage_3b_conv[0][0]']
alization) )
decoder_stage_3b_relu (Activat (None, 112, 112, 32 0 ['decoder_stage_3b_bn[0][0]']
ion) )
decoder_stage_4_upsampling (Up (None, 224, 224, 32 0 ['decoder_stage_3b_relu[0][0]']
Sampling2D) )
decoder_stage_4a_conv (Conv2D) (None, 224, 224, 16 4608 ['decoder_stage_4_upsampling[0][0
) ]']
decoder_stage_4a_bn (BatchNorm (None, 224, 224, 16 64 ['decoder_stage_4a_conv[0][0]']
alization) )
decoder_stage_4a_relu (Activat (None, 224, 224, 16 0 ['decoder_stage_4a_bn[0][0]']
ion) )
decoder_stage_4b_conv (Conv2D) (None, 224, 224, 16 2304 ['decoder_stage_4a_relu[0][0]']
)
decoder_stage_4b_bn (BatchNorm (None, 224, 224, 16 64 ['decoder_stage_4b_conv[0][0]']
alization) )
decoder_stage_4b_relu (Activat (None, 224, 224, 16 0 ['decoder_stage_4b_bn[0][0]']
ion) )
final_conv (Conv2D) (None, 224, 224, 3) 435 ['decoder_stage_4b_relu[0][0]']
==================================================================================================
Total params: 17,868,130
Trainable params: 17,778,843
Non-trainable params: 89,287
__________________________________________________________________________________________________
Inference#
def predict_single(image):
prediction = model.predict(image[tf.newaxis, ...], verbose=0)[0]
return single_hot_to_color_mask(prediction, CLASSES, COLOR_MAP, BG_COLOR)
def dataset_tuple_to_images(image, label):
return (
image,
single_hot_to_color_mask(label, CLASSES, COLOR_MAP, BG_COLOR),
tf.convert_to_tensor(predict_single(image))
)
def show_predictions(dataset):
show([dataset_tuple_to_images(image, label) for image, label in dataset])
show_predictions(test_dataset)
Tensor("BroadcastTo:0", shape=(224, 224, 3), dtype=int32)
Multi-model inference#
INFERENCE_MODEL_NAMES = [("FPN", "EfficientNetB3"), ("UNet", "EfficientNetB3"), ("FCN", "EfficientNetB3"), ("FCN", "MobileNetV2")]
inference_models = [create_compile_and_load_model(encoder, decoder) for decoder, encoder in INFERENCE_MODEL_NAMES]
Preprocessed input shape: (None, 224, 224, 3)
SAMPLE_IMAGE_INDICES = [83, 80, 63]
sample_images = [list(test_dataset)[i] for i in SAMPLE_IMAGE_INDICES]
rows_cols = (len(sample_images), len(inference_models) + 2) # + 2 for image and label
fig, axes = plt.subplots(
*rows_cols,
dpi=150,
gridspec_kw = {'wspace':0.05, 'hspace':0.05}
)
fig.set_figwidth(9.6)
TITLE_Y_OFFSET = -0.3
for i, (row, (image, label)) in enumerate(zip(axes, sample_images)):
ax_image, ax_label, *ax_predictions = row
imshow_rgb(ax_image, image)
ax_image.axis("off")
imshow_mask(ax_label, single_hot_to_color_mask(label, CLASSES, COLOR_MAP, BG_COLOR))
ax_label.axis("off")
if i == len(sample_images) - 1:
ax_image.set_title("Image", y=TITLE_Y_OFFSET)
ax_label.set_title("GT", y=TITLE_Y_OFFSET)
for j, (ax, (decoder, encoder), model) in enumerate(zip(ax_predictions, INFERENCE_MODEL_NAMES, inference_models)):
prediction = tf.convert_to_tensor(single_hot_to_color_mask(
model.predict(image[tf.newaxis, ...], verbose=0)[0],
CLASSES, COLOR_MAP, BG_COLOR
))
imshow_mask(ax, prediction)
ax.axis("off")
if i == len(sample_images) - 1:
ax.set_title(f"{decoder}+{re.sub(r'[a-z]', '', encoder)}", y=TITLE_Y_OFFSET)
plt.show()
Metrics#
ENCODERS = ("MobileNetV2", "InceptionResNetV2", "EfficientNetB3")
DECODERS = ("UNet", "FCN", "FPN")
models_train_metrics = {}
for encoder in ENCODERS:
for decoder in DECODERS:
with open(str(MODELS_FOLDER + f"/{decoder}_{encoder}_metrics.json"), "r") as f:
models_train_metrics[(decoder, encoder)] = json.load(f)
COLORS = ['c', 'y', 'm', 'g']
LINESTYLES = [(0, (7, 2.5)), (0, (3, 2.5, 1.7, 2.5)), (0, (3, 2.5, 1.7, 2.5)), (0, (1, 1.7))]
EPOCHS = list(range(0, 40))
VAL_TRAIN_LOSS_MODEL = ("FPN", "EfficientNetB3")
VAL_LOSS_MODELS = [("FPN", "EfficientNetB3"), ("UNet", "EfficientNetB3"), ("FCN", "EfficientNetB3"), ("FCN", "MobileNetV2")]
def ax_common(ax):
ax.set_ylim(0, 1.2)
ax.set_xlim(-1, 40)
ax.set_xlabel('Epoch')
ax.set_ylabel('Loss')
def plot_all_val_loss(ax):
ax.set_title('(a)')
for i, (decoder, encoder) in enumerate(VAL_LOSS_MODELS):
metrics = models_train_metrics[(decoder, encoder)]
y = metrics["val_loss"]
ax.plot(EPOCHS[:len(y)], y, label=f'{decoder} {encoder}', marker=',', color=COLORS[i], linestyle=LINESTYLES[i])
ax.legend(loc='upper right', title='Validation loss')
def plot_val_train_loss(ax):
ax.set_title('(b)')
metrics = models_train_metrics[VAL_TRAIN_LOSS_MODEL]
y_train = metrics["loss"]
y_val = metrics["val_loss"]
ax.set_xlim(-1, len(y_train))
val_color = COLORS[0]
val_linestyle = LINESTYLES[0]
ax.plot(EPOCHS[:len(y_val)], y_val, color=val_color, marker=',', linestyle=val_linestyle, label='Validation')
ax.plot(EPOCHS[:len(y_train)], y_train , color='orange', marker=',', linestyle=(0, (1, 1.7)), label='Train')
ax.legend(loc='upper right', title='FPN + EfficientNetB3 loss')
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4), dpi=150)
ax_common(ax1)
ax_common(ax2)
plot_all_val_loss(ax1)
plot_val_train_loss(ax2)
plt.show()
if IS_NOTEBOOK:
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'validation'], loc='upper left')
plt.show()
Create LaTeX table
out = """
\\begin{tabular}{c c c c c c c}
\t\\specialrule{.1em}{.1em}{.3em}
\tDecoder & Encoder & Acc. & Precision & Recall & F1-score & MIoU \\\\
"""
for decoder in DECODERS:
out += "\t\\specialrule{.05em}{.3em}{.3em}\n"
for encoder_i, encoder in enumerate(ENCODERS):
model_metrics = models_train_metrics[(decoder, encoder)]
i_lowest_loss = np.argmin(model_metrics["val_loss"])
print(f"Lowest loss for {decoder} {encoder} is at epoch {i_lowest_loss}")
accuracy = to_str_2_dec(model_metrics['val_accuracy'][i_lowest_loss])
precision = to_str_2_dec(model_metrics['val_precision_1'][i_lowest_loss])
recall = to_str_2_dec(model_metrics['val_recall_1'][i_lowest_loss])
f1_score = to_str_2_dec(model_metrics['val_f1-score'][i_lowest_loss])
miou = to_str_2_dec(model_metrics['val_mean_iou'][i_lowest_loss])
if encoder_i == len(ENCODERS) // 2:
out += f"\t{decoder}\t"
else:
out += "\t\t\t"
out += f"& {encoder.ljust(max(len(e) for e in ENCODERS))} & "
out += f"{accuracy} & {precision} & {recall} & {f1_score} & {miou} \\\\\n"
out += """
\t\\specialrule{.1em}{.3em}{.1em}
\\end{tabular}
"""
print(out)
Lowest loss for UNet MobileNetV2 is at epoch 20
Lowest loss for UNet InceptionResNetV2 is at epoch 18
Lowest loss for UNet EfficientNetB3 is at epoch 36
Lowest loss for FCN MobileNetV2 is at epoch 13
Lowest loss for FCN InceptionResNetV2 is at epoch 35
Lowest loss for FCN EfficientNetB3 is at epoch 39
Lowest loss for FPN MobileNetV2 is at epoch 10
Lowest loss for FPN InceptionResNetV2 is at epoch 5
Lowest loss for FPN EfficientNetB3 is at epoch 19
\begin{tabular}{c c c c c c c}
\specialrule{.1em}{.1em}{.3em}
Decoder & Encoder & Acc. & Precision & Recall & F1-score & MIoU \\
\specialrule{.05em}{.3em}{.3em}
& MobileNetV2 & 0.92 & 0.92 & 0.91 & 0.91 & 0.72 \\
UNet & InceptionResNetV2 & 0.94 & 0.94 & 0.94 & 0.94 & 0.77 \\
& EfficientNetB3 & 0.96 & 0.96 & 0.95 & 0.96 & 0.81 \\
\specialrule{.05em}{.3em}{.3em}
& MobileNetV2 & 0.69 & 0.47 & 0.15 & 0.22 & 0.49 \\
FCN & InceptionResNetV2 & 0.92 & 0.93 & 0.90 & 0.92 & 0.75 \\
& EfficientNetB3 & 0.95 & 0.96 & 0.93 & 0.94 & 0.81 \\
\specialrule{.05em}{.3em}{.3em}
& MobileNetV2 & 0.91 & 0.92 & 0.91 & 0.91 & 0.71 \\
FPN & InceptionResNetV2 & 0.94 & 0.94 & 0.94 & 0.94 & 0.76 \\
& EfficientNetB3 & 0.96 & 0.96 & 0.96 & 0.96 & 0.83 \\
\specialrule{.1em}{.3em}{.1em}
\end{tabular}