People

@article{521,
  author = {Aldrin Ngorima, Albert Helberg, Marelie Davel},
  title = {Simplified Temporal Convolutional-Based Channel Estimation for a WiFi Vehicular Communication Channel},
  abstract = {},
  year = {2025},
  journal = {IEEE 3rd Wireless Africa Conference (WAC)},
  pages = {1 - 5},
  month = {02/2025},
  publisher = {IEEE},
  address = {Pretoria, South Africa},
  isbn = {979-8-3315-1758-8},
  doi = {10.1109/WAC63911.2025.10992609},
}

@article{520,
  author = {William Brooks, Marelie Davel, Coenraad Mouton},
  title = {Does Simple Trump Complex? Comparing Strategies for Adversarial Robustness in DNNs},
  abstract = {},
  year = {2024},
  journal = {Artificial Intelligence Research. SACAIR 2024. Communications in Computer and Information Science},
  volume = {vol 2326},
  pages = {253 - 269},
  month = {12/2024},
  publisher = {Springer Nature Switzerland},
  address = {Cham},
  doi = {https://doi.org/10.1007/978-3-031-78255-8_15},
}

<p>Batch normalization (BatchNorm) is a popular layer normalization technique used when training deep neural networks. It has been shown to enhance the training speed and accuracy of deep learning models. However, the mechanics by which BatchNorm achieves these benefits is an active area of research, and different perspectives have been proposed. In this paper, we investigate the effect of BatchNorm on the resulting hidden representations, that is, the vectors of activation values formed as samples are processed at each hidden layer. Specifically, we consider the sparsity of these representations, as well as their implicit clustering – the creation of groups of representations that are similar to some extent. We contrast image classification models trained with and without batch normalization and highlight consistent differences observed. These findings highlight that BatchNorm’s effect on representational sparsity is not a significant factor affecting generalization, while the representations of models trained with BatchNorm tend to show more advantageous clustering characteristics.</p>

@article{518,
  author = {Harmen Potgieter, Coenraad Mouton, Marelie Davel},
  title = {Impact of Batch Normalization on Convolutional Network Representations},
  abstract = {<p>Batch normalization (BatchNorm) is a popular layer normalization technique used when training deep neural networks. It has been shown to enhance the training speed and accuracy of deep learning models. However, the mechanics by which BatchNorm achieves these benefits is an active area of research, and different perspectives have been proposed. In this paper, we investigate the effect of BatchNorm on the resulting hidden representations, that is, the vectors of activation values formed as samples are processed at each hidden layer. Specifically, we consider the sparsity of these representations, as well as their implicit clustering – the creation of groups of representations that are similar to some extent. We contrast image classification models trained with and without batch normalization and highlight consistent differences observed. These findings highlight that BatchNorm’s effect on representational sparsity is not a significant factor affecting generalization, while the representations of models trained with BatchNorm tend to show more advantageous clustering characteristics.</p>},
  year = {2024},
  journal = {Artificial Intelligence Research (SACAIR 2024)},
  volume = {vol 2326},
  pages = {235 - 252},
  month = {12/2024},
  publisher = {Springer Nature Switzerland},
  address = {Cham},
  doi = {https://doi.org/10.1007/978-3-031-78255-8_14},
}

<p><span style="-webkit-text-stroke-width:0px;background-color:rgb(255, 255, 255);color:rgb(34, 34, 34);display:inline !important;float:none;font-family:Merriweather, serif;font-size:18px;font-style:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-weight:400;letter-spacing:normal;orphans:2;text-align:start;text-decoration-color:initial;text-decoration-style:initial;text-decoration-thickness:initial;text-indent:0px;text-transform:none;white-space:normal;widows:2;word-spacing:0px;">Due to the scarcity of data in low-resourced languages, the development of language models for these languages has been very slow. Currently, pre-trained language models have gained popularity in natural language processing, especially, in developing domain-specific models for low-resourced languages. In this study, we experiment with the impact of using occlusion-based techniques when training a language model for a text generation task. We curate 2 new datasets, the Sepedi monolingual (SepMono) dataset from several South African resources and the Sepedi radio news (SepNews) dataset from the radio news domain. We use the SepMono dataset to pre-train transformer-based models using the occlusion and non-occlusion pre-training techniques and compare performance. The SepNews dataset is specifically used for fine-tuning. Our results show that the non-occlusion models perform better compared to the occlusion-based models when measuring validation loss and perplexity. However, analysis of the generated text using the BLEU score metric, which measures the quality of the generated text, shows a slightly higher BLEU score for the occlusion-based models compared to the non-occlusion models.</span></p>

@article{517,
  author = {Simon Ramalepe, Thipe Modipa, Marelie Davel},
  title = {Pre-training a Transformer-Based Generative Model Using a Small Sepedi Dataset},
  abstract = {<p><span style="-webkit-text-stroke-width:0px;background-color:rgb(255, 255, 255);color:rgb(34, 34, 34);display:inline !important;float:none;font-family:Merriweather, serif;font-size:18px;font-style:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-weight:400;letter-spacing:normal;orphans:2;text-align:start;text-decoration-color:initial;text-decoration-style:initial;text-decoration-thickness:initial;text-indent:0px;text-transform:none;white-space:normal;widows:2;word-spacing:0px;">Due to the scarcity of data in low-resourced languages, the development of language models for these languages has been very slow. Currently, pre-trained language models have gained popularity in natural language processing, especially, in developing domain-specific models for low-resourced languages. In this study, we experiment with the impact of using occlusion-based techniques when training a language model for a text generation task. We curate 2 new datasets, the Sepedi monolingual (SepMono) dataset from several South African resources and the Sepedi radio news (SepNews) dataset from the radio news domain. We use the SepMono dataset to pre-train transformer-based models using the occlusion and non-occlusion pre-training techniques and compare performance. The SepNews dataset is specifically used for fine-tuning. Our results show that the non-occlusion models perform better compared to the occlusion-based models when measuring validation loss and perplexity. However, analysis of the generated text using the BLEU score metric, which measures the quality of the generated text, shows a slightly higher BLEU score for the occlusion-based models compared to the non-occlusion models.</span></p>},
  year = {2024},
  journal = {Artificial Intelligence Research. SACAIR 2024. Communications in Computer and Information Science},
  volume = {vol 2326},
  pages = {319-333},
  month = {12/2024},
  publisher = {Springer Nature Switzerland},
  address = {Cham},
  doi = {https://doi.org/10.1007/978-3-031-78255-8_19},
}

<p><span style="-webkit-text-stroke-width:0px;background-color:rgb(255, 255, 255);color:rgb(34, 34, 34);display:inline !important;float:none;font-family:Merriweather, serif;font-size:18px;font-style:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-weight:400;letter-spacing:normal;orphans:2;text-align:start;text-decoration-color:initial;text-decoration-style:initial;text-decoration-thickness:initial;text-indent:0px;text-transform:none;white-space:normal;widows:2;word-spacing:0px;">Vehicular communication systems face significant challenges due to high mobility and rapidly changing environments, which affect the channel over which the signals travel. To address these challenges, neural network (NN)-based channel estimation methods have been suggested. These methods are primarily trained&nbsp;on high signal-to-noise ratio (SNR) with the assumption that training&nbsp;a NN in less noisy conditions can result in good generalisation.&nbsp;This study examines the effectiveness of training NN-based channel estimators on mixed SNR datasets compared to training solely on&nbsp;high SNR datasets, as seen in several related works. Estimators evaluated in this work include an architecture that uses convolutional layers and self-attention mechanisms; a method that employs temporal convolutional networks and data pilot-aided estimation; two methods that combine classical methods with multilayer perceptrons; and&nbsp;the current state-of-the-art model that combines Long-Short-Term Memory networks with data pilot-aided and temporal averaging methods&nbsp;as post processing. Our results indicate that using only high SNR&nbsp;data for training is not always optimal, and the SNR range in&nbsp;the training dataset should be treated as a hyperparameter that can&nbsp;be adjusted for better performance. This is illustrated by the better performance of some models in low SNR conditions when trained on&nbsp;the mixed SNR dataset, as opposed to when trained exclusively on&nbsp;high SNR data.</span></p>

@article{516,
  author = {Aldrin Ngorima, Albert Helberg, Marelie Davel},
  title = {Neural Network-Based Vehicular Channel Estimation Performance: Effect of Noise in the Training Set},
  abstract = {<p><span style="-webkit-text-stroke-width:0px;background-color:rgb(255, 255, 255);color:rgb(34, 34, 34);display:inline !important;float:none;font-family:Merriweather, serif;font-size:18px;font-style:normal;font-variant-caps:normal;font-variant-ligatures:normal;font-weight:400;letter-spacing:normal;orphans:2;text-align:start;text-decoration-color:initial;text-decoration-style:initial;text-decoration-thickness:initial;text-indent:0px;text-transform:none;white-space:normal;widows:2;word-spacing:0px;">Vehicular communication systems face significant challenges due to high mobility and rapidly changing environments, which affect the channel over which the signals travel. To address these challenges, neural network (NN)-based channel estimation methods have been suggested. These methods are primarily trained on high signal-to-noise ratio (SNR) with the assumption that training a NN in less noisy conditions can result in good generalisation. This study examines the effectiveness of training NN-based channel estimators on mixed SNR datasets compared to training solely on high SNR datasets, as seen in several related works. Estimators evaluated in this work include an architecture that uses convolutional layers and self-attention mechanisms; a method that employs temporal convolutional networks and data pilot-aided estimation; two methods that combine classical methods with multilayer perceptrons; and the current state-of-the-art model that combines Long-Short-Term Memory networks with data pilot-aided and temporal averaging methods as post processing. Our results indicate that using only high SNR data for training is not always optimal, and the SNR range in the training dataset should be treated as a hyperparameter that can be adjusted for better performance. This is illustrated by the better performance of some models in low SNR conditions when trained on the mixed SNR dataset, as opposed to when trained exclusively on high SNR data.</span></p>},
  year = {2024},
  journal = {Artificial Intelligence Research. SACAIR 2024. Communications in Computer and Information Science},
  volume = {vol 2326},
  pages = {192 - 206},
  month = {12/2024},
  publisher = {Springer Nature Switzerland},
  address = {Cham},
  isbn = {978-3-031-78255-8},
  doi = {https://doi.org/10.1007/978-3-031-78255-8_12},
}