People

The design of minimum phase finite impulse response (FIR) filters is considered. The study demonstrates that the residual errors achieved by current state-of-the-art design methods are nowhere near the smallest error possible on a finite resolution digital computer. This is shown to be due to conceptual errors in the literature pertaining to what constitutes a factorable linear phase filter. This study shows that factorisation is possible with a zero residual error (in the absence of machine finite resolution error) if the linear operator or matrix representing the linear phase filter is positive definite. Methodology is proposed able to design a minimum phase filter that is optimal—in the sense that the residual error is limited only by the finite precision of the digital computer, with no systematic error. The study presents practical application of the proposed methodology by designing two minimum phase Chebyshev FIR filters. Results are compared to state-of-the-art methods from the literature, and it is shown that the proposed methodology is able to reduce currently achievable residual errors by several orders of magnitude.

@article{506,
  author = {Jan Olivier, Etienne Barnard},
  title = {Minimum phase finite impulse response filter design},
  abstract = {The design of minimum phase finite impulse response (FIR) filters is considered. The study demonstrates that the residual errors achieved by current state-of-the-art design methods are nowhere near the smallest error possible on a finite resolution digital computer. This is shown to be due to conceptual errors in the literature pertaining to what constitutes a factorable linear phase filter. This study shows that factorisation is possible with a zero residual error (in the absence of machine finite resolution error) if the linear operator or matrix representing the linear phase filter is positive definite. Methodology is proposed able to design a minimum phase filter that is optimal—in the sense that the residual error is limited only by the finite precision of the digital computer, with no systematic error. The study presents practical application of the proposed methodology by designing two minimum phase Chebyshev FIR filters. Results are compared to state-of-the-art methods from the literature, and it is shown that the proposed methodology is able to reduce currently achievable residual errors by several orders of magnitude.},
  year = {2023},
  journal = {The Institute of Engineering and Technology},
  volume = {17},
  edition = {7},
  month = {July 2023},
  doi = {https://doi.org/10.1049/sil2.12166},
}

A robust theoretical framework that can describe and predict the generalization ability of deep neural networks (DNNs) in general circumstances remains elusive. Classical attempts have produced complexity metrics that rely heavily on global measures of compactness and capacity with little investigation into the effects of sub-component collaboration. We demonstrate intriguing regularities in the activation patterns of the hidden nodes within fully-connected feedforward networks. By tracing the origin of these patterns, we show how such networks can be viewed as the combination of two information processing systems: one continuous and one discrete. We describe how these two systems arise naturally from the gradient-based optimization process, and demonstrate the classification ability of the two systems, individually and in collaboration. This perspective on DNN classification offers a novel way to think about generalization, in which different subsets of the training data are used to train distinct classifiers; those classifiers are then combined to perform the classification task, and their consistency is crucial for accurate classification.

@{236,
  author = {Marelie Davel, Marthinus Theunissen, Arnold Pretorius, Etienne Barnard},
  title = {DNNs as layers of cooperating classifiers},
  abstract = {A robust theoretical framework that can describe and predict the generalization ability of deep neural networks (DNNs) in general circumstances remains elusive. Classical attempts have produced complexity metrics that rely heavily on global measures of compactness and capacity with little investigation into the effects of sub-component collaboration. We demonstrate intriguing regularities in the activation patterns of the hidden nodes within fully-connected feedforward networks. By tracing the origin of these patterns, we show how such networks can be viewed as the combination of two information processing systems: one continuous and one discrete. We describe how these two systems arise naturally from the gradient-based optimization process, and demonstrate the classification ability of the two systems, individually and in collaboration. This perspective on DNN classification offers a novel way to think about generalization, in which different subsets of the training data are used to train distinct classifiers; those classifiers are then combined to perform the classification task, and their consistency is crucial for accurate classification.},
  year = {2020},
  journal = {The Thirty-Fourth AAAI Conference on Artificial Intelligence (AAAI-20)},
  pages = {3725 - 3732},
  month = {07/02-12/02/2020},
  address = {New York},
}

The understanding of generalization in machine learning is in a state of flux. This is partly due to the elatively recent revelation that deep learning models are able to completely memorize training data and still perform appropriately on out-of-sample data, thereby contradicting long-held intuitions about generalization. The phenomenon was brought to light and discussed in a seminal paper by Zhang et al. [24]. We expand upon this work by discussing local attributes of neural network training within the context of a relatively simple and generalizable framework. We describe how various types of noise can be compensated for within the proposed framework in order to allow the global deep learning model to generalize in spite of interpolating spurious function descriptors. Empirically, we support our postulates with experiments involving overparameterized multilayer perceptrons and controlled noise in the training data. The main insights are that deep learning models are optimized for training data modularly, with different regions in the function space dedicated to fitting distinct kinds of sample information. Detrimental overfitting is largely prevented by the fact that different regions in the function space are used for prediction based on the similarity between new input data and that which has been optimized for.

@{284,
  author = {Marthinus Theunissen, Marelie Davel, Etienne Barnard},
  title = {Insights regarding overfitting on noise in deep learning},
  abstract = {The understanding of generalization in machine learning is in a state of flux. This is partly due to the elatively recent revelation that deep learning models are able to completely memorize training data and still perform appropriately on out-of-sample data, thereby contradicting long-held intuitions about generalization. The phenomenon was brought to light and discussed in a seminal paper by Zhang et al. [24]. We expand upon this work by discussing local attributes of neural network training within the context of a relatively simple and generalizable framework. We describe how various types of noise can be compensated for within the proposed framework in order to allow the global deep learning model to generalize in spite of interpolating spurious function descriptors. Empirically, we support our postulates with experiments involving overparameterized multilayer perceptrons and controlled noise in the training data. The main insights are that deep learning models are optimized for training data modularly, with different regions in the function space dedicated to fitting distinct kinds of sample information. Detrimental overfitting is largely prevented by the fact that different regions in the function space are used for prediction based on the similarity between new input data and that which has been optimized for.},
  year = {2019},
  journal = {South African Forum for Artificial Intelligence Research (FAIR)},
  pages = {49-63},
  address = {Cape Town, South Africa},
}

The generalization capabilities of deep neural networks are not well understood, and in particular, the influence of activation functions on generalization has received little theoretical attention. Phenomena such as vanishing gradients, node saturation and network sparsity have been identified as possible factors when comparing different activation functions [1]. We investigate these factors using fully connected feedforward networks on two standard benchmark problems, and find that the most salient differences between networks with sigmoidal and ReLU activations relate to the way that class-distinctive information is propagated through a network.

@{279,
  author = {Arnold Pretorius, Etienne Barnard, Marelie Davel},
  title = {ReLU and sigmoidal activation functions},
  abstract = {The generalization capabilities of deep neural networks are not well understood, and in particular, the influence of activation functions on generalization has received little theoretical attention. Phenomena such as vanishing gradients, node saturation and network sparsity have been identified as possible factors when comparing different activation functions [1]. We investigate these factors using fully connected feedforward networks on two standard benchmark problems, and find that the most salient differences between networks with sigmoidal and ReLU activations relate to the way that class-distinctive information is propagated through a network.},
  year = {2019},
  journal = {South African Forum for Artificial Intelligence Research (FAIR)},
  pages = {37-48},
  month = {04/12-07/12},
  publisher = {CEUR Workshop Proceedings},
  address = {Cape Town, South Africa},
}

Estimating the joint probability density function of a dataset is a central task in many machine learning applications. In this work we address the fundamental problem of kernel bandwidth estimation for variable kernel density estimation in high-dimensional feature spaces. We derive a variable kernel bandwidth estimator by minimizing the leave-one-out entropy objective function and show that this estimator is capable of performing estimation in high-dimensional feature spaces with great success. We compare the performance of this estimator to state-of-the art maximum likelihood estimators on a number of representative high-dimensional machine learning tasks and show that the newly introduced minimum leave-one-out entropy estimator performs optimally on a number of high-dimensional datasets considered.

@{275,
  author = {Christiaan Van der Walt, Etienne Barnard},
  title = {Variable Kernel Density Estimation in High-dimensional Feature Spaces},
  abstract = {Estimating the joint probability density function of a dataset is a central task in many machine learning applications. In this work we address the fundamental problem of kernel bandwidth estimation for variable kernel density estimation in high-dimensional feature spaces. We derive a variable kernel bandwidth estimator by minimizing the leave-one-out entropy objective function and show that this estimator is capable of performing estimation in high-dimensional feature spaces with great success. We compare the performance of this estimator to state-of-the art maximum likelihood estimators on a number of representative high-dimensional machine learning tasks and show that the newly introduced minimum leave-one-out entropy estimator performs optimally on a number of high-dimensional datasets considered.},
  year = {2017},
  journal = {AAAI Conf. on Artificial Intelligence (AAAI-17)},
  pages = {2674-2680},
  month = {04/02-09/04},
}