2019
No Abstract
@{368,
author = {Stefan Lotz and Jacques Beukes and Marelie Davel},
title = {A neural network based method for input parameter selection (Poster)},
abstract = {No Abstract},
year = {2019},
journal = {Machine Learning in Heliophysics},
address = {Amsterdam, The Netherlands},
}
The understanding of generalization in machine learning is in a state of flux. This is partly due to the elatively recent revelation that deep learning models are able to completely memorize training data and still perform appropriately on out-of-sample data, thereby contradicting long-held intuitions about generalization. The phenomenon was brought to light and discussed in a seminal paper by Zhang et al. [24]. We expand upon this work by discussing local attributes of neural network training within the context of a relatively simple and generalizable framework. We describe how various types of noise can be compensated for within the proposed framework in order to allow the global deep learning model to generalize in spite of interpolating spurious function descriptors. Empirically, we support our postulates with experiments involving overparameterized multilayer perceptrons and controlled noise in the training data. The main insights are that deep learning models are optimized for training data modularly, with different regions in the function space dedicated to fitting distinct kinds of sample information. Detrimental overfitting is largely prevented by the fact that different regions in the function space are used for prediction based on the similarity between new input data and that which has been optimized for.
@{284,
author = {Marthinus Theunissen and Marelie Davel and Etienne Barnard},
title = {Insights regarding overfitting on noise in deep learning},
abstract = {The understanding of generalization in machine learning is in a state of flux. This is partly due to the elatively recent revelation that deep learning models are able to completely memorize training data and still perform appropriately on out-of-sample data, thereby contradicting long-held intuitions about generalization. The phenomenon was brought to light and discussed in a seminal paper by Zhang et al. [24]. We expand upon this work by discussing local attributes of neural network training within the context of a relatively simple and generalizable framework. We describe how various types of noise can be compensated for within the proposed framework in order to allow the global deep learning model to generalize in spite of interpolating spurious function descriptors. Empirically, we support our postulates with experiments involving overparameterized multilayer perceptrons and controlled noise in the training data. The main insights are that deep learning models are optimized for training data modularly, with different regions in the function space dedicated to fitting distinct kinds of sample information. Detrimental overfitting is largely prevented by the fact that different regions in the function space are used for prediction based on the similarity between new input data and that which has been optimized for.},
year = {2019},
journal = {South African Forum for Artificial Intelligence Research (FAIR)},
pages = {49-63},
address = {Cape Town, South Africa},
}
The generalization capabilities of deep neural networks are not well understood, and in particular, the influence of activation functions on generalization has received little theoretical attention. Phenomena such as vanishing gradients, node saturation and network sparsity have been identified as possible factors when comparing different activation functions [1]. We investigate these factors using fully connected feedforward networks on two standard benchmark problems, and find that the most salient differences between networks with sigmoidal and ReLU activations relate to the way that class-distinctive information is propagated through a network.
@{279,
author = {Arnold Pretorius and Etienne Barnard and Marelie Davel},
title = {ReLU and sigmoidal activation functions},
abstract = {The generalization capabilities of deep neural networks are not well understood, and in particular, the influence of activation functions on generalization has received little theoretical attention. Phenomena such as vanishing gradients, node saturation and network sparsity have been identified as possible factors when comparing different activation functions [1]. We investigate these factors using fully connected feedforward networks on two standard benchmark problems, and find that the most salient differences between networks with sigmoidal and ReLU activations relate to the way that class-distinctive information is propagated through a network.},
year = {2019},
journal = {South African Forum for Artificial Intelligence Research (FAIR)},
pages = {37-48},
month = {04/12-07/12},
publisher = {CEUR Workshop Proceedings},
address = {Cape Town, South Africa},
}
Geomagnetic storms are multi-day events characterised by significant perturbations to the magnetic field of the Earth, driven by solar activity. Numerous efforts have been undertaken to utilise in-situ measurements of the solar wind plasma to predict perturbations to the geomagnetic field measured on the ground. Typically, solar wind measurements are used as input parameters to a regression problem tasked with predicting a perturbation index such as the 1-minute cadence symmetric-H (Sym-H) index. We re-visit this problem, with two important twists: (i) An adapted feedforward neural network topology is designed to enable the pairwise analysis of input parameter weights. This enables the ranking of input parameters in terms of importance to output accuracy, without the need to train numerous models. (ii) Geomagnetic storm phase information is incorporated as model inputs and shown to increase performance. This is motivated by the fact that different physical phenomena are at play during different phases of a geomagnetic storm.
@{283,
author = {Stefan Lotz and Jacques Beukes and Marelie Davel},
title = {Input parameter ranking for neural networks in a space weather regression problem},
abstract = {Geomagnetic storms are multi-day events characterised by significant perturbations to the magnetic field of the Earth, driven by solar activity. Numerous efforts have been undertaken to utilise in-situ measurements of the solar wind plasma to predict perturbations to the geomagnetic field measured on the ground. Typically, solar wind measurements are used as input parameters to a regression problem tasked with predicting a perturbation index such as the 1-minute cadence symmetric-H (Sym-H) index. We re-visit this problem, with two important twists:
(i) An adapted feedforward neural network topology is designed to enable the pairwise analysis of input parameter weights. This enables the ranking of input parameters in terms of importance to output accuracy, without the need to train numerous models. (ii) Geomagnetic storm phase information is incorporated as model inputs and shown to increase performance. This is motivated by the fact that different physical phenomena are at play during different phases of a geomagnetic storm.},
year = {2019},
journal = {South African Forum for Artificial Intelligence Research (FAIR)},
pages = {133-144},
publisher = {CEUR workshop proceedings},
address = {Cape Town, South Africa},
}
Sequences are typically modelled with recurrent architectures, but growing research is finding convolutional architectures to also work well for sequence modelling [1]. We explore the performance of Temporal Convolutional Networks (TCNs) when applied to an important sequence modelling task: solar flare prediction. We take this approach, as our future goal is to apply techniques developed for probing and interpreting general convolutional neural networks (CNNs) to solar flare prediction.
@{282,
author = {Dewald Krynauw and Marelie Davel and Stefan Lotz},
title = {Solar flare prediction with temporal convolutional networks (Work in progress)},
abstract = {Sequences are typically modelled with recurrent architectures, but growing research is finding convolutional architectures to also work well for sequence modelling [1]. We explore the performance of Temporal Convolutional Networks (TCNs) when applied to an important sequence modelling task: solar flare prediction. We take this approach, as our future goal is to apply techniques developed for probing and interpreting general convolutional neural networks (CNNs) to solar flare prediction.},
year = {2019},
journal = {South African Forum for Artificial Intelligence Research (FAIR)},
pages = {Work in progress},
publisher = {CEUR workshop proceedings},
isbn = {1613-0073},
}
No framework exists that can explain and predict the generalisation ability of DNNs in general circumstances. In fact, this question has not been addressed for some of the least complicated of neural network architectures: fully-connected feedforward networks with ReLU activations and a limited number of hidden layers. Building on recent work [2] that demonstrates the ability of individual nodes in a hidden layer to draw class-specific activation distributions apart, we show how a simplified network architecture can be analysed in terms of these activation distributions, and more specifically, the sample distances or activation gaps each node produces. We provide a theoretical perspective on the utility of viewing nodes as activation gap generators, and define the gap conditions that are guaranteed to result in perfect classification of a set of samples. We support these conclusions with empirical results.
@{230,
author = {Marelie Davel},
title = {Activation gap generators in neural networks},
abstract = {No framework exists that can explain and predict the generalisation ability of DNNs in general circumstances. In fact, this question has not been addressed for some of the least complicated of neural network architectures: fully-connected feedforward networks with ReLU activations and a limited number of hidden layers. Building on recent work [2] that demonstrates the ability of individual nodes in a hidden layer to draw class-specific activation distributions apart, we show how a simplified network architecture can be analysed in terms of these activation distributions, and more specifically, the sample distances or activation gaps each node produces. We provide a theoretical perspective on the utility of viewing nodes as activation gap generators, and define the gap conditions that are guaranteed to result in perfect classification of a set of samples. We support these conclusions with empirical results.},
year = {2019},
journal = {South African Forum for Artificial Intelligence Research (FAIR)},
pages = {64-76},
month = {04/12-06/12/2019},
publisher = {CEUR workshop proceedings},
address = {Cape Town, South Africa},
}
2017
This paper describes the development of text-to-speech corpora for four South African languages. The approach followed investigated the possibility of using low-cost methods including informal recording environments and untrained volunteer speakers. This objective and the additional future goal of expanding the corpus to increase coverage of South Africa’s 11 official languages necessitated experimenting with multi-speaker and code-switched data. The process and relevant observations are detailed throughout. The latest version of the corpora are available for download under an open-source licence and will likely see further development and refinement in future.
@{278,
author = {Daniel Van Niekerk and Charl Van Heerden and Marelie Davel and Neil Kleynhans and Oddur Kjartansson and Martin Jansche and Linne Ha},
title = {Rapid development of TTS corpora for four South African languages},
abstract = {This paper describes the development of text-to-speech corpora for four South African languages. The approach followed investigated the possibility of using low-cost methods including informal recording environments and untrained volunteer speakers. This objective and the additional future goal of expanding the corpus to increase coverage of South Africa’s 11 official languages necessitated experimenting with multi-speaker and code-switched data. The process and relevant observations are detailed throughout. The latest version of the corpora are available for download under an open-source licence and will likely see further development and refinement in future.},
year = {2017},
journal = {Interspeech},
pages = {2178-2182},
address = {Stockholm, Sweden},
doi = {10.21437/Interspeech.2017-1139},
}
An explicit lexical stress feature is investigated for statistical parametric speech synthesis in Afrikaans: Firstly, objective measures are used to assess proposed annotation protocols and dictionaries compared to the baseline (implicit modelling) on the Lwazi 2 text-to-speech corpus. Secondly, the best candidates are evaluated on additional corpora. Finally, a comparative subjective evaluation is conducted to determine the perceptual impact on text-to-speech synthesis. The best candidate dictionary is associated with favourable objective results obtained on all corpora and was preferred in the subjective test. This suggests that it may form a basis for further refinement and work on improved prosodic models.
@{277,
author = {Daniel Van Niekerk},
title = {Evaluating acoustic modelling of lexical stress for Afrikaans speech synthesis},
abstract = {An explicit lexical stress feature is investigated for statistical parametric speech synthesis in Afrikaans: Firstly, objective measures are used to assess proposed annotation protocols and dictionaries compared to the baseline (implicit modelling) on the Lwazi 2 text-to-speech corpus. Secondly, the best candidates are evaluated on additional corpora. Finally, a comparative subjective evaluation is conducted to determine the perceptual impact on text-to-speech synthesis. The best candidate dictionary is associated with favourable objective results obtained on all corpora and was preferred in the subjective test. This suggests that it may form a basis for further refinement and work on improved prosodic models.},
year = {2017},
journal = {Pattern Recognition Association of South Africa and Mechatronics International Conference (PRASA-RobMech)},
pages = {86-91},
address = {Bloemfontein, South Africa},
isbn = {978-1-5386-2314-5, 978-1-5386-2313-8},
doi = {10.1109/RoboMech.2017.8261128},
}
Spoken term detection, especially of out-of-vocabulary (OOV) key-words, benefits from the use of sub-word systems. We experiment with different language-dependent approaches to sub-word unit generation, generating both syllable-like and morpheme-like units, and demonstrate how the performance of syllable-like units can be improved by artificially increasing the number of unique units. The effect of unit choice is empirically evaluated using the eight languages from the 2016 IARPA BABEL evaluation.
@{276,
author = {Charl Van Heerden and Damianos Karakos and Karthik Narasimhan and Marelie Davel and Richard Schwartz},
title = {Constructing Sub-Word Units for Spoken Term Detection},
abstract = {Spoken term detection, especially of out-of-vocabulary (OOV) key-words, benefits from the use of sub-word systems. We experiment with different language-dependent approaches to sub-word unit generation, generating both syllable-like and morpheme-like units, and demonstrate how the performance of syllable-like units can be improved by artificially increasing the number of unique units. The effect of unit choice is empirically evaluated using the eight languages from the 2016 IARPA BABEL evaluation.},
year = {2017},
journal = {IEEE Int. Conf. on Acoustics, Speech and Signal Processing (ICASSP)},
pages = {5780-5784},
address = {New Orleans, Louisiana},
isbn = {9781509041176},
doi = {10.1109/ICASSP.2017.7953264},
}
Estimating the joint probability density function of a dataset is a central task in many machine learning applications. In this work we address the fundamental problem of kernel bandwidth estimation for variable kernel density estimation in high-dimensional feature spaces. We derive a variable kernel bandwidth estimator by minimizing the leave-one-out entropy objective function and show that this estimator is capable of performing estimation in high-dimensional feature spaces with great success. We compare the performance of this estimator to state-of-the art maximum likelihood estimators on a number of representative high-dimensional machine learning tasks and show that the newly introduced minimum leave-one-out entropy estimator performs optimally on a number of high-dimensional datasets considered.
@{275,
author = {Christiaan Van der Walt and Etienne Barnard},
title = {Variable Kernel Density Estimation in High-dimensional Feature Spaces},
abstract = {Estimating the joint probability density function of a dataset is a central task in many machine learning applications. In this work we address the fundamental problem of kernel bandwidth estimation for variable kernel density estimation in high-dimensional feature spaces. We derive a variable kernel bandwidth estimator by minimizing the leave-one-out entropy objective function and show that this estimator is capable of performing estimation in high-dimensional feature spaces with great success. We compare the performance of this estimator to state-of-the art maximum likelihood estimators on a number of representative high-dimensional machine learning tasks and show that the newly introduced minimum leave-one-out entropy estimator performs optimally on a number of high-dimensional datasets considered.},
year = {2017},
journal = {AAAI Conf. on Artificial Intelligence (AAAI-17)},
pages = {2674-2680},
month = {04/02-09/04},
}
Utilizing the known language of origin of a name can be useful when predicting the pronunciation of the name. When this language is not known, automatic language identification (LID) can be used to influence which language-specific grapheme-to-phoneme (G2P) predictor is triggered to produce a pronunciation for the name. We investigate the implications when both the LID system and the G2P system generate errors: what influence does this have on a resulting speech recognition system? We experiment with different approaches to LID-based dictionary creation and report on results in four South African languages: Afrikaans, English, Sesotho and isiZulu.
@{274,
author = {Oluwapelumi Giwa and Marelie Davel},
title = {The Effect of Language Identification Accuracy on Speech Recognition Accuracy of Proper Names},
abstract = {Utilizing the known language of origin of a name can be useful when predicting the pronunciation of the name. When this language is not known, automatic language identification (LID) can be used to influence which language-specific grapheme-to-phoneme (G2P) predictor is triggered to produce a pronunciation for the name. We investigate the implications when both the LID system and the G2P system generate errors: what influence does this have on a resulting speech recognition system? We experiment with different approaches to LID-based dictionary creation and report on results in four South African languages: Afrikaans, English, Sesotho and isiZulu.},
year = {2017},
journal = {Pattern Recognition Association of South Africa and Mechatronics International Conference (PRASA-RobMech)},
pages = {187-192},
address = {Bloemfontein, South Africa},
isbn = {978-1-5386-2314-5, 978-1-5386-2313-8},
doi = {10.1109/RoboMech.2017.8261145},
}
Incorporating pronunciation variants in a dictionary is controversial, as this can be either advantageous or detrimental for a speech recognition system. Grapheme-ophoneme (G2P) accuracy can help guide this decision, but calculating the G2P accuracy of variant-based dictionaries is not fully straightforward. We propose a variant matching technique to measure G2P accuracy in a principled way, when both the reference and hypothesized dictionaries may include variants. We use the new measure to evaluate G2P accuracy and speech recognition performance of systems developed with an existing set of dictionaries, and observe a better correlation between G2P accuracy and speech recognition performance, than when utilising alternative metrics.
@{273,
author = {Oluwapelumi Giwa and Marelie Davel},
title = {Bilateral G2P Accuracy: Measuring the effect of variants},
abstract = {Incorporating pronunciation variants in a dictionary is controversial, as this can be either advantageous or detrimental for a speech recognition system. Grapheme-ophoneme (G2P) accuracy can help guide this decision, but calculating the G2P accuracy of variant-based dictionaries is not fully straightforward. We propose a variant matching technique to measure G2P accuracy in a principled way, when both the reference and hypothesized dictionaries may include variants. We use the new measure to evaluate G2P accuracy and speech recognition performance of systems developed with an existing set of dictionaries, and observe a better correlation between G2P accuracy and speech recognition performance, than when utilising alternative metrics.},
year = {2017},
journal = {Pattern Recognition Association of South Africa and Mechatronics International Conference (PRASA-RobMech)},
pages = {208-213},
address = {Bloemfontein, South Africa},
isbn = {978-1-5386-2314-5, 978-1-5386-2313-8},
doi = {10.1109/RoboMech.2017.8261149},
}
For purposes of automated speech recognition in under-resourced environments, techniques used to share acoustic data between closely related or similar languages become important. Donor languages with abundant resources can potentially be used to increase the recognition accuracy of speech systems developed in the resource poor target language. The assumption is that adding more data will increase the robustness of the statistical estimations captured by the acoustic models. In this study we investigated data sharing between Afrikaans and Flemish – an under-resourced and well-resourced language, respectively. Our approach was focused on the exploration of model adaptation and refinement techniques associated with hidden Markov model based speech recognition systems to improve the benefit of sharing data. Specifically, we focused on the use of currently available techniques, some possible combinations and the exact utilisation of the techniques during the acoustic model development process. Our findings show that simply using normal approaches to adaptation and refinement does not result in any benefits when adding Flemish data to the Afrikaans training pool. The only observed improvement was achieved when developing acoustic models on all available data but estimating model refinements and adaptations on the target data only. Significance: • Acoustic modelling for under-resourced languages • Automatic speech recognition for Afrikaans • Data sharing between Flemish and Afrikaans to improve acoustic modelling for Afrikaans
@article{272,
author = {Febe De Wet and Neil Kleynhans and Dirk Van Compernolle and Reza Sahraeian},
title = {Speech recognition for under-resourced languages: Data sharing in hidden Markov model systems},
abstract = {For purposes of automated speech recognition in under-resourced environments, techniques used to
share acoustic data between closely related or similar languages become important. Donor languages
with abundant resources can potentially be used to increase the recognition accuracy of speech
systems developed in the resource poor target language. The assumption is that adding more data will
increase the robustness of the statistical estimations captured by the acoustic models. In this study
we investigated data sharing between Afrikaans and Flemish – an under-resourced and well-resourced
language, respectively. Our approach was focused on the exploration of model adaptation and refinement
techniques associated with hidden Markov model based speech recognition systems to improve the
benefit of sharing data. Specifically, we focused on the use of currently available techniques, some
possible combinations and the exact utilisation of the techniques during the acoustic model development
process. Our findings show that simply using normal approaches to adaptation and refinement does
not result in any benefits when adding Flemish data to the Afrikaans training pool. The only observed
improvement was achieved when developing acoustic models on all available data but estimating model
refinements and adaptations on the target data only.
Significance:
• Acoustic modelling for under-resourced languages
• Automatic speech recognition for Afrikaans
• Data sharing between Flemish and Afrikaans to improve acoustic modelling for Afrikaans},
year = {2017},
journal = {South African Journal of Science},
volume = {113},
pages = {25-33},
issue = {1/2},
publisher = {Academy of Science for South Africa (ASSAf)},
doi = {https://doi.org/10.17159/sajs.2017/20160038},
}
2016
This paper describes the continuing development of a pronunciation resource for speech synthesis of Afrikaans by augmenting an existing pronunciation dictionary to include syllable boundaries and stress. Furthermore, different approaches for grapheme to phoneme conversion and syllabification derived from the dictionary are evaluated. Cross-validation experiments suggest that joint sequence models are effective at directly modelling pronunciations including syllable boundaries. Finally, some informal observations and demonstrations are presented regarding the integration of this work into a typical text-to-speech system.
@{285,
author = {Daniel Van Niekerk},
title = {Syllabification for Afrikaans speech synthesis},
abstract = {This paper describes the continuing development of a pronunciation resource for speech synthesis of Afrikaans by augmenting an existing pronunciation dictionary to include syllable boundaries and stress. Furthermore, different approaches for grapheme to phoneme conversion and syllabification derived from the dictionary are evaluated. Cross-validation experiments suggest that joint sequence models are effective at directly modelling pronunciations including syllable boundaries. Finally, some informal observations and demonstrations are presented regarding the integration of this work into a typical text-to-speech system.},
year = {2016},
journal = {Pattern Recognition Association of South Africa and Robotics and Mechatronics International Conference (PRASA-RobMech)},
pages = {31-36},
address = {Stellenbosch, South Africa},
isbn = {978-1-5090-3335-5},
doi = {10.1109/RoboMech.2016.7813143},
}
We investigate modeling strategies for English code-switched words as found in a Swahili spoken term detection system. Code switching, where speakers switch language in a conversation, occurs frequently in multilingual environments, and typically deteriorates STD performance. Analysis is performed in the context of the IARPA Babel program which focuses on rapid STD system development for under-resourced languages. Our results show that approaches that specifically target the modeling of code-switched words, significantly improve the detection performance of these words.
@article{271,
author = {Neil Kleynhans and William Hartman and Daniel Van Niekerk and Charl Van Heerden and Richard Schwartz and Stavros Tsakalidis and Marelie Davel},
title = {Code-switched English Pronunciation Modeling for Swahili Spoken Term Detection},
abstract = {We investigate modeling strategies for English code-switched words as found in a Swahili spoken term detection system. Code
switching, where speakers switch language in a conversation, occurs frequently in multilingual environments, and typically deteriorates STD performance. Analysis is performed in the context of the IARPA Babel program which focuses on rapid STD
system development for under-resourced languages. Our results show that approaches that specifically target the modeling of
code-switched words, significantly improve the detection performance of these words.},
year = {2016},
journal = {Procedia Computer Science},
volume = {81},
pages = {128-135},
publisher = {Elsevier B.V.},
address = {Yogyakarta, Indonesia},
isbn = {1877-0509},
doi = {10.1016/j.procs.2016.04.040},
}
