by Miguel Angrick, Christian Herff, Garett Johnson, Jerry Shih, Dean Krusienski, Tanja Schultz
Abstract:
The direct synthesis of continuously spoken speech from neural activity is envisioned to enable fast and intuitive Brain-Computer Interfaces. Earlier results indicate that intracranial recordings reveal very suitable signal characteristics for direct synthesis. To map the complex dynamics of neural activity to spectral representations of speech, Convolutional Neural Networks (CNNs) can be trained. However, the resulting networks are hard to interpret and thus provide little opportunity to gain insights on neural processes underlying speech. Here, we show that CNNs are useful to reconstruct speech from intracranial recordings of brain activity and propose an approach to interpret the trained CNNs.
Reference:
Interpretation of Convolutional Neural Networks for Speech Regression from Electrocorticography (Miguel Angrick, Christian Herff, Garett Johnson, Jerry Shih, Dean Krusienski, Tanja Schultz), In ESANN 2018 – 26th European Symposium on Artificial Neural Networks, Computational Intelligence and Machine Learning, 2018.
Bibtex Entry:
@inproceedings{angrick_2018esann,
author={Angrick, Miguel and Herff, Christian and Johnson, Garett and Shih, Jerry and Krusienski, Dean and Schultz, Tanja},
title={{Interpretation of Convolutional Neural Networks for Speech Regression from Electrocorticography}},
booktitle={{ESANN} 2018 -- 26th European Symposium on Artificial Neural Networks, Computational Intelligence and Machine Learning},
isbn = {978-2-87587-047-6},
pages = {7--12},
numpages = {6},
url={https://www.csl.uni-bremen.de/cms/images/documents/publications/angrick_2018esann.pdf},
address = {Brugge, Belgium},
year={2018},
abstract={The direct synthesis of continuously spoken speech from neural activity is envisioned to enable fast and intuitive Brain-Computer Interfaces. Earlier results indicate that intracranial recordings reveal very suitable signal characteristics for direct synthesis. To map the complex dynamics of neural activity to spectral representations of speech, Convolutional Neural Networks (CNNs) can be trained. However, the resulting networks are hard to interpret and thus provide little opportunity to gain insights on neural processes underlying speech. Here, we show that CNNs are useful to reconstruct speech from intracranial recordings of brain activity and propose an approach to interpret the trained CNNs.},
}