@article {69290, title = {Whole-body simulation of realistic fruit fly locomotion with deep reinforcement learning}, journal = {bioRxiv}, year = {2024}, month = {2024 Mar 14}, abstract = {

The body of an animal determines how the nervous system produces behavior. Therefore, detailed modeling of the neural control of sensorimotor behavior requires a detailed model of the body. Here we contribute an anatomically-detailed biomechanical whole-body model of the fruit fly Drosophila melanogaster in the MuJoCo physics engine. Our model is general-purpose, enabling the simulation of diverse fly behaviors, both on land and in the air. We demonstrate the generality of our model by simulating realistic locomotion, both flight and walking. To support these behaviors, we have extended MuJoCo with phenomenological models of fluid forces and adhesion forces. Through data-driven end-to-end reinforcement learning, we demonstrate that these advances enable the training of neural network controllers capable of realistic locomotion along complex trajectories based on high-level steering control signals. With a visually guided flight task, we demonstrate a neural controller that can use the vision sensors of the body model to control and steer flight. Our project is an open-source platform for modeling neural control of sensorimotor behavior in an embodied context.Competing Interest StatementThe authors have declared no competing interest.

}, doi = {10.1101/2024.03.11.584515}, url = {https://www.biorxiv.org/content/early/2024/03/14/2024.03.11.584515}, author = {Roman Vaxenburg and Igor Siwanowicz and Josh Merel and Alice A Robie and Carmen Morrow and Guido Novati and Zinovia Stefanidi and Gwyneth M Card and Michael B Reiser and Matthew M Botvinick and Kristin M Branson and Yuval Tassa and Srinivas C Turaga} } @conference {67973, title = {Connectome-constrained Latent Variable Model of Whole-Brain Neural Activity}, booktitle = {International Conference on Learning Representations}, year = {2022}, month = {09/2021}, abstract = {

The availability of both anatomical connectivity and brain-wide neural activity measurements in C. elegans make the worm a promising system for learning detailed, mechanistic models of an entire nervous system in a data-driven way. However, one faces several challenges when constructing such a model. We often do not have direct experimental access to important modeling details such as single-neuron dynamics and the signs and strengths of the synaptic connectivity. Further, neural activity can only be measured in a subset of neurons, often indirectly via calcium imaging, and significant trial-to-trial variability has been observed. To address these challenges, we introduce a connectome-constrained latent variable model (CC-LVM) of the unobserved voltage dynamics of the entire C. elegans nervous system and the observed calcium signals. We used the framework of variational autoencoders to fit parameters of the mechanistic simulation constituting the generative model of the LVM to calcium imaging observations. A variational approximate posterior distribution over latent voltage traces for all neurons is efficiently inferred using an inference network, and constrained by a prior distribution given by the biophysical simulation of neural dynamics. We applied this model to an experimental whole-brain dataset, and found that connectomic constraints enable our LVM to predict the activity of neurons whose activity were withheld significantly better than models unconstrained by a connectome. We explored models with different degrees of biophysical detail, and found that models with realistic conductance-based synapses provide markedly better predictions than current-based synapses for this system.

}, url = {https://openreview.net/forum?id=CJzi3dRlJE-}, author = {Lu Mi and Richard Xu and Sridhama Prakhya and Albert Lin and Nir Shavit and Aravinthan Samuel and Srinivas C Turaga} } @conference {68678, title = {FourierNets enable the design of highly non-local optical encoders for computational imaging}, booktitle = {Advances in Neural Information Processing Systems}, year = {2022}, month = {10/2022}, abstract = {

Differentiable simulations of optical systems can be combined with deep learning-based reconstruction networks to enable high performance computational imaging via end-to-end (E2E) optimization of both the optical encoder and the deep decoder. This has enabled imaging applications such as 3D localization microscopy, depth estimation, and lensless photography via the optimization of local optical encoders. More challenging computational imaging applications, such as 3D snapshot microscopy which compresses 3D volumes into single 2D images, require a highly non-local optical encoder. We show that existing deep network decoders have a locality bias which prevents the optimization of such highly non-local optical encoders. We address this with a decoder based on a shallow neural network architecture using global kernel Fourier convolutional neural networks (FourierNets). We show that FourierNets surpass existing deep network based decoders at reconstructing photographs captured by the highly non-local DiffuserCam optical encoder. Further, we show that FourierNets enable E2E optimization of highly non-local optical encoders for 3D snapshot microscopy. By combining FourierNets with a large-scale multi-GPU differentiable optical simulation, we are able to optimize non-local optical encoders 170{\texttimes}\ to 7372{\texttimes}\ larger than prior state of the art, and demonstrate the potential for ROI-type specific optical encoding with a programmable microscope.

}, keywords = {Computer Vision and Pattern Recognition (cs.CV), electronic engineering, FOS: Computer and information sciences, FOS: Electrical engineering, Image and Video Processing (eess.IV), information engineering, Machine Learning (cs.LG)}, doi = {https://doi.org/10.48550/arXiv.2104.10611}, url = {https://openreview.net/forum?id=Kx1VCs1treH}, author = {Diptodip Deb and Zhenfei Jiao and Ruth R Sims and Alex Bo-Yuan Chen and Michael Broxton and Misha Ahrens and Kaspar Podgorski and Srinivas C Turaga}, editor = {Alice H. Oh and Alekh Agarwal and Danielle Belgrave and Kyunghyun Cho} }