% IMPORTANT: The following is UTF-8 encoded.  This means that in the presence
% of non-ASCII characters, it will not work with BibTeX 0.99 or older.
% Instead, you should use an up-to-date BibTeX implementation like “bibtex8” or
% “biber”.

@INPROCEEDINGS{Lober:1022261,
      author       = {Lober, Melissa and Diesmann, Markus and Kunkel, Susanne},
      title        = {{E}xploiting network topology in brain-scale multi-area
                      model simulations},
      school       = {RWTH Aachen},
      reportid     = {FZJ-2024-01379},
      year         = {2023},
      abstract     = {The communication of spike events constitutes a major
                      bottleneck in simulations of brain-scale networks with
                      realistic connectivity, as for example, the multi-area model
                      of macaque visual cortex [1], [2]. The model consists of 32
                      cortical microcircuits, each representing a downscaled area
                      of the visual cortex at the resolution of single neurons and
                      synapses. Such models not only have a dense connectivity
                      within areas but also between areas, as approximately half
                      of a neuron’s connections reach beyond its own brain
                      region. Synaptic transmission delays within an area can be
                      as short as 0.1 ms and therefore simulations require
                      frequent spike communication between compute nodes to
                      maintain causality in the network dynamics [3]. This poses a
                      challenge to the conventional round-robin scheme used to
                      distribute neurons uniformly across compute nodes
                      disregarding the network’s specific topology. With this
                      scheme, short-delay connections are present between any pair
                      of compute nodes which significantly impairs communication
                      efficiency.We target this challenge and propose a
                      structure-aware neuron distribution scheme along with a
                      novel spike-communication framework that exploits this
                      approach in order to make communication in large-scale
                      distributed simulations more efficient. In the
                      structure-aware neuron distribution scheme, neurons are
                      placed on the hardware in a way that mimics the network’s
                      topology. Neurons of the same area are placed closer
                      together on the hardware, i.e. onto the same or only few
                      different compute nodes. Investigations of the load
                      balancing in simulation phases where compute nodes operate
                      independently (e.g. update of neuronal state variables)
                      showed that such neuron distributions do not negatively
                      affect compute time. Paired with a communication framework
                      that distinguishes short delay intra-area communication
                      between a small group of compute nodes and long delay
                      inter-area communication between all available compute
                      nodes, the structure-aware approach requires less of the
                      costly global communication and thereby reduces
                      communication time. Our prototype implementation is fully
                      tested and was developed within the neuronal simulator tool
                      NEST [4], [5].For the benchmarking of our approach, we
                      developed a multi-area model that resembles the macaque
                      multi-area model in terms of connectivity and work load,
                      while being more easily scalable as it retains constant
                      activity levels. First weak-scaling experiments showed that
                      our implementation significantly reduces communication time
                      and the effect increases with a rising number of compute
                      nodes. For further analysis, we plan to study how the novel
                      structure-aware communication scheme performs for larger
                      networks, various inter- and intra-area delay distributions,
                      as well as multi-area networks of unbalanced activity or
                      size across areas. [1] Schmidt M, Bakker R, Hilgetag CC,
                      Diesmann M $\&$ van Albada SJ (2018) Multi-scale account of
                      the network structure of macaque visual cortex. Brain
                      Structure and Function, 223: 1409
                      https://doi.org/10.1007/s00429-017-1554-4[2] Schmidt M,
                      Bakker R, Shen K, Bezgin B, Diesmann M $\&$ van Albada SJ
                      (2018) A multi-scale layer-resolved spiking network model of
                      resting-state dynamics in macaque cortex. PLOS Computational
                      Biology, 14(9): e1006359.
                      https://doi.org/10.1371/journal.pcbi.1006359[3] Morrison A,
                      Diesmann M (2008) Maintaining Causality in Discrete Time
                      Neuronal Network Simulations. Springer Berlin Heidelberg, pp
                      267-278. $https://doi.org/10.1007/978-3-540-73159-7_10$ [4]
                      https://nest-simulator.readthedocs.io/enlatest[5] Gewaltig
                      M-O $\&$ Diesmann M (2007) NEST (Neural Simulation Tool)
                      Scholarpedia 2(4):1430},
      month         = {Oct},
      date          = {2023-10-17},
      organization  = {INM Retreat, Research Center Jülich
                       (Germany), 17 Oct 2023 - 18 Oct 2023},
      subtyp        = {After Call},
      cin          = {INM-6 / IAS-6 / INM-10},
      cid          = {I:(DE-Juel1)INM-6-20090406 / I:(DE-Juel1)IAS-6-20130828 /
                      I:(DE-Juel1)INM-10-20170113},
      pnm          = {5231 - Neuroscientific Foundations (POF4-523) / 5232 -
                      Computational Principles (POF4-523) / JL SMHB - Joint Lab
                      Supercomputing and Modeling for the Human Brain (JL
                      SMHB-2021-2027) / BMBF 03ZU1106CB - NeuroSys:
                      Algorithm-Hardware Co-Design (Projekt C) - B
                      (BMBF-03ZU1106CB)},
      pid          = {G:(DE-HGF)POF4-5231 / G:(DE-HGF)POF4-5232 / G:(DE-Juel1)JL
                      SMHB-2021-2027 / G:(DE-Juel1)BMBF-03ZU1106CB},
      typ          = {PUB:(DE-HGF)24},
      doi          = {10.34734/FZJ-2024-01379},
      url          = {https://juser.fz-juelich.de/record/1022261},
}