% IMPORTANT: The following is UTF-8 encoded.  This means that in the presence
% of non-ASCII characters, it will not work with BibTeX 0.99 or older.
% Instead, you should use an up-to-date BibTeX implementation like “bibtex8” or
% “biber”.

@INPROCEEDINGS{Krajsek:1019553,
      author       = {Krajsek, Kai},
      title        = {{A}djoint {MPI} for {N}on-{A}dditive {S}eparable {L}oss
                      {F}unctions},
      reportid     = {FZJ-2023-05493},
      year         = {2023},
      abstract     = {Most contemporary frameworks for parallelizing deep
                      learning models follow a fixed design pattern that favours a
                      specific parallelization paradigm. More flexible libraries,
                      such as PyTorch.distributed provides communication
                      primitives but lack the flexibility of the capabilities of
                      the MPI standard. PyTorch.distributed currently supports
                      automatic differentiation for point-to-point communication,
                      but not for collective communication. In contrast, flexible
                      and effective communication patterns are useful for
                      effective non-additive separable loss functions encountered
                      in self-supervised contrastive learning approaches. This
                      poster explores the implementation of the adjoint MPI
                      concept in distributed deep learning models. It begins with
                      an introduction to the fundamental principles of adjoint
                      modelling, followed by an adjoint MPI concept for
                      distributed Deep Learning which enables flexible
                      parallelization in conjunction with existing libraries. The
                      poster also covers implementation details and usage of the
                      approach in contrastive self-supervised learning.},
      month         = {May},
      date          = {2023-05-21},
      organization  = {ISC High Performance 2023, Hamburg
                       (Germany), 21 May 2023 - 25 May 2023},
      subtyp        = {After Call},
      cin          = {JSC},
      cid          = {I:(DE-Juel1)JSC-20090406},
      pnm          = {5111 - Domain-Specific Simulation $\&$ Data Life Cycle Labs
                      (SDLs) and Research Groups (POF4-511) / SLNS - SimLab
                      Neuroscience (Helmholtz-SLNS)},
      pid          = {G:(DE-HGF)POF4-5111 / G:(DE-Juel1)Helmholtz-SLNS},
      typ          = {PUB:(DE-HGF)24},
      url          = {https://juser.fz-juelich.de/record/1019553},
}