% IMPORTANT: The following is UTF-8 encoded.  This means that in the presence
% of non-ASCII characters, it will not work with BibTeX 0.99 or older.
% Instead, you should use an up-to-date BibTeX implementation like “bibtex8” or
% “biber”.

@INPROCEEDINGS{Penke:1034067,
      author       = {Penke, Carolin},
      title        = {{M}athematical {T}echniques to {R}educe {M}emory
                      {R}equirements in {D}eep {L}earning},
      reportid     = {FZJ-2024-06888},
      year         = {2024},
      abstract     = {We present a method to substantially lower memory
                      requirements during the training of deep neural networks,
                      based on the GaLore (Gradient Low-Rank Projection) training
                      framework. A rapid decay of singular values in gradient
                      matrices permits the use of low-rank bases to encapsulate
                      the relevant subspaces, reducing the memory requirements for
                      storing optimizer states between iterations. A novel,
                      rank-adaptive, GPU-optimized version of the randomized range
                      finder algorithm is employed to exploit this property and
                      future research directions are discussed.},
      month         = {Nov},
      date          = {2024-11-05},
      organization  = {OpenGPT-X Forum 2024, Berlin
                       (Germany), 5 Nov 2024 - 5 Nov 2024},
      subtyp        = {Other},
      cin          = {JSC},
      cid          = {I:(DE-Juel1)JSC-20090406},
      pnm          = {5112 - Cross-Domain Algorithms, Tools, Methods Labs (ATMLs)
                      and Research Groups (POF4-511) / OpenGPT-X - Aufbau eines
                      Gaia-X Knotens für große KI-Sprachmodelle und innovative
                      Sprachapplikations-Services; Teilvorhaben: Optimierung und
                      Skalierung auf großen HPC-Systemen (68GX21007F)},
      pid          = {G:(DE-HGF)POF4-5112 / G:(DE-Juel-1)68GX21007F},
      typ          = {PUB:(DE-HGF)6},
      doi          = {10.34734/FZJ-2024-06888},
      url          = {https://juser.fz-juelich.de/record/1034067},
}