% IMPORTANT: The following is UTF-8 encoded.  This means that in the presence
% of non-ASCII characters, it will not work with BibTeX 0.99 or older.
% Instead, you should use an up-to-date BibTeX implementation like “bibtex8” or
% “biber”.

@INPROCEEDINGS{Quercia:1046535,
      author       = {Quercia, Alessio and Cao, Zhuo and Bangun, Arya and Paul,
                      Richard Dominik and Morrison, Abigail and Assent, Ira and
                      Scharr, Hanno},
      title        = {{S}ummation {C}ompression for {V}ery-{L}ow {R}ank
                      {A}daptation},
      school       = {RWTH Aachen},
      reportid     = {FZJ-2025-03853},
      year         = {2025},
      abstract     = {Parameter-Efficient Fine-Tuning (PEFT) methods have
                      transformed the approach to fine-tuning large models for
                      downstream tasks by enabling the adjustment of significantly
                      fewer parameters than those in the original model matrices.
                      In this work, we study the "very low rank regime", where we
                      fine-tune the lowest amount of parameters per linear layer
                      for each considered PEFT method. We propose 1LoRA (Summation
                      Low-Rank Adaptation), a compute, parameter and memory
                      efficient fine-tuning method which uses the feature sum as
                      fixed compression and a single trainable vector as
                      decompression. Differently from state-of-the-art PEFT
                      methods like LoRA, VeRA, and the recent MoRA, 1LoRA uses
                      fewer parameters per layer, reducing the memory footprint
                      and the computational cost. We extensively evaluate our
                      method against state-of-the-art PEFT methods on multiple
                      fine-tuning tasks, and show that our method not only
                      outperforms them, but is also more parameter, memory and
                      computationally efficient. Moreover, thanks to its memory
                      efficiency, 1LoRA allows to fine-tune more evenly across
                      layers, instead of focusing on specific ones (e.g. attention
                      layers), improving performance further.},
      month         = {Jun},
      date          = {2025-06-03},
      organization  = {Helmholtz AI Conference, Karlsruhe
                       (Germany), 3 Jun 2025 - 5 Jun 2025},
      subtyp        = {After Call},
      cin          = {IAS-8 / IAS-6},
      cid          = {I:(DE-Juel1)IAS-8-20210421 / I:(DE-Juel1)IAS-6-20130828},
      pnm          = {5112 - Cross-Domain Algorithms, Tools, Methods Labs (ATMLs)
                      and Research Groups (POF4-511)},
      pid          = {G:(DE-HGF)POF4-5112},
      typ          = {PUB:(DE-HGF)6},
      url          = {https://juser.fz-juelich.de/record/1046535},
}