% IMPORTANT: The following is UTF-8 encoded.  This means that in the presence
% of non-ASCII characters, it will not work with BibTeX 0.99 or older.
% Instead, you should use an up-to-date BibTeX implementation like “bibtex8” or
% “biber”.

@INPROCEEDINGS{Cherti:905638,
      author       = {Cherti, Mehdi and Jitsev, Jenia},
      title        = {{E}ffect of {P}re-{T}raining {S}cale on {I}ntra- and
                      {I}nter-{D}omain {F}ull and {F}ew-{S}hot {T}ransfer
                      {L}earning for {N}atural and {M}edical {X}-{R}ay {C}hest
                      {I}mages},
      reportid     = {FZJ-2022-00865},
      pages        = {1-6},
      year         = {2021},
      abstract     = {Transfer learning aims to exploit pre-trained models for
                      more efficient follow-up training on wide range of
                      downstream tasks and datasets, enabling successful training
                      also on small data. Recently, strong improvement was shown
                      for transfer learning and model generalization when
                      increasing model, data and compute budget scale in the
                      pre-training. To compare effect of scale both in intra- and
                      inter-domain full and few-shot transfer, in this study we
                      combine for the first time large openly available medical
                      X-Ray chest imaging datasets to reach a dataset scale
                      comparable to ImageNet-1k. We then conduct pre-training and
                      transfer to different natural or medical targets while
                      varying network size and source data scale and domain, being
                      either large natural (ImageNet-1k/21k) or large medical
                      chest X-Ray datasets. We observe strong improvement due to
                      larger pre-training scale for intra-domain natural-natural
                      and medical-medical transfer. For inter-domain
                      natural-medical transfer, we find improvements due to larger
                      pre-training scale on larger X-Ray targets in full shot
                      regime, while for smaller targets and for few-shot regime
                      the improvement is not visible. Remarkably, large networks
                      pre-trained on very large natural ImageNet-21k are as good
                      or better than networks pre-trained on largest available
                      medical X-Ray data when performing transfer to large X-Ray
                      targets. We conclude that high quality models for
                      inter-domain transfer can be also obtained by substantially
                      increasing scale of model and generic natural source data,
                      removing necessity for large domain-specific medical source
                      data in the pre-training. Code is available at:
                      $\url{https://github.com/SLAMPAI/large-scale-pretraining-transfer}}$},
      month         = {Dec},
      date          = {2021-12-06},
      organization  = {Medical Imaging Meets NeurIPS
                       (MedNeurIPS), Sydney / online
                       (Australia), 6 Dec 2021 - 14 Dec 2021},
      cin          = {JSC},
      cid          = {I:(DE-Juel1)JSC-20090406},
      pnm          = {5112 - Cross-Domain Algorithms, Tools, Methods Labs (ATMLs)
                      and Research Groups (POF4-511)},
      pid          = {G:(DE-HGF)POF4-5112},
      typ          = {PUB:(DE-HGF)8},
      eprint       = {2106.00116},
      howpublished = {arXiv:2106.00116},
      archivePrefix = {arXiv},
      SLACcitation = {$\%\%CITATION$ = $arXiv:2106.00116;\%\%$},
      url          = {https://juser.fz-juelich.de/record/905638},
}