% IMPORTANT: The following is UTF-8 encoded. This means that in the presence
% of non-ASCII characters, it will not work with BibTeX 0.99 or older.
% Instead, you should use an up-to-date BibTeX implementation like “bibtex8” or
% “biber”.
@MISC{Clausen:1043683,
author = {Clausen, Alexander and Weber, Dieter and Bryan, Matthew and
Ruzaeva, Karina and Migunov, Vadim and Dagenborg, Sivert and
Baburajan, Anand and Bahuleyan, Abijith and Caron, Jan and
Chandra, Rahul and Dey, Shankhadeep and Halder, Sayandip and
Katz, Daniel S. and Levin, Barnaby D. A. and Nord, Magnus
and Ophus, Colin and Peter, Simon and Puskás, Levente and
Schyndel van, Jay and Shin, Jaeweon and Sunku, Sai and
Ånes, Håkon W. and Müller-Caspary, Knut and
Dunin-Borkowski, Rafal E. and Huang, Chen},
othercontributors = {Barthel, Juri and Bauer, Reimar and Becker, Julian and
Beckmann, Andreas and Bücker, Robert and zu Castell,
Wolfgang and Fedorov, Andrey and Jensen, Mark and Krajnak,
Matus and Crout, Phillip and Ehrig, Simeon and Eljarrat
Ascunce, Alberto and Ercius, Peter and Fuery, Caroline and
Furhmann, Patrick and Gaida, John and Goscinski, Wojtek
James and Guzzinati, Giulio and Haas, Benedikt and Hines,
Chris and Houben, Lothar and Huth, Martin and Jemian, Pete
and Johnstone, Duncan N. and Koch, Christoph and Krings,
Alexander and Lam, Siu Kwan and Lei, Weng I and Lesnichaia,
Anastasiia and Liberti, Emanuela and Lu, Penghan and
MacLaren, Ian and Mahr, Christoph and McCartan, Shane and
Meissner, Heide and Mittelberger, Andreas and Moldovan,
Grigore and Müller, Johannes and Nebot, Eduardo and Oda,
Terri and O'Ryan, Liam and Oster, Marco and Pakzad, Ana and
Pekin, Thomas C. and Peña, Francisco de la and Richter,
Tobias and Ritz, Robert and Sander, Kunt and Schloz, Marcel
and Schuh, Michael and Shabih, Sherjeel and Simson, Martin
and Sparlinek, Peter and Stewart, Andy and Sukumaran, Murali
and Sweeney, Eugene and Verbeeck, Jo and Voyles, Paul and
Watts, Benjamin and Wepf, Roger and Wilbrink, Jacob and
Wilson, Lance and Winkler, Florian and Wollgarten, Markus
and @theassassin and Ozsoy-Keskinbora, Cigdem},
title = {{L}iber{TEM}/{L}iber{TEM}: 0.14.2},
reportid = {FZJ-2025-02981},
year = {2025},
abstract = {Homepage: https://libertem.github.io/LiberTEM/GitHub
repository: https://github.com/LiberTEM/LiberTEM/PyPI:
https://pypi.org/project/libertem/ LiberTEM is an open
source platform for high-throughput distributed processing
of large-scale binary data sets and live data streams using
a modified MapReduce programming model. The current focus is
pixelated scanning transmission electron microscopy (STEM)
and scanning electron beam diffraction data. MapReduce-like
processing allows to specify an algorithm through two
functions: One function that is mapped on portions of the
input data, and another function that merges (reduces) a
partial result from this mapping step into the complete
result. A wide range of TEM and 4D STEM processing tasks can
be expressed in this fashion, see Applications. The UDF
interface of LiberTEM offers a standardized, versatile API
to decouple the mathematical core of an algorithm from
details of data source, parallelism, and use of results.
Mapping and merging can be performed in any order and with
different subdivisions of the input data, including running
parts of the calculation concurrently. That means the same
implementation can be used in a wide range of modalities,
including massive scaling on clusters. Since each merge step
produces an intermediate result, this style of processing is
suitable for displaying live results from a running
calculation in a GUI application and for processing live
data streams. A closed-loop feedback between processing and
instrument control can be realized as well. See User-defined
functions for more details on the LiberTEM UDF interface.
The LiberTEM back-end offers high throughput and scalability
on PCs, single server nodes, clusters and cloud services. On
clusters it can use fast distributed local storage on
high-performance SSDs. That way it achieves very high
aggregate IO performance on a compact and cost-efficient
system built from stock components. All CPU cores and CUDA
devices in a system can be used in parallel. LiberTEM is
supported on Linux, Mac OS X and Windows. Other platforms
that allow installation of Python 3.7+ and the required
packages will likely work as well. The GUI is running in a
web browser. Installation The short version: $ virtualenv -p
python3 ~/libertem-venv/ $ source
~/libertem-venv/bin/activate (libertem-venv) $ python -m pip
install 'libertem[torch]' # optional for GPU support # See
also https://docs.cupy.dev/en/stable/install.html
(libertem-venv) $ python -m pip install cupy Please see our
documentation for details! Alternatively, to run the
LiberTEM Docker image: $ docker run -p localhost:9000:9000
--mount type=bind,source=/path/to/your/data/,dst=/data/,ro
ghcr.io/libertem/libertem or $ singularity exec
docker://ghcr.io/libertem/libertem /venv/bin/libertem-server
Deployment for offline data processing on a single-node
system for a local user is thoroughly tested and can be
considered stable. Deployment on a cluster is experimental
and still requires some additional work, see Issue #105.
Back-end support for live data processing is still
experimental as well, see
https://github.com/LiberTEM/LiberTEM-live. Applications
Since LiberTEM is programmable through user-defined
functions (UDFs), it can be used for a wide range of
processing tasks on array-like data and data streams. The
following applications have been implemented already:
Virtual detectors (virtual bright field, virtual HAADF,
center of mass , custom shapes via masks) Analysis of
amorphous materials Strain mapping Off-axis electron
holography reconstruction Single Side Band ptychography Some
of these applications are available through an interactive
web GUI. Please see the applications section of our
documentation for details! The Python API and user-defined
functions (UDFs) can be used for complex operations such as
arbitrary linear operations and other features like data
export. Example Jupyter notebooks are available in the
examples directory. If you are having trouble running the
examples, please let us know by filing an issue or by
joining our Gitter chat. LiberTEM is suitable as a
high-performance processing backend for other applications,
including live data streams. Contact us if you are
interested! LiberTEM is evolving rapidly and prioritizes
features following user demand and contributions. Currently
we are working on live data processing, improving
application support for sparse data and event-based
detectors, performance improvements for GPU processing, and
implementing analysis methods for various applications of
pixelated STEM and other large-scale detector data. If you
like to influence the direction this project is taking, or
if you'd like to contribute, please join our gitter chat and
our general mailing list. File formats LiberTEM currently
opens most file formats used for pixelated STEM. See our
general information on loading data and format-specific
documentation for more information! Raw binary files NumPy
.npy binary files Thermo Fisher EMPAD detector files Quantum
Detectors MIB format Nanomegas .blo block files Direct
Electron DE5 files (HDF5-based) and Norpix SEQ files for
DE-Series detectors Gatan K2 IS raw format Stacks of Gatan
DM3 and DM4 files (via openNCEM) Single-file Gatan DM4 scans
when saved using C-ordering FRMS6 from PNDetector pnCCD
cameras (currently alpha, gain correction still needs UI
changes) FEI SER files (via openNCEM) MRC (via openNCEM)
HDF5-based formats such as HyperSpy files, NeXus and EMD
TVIPS binary files Sparse data in Raw CSR (compressed sparse
row) format, as is possible to generate from event-based
detectors Please contact us if you are interested in support
for an additional format! Live processing and detectors
(experimental) See LiberTEM-live! License LiberTEM is
licensed under GPLv3. The I/O parts are also available under
the MIT license, please see LICENSE files in the
subdirectories for details. Acknowledgements We are very
grateful for your continuing support for LiberTEM! See the
acknowledgement page for a list of authors and contributors
to LiberTEM and its subprojects. See also our info on
funding and industry partners.},
keywords = {STEM (Other) / TEM (Other) / pixelated STEM (Other) / 4D
STEM (Other) / high-throughput (Other) / electron microscopy
(Other)},
cin = {ER-C-1 / ER-C-2},
cid = {I:(DE-Juel1)ER-C-1-20170209 / I:(DE-Juel1)ER-C-2-20170209},
pnm = {5351 - Platform for Correlative, In Situ and Operando
Characterization (POF4-535) / 5353 - Understanding the
Structural and Functional Behavior of Solid State Systems
(POF4-535) / moreSTEM - Momentum-resolved Scanning
Transmission Electron Microscopy (VH-NG-1317) / Ptychography
4.0 - Proposal for a pilot project "Information $\&$ Data
Science" (ZT-I-0025) / AIDAS - Joint Virtual Laboratory for
AI, Data Analytics and Scalable Simulation
$(aidas_20200731)$},
pid = {G:(DE-HGF)POF4-5351 / G:(DE-HGF)POF4-5353 /
G:(DE-HGF)VH-NG-1317 / G:(DE-HGF)ZT-I-0025 /
$G:(DE-Juel-1)aidas_20200731$},
typ = {PUB:(DE-HGF)33},
doi = {10.5281/ZENODO.15191240},
url = {https://juser.fz-juelich.de/record/1043683},
}
Gast ::