% IMPORTANT: The following is UTF-8 encoded. This means that in the presence
% of non-ASCII characters, it will not work with BibTeX 0.99 or older.
% Instead, you should use an up-to-date BibTeX implementation like “bibtex8” or
% “biber”.
@MISC{Clausen:1007000,
author = {Clausen, Alexander and Weber, Dieter and Bryan, Matthew and
Ruzaeva, Karina and Migunov, Vadim and Baburajan, Anand and
Bahuleyan, Abijith and Caron, Jan and Chandra, Rahul and
Dey, Shankhadeep and Halder, Sayandip and Katz, Daniel S.
and Levin, Barnaby D. A. and Nord, Magnus and Ophus, Colin
and Peter, Simon and Puskás, Levente and Schyndel van, Jay
and Shin, Jaeweon and Sunku, Sai and Müller-Caspary, Knut
and Dunin-Borkowski, Rafal E. and Ånes, Håkon W.},
title = {{L}iber{TEM}/{L}iber{TEM}: 0.11.0},
reportid = {FZJ-2023-01939},
year = {2023},
abstract = {Homepage: https://libertem.github.io/LiberTEM/ GitHub
repository: https://github.com/LiberTEM/LiberTEM/ PyPI:
https://pypi.org/project/libertem/ LiberTEM is an open
source platform for high-throughput distributed processing
of large-scale binary data sets and live data streams using
a modified MapReduce programming model . The current focus
is pixelated scanning transmission electron microscopy (
STEM ) and scanning electron beam diffraction data.
MapReduce-like processing allows to specify an algorithm
through two functions: One function that is mapped on
portions of the input data, and another function that merges
(reduces) a partial result from this mapping step into the
complete result. A wide range of TEM and 4D STEM processing
tasks can be expressed in this fashion, see Applications .
The UDF interface of LiberTEM offers a standardized,
versatile API to decouple the mathematical core of an
algorithm from details of data source, parallelism, and use
of results. Mapping and merging can be performed in any
order and with different subdivisions of the input data,
including running parts of the calculation concurrently.
That means the same implementation can be used in a wide
range of modalities, including massive scaling on clusters.
Since each merge step produces an intermediate result, this
style of processing is suitable for displaying live results
from a running calculation in a GUI application and for
processing live data streams . A closed-loop feedback
between processing and instrument control can be realized as
well. See User-defined functions for more details on the
LiberTEM UDF interface. The LiberTEM back-end offers high
throughput and scalability on PCs, single server nodes,
clusters and cloud services. On clusters it can use fast
distributed local storage on high-performance SSDs. That way
it achieves very high aggregate IO performance on a compact
and cost-efficient system built from stock components. All
CPU cores and CUDA devices in a system can be used in
parallel. LiberTEM is supported on Linux, Mac OS X and
Windows. Other platforms that allow installation of Python
3.7+ and the required packages will likely work as well. The
GUI is running in a web browser. Installation The short
version: $ virtualenv -p python3 ~/libertem-venv/ $ source
~/libertem-venv/bin/activate ( libertem-venv ) $ python -m
pip install 'libertem[torch]' # optional for GPU support #
See also https://docs.cupy.dev/en/stable/install.html (
libertem-venv ) $ python -m pip install cupy Please see our
documentation for details! Alternatively, to run the
LiberTEM Docker image : $ docker run -p localhost:9000:9000
--mount type = bind,source = /path/to/your/data/,dst =
/data/,ro libertem/libertem or $ singularity exec
docker://libertem/libertem /venv/bin/libertem-server
Deployment for offline data processing on a single-node
system for a local user is thoroughly tested and can be
considered stable. Deployment on a cluster is experimental
and still requires some additional work, see Issue #105 .
Back-end support for live data processing is still
experimental as well, see
https://github.com/LiberTEM/LiberTEM-live . Applications
Since LiberTEM is programmable through user-defined
functions (UDFs) , it can be used for a wide range of
processing tasks on array-like data and data streams. The
following applications have been implemented already:
Virtual detectors (virtual bright field, virtual HAADF,
center of mass , custom shapes via masks) Analysis of
amorphous materials Strain mapping Off-axis electron
holography reconstruction Single Side Band ptychography Some
of these applications are available through an interactive
web GUI . Please see the applications section of our
documentation for details! The Python API and user-defined
functions (UDFs) can be used for complex operations such as
arbitrary linear operations and other features like data
export. Example Jupyter notebooks are available in the
examples directory . If you are having trouble running the
examples, please let us know by filing an issue or by
joining our Gitter chat . LiberTEM is suitable as a
high-performance processing backend for other applications,
including live data streams. Contact us if you are
interested! LiberTEM is evolving rapidly and prioritizes
features following user demand and contributions. Currently
we are working on live data processing , improving
application support for sparse data and event-based
detectors, performance improvements for GPU processing, and
implementing analysis methods for various applications of
pixelated STEM and other large-scale detector data. If you
like to influence the direction this project is taking, or
if you'd like to contribute , please join our gitter chat
and our general mailing list . File formats LiberTEM
currently opens most file formats used for pixelated STEM.
See our general information on loading data and
format-specific documentation for more information! Raw
binary files NumPy .npy binary files Thermo Fisher EMPAD
detector files Quantum Detectors MIB format Nanomegas .blo
block files Direct Electron DE5 files (HDF5-based) and
Norpix SEQ files for DE-Series detectors Gatan K2 IS raw
format Stacks of Gatan DM3 and DM4 files (via openNCEM )
Single-file Gatan DM4 scans when saved using C-ordering
FRMS6 from PNDetector pnCCD cameras (currently alpha, gain
correction still needs UI changes) FEI SER files (via
openNCEM ) MRC (via openNCEM ) HDF5-based formats such as
HyperSpy files, NeXus and EMD TVIPS binary files Sparse data
in Raw CSR (compressed sparse row) format, as is possible to
generate from event-based detectors Please contact us if you
are interested in support for an additional format! Live
processing and detectors (experimental) See LiberTEM-live !
License LiberTEM is licensed under GPLv3. The I/O parts are
also available under the MIT license, please see LICENSE
files in the subdirectories for details. Acknowledgements We
are very grateful for your continuing support for LiberTEM!
See the acknowledgement page for a list of authors and
contributors to LiberTEM and its subprojects. See also our
info on funding and industry partners .},
keywords = {STEM (Other) / TEM (Other) / pixelated STEM (Other) / 4D
STEM (Other) / high-throughput (Other) / electron microscopy
(Other)},
cin = {ER-C-1 / ER-C-2},
cid = {I:(DE-Juel1)ER-C-1-20170209 / I:(DE-Juel1)ER-C-2-20170209},
pnm = {5351 - Platform for Correlative, In Situ and Operando
Characterization (POF4-535) / 5353 - Understanding the
Structural and Functional Behavior of Solid State Systems
(POF4-535) / VIDEO - Versatile and Innovative Detector for
Electron Optics (780487) / CritCat - Towards Replacement of
Critical Catalyst Materials by Improved Nanoparticle Control
and Rational Design (686053) / ESTEEM3 - Enabling Science
and Technology through European Electron Microscopy (823717)
/ 3D MAGiC - Three-dimensional magnetization textures:
Discovery and control on the nanoscale (856538) / moreSTEM -
Momentum-resolved Scanning Transmission Electron Microscopy
(VH-NG-1317) / Ptychography 4.0 - Proposal for a pilot
project "Information $\&$ Data Science" (ZT-I-0025) / AIDAS
- Joint Virtual Laboratory for AI, Data Analytics and
Scalable Simulation $(aidas_20200731)$},
pid = {G:(DE-HGF)POF4-5351 / G:(DE-HGF)POF4-5353 /
G:(EU-Grant)780487 / G:(EU-Grant)686053 / G:(EU-Grant)823717
/ G:(EU-Grant)856538 / G:(DE-HGF)VH-NG-1317 /
G:(DE-HGF)ZT-I-0025 / $G:(DE-Juel-1)aidas_20200731$},
typ = {PUB:(DE-HGF)33},
doi = {10.5281/ZENODO.7853070},
url = {https://juser.fz-juelich.de/record/1007000},
}
guest ::