% IMPORTANT: The following is UTF-8 encoded. This means that in the presence
% of non-ASCII characters, it will not work with BibTeX 0.99 or older.
% Instead, you should use an up-to-date BibTeX implementation like “bibtex8” or
% “biber”.
@MISC{Clausen:897390,
author = {Clausen, Alexander and Weber, Dieter and Ruzaeva, Karina
and Migunov, Vadim and Baburajan, Anand and Bahuleyan,
Abijith and Bryan, Matthew and Caron, Jan and Chandra, Rahul
and Dey, Shankhadeep and Halder, Sayandip and Katz, Daniel
S. and Levin, Barnaby D. A. and Nord, Magnus and Ophus,
Colin and Peter, Simon and Puskás, Levente and Schyndel
van, Jay and Shin, Jaeweon and Sunku, Sai and
Müller-Caspary, Knut and Dunin-Borkowski, Rafal E.},
title = {{L}iber{TEM}/{L}iber{TEM}: 0.8.0},
reportid = {FZJ-2021-03756},
year = {2021},
abstract = {<div class='document'> <strong>Homepage: <a
href='https://libertem.github.io/LiberTEM/'>https://libertem.github.io/LiberTEM/</a><br></strong><strong>GitHub
repository: <a
href='https://github.com/LiberTEM/LiberTEM/'>https://github.com/LiberTEM/LiberTEM/</a><br></strong><strong>PyPI:
<a
href='https://pypi.org/project/libertem/'>https://pypi.org/project/libertem/</a><br></strong><p>LiberTEM
is an open source platform for high-throughput distributed
processing of large-scale binary data sets and live data
streams using a modified <a class='reference external'
href='https://en.wikipedia.org/wiki/MapReduce'>MapReduce
programming model</a>. The current focus is <a
class='reference external'
$href='https://en.wikipedia.org/wiki/Scanning_transmission_electron_microscopy#Universal_detectors_(4D_STEM)'>pixelated</a>$
scanning transmission electron microscopy (<a
class='reference external'
$href='https://en.wikipedia.org/wiki/Scanning_transmission_electron_microscopy'>STEM</a>)$
and scanning electron beam diffraction data.</p>
<p>MapReduce-like processing allows to specify an algorithm
through two functions: One function that is mapped on
portions of the input data, and another function that merges
(reduces) a partial result from this mapping step into the
complete result. A wide range of TEM and 4D STEM processing
tasks can be expressed in this fashion, see <a
class='reference internal'
href='#applications'>Applications</a>.</p> <p>The UDF
interface of LiberTEM offers a standardized, versatile API
to decouple the mathematical core of an algorithm from
details of data source, parallelism, and use of results.
Mapping and merging can be performed in any order and with
different subdivisions of the input data, including running
parts of the calculation concurrently. That means the same
implementation can be used in a wide range of modalities,
including massive scaling on clusters. Since each merge step
produces an intermediate result, this style of processing is
suitable for displaying live results from a running
calculation in a GUI application and for <a class='reference
external'
href='https://github.com/LiberTEM/LiberTEM-live'>processing
live data streams</a>. A closed-loop feedback between
processing and instrument control can be realized as well.
See <a class='reference external'
href='https://libertem.github.io/LiberTEM/udf.html'>User-defined
functions</a> for more details on the LiberTEM UDF
interface.</p> <p>The LiberTEM back-end offers <a
class='reference external'
href='https://libertem.github.io/LiberTEM/architecture.html'>high
throughput and scalability</a> on PCs, single server nodes,
clusters and cloud services. On clusters it can use fast
distributed local storage on high-performance SSDs. That way
it achieves <a class='reference external'
href='https://libertem.github.io/LiberTEM/performance.html'>very
high aggregate IO performance</a> on a compact and
cost-efficient system built from stock components. All CPU
cores and CUDA devices in a system can be used in
parallel.</p> <p>LiberTEM is supported on Linux, Mac OS X
and Windows. Other platforms that allow installation of
Python 3.6+ and the required packages will likely work as
well. The GUI is running in a web browser.</p> <div
class='section'> <p><strong>Installation</strong></p><p>The
short version:</p> <pre class='code shell literal-block'>$
virtualenv -p python3 ~/libertem-venv/ $ <span class='name
builtin'>source</span> ~/libertem-venv/bin/activate <span
class='operator'>(</span>libertem-venv<span
class='operator'>)</span> $ python -m pip install <span
class='literal string double'>'libertem[torch]'</span> <span
class='comment single'># optional for GPU support # See also
https://docs.cupy.dev/en/stable/install.html </span><span
class='operator'>(</span>libertem-venv<span
class='operator'>)</span> $ python -m pip install cupy
</pre> <p>Please see <a class='reference external'
href='https://libertem.github.io/LiberTEM/install.html'>our
documentation</a> for details!</p> <p>Deployment for offline
data processing on a single-node system for a local user is
thoroughly tested and can be considered stable. Deployment
on a cluster is experimental and still requires some
additional work, see <a class='reference external'
href='https://github.com/LiberTEM/LiberTEM/issues/105'>Issue
#105</a>. Back-end support for live data processing is still
experimental as well, see <a class='reference external'
href='https://github.com/LiberTEM/LiberTEM-live'>https://github.com/LiberTEM/LiberTEM-live</a>.</p>
</div> <div class='section'>
<p><strong>Applications</strong></p><p>Since LiberTEM is
programmable through <a class='reference external'
href='https://libertem.github.io/LiberTEM/udf.html'>user-defined
functions (UDFs)</a>, it can be used for a wide range of
processing tasks on array-like data and data streams. The
following applications have been implemented already:</p>
<ul class='simple'> <li>Virtual detectors (virtual bright
field, virtual HAADF, center of mass , custom shapes via
masks)</li> <li><a class='reference external'
href='https://libertem.github.io/LiberTEM/app/amorphous.html'>Analysis
of amorphous materials</a></li> <li><a class='reference
external'
href='https://libertem.github.io/LiberTEM-blobfinder/'>Strain
mapping</a></li> <li><a class='reference external'
href='https://libertem.github.io/LiberTEM/app/holography.html'>Off-axis
electron holography reconstruction</a></li> <li><a
class='reference external'
href='https://ptychography-4-0.github.io/ptychography/'>Single
Side Band ptychography</a></li> </ul> <p>Some of these
applications are available through an <a class='reference
external'
href='https://libertem.github.io/LiberTEM/usage.html#gui-usage'>interactive
web GUI</a>. Please see <a class='reference external'
href='https://libertem.github.io/LiberTEM/applications.html'>the
applications section</a> of our documentation for
details!</p> <p>The Python API and user-defined functions
(UDFs) can be used for complex operations such as arbitrary
linear operations and other features like data export.
Example Jupyter notebooks are available in the <a
class='reference external'
href='https://github.com/LiberTEM/LiberTEM/tree/master/examples'>examples
directory</a>. If you are having trouble running the
examples, please let us know by filing an issue or by <a
class='reference external'
href='https://gitter.im/LiberTEM/Lobby'>joining our Gitter
chat</a>.</p> <p>LiberTEM is suitable as a high-performance
processing backend for other applications, including live
data streams. <a class='reference external'
href='https://gitter.im/LiberTEM/Lobby'>Contact us</a> if
you are interested!</p> <p>LiberTEM is evolving rapidly and
prioritizes features following user demand and
contributions. Currently we are working on <a
class='reference external'
href='https://github.com/LiberTEM/LiberTEM-live'>live data
processing</a>, <a class='reference external'
href='https://github.com/LiberTEM/LiberTEM/issues/922'>integration
with Dask arrays and Hyperspy</a>, support for sparse data,
and implementing analysis methods for various applications
of pixelated STEM and other large-scale detector data. If
you like to influence the direction this project is taking,
or if you'd like to <a class='reference external'
href='https://libertem.github.io/LiberTEM/contributing.html'>contribute</a>,
please join our <a class='reference external'
href='https://gitter.im/LiberTEM/Lobby'>gitter chat</a> and
our <a class='reference external'
href='https://groups.google.com/forum/#!forum/libertem'>general
mailing list</a>.</p> </div> <div class='section'>
<p><strong>File formats</strong></p><p>LiberTEM currently
opens most file formats used for pixelated STEM. See <a
class='reference external'
href='https://libertem.github.io/LiberTEM/formats.html'>our
general information on loading data</a> and <a
class='reference external'
href='https://libertem.github.io/LiberTEM/reference/dataset.html#formats'>format-specific
documentation</a> for more information!</p> <ul
class='simple'> <li>Raw binary files</li> <li>Thermo Fisher
EMPAD detector files</li> <li><a class='reference external'
href='http://quantumdetectors.com/wp-content/uploads/2017/01/1532-Merlin-for-EM-Technical-Datasheet-v2.pdf'>Quantum
Detectors MIB format</a></li> <li>Nanomegas .blo block
files</li> <li>Direct Electron DE5 files (HDF5-based) and
Norpix SEQ files for <a class='reference external'
href='http://www.directelectron.com/de-series/'>DE-Series</a>
detectors</li> <li><a class='reference external'
href='https://web.archive.org/web/20180809021832/http://www.gatan.com/products/tem-imaging-spectroscopy/k2-camera'>Gatan
K2 IS</a> raw format</li> <li>Stacks of Gatan DM3 and DM4
files (via <a class='reference external'
href='https://github.com/ercius/openNCEM'>openNCEM</a>)</li>
<li>FRMS6 from PNDetector pnCCD cameras (currently alpha,
gain correction still needs UI changes)</li> <li>FEI SER
files (via <a class='reference external'
href='https://github.com/ercius/openNCEM'>openNCEM</a>)</li>
<li>MRC (via <a class='reference external'
href='https://github.com/ercius/openNCEM'>openNCEM</a>)</li>
<li>HDF5-based formats such as Hyperspy files, NeXus and
EMD</li> <li>Please contact us if you are interested in
support for an additional format!</li> </ul> </div> <div
class='section'> <p><strong>Detectors
(experimental)</strong></p><p>Currently the Quantum
Detectors Merlin camera is supported for live processing.
Support for the Gatan K2 IS camera is in a prototype state.
Please <a class='reference external'
href='https://gitter.im/LiberTEM/Lobby'>contact us</a> if
you are interested in this feature! See <a class='reference
external'
href='https://github.com/LiberTEM/LiberTEM-live'>https://github.com/LiberTEM/LiberTEM-live</a>
for more details on live processing.</p> </div> <div
class='section'> <p><strong>License</strong></p><p>LiberTEM
is licensed under GPLv3. The I/O parts are also available
under the MIT license, please see LICENSE files in the
subdirectories for details.</p> </div> </div>},
keywords = {STEM (Other) / TEM (Other) / pixelated STEM (Other) / 4D
STEM (Other) / high-throughput (Other) / electron microscopy
(Other)},
cin = {ER-C-1},
cid = {I:(DE-Juel1)ER-C-1-20170209},
pnm = {5351 - Platform for Correlative, In Situ and Operando
Characterization (POF4-535) / VIDEO - Versatile and
Innovative Detector for Electron Optics (780487) / CritCat -
Towards Replacement of Critical Catalyst Materials by
Improved Nanoparticle Control and Rational Design (686053) /
3D MAGiC - Three-dimensional magnetization textures:
Discovery and control on the nanoscale (856538) / ESTEEM3 -
Enabling Science and Technology through European Electron
Microscopy (823717)},
pid = {G:(DE-HGF)POF4-5351 / G:(EU-Grant)780487 /
G:(EU-Grant)686053 / G:(EU-Grant)856538 /
G:(EU-Grant)823717},
typ = {PUB:(DE-HGF)33},
doi = {10.5281/ZENODO.5547992},
url = {https://juser.fz-juelich.de/record/897390},
}
guest ::