001017950 001__ 1017950
001017950 005__ 20240403082756.0
001017950 0247_ $$2doi$$a10.1109/IGARSS52108.2023.10283416
001017950 0247_ $$2datacite_doi$$a10.34734/FZJ-2023-04455
001017950 0247_ $$2WOS$$aWOS:001098971601004
001017950 037__ $$aFZJ-2023-04455
001017950 1001_ $$0P:(DE-HGF)0$$aTian, Liang$$b0
001017950 1112_ $$aIEEE International Geoscience and Remote Sensing Symposium (IGARSS)$$cPasadena$$d2023-07-16 - 2023-07-21$$wCA
001017950 245__ $$aEnd-to-End Process Orchestration of Earth Observation Data Workflows with Apache Airflow on High Performance Computing
001017950 260__ $$bIEEE$$c2023
001017950 29510 $$aIGARSS 2023 - 2023 IEEE International Geoscience and Remote Sensing Symposium : [Proceedings] - IEEE, 2023. - ISBN 979-8-3503-2010-7 - doi:10.1109/IGARSS52108.2023.10283416
001017950 300__ $$a711-714
001017950 3367_ $$2ORCID$$aCONFERENCE_PAPER
001017950 3367_ $$033$$2EndNote$$aConference Paper
001017950 3367_ $$2BibTeX$$aINPROCEEDINGS
001017950 3367_ $$2DRIVER$$aconferenceObject
001017950 3367_ $$2DataCite$$aOutput Types/Conference Paper
001017950 3367_ $$0PUB:(DE-HGF)8$$2PUB:(DE-HGF)$$aContribution to a conference proceedings$$bcontrib$$mcontrib$$s1702456531_7741
001017950 3367_ $$0PUB:(DE-HGF)7$$2PUB:(DE-HGF)$$aContribution to a book$$mcontb
001017950 520__ $$aEarth Observation (EO) data processing faces challenges due to large volumes, multiple sources, and diverse formats. To address this issue, this paper presents a scalable and parallelizable workflow using Apache Airflow, capable of integrating Machine Learning (ML) and Deep Learning (DL) models with Modular Supercomputing Architecture (MSA) systems. To test the workflow, we considered the production of large-scale Land-Cover (LC) maps as a case study. The workflow manager, Airflow, offers scalability, extensibility, and programmable task definition in Python. It allows us to execute different steps of the workflow in different High-Performance Computing (HPC) systems. The workflow is demonstrated on the Dynamical Exascale Entry Platform (DEEP) and Jülich Research on Exascale Cluster Architectures (JURECA) hosted at the Jülich Supercomputing Centre (JSC), a platform that incorporates heterogeneous JSC systems.
001017950 536__ $$0G:(DE-HGF)POF4-5111$$a5111 - Domain-Specific Simulation & Data Life Cycle Labs (SDLs) and Research Groups (POF4-511)$$cPOF4-511$$fPOF IV$$x0
001017950 536__ $$0G:(DE-HGF)POF4-5112$$a5112 - Cross-Domain Algorithms, Tools, Methods Labs (ATMLs) and Research Groups (POF4-511)$$cPOF4-511$$fPOF IV$$x1
001017950 536__ $$0G:(EU-Grant)951733$$aRAISE - Research on AI- and Simulation-Based Engineering at Exascale (951733)$$c951733$$fH2020-INFRAEDI-2019-1$$x2
001017950 536__ $$0G:(DE-Juel-1)DEA02266$$aEUROCC-2 (DEA02266)$$cDEA02266$$x3
001017950 588__ $$aDataset connected to CrossRef Conference
001017950 7001_ $$0P:(DE-Juel1)178695$$aSedona, Rocco$$b1$$ufzj
001017950 7001_ $$0P:(DE-Juel1)166264$$aMozaffari, Amirpasha$$b2$$ufzj
001017950 7001_ $$0P:(DE-Juel1)188445$$aKreshpa, Enxhi$$b3$$ufzj
001017950 7001_ $$0P:(DE-HGF)0$$aParis, Claudia$$b4
001017950 7001_ $$0P:(DE-Juel1)132239$$aRiedel, Morris$$b5$$ufzj
001017950 7001_ $$0P:(DE-Juel1)6952$$aSchultz, Martin G.$$b6$$ufzj
001017950 7001_ $$0P:(DE-Juel1)171343$$aCavallaro, Gabriele$$b7$$ufzj
001017950 773__ $$a10.1109/IGARSS52108.2023.10283416
001017950 8564_ $$uhttps://juser.fz-juelich.de/record/1017950/files/Liang_Tian_IGARSS_2023.pdf$$yOpenAccess
001017950 909CO $$ooai:juser.fz-juelich.de:1017950$$pdnbdelivery$$pec_fundedresources$$pVDB$$pdriver$$popen_access$$popenaire
001017950 9101_ $$0I:(DE-588b)5008462-8$$6P:(DE-Juel1)178695$$aForschungszentrum Jülich$$b1$$kFZJ
001017950 9101_ $$0I:(DE-588b)5008462-8$$6P:(DE-Juel1)166264$$aForschungszentrum Jülich$$b2$$kFZJ
001017950 9101_ $$0I:(DE-588b)5008462-8$$6P:(DE-Juel1)188445$$aForschungszentrum Jülich$$b3$$kFZJ
001017950 9101_ $$0I:(DE-588b)5008462-8$$6P:(DE-Juel1)132239$$aForschungszentrum Jülich$$b5$$kFZJ
001017950 9101_ $$0I:(DE-588b)5008462-8$$6P:(DE-Juel1)6952$$aForschungszentrum Jülich$$b6$$kFZJ
001017950 9101_ $$0I:(DE-588b)5008462-8$$6P:(DE-Juel1)171343$$aForschungszentrum Jülich$$b7$$kFZJ
001017950 9131_ $$0G:(DE-HGF)POF4-511$$1G:(DE-HGF)POF4-510$$2G:(DE-HGF)POF4-500$$3G:(DE-HGF)POF4$$4G:(DE-HGF)POF$$9G:(DE-HGF)POF4-5111$$aDE-HGF$$bKey Technologies$$lEngineering Digital Futures – Supercomputing, Data Management and Information Security for Knowledge and Action$$vEnabling Computational- & Data-Intensive Science and Engineering$$x0
001017950 9131_ $$0G:(DE-HGF)POF4-511$$1G:(DE-HGF)POF4-510$$2G:(DE-HGF)POF4-500$$3G:(DE-HGF)POF4$$4G:(DE-HGF)POF$$9G:(DE-HGF)POF4-5112$$aDE-HGF$$bKey Technologies$$lEngineering Digital Futures – Supercomputing, Data Management and Information Security for Knowledge and Action$$vEnabling Computational- & Data-Intensive Science and Engineering$$x1
001017950 9141_ $$y2023
001017950 915__ $$0StatID:(DE-HGF)0510$$2StatID$$aOpenAccess
001017950 9201_ $$0I:(DE-Juel1)JSC-20090406$$kJSC$$lJülich Supercomputing Center$$x0
001017950 980__ $$acontrib
001017950 980__ $$aVDB
001017950 980__ $$aUNRESTRICTED
001017950 980__ $$acontb
001017950 980__ $$aI:(DE-Juel1)JSC-20090406
001017950 9801_ $$aFullTexts