Publications

@inproceedings{guo_digital_2025,

title = {Digital Human Techniques for Education Reform},

author = {P. Guo and Q. Zhang and C. Tian and W. Xue and X. Feng},

url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-105001671326&doi=10.1145%2f3711403.3711428&partnerID=40&md5=dd96647315af9409d119f68f9cf4e980},

doi = {10.1145/3711403.3711428},

isbn = {979-840071746-8 (ISBN)},

year  = {2025},

date = {2025-01-01},

booktitle = {ICETM - Proc. Int. Conf. Educ. Technol. Manag.},

pages = {173–178},

publisher = {Association for Computing Machinery, Inc},

abstract = {The rapid evolution of artificial intelligence, big data, and generative AI models has ushered in significant transformations across various sectors, including education. Digital Human Technique, an innovative technology grounded in advanced computer science and artificial intelligence, is reshaping educational paradigms by enabling virtual humans to simulate human behavior, express emotions, and interact with users. This paper explores the application of Digital Human Technique in education reform, focusing on creating immersive, intelligent classroom experiences that foster meaningful interactions between teachers and students. We define Digital Human Technique and delve into its key technical components such as character modeling and rendering, natural language processing, computer vision, and augmented reality technologies. Our methodology involves analyzing the role of educational digital humans created through these technologies, assessing their impact on educational processes, and examining various application scenarios in educational reform. Results indicate that Digital Human Technique significantly enhances the learning experience by enabling personalized teaching, increasing engagement, and fostering emotional connections. Educational digital humans serve as virtual teachers, interactive learning aids, and facilitators of emotional interaction, effectively addressing the challenges of traditional educational methods. They also promote a deeper understanding of complex concepts through simulated environments and interactive digital content. © 2024 Copyright held by the owner/author(s).},

keywords = {Augmented Reality, Contrastive Learning, Digital elevation model, Digital human technique, Digital Human Techniques, Digital humans, Education Reform, Education reforms, Educational Technology, Express emotions, Federated learning, Human behaviors, Human form models, Human techniques, Immersive, Innovative technology, Modeling languages, Natural language processing systems, Teachers', Teaching, Virtual environments, Virtual humans},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{lai_-demand_2024,

title = {On-demand Quantization for Green Federated Generative Diffusion in Mobile Edge Networks},

author = {B. Lai and J. He and J. Kang and G. Li and M. Xu and T. Zhang and S. Xie},

editor = {Valenti M. and Reed D. and Torres M.},

url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-85202836582&doi=10.1109%2fICC51166.2024.10622695&partnerID=40&md5=8318ef451bb946470eb34ef50d09b5a1},

doi = {10.1109/ICC51166.2024.10622695},

isbn = {15503607 (ISSN); 978-172819054-9 (ISBN)},

year  = {2024},

date = {2024-01-01},

booktitle = {IEEE Int Conf Commun},

pages = {2883–2888},

publisher = {Institute of Electrical and Electronics Engineers Inc.},

abstract = {Generative Artificial Intelligence (GAI) shows remarkable productivity and creativity in Mobile Edge Networks, such as the metaverse and the Industrial Internet of Things. Federated learning is a promising technique for effectively training GAI models in mobile edge networks due to its data distribution. However, there is a notable issue with communication consumption when training large GAI models like generative diffusion models in mobile edge networks. Additionally, the substantial energy consumption associated with training diffusion-based models, along with the limited resources of edge devices and complexities of network environments, pose challenges for improving the training efficiency of GAI models. To address this challenge, we propose an on-demand quantized energy-efficient federated diffusion approach for mobile edge networks. Specifically, we first design a dynamic quantized federated diffusion training scheme considering various demands from the edge devices. Then, we study an energy efficiency problem based on specific quantization requirements. Numerical results show that our proposed method significantly reduces system energy consumption and transmitted model size compared to both baseline federated diffusion and fixed quantized federated diffusion methods while effectively maintaining reasonable quality and diversity of generated data. © 2024 IEEE.},

keywords = {EDGE Networks, Energy Efficient, Federated Diffusion, Federated learning, Generative adversarial networks, Generative AI, Generative Diffusion, Intelligence models, Metaverses, On demands, On-demand Quantization, Quantisation},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{noauthor_fostering_2024,

title = {Fostering Personalized Learning in Data Science: Integrating Innovative Tools and Strategies for Diverse Pathways},

url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-105001849041&doi=10.1109%2fICEED62316.2024.10923798&partnerID=40&md5=cfec507f601df5ffc3b07db0df6d80a7},

doi = {10.1109/ICEED62316.2024.10923798},

isbn = {979-835036741-6 (ISBN)},

year  = {2024},

date = {2024-01-01},

booktitle = {IEEE Int. Conf. Eng. Educ.: Dissem. Adv. Eng. Educ. using Artif. Intell., ICEED},

publisher = {Institute of Electrical and Electronics Engineers Inc.},

abstract = {This paper introduces an innovative teaching approach in data science tailored for students in non-computer science pathways, specifically Business Information Technology (BIT) and Computing and Information Technology (CIT). Over a five-year period, a unique teaching approach has been developed incorporating a virtual reality (VR) game event and ChatGPT-4 as a generative artificial intelligence (AI) tool. To address the inherent complexities of learning data science, particularly the diverse prerequisite skills, this study introduces a framework including a diagnostic assessment centered around a specific education research question: 'How can the learning experiences of individual students be customized to address the multifaceted challenges of data science education?' Through a diagnostic assessment process, conducted via a survey completed by students, this framework identifies students' unique requirements and skill areas facilitating the delivery of personalized content recommendations within the initial week of teaching. By fostering a culture of self-directed learning, the approach aims to enable students to concentrate on essential customized learning materials. This paper also highlights the overall student satisfaction with the module averaged 4.5 out of 5 with a standard deviation of 0.9 indicating a high level of contentment with the teaching approach. The discussion encompasses the framework's implications for teaching and its alignment with educational theories. This paper contributes to the computing education field by addressing the research question and offering insights for future research and teaching practices. © 2024 IEEE.},

keywords = {Adversarial machine learning, ChatGPT-4, Content recommendation, Content recommendations, Contrastive Learning, Data Science, Data science education, Federated learning, Individualized learning, Individualized learning experience framework, Learning experiences, Prerequisite skill identification, Science education, Self-directed learning, Teaching approaches, Virtual environments, Virtual Reality},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{taheri_enhancing_2024,

title = {Enhancing Presentation Skills: A Virtual Reality-Based Simulator with Integrated Generative AI for Dynamic Pitch Presentations and Interviews},

author = {M. Taheri and K. Tan},

editor = {De Paolis L.T. and Arpaia P. and Sacco M.},

url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-85204618832&doi=10.1007%2f978-3-031-71707-9_30&partnerID=40&md5=fd649ec5c0e2ce96593fe8a129e94449},

doi = {10.1007/978-3-031-71707-9_30},

isbn = {03029743 (ISSN); 978-303171706-2 (ISBN)},

year  = {2024},

date = {2024-01-01},

booktitle = {Lect. Notes Comput. Sci.},

volume = {15027 LNCS},

pages = {360–366},

publisher = {Springer Science and Business Media Deutschland GmbH},

abstract = {Presenting before an audience presents challenges throughout preparation and delivery, necessitating tools to securely refine skills securely. Interviews mirror presentations, showcasing oneself to convey qualifications. Virtual environments offer safe spaces for trial and error, crucial for iterative practice without emotional distress. This research proposes a Virtual Reality-Based Dynamic Pitch Simulation with Integrated Generative AI to effectively enhance presentation skills. The simulation converts spoken words to text, then uses AI to generate relevant questions for practice. Benefits include realistic feedback and adaptability to user proficiency. Open-source language models evaluate content, coherence, and delivery, offering personalized challenges. This approach supplements learning, enhancing presentation skills effectively. Voice-to-text conversion and AI feedback create a potent pedagogical tool, fostering a prompt feedback loop vital for learning effectiveness. Challenges in simulation design must be addressed for robustness and efficacy. The study validates these concepts by proposing a real-time 3D dialogue simulator, emphasizing the importance of continual improvement in presentation skill development. © The Author(s), under exclusive license to Springer Nature Switzerland AG 2024.},

keywords = {Adversarial machine learning, AI feedback, Contrastive Learning, Digital elevation model, Dynamic pitch, Federated learning, feedback, Generative adversarial networks, Iterative practice, Language Model, Open source language, Open source software, Presentation skills, Simulation Design, Spoken words, Trial and error, Virtual environments, Virtual reality based simulators},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{arrigo_task-interaction_2024,

title = {A Task-Interaction Framework to Monitor Mobile Learning Activities Based on Artificial Intelligence and Augmented Reality},

author = {M. Arrigo and M. Farella and G. Fulantelli and D. Schicchi and D. Taibi},

editor = {De Paolis L.T. and Arpaia P. and Sacco M.},

url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-85204618733&doi=10.1007%2f978-3-031-71707-9_26&partnerID=40&md5=8969f18ab0f10dcddf37e54265d10518},

doi = {10.1007/978-3-031-71707-9_26},

isbn = {03029743 (ISSN); 978-303171706-2 (ISBN)},

year  = {2024},

date = {2024-01-01},

booktitle = {Lect. Notes Comput. Sci.},

volume = {15027 LNCS},

pages = {325–333},

publisher = {Springer Science and Business Media Deutschland GmbH},

abstract = {The complexity behind the analysis of mobile learning activities has requested the development of specifically designed frameworks. When students are involved in mobile learning experiences, they interact with the context in which the activities occur, the content they have access to, with peers and their teachers. The wider adoption of generative artificial intelligence introduces new interactions that researchers have to look at when learning analytics techniques are applied to monitor learning patterns. The task interaction framework proposed in this paper explores how AI-based tools affect student-content and student-context interactions during mobile learning activities, thus focusing on the interplay of Learning Analytics and Artificial Intelligence advances in the educational domain. A use case scenario that explores the framework’s application in a real educational context is also presented. Finally, we describe the architectural design of an environment that leverages the task interaction framework to analyze enhanced mobile learning experiences in which structured content extracted from a Knowledge Graph is elaborated by a large language model to provide students with personalized content. © The Author(s), under exclusive license to Springer Nature Switzerland AG 2024.},

keywords = {Activity-based, Adversarial machine learning, Analytic technique, Augmented Reality, Contrastive Learning, Federated learning, Generative AI, Interaction framework, Learning Activity, Learning analytic framework, Learning Analytics Framework, Learning experiences, Learning patterns, Mobile Learning, Teachers'},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{gao_datalivr_2024,

title = {DataliVR: Transformation of Data Literacy Education through Virtual Reality with ChatGPT-Powered Enhancements},

author = {H. Gao and H. Huai and S. Yildiz-Degirmenci and M. Bannert and E. Kasneci},

editor = {Eck U. and Sra M. and Stefanucci J. and Sugimoto M. and Tatzgern M. and Williams I.},

url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-85213525613&doi=10.1109%2fISMAR62088.2024.00026&partnerID=40&md5=abdeba7ecfecc8b1d715d633a29bd11d},

doi = {10.1109/ISMAR62088.2024.00026},

isbn = {979-833151647-5 (ISBN)},

year  = {2024},

date = {2024-01-01},

booktitle = {Proc. - IEEE Int. Symp. Mixed Augment. Real., ISMAR},

pages = {120–129},

publisher = {Institute of Electrical and Electronics Engineers Inc.},

abstract = {Data literacy is essential in today's data-driven world, emphasizing individuals' abilities to effectively manage data and extract meaningful insights. However, traditional classroom-based educational approaches often struggle to fully address the multifaceted nature of data literacy. As education undergoes digital transformation, innovative technologies such as Virtual Reality (VR) offer promising avenues for immersive and engaging learning experiences. This paper introduces DataliVR, a pioneering VR application aimed at enhancing the data literacy skills of university students within a contextual and gamified virtual learning environment. By integrating Large Language Models (LLMs) like ChatGPT as a conversational artificial intelligence (AI) chatbot embodied within a virtual avatar, DataliVR provides personalized learning assistance, enriching user learning experiences. Our study employed an experimental approach, with chatbot availability as the independent variable, analyzing learning experiences and outcomes as dependent variables with a sample of thirty participants. Our approach underscores the effectiveness and user-friendliness of ChatGPT-powered DataliVR in fostering data literacy skills. Moreover, our study examines the impact of the ChatGPT-based AI chatbot on users' learning, revealing significant effects on both learning experiences and outcomes. Our study presents a robust tool for fostering data literacy skills, contributing significantly to the digital advancement of data literacy education through cutting-edge VR and AI technologies. Moreover, our research provides valuable insights and implications for future research endeavors aiming to integrate LLMs (e.g., ChatGPT) into educational VR platforms. © 2024 IEEE.},

keywords = {Adversarial machine learning, Chatbots, ChatGPT, Contrastive Learning, Data driven, Data literacy, Digital transformation, Federated learning, Immersive learning, Language Model, Large language model, Learning experiences, Learning outcome, LLMs, Virtual environments, Virtual Reality},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{velev_challenges_2024,

title = {Challenges of Merging Generative AI with Metaverse for Next-Gen Education},

author = {D. Velev and L. Steshina and I. Petukhov and P. Zlateva},

editor = {Tallon-Ballesteros A.J.},

url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-85215823646&doi=10.3233%2fFAIA241462&partnerID=40&md5=a3ed4e8486e2e32d0856a71a3a87496c},

doi = {10.3233/FAIA241462},

isbn = {09226389 (ISSN); 978-164368569-4 (ISBN)},

year  = {2024},

date = {2024-01-01},

booktitle = {Front. Artif. Intell. Appl.},

volume = {398},

pages = {606–616},

publisher = {IOS Press BV},

abstract = {The integration of Generative Artificial Intelligence (GenAI) with the Metaverse for a next-generation education is a complex but challenging task. The GenAI-enhanced Metaverse classrooms require innovative instructional designs that use virtual reality and augmented reality to enhance engagement and personalized learning. Educators must adapt to new roles over traditional teaching methods, while learners need to develop digital literacy skills that are essential for navigating and inhabiting in these environments. Such learning environments require significant advancements in real-time processing, scalability and interoperability of different platforms, while ensuring data privacy and security. The equity of access to high-speed internet and advanced devices still remains a serious barrier, which can increase the potential existing inequalities between different educational environments. Ethical considerations, including the responsible use of GenAI, the creation of unbiased educational content, and the psychological impacts of extended usage of virtual reality, are also of important consideration. The aim of the paper is to explore in detail the different challenges through a comprehensive analysis of the obstacles and potential solutions and to propose a collaborative framework involving educators, technologists, policymakers and industry stakeholders to address the effective implementation of the integration of GenAI and the Metaverse for a next generation education. © 2024 The Authors.},

keywords = {Adversarial machine learning, Augmented Reality, Contrastive Learning, Data privacy and securities, Digital literacies, Education, Federated learning, Generative adversarial networks, Generative AI, High speed internet, Instructional designs, Learning Environments, Metaverse, Metaverses, Personalized learning, Realtime processing, Teaching methods, Virtual environments, Virtual Reality},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{jia_nivta_2024,

title = {NivTA: Towards a Naturally Interactable Edu-Metaverse Teaching Assistant for CAVE},

author = {Y. Jia and Z. P. T. Sin and X. E. Wang and C. Li and P. H. F. Ng and X. Huang and J. Dong and Y. Wang and G. Baciu and J. Cao and Q. Li},

url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-85211447638&doi=10.1109%2fMetaCom62920.2024.00023&partnerID=40&md5=efefd453c426e74705518254bdc49e87},

doi = {10.1109/MetaCom62920.2024.00023},

isbn = {979-833151599-7 (ISBN)},

year  = {2024},

date = {2024-01-01},

booktitle = {Proc. - IEEE Int. Conf. Metaverse Comput., Netw., Appl., MetaCom},

pages = {57–64},

publisher = {Institute of Electrical and Electronics Engineers Inc.},

abstract = {Edu-metaverse is a specialized metaverse dedicated for interactive education in an immersive environment. Its main purpose is to immerse the learners in a digital environment and conduct learning activities that could mirror reality. Not only does it enable activities that may be difficult to perform in the real world, but it also extends the interaction to personalized and CL. This is a more effective pedagogical approach as it tends to enhance the motivation and engagement of students and it increases their active participation in lessons delivered. To this extend, we propose to realize an interactive virtual teaching assistant called NivTA. To make NivTA easily accessible and engaging by multiple users simultaneously, we also propose to use a CAVE virtual environment (CAVE-VR) as a "metaverse window"into concepts, ideas, topics, and learning activities. The students simply need to step into the CAVE-VR and interact with a life-size teaching assistant that they can engage with naturally, as if they are approaching a real person. Instead of text-based interaction currently developed for large language models (LLM), NivTA is given additional cues regarding the users so it can react more naturally via a specific prompt design. For example, the user can simply point to an educational concept and ask NivTA to explain what it is. To guide NivTA onto the educational concept, the prompt is also designed to feed in an educational KG to provide NivTA with the context of the student's question. The NivTA system is an integration of several components that are discussed in this paper. We further describe how the system is designed and implemented, along with potential applications and future work on interactive collaborative edu-metaverse environments dedicated for teaching and learning. © 2024 IEEE.},

keywords = {Active learning, Adversarial machine learning, cave automatic virtual environment, Cave automatic virtual environments, Caves, Chatbots, Contrastive Learning, Digital elevation model, Federated learning, Interactive education, Language Model, Large language model agent, Learning Activity, LLM agents, Metaverses, Model agents, Natural user interface, Students, Teaching, Teaching assistants, Virtual environments, Virtual Reality, virtual teaching assistant, Virtual teaching assistants},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{bandara_generative-aicustom-trained_2024,

title = {Generative-AI(with Custom-Trained Meta's Llama2 LLM), Blockchain, NFT, Federated Learning and PBOM Enabled Data Security Architecture for Metaverse on 5G/6G Environment},

author = {E. Bandara and P. Foytik and S. Shetty and A. Hassanzadeh},

url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-85210243120&doi=10.1109%2fMASS62177.2024.00026&partnerID=40&md5=70d21ac1e9c7b886da14825376919cac},

doi = {10.1109/MASS62177.2024.00026},

isbn = {979-835036399-9 (ISBN)},

year  = {2024},

date = {2024-01-01},

booktitle = {Proc. - IEEE Int. Conf. Mob. Ad-Hoc Smart Syst., MASS},

pages = {118–124},

publisher = {Institute of Electrical and Electronics Engineers Inc.},

abstract = {The Metaverse is an integrated network of 3D virtual worlds accessible through a virtual reality headset. Its impact on data privacy and security is increasingly recognized as a major concern. There is a growing interest in developing a reference architecture that describes the four core aspects of its data: acquisition, storage, sharing, and interoperability. Establishing a secure data architecture is imperative to manage users' personal data and facilitate trusted AR/VR and AI/ML solutions within the Metaverse. This paper details a reference architecture empowered by Generative-AI, Blockchain, Federated Learning, and Non-Fungible Tokens (NFTs). Within this archi-tecture, various resource providers collaborate via the blockchain network. Handling personal user data and resource provider identities is executed through a Self-Sovereign Identity-enabled privacy-preserving framework. AR/NR devices in the Metaverse are represented as NFT tokens available for user purchase. Software updates and supply-chain verification for these devices are managed using a Software Bill of Materials (SBOM) and a Pipeline Bill of Materials (PBOM) verification system. Moreover, a custom-trained Llama2 LLM from Meta has been integrated to generate PBOMs for AR/NR devices' software updates, thereby preventing malware intrusions and data breaches. This Llama2-13B LLM has been quantized and fine-tuned using Qlora to ensure optimal performance on consumer-grade hardware. The provenance of AI/ML models used in the Metaverse is encapsu-lated as Model Card objects, allowing external parties to audit and verify them, thus mitigating adversarial learning attacks within these models. To the best of our knowledge, this is the very first research effort aimed at standardizing PBOM schemas and integrating Language Model algorithms for the generation of PBOMs. Additionally, a proposed mechanism facilitates different AI/ML providers in training their machine learning models using a privacy-preserving federated learning approach. Authorization of communications among AR/VR devices in the Metaverse is conducted through a Zero-Trust security-enabled rule engine. A system testbed has been implemented within a 5G environment, utilizing Ericsson new Radio with Open5GS 5G core. © 2024 IEEE.},

keywords = {5G, 6G, Adversarial machine learning, Bill of materials, Block-chain, Blockchain, Curricula, Data privacy, Distance education, Federated learning, Generative adversarial networks, Generative-AI, Hardware security, Llama2, LLM, Medium access control, Metaverse, Metaverses, Network Security, Nft, Non-fungible token, Personnel training, Problem oriented languages, Reference architecture, Steganography},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{park_enabling_2023,

title = {Enabling the Wireless Metaverse via Semantic Multiverse Communication},

author = {J. Park and J. Choi and S. -L. Kim and M. Bennis},

url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-85177465286&doi=10.1109%2fSECON58729.2023.10287438&partnerID=40&md5=b052572fb2f78ce0694c7ae5726c8daf},

doi = {10.1109/SECON58729.2023.10287438},

isbn = {21555486 (ISSN); 979-835030052-9 (ISBN)},

year  = {2023},

date = {2023-01-01},

booktitle = {Annu. IEEE Commun.Soc. Conf. Sens., Mesh Ad Hoc Commun. Netw. workshops},

volume = {2023-September},

pages = {85–90},

publisher = {IEEE Computer Society},

abstract = {Metaverse over wireless networks is an emerging use case of the sixth generation (6G) wireless systems, posing unprecedented challenges in terms of its multi-modal data transmissions with stringent latency and reliability requirements. Towards enabling this wireless metaverse, in this article we propose a novel semantic communication (SC) framework by decomposing the metaverse into human/machine agent-specific semantic multiverses (SMs). An SM stored at each agent comprises a semantic encoder and a generator, leveraging recent advances in generative artificial intelligence (AI). To improve communication efficiency, the encoder learns the semantic representations (SRs) of multi-modal data, while the generator learns how to manipulate them for locally rendering scenes and interactions in the metaverse. Since these learned SMs are biased towards local environments, their success hinges on synchronizing heterogeneous SMs in the background while communicating SRs in the foreground, turning the wireless metaverse problem into the problem of semantic multiverse communication (SMC). Based on this SMC architecture, we propose several promising algorithmic and analytic tools for modeling and designing SMC, ranging from distributed learning and multi-agent reinforcement learning (MARL) to signaling games and symbolic AI. © 2023 IEEE.},

keywords = {Deep learning, Extended reality (XR), Federated learning, Fertilizers, Learn+, Learning systems, Metaverse, Metaverses, Modal analysis, Multi agent systems, Multi-agent reinforcement learning, Multi-modal data, Reinforcement Learning, Semantic communication, Semantics, Signal encoding, Signaling game, Split learning, Symbolic artificial intelligence},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{feng_rono_2023,

title = {RONO: Robust Discriminative Learning with Noisy Labels for 2D-3D Cross-Modal Retrieval},

author = {Y. Feng and H. Zhu and D. Peng and X. Peng and P. Hu},

url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-85170845124&doi=10.1109%2fCVPR52729.2023.01117&partnerID=40&md5=2eee285207ff3ea8e774480e29d96ec1},

doi = {10.1109/CVPR52729.2023.01117},

isbn = {10636919 (ISSN)},

year  = {2023},

date = {2023-01-01},

booktitle = {Proc IEEE Comput Soc Conf Comput Vision Pattern Recognit},

volume = {2023-June},

pages = {11610–11619},

publisher = {IEEE Computer Society},

abstract = {Recently, with the advent of Metaverse and AI Generated Content, cross-modal retrieval becomes popular with a burst of 2D and 3D data. However, this problem is challenging given the heterogeneous structure and semantic discrepancies. Moreover, imperfect annotations are ubiquitous given the ambiguous 2D and 3D content, thus inevitably producing noisy labels to degrade the learning performance. To tackle the problem, this paper proposes a robust 2D-3D retrieval framework (RONO) to robustly learn from noisy multimodal data. Specifically, one novel Robust Discriminative Center Learning mechanism (RDCL) is proposed in RONO to adaptively distinguish clean and noisy samples for respectively providing them with positive and negative optimization directions, thus mitigating the negative impact of noisy labels. Besides, we present a Shared Space Consistency Learning mechanism (SSCL) to capture the intrinsic information inside the noisy data by minimizing the cross-modal and semantic discrepancy between common space and label space simultaneously. Comprehensive mathematical analyses are given to theoretically prove the noise tolerance of the proposed method. Furthermore, we conduct extensive experiments on four 3D-model multimodal datasets to verify the effectiveness of our method by comparing it with 15 state-of-the-art methods. © 2023 IEEE.},

keywords = {3D content, 3D data, 3D modeling, Adversarial machine learning, Contrastive Learning, Cross-modal, Discriminative learning, Federated learning, Heterogeneous structures, Learning mechanism, Learning performance, Metaverses, Multi-modal learning, Noisy labels, Spatio-temporal data},

pubstate = {published},

tppubtype = {inproceedings}

}