Publications

@inproceedings{hu_avatar_2025,

title = {Avatar Motion Generation Pipeline for the Metaverse via Synthesis of Generative Models of Text and Video},

author = {Y. -H. Hu and A. Matsumoto and K. Ito and T. Narumi and H. Kuzuoka and T. Amemiya},

url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-105005158851&doi=10.1109%2fVRW66409.2025.00155&partnerID=40&md5=2bc9a6390e1cf710206835722ca8dbbf},

doi = {10.1109/VRW66409.2025.00155},

isbn = {979-833151484-6 (ISBN)},

year  = {2025},

date = {2025-01-01},

booktitle = {Proc. - IEEE Conf. Virtual Real. 3D User Interfaces Abstr. Workshops, VRW},

pages = {767–771},

publisher = {Institute of Electrical and Electronics Engineers Inc.},

abstract = {Efforts to integrate AI avatars into the metaverse to enhance interactivity have progressed in both research and commercial domains. AI avatars in the metaverse are expected to exhibit not only verbal responses but also avatar motions, such as non-verbal gestures, to enable seamless communication with users. Large Language Models (LLMs) are known for their advanced text processing capabilities, such as user input, avatar actions, and even entire virtual environments as text, making them a promising approach for planning avatar motions. However, generating the avatar motions solely from the textual information often requires extensive training data whereas the configuration is very challenging, with results that often lack diversity and fail to match user expectations. On the other hand, AI technologies for generating videos have progressed to the point where they can depict diverse and natural human movements based on prompts. Therefore, this paper introduces a novel pipeline, TVMP, that synthesizes LLMs with advanced text processing capabilities and video generation models with the ability to generate videos containing a variety of motions. The pipeline first generates videos from text input, then estimates the motions from the generated videos, and lastly exports the estimated motion data into the avatars in the metaverse. Feedback on the TVMP prototype suggests further refinement is needed, such as speed control, display of the progress, and direct edition for contextual relevance and usability enhancements. The proposed method enables AI avatars to perform highly adaptive and diverse movements to fulfill user expectations and contributes to developing a more immersive metaverse. © 2025 IEEE.},

keywords = {Ambient intelligence, Design and evaluation methods, Distributed computer systems, Human-centered computing, Language Model, Metaverses, Processing capability, Text-processing, Treemap, Treemaps, Visualization, Visualization design and evaluation method, Visualization design and evaluation methods, Visualization designs, Visualization technique, Visualization techniques},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{ding_rag-vr_2025,

title = {RAG-VR: Leveraging Retrieval-Augmented Generation for 3D Question Answering in VR Environments},

author = {S. Ding and Y. Chen},

url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-105005140593&doi=10.1109%2fVRW66409.2025.00034&partnerID=40&md5=36dc5fef97aeea4d6e183c83ce9fcd89},

doi = {10.1109/VRW66409.2025.00034},

isbn = {979-833151484-6 (ISBN)},

year  = {2025},

date = {2025-01-01},

booktitle = {Proc. - IEEE Conf. Virtual Real. 3D User Interfaces Abstr. Workshops, VRW},

pages = {131–136},

publisher = {Institute of Electrical and Electronics Engineers Inc.},

abstract = {Recent advances in large language models (LLMs) provide new opportunities for context understanding in virtual reality (VR). However, VR contexts are often highly localized and personalized, limiting the effectiveness of general-purpose LLMs. To address this challenge, we present RAG-VR, the first 3D question-answering system for VR that incorporates retrieval-augmented generation (RAG), which augments an LLM with external knowledge retrieved from a localized knowledge database to improve the answer quality. RAG-VR includes a pipeline for extracting comprehensive knowledge about virtual environments and user conditions for accurate answer generation. To ensure efficient retrieval, RAG-VR offloads the retrieval process to a nearby edge server and uses only essential information during retrieval. Moreover, we train the retriever to effectively distinguish among relevant, irrelevant, and hard-to-differentiate information in relation to questions. RAG-VR improves answer accuracy by 17.9%-41.8% and reduces end-to-end latency by 34.5%-47.3% compared with two baseline systems. © 2025 IEEE.},

keywords = {Ambient intelligence, Computational Linguistics, Computer interaction, Computing methodologies, Computing methodologies-Artificial intelligence-Natural language processing-Natural language generation, Computing methodology-artificial intelligence-natural language processing-natural language generation, Data handling, Formal languages, Human computer interaction, Human computer interaction (HCI), Human-centered computing, Interaction paradigm, Interaction paradigms, Language Model, Language processing, Natural language generation, Natural language processing systems, Natural languages, Virtual Reality, Word processing},

pubstate = {published},

tppubtype = {inproceedings}

}