Publications

@inproceedings{ding_rag-vr_2025,

title = {RAG-VR: Leveraging Retrieval-Augmented Generation for 3D Question Answering in VR Environments},

author = {S. Ding and Y. Chen},

url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-105005140593&doi=10.1109%2fVRW66409.2025.00034&partnerID=40&md5=36dc5fef97aeea4d6e183c83ce9fcd89},

doi = {10.1109/VRW66409.2025.00034},

isbn = {979-833151484-6 (ISBN)},

year  = {2025},

date = {2025-01-01},

booktitle = {Proc. - IEEE Conf. Virtual Real. 3D User Interfaces Abstr. Workshops, VRW},

pages = {131–136},

publisher = {Institute of Electrical and Electronics Engineers Inc.},

abstract = {Recent advances in large language models (LLMs) provide new opportunities for context understanding in virtual reality (VR). However, VR contexts are often highly localized and personalized, limiting the effectiveness of general-purpose LLMs. To address this challenge, we present RAG-VR, the first 3D question-answering system for VR that incorporates retrieval-augmented generation (RAG), which augments an LLM with external knowledge retrieved from a localized knowledge database to improve the answer quality. RAG-VR includes a pipeline for extracting comprehensive knowledge about virtual environments and user conditions for accurate answer generation. To ensure efficient retrieval, RAG-VR offloads the retrieval process to a nearby edge server and uses only essential information during retrieval. Moreover, we train the retriever to effectively distinguish among relevant, irrelevant, and hard-to-differentiate information in relation to questions. RAG-VR improves answer accuracy by 17.9%-41.8% and reduces end-to-end latency by 34.5%-47.3% compared with two baseline systems. © 2025 IEEE.},

keywords = {Ambient intelligence, Computational Linguistics, Computer interaction, Computing methodologies, Computing methodologies-Artificial intelligence-Natural language processing-Natural language generation, Computing methodology-artificial intelligence-natural language processing-natural language generation, Data handling, Formal languages, Human computer interaction, Human computer interaction (HCI), Human-centered computing, Interaction paradigm, Interaction paradigms, Language Model, Language processing, Natural language generation, Natural language processing systems, Natural languages, Virtual Reality, Word processing},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{jeong_function-adaptive_2024,

title = {Function-Adaptive Affordance Extraction from 3D Objects Using LLM for Interaction Authoring with Augmented Artifacts},

author = {E. Jeong and H. Kim and S. Park and S. Yoon and J. Ahn and W. Woo},

editor = {Eck U. and Sra M. and Stefanucci J. and Sugimoto M. and Tatzgern M. and Williams I.},

url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-85214379963&doi=10.1109%2fISMAR-Adjunct64951.2024.00050&partnerID=40&md5=7222e0599a7e2aa0adaea38e4b9e13cc},

doi = {10.1109/ISMAR-Adjunct64951.2024.00050},

isbn = {979-833150691-9 (ISBN)},

year  = {2024},

date = {2024-01-01},

booktitle = {Proc. - IEEE Int. Symp. Mixed Augment. Real. Adjunct, ISMAR-Adjunct},

pages = {205–208},

publisher = {Institute of Electrical and Electronics Engineers Inc.},

abstract = {We propose an algorithm that extracts the most suitable affordances, interaction targets, and corresponding coordinates adaptively from 3D models of various artifacts based on their functional context for efficient authoring of XR content with artifacts. Traditionally, authoring AR scenes to convey artifact context required one-to-one manual work. Our approach leverages a Large Language Model (LLM) to extract interaction types, positions, and subjects based on the artifact's name and usage context. This enables templated XR experience creation, replacing repetitive manual labor. Consequently, our system streamlines the XR authoring process, making it more efficient and scalable. © 2024 IEEE.},

keywords = {3D modeling, Applied computing, Art and humanity, Artificial intelligence, Arts and humanities, Augmented Reality, Computer interaction, Computer vision, Computing methodologies, computing methodology, Human computer interaction, Human computer interaction (HCI), Human-centered computing, Humanities computing, Interaction paradigm, Interaction paradigms, Language processing, Mixed / augmented reality, Mixed reality, Modeling languages, Natural Language Processing, Natural language processing systems, Natural languages, Three dimensional computer graphics},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{rozo-torres_coffee_2024,

title = {Coffee Masterclass: An Experience of Co-Creation with Prompt Engineering and Generative AI for Immersive Environments Development},

author = {A. Rozo-Torres and W. J. Sarmiento},

url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-85195576520&doi=10.1109%2fVRW62533.2024.00379&partnerID=40&md5=f07b66f849aaa1c4d9b2e2ea79c57cf8},

doi = {10.1109/VRW62533.2024.00379},

isbn = {979-835037449-0 (ISBN)},

year  = {2024},

date = {2024-01-01},

booktitle = {Proc. - IEEE Conf. Virtual Real. 3D User Interfaces Abstr. Workshops, VRW},

pages = {1170–1171},

publisher = {Institute of Electrical and Electronics Engineers Inc.},

abstract = {This work presents the design and development process of an immersive experience applying a co-creation approach between humans and generative artificial intelligence tools. From the point of view of any user, Coffee Masterclass is an immersive experience that brings anyone to the art and pleasure of preparing specialty coffees. However, the Coffee Masterclass is the result of the inclusion of prompt engineering outputs in each stage of the building process. The co-creation approach is included in all development processes, i.e., from the narrative to the visual content generated through code writing, which has been co-created between the creative team and GenAI. This work tells details of this approach, including how the generative artificial intelligence tools were used in each stage of immersive experience development. This work shows the advantage of involvement in a development team of people with skills in prompt engineering and interaction with Large Language Models. Also, it includes recommendations to other development teams, including generative artificial intelligence tools by future developments. © 2024 IEEE.},

keywords = {Artificial intelligence, Artificial intelligence tools, Co-creation, Computer graphics, Computing methodologies, Design and development process, Development teams, Graphic system, Graphics interface, Graphics systems and interfaces, Immersive, Immersive environment, Mixed/augmented reality},

pubstate = {published},

tppubtype = {inproceedings}

}