AHCI RESEARCH GROUP
Publications
Papers published in international journals,
proceedings of conferences, workshops and books.
OUR RESEARCH
Scientific Publications
How to
You can use the tag cloud to select only the papers dealing with specific research topics.
You can expand the Abstract, Links and BibTex record for each paper.
2025
Li, K.; Mostajeran, F.; Rings, S.; Kruse, L.; Schmidt, S.; Arz, M.; Wolf, E.; Steinicke, F.
I Hear, See, Speak & Do: Bringing Multimodal Information Processing to Intelligent Virtual Agents for Natural Human-AI Communication Proceedings Article
In: Proc. - IEEE Conf. Virtual Real. 3D User Interfaces Abstr. Workshops, VRW, pp. 1648–1649, Institute of Electrical and Electronics Engineers Inc., 2025, ISBN: 979-833151484-6 (ISBN).
Abstract | Links | BibTeX | Tags: Artificial intelligence tools, Cloud services, Embodied AI, Embodied artificial intelligence, Extended reality, Human computer interaction, Human-AI Interaction, Human-artificial intelligence interaction, Information processing capability, Intelligent virtual agents, Language Model, Multi-modal information, Virtual agent, Work-flows
@inproceedings{li_i_2025,
title = {I Hear, See, Speak & Do: Bringing Multimodal Information Processing to Intelligent Virtual Agents for Natural Human-AI Communication},
author = {K. Li and F. Mostajeran and S. Rings and L. Kruse and S. Schmidt and M. Arz and E. Wolf and F. Steinicke},
url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-105005146647&doi=10.1109%2fVRW66409.2025.00469&partnerID=40&md5=77e755f6a059f81e81c18987f58d00cc},
doi = {10.1109/VRW66409.2025.00469},
isbn = {979-833151484-6 (ISBN)},
year = {2025},
date = {2025-01-01},
booktitle = {Proc. - IEEE Conf. Virtual Real. 3D User Interfaces Abstr. Workshops, VRW},
pages = {1648–1649},
publisher = {Institute of Electrical and Electronics Engineers Inc.},
abstract = {In this demo paper, we present an Extended Reality (XR) framework providing a streamlined workflow for creating and interacting with intelligent virtual agents (IVAs) with multimodal information processing capabilities using commercially available artificial intelligence (AI) tools and cloud services such as large language and vision models. The system supports (i) the integration of high-quality, customizable virtual 3D human models for visual representations of IVAs and (ii) multimodal communication with generative AI-driven IVAs in immersive XR, featuring realistic human behavior simulations. Our demo showcases the enormous potential and vast design space of embodied IVAs for various XR applications. © 2025 IEEE.},
keywords = {Artificial intelligence tools, Cloud services, Embodied AI, Embodied artificial intelligence, Extended reality, Human computer interaction, Human-AI Interaction, Human-artificial intelligence interaction, Information processing capability, Intelligent virtual agents, Language Model, Multi-modal information, Virtual agent, Work-flows},
pubstate = {published},
tppubtype = {inproceedings}
}
Leininger, P.; Weber, C. J.; Rothe, S.
Understanding Creative Potential and Use Cases of AI-Generated Environments for Virtual Film Productions: Insights from Industry Professionals Proceedings Article
In: IMX - Proc. ACM Int. Conf. Interact. Media Experiences, pp. 60–78, Association for Computing Machinery, Inc, 2025, ISBN: 979-840071391-0 (ISBN).
Abstract | Links | BibTeX | Tags: 3-D environments, 3D reconstruction, 3D Scene Reconstruction, 3d scenes reconstruction, AI-generated 3d environment, AI-Generated 3D Environments, Computer interaction, Creative Collaboration, Creatives, Digital content creation, Digital Content Creation., Filmmaking workflow, Filmmaking Workflows, Gaussian distribution, Gaussian Splatting, Gaussians, Generative AI, Graphical user interface, Graphical User Interface (GUI), Graphical user interfaces, Human computer interaction, human-computer interaction, Human-Computer Interaction (HCI), Immersive, Immersive Storytelling, Interactive computer graphics, Interactive computer systems, Interactive media, Mesh generation, Previsualization, Real-Time Rendering, Splatting, Three dimensional computer graphics, Virtual production, Virtual Production (VP), Virtual Reality, Work-flows
@inproceedings{leininger_understanding_2025,
title = {Understanding Creative Potential and Use Cases of AI-Generated Environments for Virtual Film Productions: Insights from Industry Professionals},
author = {P. Leininger and C. J. Weber and S. Rothe},
url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-105007976841&doi=10.1145%2f3706370.3727853&partnerID=40&md5=0d4cf7a2398d12d04e4f0ab182474a10},
doi = {10.1145/3706370.3727853},
isbn = {979-840071391-0 (ISBN)},
year = {2025},
date = {2025-01-01},
booktitle = {IMX - Proc. ACM Int. Conf. Interact. Media Experiences},
pages = {60–78},
publisher = {Association for Computing Machinery, Inc},
abstract = {Virtual production (VP) is transforming filmmaking by integrating real-time digital elements with live-action footage, offering new creative possibilities and streamlined workflows. While industry experts recognize AI's potential to revolutionize VP, its practical applications and value across different production phases and user groups remain underexplored. Building on initial research into generative and data-driven approaches, this paper presents the first systematic pilot study evaluating three types of AI-generated 3D environments - Depth Mesh, 360° Panoramic Meshes, and Gaussian Splatting - through the participation of 15 filmmaking professionals from diverse roles. Unlike commonly used 2D AI-generated visuals, our approach introduces navigable 3D environments that offer greater control and flexibility, aligning more closely with established VP workflows. Through expert interviews and literature research, we developed evaluation criteria to assess their usefulness beyond concept development, extending to previsualization, scene exploration, and interdisciplinary collaboration. Our findings indicate that different environments cater to distinct production needs, from early ideation to detailed visualization. Gaussian Splatting proved effective for high-fidelity previsualization, while 360° Panoramic Meshes excelled in rapid concept ideation. Despite their promise, challenges such as limited interactivity and customization highlight areas for improvement. Our prototype, EnVisualAIzer, built in Unreal Engine 5, provides an accessible platform for diverse filmmakers to engage with AI-generated environments, fostering a more inclusive production process. By lowering technical barriers, these environments have the potential to make advanced VP tools more widely available. This study offers valuable insights into the evolving role of AI in VP and sets the stage for future research and development. © 2025 Copyright held by the owner/author(s). Publication rights licensed to ACM.},
keywords = {3-D environments, 3D reconstruction, 3D Scene Reconstruction, 3d scenes reconstruction, AI-generated 3d environment, AI-Generated 3D Environments, Computer interaction, Creative Collaboration, Creatives, Digital content creation, Digital Content Creation., Filmmaking workflow, Filmmaking Workflows, Gaussian distribution, Gaussian Splatting, Gaussians, Generative AI, Graphical user interface, Graphical User Interface (GUI), Graphical user interfaces, Human computer interaction, human-computer interaction, Human-Computer Interaction (HCI), Immersive, Immersive Storytelling, Interactive computer graphics, Interactive computer systems, Interactive media, Mesh generation, Previsualization, Real-Time Rendering, Splatting, Three dimensional computer graphics, Virtual production, Virtual Production (VP), Virtual Reality, Work-flows},
pubstate = {published},
tppubtype = {inproceedings}
}
2024
Xi, M.; Perera, M.; Matthews, B.; Wang, R.; Weiley, V.; Somarathna, R.; Maqbool, H.; Chen, J.; Engelke, U.; Anderson, S.; Adcock, M.; Thomas, B. H.
Towards Immersive AI Proceedings Article
In: U., Eck; M., Sra; J., Stefanucci; M., Sugimoto; M., Tatzgern; I., Williams (Ed.): Proc. - IEEE Int. Symp. Mixed Augment. Real. Adjunct, ISMAR-Adjunct, pp. 260–264, Institute of Electrical and Electronics Engineers Inc., 2024, ISBN: 979-833150691-9 (ISBN).
Abstract | Links | BibTeX | Tags: Artificial intelligence, Augmented Reality, Data visualization, Decision making, Heterogenous data, Immersive, Immersive analytic, Immersive analytics, Industrial research, Mixed reality, Neuro-symbolic system, Real- time, Scientific paradigm, Situated imaging., Time-interleaved, Visual analytics, Work-flows
@inproceedings{xi_towards_2024,
title = {Towards Immersive AI},
author = {M. Xi and M. Perera and B. Matthews and R. Wang and V. Weiley and R. Somarathna and H. Maqbool and J. Chen and U. Engelke and S. Anderson and M. Adcock and B. H. Thomas},
editor = {Eck U. and Sra M. and Stefanucci J. and Sugimoto M. and Tatzgern M. and Williams I.},
url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-85214375967&doi=10.1109%2fISMAR-Adjunct64951.2024.00062&partnerID=40&md5=fd07c97119d71418bb4365582b1d188c},
doi = {10.1109/ISMAR-Adjunct64951.2024.00062},
isbn = {979-833150691-9 (ISBN)},
year = {2024},
date = {2024-01-01},
booktitle = {Proc. - IEEE Int. Symp. Mixed Augment. Real. Adjunct, ISMAR-Adjunct},
pages = {260–264},
publisher = {Institute of Electrical and Electronics Engineers Inc.},
abstract = {With every shift in scientific paradigms comes not only a new way of seeing the world, but as Kunh argues, new tools for seeing [13]. Today, generative AI and neuro-symbolic systems show signs of changing how science functions, making it possible to synthesise complex heterogenous data in real time, interleaved with complex and situated workflows. But the new tools are not yet fully formed. To realise the opportunities and meet the challenges posed by the growth of generative AI for science and other knowledge work requires us to look beyond improvements in algorithms. The decision-making landscape for information workers has drastically changed, and the pressing need for analysts and experts to collaborate with AI in complex, high-tempo data environments has never been more evident.To bring strategic focus to these challenges in ways that will enable social, environmental and economic benefits for all, CSIRO's Data61 (the data and digital specialist arm of the Commonwealth Scientific and Industrial Research Organisation - Australia's national science agency) has established the Immersive AI Research Cluster. The cluster allows more than 30 research scientists and engineers to focus on defining a broad range of scientific disciplines for people to work with and understand the information provided by AI, such as data visualisation, visual analytics, connecting remote people, through immersive technologies like virtual and augmented reality. This workshop paper presents the trending research directions and challenges that emerged from this research cluster, which are closely linked to the scientific domains and illustrated through use cases. © 2024 IEEE.},
keywords = {Artificial intelligence, Augmented Reality, Data visualization, Decision making, Heterogenous data, Immersive, Immersive analytic, Immersive analytics, Industrial research, Mixed reality, Neuro-symbolic system, Real- time, Scientific paradigm, Situated imaging., Time-interleaved, Visual analytics, Work-flows},
pubstate = {published},
tppubtype = {inproceedings}
}
Rosati, R.; Senesi, P.; Lonzi, B.; Mancini, A.; Mandolini, M.
An automated CAD-to-XR framework based on generative AI and Shrinkwrap modelling for a User-Centred design approach Journal Article
In: Advanced Engineering Informatics, vol. 62, 2024, ISSN: 14740346 (ISSN).
Abstract | Links | BibTeX | Tags: Adversarial networks, Artificial intelligence, CAD-to-XR, Computer aided design models, Computer aided logic design, Computer-aided design, Computer-aided design-to-XR, Design simplification, Digital elevation model, Digital storage, Extended reality, Flow visualization, Generative adversarial networks, Guns (armament), Helmet mounted displays, Intellectual property core, Mixed reality, Photo-realistic, Shrinkfitting, Structural dynamics, User centered design, User-centered design, User-centered design approaches, User-centred, Virtual Prototyping, Work-flows
@article{rosati_automated_2024,
title = {An automated CAD-to-XR framework based on generative AI and Shrinkwrap modelling for a User-Centred design approach},
author = {R. Rosati and P. Senesi and B. Lonzi and A. Mancini and M. Mandolini},
url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-85204897460&doi=10.1016%2fj.aei.2024.102848&partnerID=40&md5=3acce73b986bed7a9de42e6336d637ad},
doi = {10.1016/j.aei.2024.102848},
issn = {14740346 (ISSN)},
year = {2024},
date = {2024-01-01},
journal = {Advanced Engineering Informatics},
volume = {62},
abstract = {CAD-to-XR is the workflow to generate interactive Photorealistic Virtual Prototypes (iPVPs) for Extended Reality (XR) apps from Computer-Aided Design (CAD) models. This process entails modelling, texturing, and XR programming. In the literature, no automatic CAD-to-XR frameworks simultaneously manage CAD simplification and texturing. There are no examples of their adoption for User-Centered Design (UCD). Moreover, such CAD-to-XR workflows do not seize the potentialities of generative algorithms to produce synthetic images (textures). The paper presents a framework for implementing the CAD-to-XR workflow. The solution consists of a module for texture generation based on Generative Adversarial Networks (GANs). The generated texture is then managed by another module (based on Shrinkwrap modelling) to develop the iPVP by simplifying the 3D model and UV mapping the generated texture. The geometric and material data is integrated into a graphic engine, which allows for programming an interactive experience with the iPVP in XR. The CAD-to-XR framework was validated on two components (rifle stock and forend) of a sporting rifle. The solution can automate the texturing process of different product versions in shorter times (compared to a manual procedure). After each product revision, it avoids tedious and manual activities required to generate a new iPVP. The image quality metrics highlight that images are generated in a “realistic” manner (the perceived quality of generated textures is highly comparable to real images). The quality of the iPVPs, generated through the proposed framework and visualised by users through a mixed reality head-mounted display, is equivalent to traditionally designed prototypes. © 2024 The Author(s)},
keywords = {Adversarial networks, Artificial intelligence, CAD-to-XR, Computer aided design models, Computer aided logic design, Computer-aided design, Computer-aided design-to-XR, Design simplification, Digital elevation model, Digital storage, Extended reality, Flow visualization, Generative adversarial networks, Guns (armament), Helmet mounted displays, Intellectual property core, Mixed reality, Photo-realistic, Shrinkfitting, Structural dynamics, User centered design, User-centered design, User-centered design approaches, User-centred, Virtual Prototyping, Work-flows},
pubstate = {published},
tppubtype = {article}
}
Chen, X.; Gao, W.; Chu, Y.; Song, Y.
Enhancing interaction in virtual-real architectural environments: A comparative analysis of generative AI-driven reality approaches Journal Article
In: Building and Environment, vol. 266, 2024, ISSN: 03601323 (ISSN).
Abstract | Links | BibTeX | Tags: Architectural design, Architectural environment, Architectural environments, Artificial intelligence, cluster analysis, Comparative analyzes, comparative study, Computational design, Generative adversarial networks, Generative AI, generative artificial intelligence, Mixed reality, Real time interactions, Real-space, Unity3d, Virtual addresses, Virtual environments, Virtual Reality, Virtual spaces, Work-flows
@article{chen_enhancing_2024,
title = {Enhancing interaction in virtual-real architectural environments: A comparative analysis of generative AI-driven reality approaches},
author = {X. Chen and W. Gao and Y. Chu and Y. Song},
url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-85205298350&doi=10.1016%2fj.buildenv.2024.112113&partnerID=40&md5=8c7d4f5477e25b021dfc5e013a851620},
doi = {10.1016/j.buildenv.2024.112113},
issn = {03601323 (ISSN)},
year = {2024},
date = {2024-01-01},
journal = {Building and Environment},
volume = {266},
abstract = {The architectural environment is expanding into digital, virtual, and informational dimensions, introducing challenges in virtual-real space interaction. Traditional design methods struggle with real-time interaction, integration with existing workflows, and rapid space modification. To address these issues, we present a generative design method that enables symbiotic interaction between virtual and real spaces using Mixed Reality (MR) and Generative Artificial Intelligence (AI) technologies. We developed two approaches: one using the Rhino modeling platform and the other based on the Unity3D game engine, tailored to different application needs. User experience testing in exhibition, leisure, and residential spaces evaluated our method's effectiveness. Results showed significant improvements in design flexibility, interactive efficiency, and user satisfaction. In the exhibition scenario, the Unity3D-based method excelled in rapid design modifications and immersive experiences. Questionnaire data indicated that MR offers good visual comfort and higher immersion than VR, effectively supporting architects in interface and scale design. Clustering analysis of participants' position and gaze data revealed diverse behavioral patterns in the virtual-physical exhibition space, providing insights for optimizing spatial layouts and interaction methods. Our findings suggest that the generative AI-driven MR method simplifies traditional design processes by enabling real-time modification and interaction with spatial interfaces through simple verbal and motion interactions. This approach streamlines workflows by reducing steps like measuring, modeling, and rendering, while enhancing user engagement and creativity. Overall, this method offers new possibilities for experiential exhibition and architectural design, contributing to future environments where virtual and real spaces coexist seamlessly. © 2024},
keywords = {Architectural design, Architectural environment, Architectural environments, Artificial intelligence, cluster analysis, Comparative analyzes, comparative study, Computational design, Generative adversarial networks, Generative AI, generative artificial intelligence, Mixed reality, Real time interactions, Real-space, Unity3d, Virtual addresses, Virtual environments, Virtual Reality, Virtual spaces, Work-flows},
pubstate = {published},
tppubtype = {article}
}
2023
Horvath, I.; Csapo, A. B.
Structured Template Language and Generative AI Driven Content Management for Personalized Workspace Reconfiguration Proceedings Article
In: IEEE Int. Conf. Cogn. Asp. Virtual Real., CVR, Institute of Electrical and Electronics Engineers Inc., 2023, ISBN: 979-835033863-8 (ISBN).
Abstract | Links | BibTeX | Tags: 3D spaces, 3D virtual reality, Cognitive infocommunications, Content management, Content management solutions, Geometric layout, Knowledge engineering, Semantic content, Semantic content management, Semantics, Simple++, Virtual Reality, Work-flows
@inproceedings{horvath_structured_2023,
title = {Structured Template Language and Generative AI Driven Content Management for Personalized Workspace Reconfiguration},
author = {I. Horvath and A. B. Csapo},
url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-85184849596&doi=10.1109%2fCVR58941.2023.10395520&partnerID=40&md5=c3890e80798c9ec542fe453875dde253},
doi = {10.1109/CVR58941.2023.10395520},
isbn = {979-835033863-8 (ISBN)},
year = {2023},
date = {2023-01-01},
booktitle = {IEEE Int. Conf. Cogn. Asp. Virtual Real., CVR},
publisher = {Institute of Electrical and Electronics Engineers Inc.},
abstract = {This work presents a systematic approach towards personalized workspace management and reconfiguration in 3D Virtual Reality (VR) spaces, focusing on a structured template language for defining and manipulating content layout schemas, as well as a generative AI supported content management solution. Recognizing the varying requirements of different tasks and workflows, on the one hand we propose a hierarchical template language that enables, through simple steps, the adaptation of number and variety of documents within geometric layout schemas in digital 3D spaces. In the second half of the paper, we present a generative AI driven framework for integrating associative-semantic content management into such workspaces, thereby enhancing the relevance and contextuality of workspace configurations. The proposed approach aids in identifying content that is semantically linked to a given spatial, temporal and topical environment, enabling workspace designers and users to create more efficient and personalized workspace layouts. © 2023 IEEE.},
keywords = {3D spaces, 3D virtual reality, Cognitive infocommunications, Content management, Content management solutions, Geometric layout, Knowledge engineering, Semantic content, Semantic content management, Semantics, Simple++, Virtual Reality, Work-flows},
pubstate = {published},
tppubtype = {inproceedings}
}