Publications

@inproceedings{leng_generating_2023,

title = {Generating Virtual On-body Accelerometer Data from Virtual Textual Descriptions for Human Activity Recognition},

author = {Z. Leng and H. Kwon and T. Ploetz},

url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-85175788497&doi=10.1145%2f3594738.3611361&partnerID=40&md5=ddecaf6d81f71511c8152ca14f33cd7f},

doi = {10.1145/3594738.3611361},

isbn = {979-840070199-3 (ISBN)},

year  = {2023},

date = {2023-01-01},

booktitle = {ISWC - Proc. Int. Symp. Wearable Comput.},

pages = {39–43},

publisher = {Association for Computing Machinery, Inc},

abstract = {The development of robust, generalized models for human activity recognition (HAR) has been hindered by the scarcity of large-scale, labeled data sets. Recent work has shown that virtual IMU data extracted from videos using computer vision techniques can lead to substantial performance improvements when training HAR models combined with small portions of real IMU data. Inspired by recent advances in motion synthesis from textual descriptions and connecting Large Language Models (LLMs) to various AI models, we introduce an automated pipeline that first uses ChatGPT to generate diverse textual descriptions of activities. These textual descriptions are then used to generate 3D human motion sequences via a motion synthesis model, T2M-GPT, and later converted to streams of virtual IMU data. We benchmarked our approach on three HAR datasets (RealWorld, PAMAP2, and USC-HAD) and demonstrate that the use of virtual IMU training data generated using our new approach leads to significantly improved HAR model performance compared to only using real IMU data. Our approach contributes to the growing field of cross-modality transfer methods and illustrate how HAR models can be improved through the generation of virtual training data that do not require any manual effort. © 2023 Owner/Author.},

keywords = {Activity recognition, Computational Linguistics, E-Learning, Human activity recognition, Language Model, Large language model, large language models, Motion estimation, Motion Synthesis, On-body, Pattern recognition, Recognition models, Textual description, Training data, Virtual IMU Data, Virtual Reality, Wearable Sensors},

pubstate = {published},

tppubtype = {inproceedings}

}

@inproceedings{wang_elevating_2023,

title = {Elevating Perception: Unified Recognition Framework and Vision-Language Pre-Training Using Three-Dimensional Image Reconstruction},

author = {Z. Wang and A. Joshi and G. Zhang and W. Ren and F. Jia and X. Sun},

url = {https://www.scopus.com/inward/record.uri?eid=2-s2.0-85192837757&doi=10.1109%2fAIHCIR61661.2023.00105&partnerID=40&md5=0fe17cc622a9aa90e88b8c3e6a3bed3b},

doi = {10.1109/AIHCIR61661.2023.00105},

isbn = {979-835036036-3 (ISBN)},

year  = {2023},

date = {2023-01-01},

booktitle = {Proc. - Int. Conf. Artif. Intell., Human-Comput. Interact. Robot., AIHCIR},

pages = {592–596},

publisher = {Institute of Electrical and Electronics Engineers Inc.},

abstract = {This research project explores a paradigm shift in perceptual enhancement by integrating a Unified Recognition Framework and Vision-Language Pre-Training in three-dimensional image reconstruction. Through the synergy of advanced algorithms from computer vision & language processing, the project tries to enhance the precision and depth of perception in reconstructed images. This innovative approach holds the potential to revolutionize fields such as medical imaging, virtual reality, and computer-aided design, providing a comprehensive perspective on the intersection of multimodal data processing and perceptual advancement. The anticipated research outcomes are expected to significantly contribute to the evolution of technologies that rely on accurate and contextually rich three-dimensional reconstructions. Moreover, the research aims to reduce the constant need for new datasets by improving pattern recognition through 3D image patterning on backpropagation. This continuous improvement of vectors is envisioned to enhance the efficiency and accuracy of pattern recognition, contributing to the optimization of perceptual systems over time. © 2023 IEEE.},

keywords = {3D Model LLM, 3D modeling, 3D models, 3D Tech, 3d-modeling, Augmented Reality, Character recognition, Component, Computer aided design, Computer vision, Continuous time systems, Data handling, Generative AI, Image enhancement, Image Reconstruction, Image to Text Generation, Medical Imaging, Pattern recognition, Pre-training, Reconstructive Training, Text generations, Three dimensional computer graphics, Virtual Reality},

pubstate = {published},

tppubtype = {inproceedings}

}