%Version 3 October 2023
% See section 11 of the User Manual for version history
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%                                                                 %%
%% Please do not use \input{...} to include other tex files.       %%
%% Submit your LaTeX manuscript as one .tex document.              %%
%%                                                                 %%
%% All additional figures and files should be attached             %%
%% separately and not embedded in the \TeX\ document itself.       %%
%%                                                                 %%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

%%\documentclass[referee,sn-basic]{sn-jnl}% referee option is meant for double line spacing

%%=======================================================%%
%% to print line numbers in the margin use lineno option %%
%%=======================================================%%

%%\documentclass[lineno,sn-basic]{sn-jnl}% Basic Springer Nature Reference Style/Chemistry Reference Style

%%======================================================%%
%% to compile with pdflatex/xelatex use pdflatex option %%
%%======================================================%%

%%\documentclass[pdflatex,sn-basic]{sn-jnl}% Basic Springer Nature Reference Style/Chemistry Reference Style


%%Note: the following reference styles support Namedate and Numbered referencing. By default the style follows the most common style. To switch between the options you can add or remove Numbered in the optional parenthesis. 
%%The option is available for: sn-basic.bst, sn-vancouver.bst, sn-chicago.bst%  
 
%%\documentclass[sn-nature]{sn-jnl}% Style for submissions to Nature Portfolio journals
%%\documentclass[sn-basic]{sn-jnl}% Basic Springer Nature Reference Style/Chemistry Reference Style
\documentclass[sn-mathphys-num]{sn-jnl}
%%\documentclass[sn-mathphys-ay]{sn-jnl}% Math and Physical Sciences Author Year Reference Style
%%\documentclass[sn-aps]{sn-jnl}% American Physical Society (APS) Reference Style
%%\documentclass[sn-vancouver,Numbered]{sn-jnl}% Vancouver Reference Style
%%\documentclass[sn-apa]{sn-jnl}% APA Reference Style 
%%\documentclass[sn-chicago]{sn-jnl}% Chicago-based Humanities Reference Style

%%%% Standard Packages
%%<additional latex packages if required can be included here>

\usepackage{graphicx}%
\usepackage{multirow}%
\usepackage{amsmath,amssymb,amsfonts}%
\usepackage{amsthm}%
\usepackage{mathrsfs}%
\usepackage[title]{appendix}%
\usepackage{xcolor}%
\usepackage{textcomp}%
\usepackage{manyfoot}%
\usepackage{booktabs}%
\usepackage{algorithm}%
\usepackage{algorithmicx}%
\usepackage{algpseudocode}%
\usepackage{listings}%
%%%%

%\definecolor{blue}{rgb}{0,0,0}

%%%%%=============================================================================%%%%
%%%%  Remarks: This template is provided to aid authors with the preparation
%%%%  of original research articles intended for submission to journals published 
%%%%  by Springer Nature. The guidance has been prepared in partnership with 
%%%%  production teams to conform to Springer Nature technical requirements. 
%%%%  Editorial and presentation requirements differ among journal portfolios and 
%%%%  research disciplines. You may find sections in this template are irrelevant 
%%%%  to your work and are empowered to omit any such section if allowed by the 
%%%%  journal you intend to submit to. The submission guidelines and policies 
%%%%  of the journal take precedence. A detailed User Manual is available in the 
%%%%  template package for technical guidance.
%%%%%=============================================================================%%%%

%% as per the requirement new theorem styles can be included as shown below
\theoremstyle{thmstyleone}%
\newtheorem{theorem}{Theorem}%  meant for continuous numbers
%%\newtheorem{theorem}{Theorem}[section]% meant for sectionwise numbers
%% optional argument [theorem] produces theorem numbering sequence instead of independent numbers for Proposition
\newtheorem{proposition}[theorem]{Proposition}% 
%%\newtheorem{proposition}{Proposition}% to get separate numbers for theorem and proposition etc.

\theoremstyle{thmstyletwo}%
\newtheorem{example}{Example}%
\newtheorem{remark}{Remark}%

\theoremstyle{thmstylethree}%
\newtheorem{definition}{Definition}%

\raggedbottom
%%\unnumbered% uncomment this for unnumbered level heads

\begin{document}

\title[Article Title]{Dr.VCoach: Employment of Advanced Deep Learning and Human-Robot Interaction for Virtual Coaching}

%\author[1]{\fnm{Nino} \sur{Cauli}}\email{nino.cauli@unica.it}
%%\equalcont{These authors contributed equally to this work.}

%\author*[1]{\fnm{Diego} \sur{Reforgiato Recupero}}\email{diego.reforgiato@unica.it}
%%\equalcont{These authors contributed equally to this work.}

%\affil[1]{\orgdiv{Department of Mathematics and Computer Science}, \orgname{University of Cagliari}, \orgaddress{\street{Via Ospedale 72}, \city{Cagliari}, \postcode{09124}, \state{Sardinia}, \country{Italy}}}

\abstract{Ensuring healthy aging and physical fitness is crucial for leading a happy, independent, and fulfilling life. Daily gentle gymnastic training at home can significantly contribute to maintaining physical health among older adults. Virtual and robotic coaches serve as valuable tools to assist elders during their daily exercise routines. In this paper, we introduce the design of a comprehensive robotic coach capable of guiding patients throughout their training sessions. Additionally, we propose an innovative synthetic dataset generator to train the video action recognition module of the robot effectively. To evaluate the efficacy of our robotic coach, we conducted tests with nine patients aged over 50, who were asked to complete a System Usability Scale (SUS) survey.}

\keywords{Robotic coach, Active ageing, Video action recognition, Transformers, Human robot interaction, Assistive robotics}

%%\pacs[JEL Classification]{D8, H51}

%%\pacs[MSC Classification]{35A01, 65L10, 65L12, 65L20, 65L70}

\maketitle

\section{Introduction}

The world we live in is becoming increasingly frenetic with rapid population growth and the proliferation of densely populated cities worldwide. Job opportunities are abundant in these urban centers, leading many people to settle in metropolitan areas. However, city life often fosters a sedentary lifestyle, with individuals spending much of their day seated at desks or commuting between home and work. Conversely, advancements in healthcare, reduced family sizes, and improved access to food have contributed to longer life expectancies and a growing percentage of elderly individuals in the global population. \\
Maintaining independence throughout life is crucial for older individuals. However, aging often leads to a gradual decline in physical and mental capabilities, necessitating external support for daily activities. Engaging in a healthy and active lifestyle, including regular physical activity and walking, can delay the need for assistance later in life. Unfortunately, the sedentary nature of modern life necessitates frequent physical training sessions for older adults to maintain optimal activity levels. To ensure safety, motivation, and effective training, older adults often require the guidance of a personal therapist or trainer to schedule, monitor, and supervise their exercise routines. \\
The high number of elders in need of assistance in their daily training routine has led to an increased demand for dedicated personal therapists. Furthermore, some elders are unable to perform their daily training at care center facilities, requiring the therapist to visit their homes for sessions. These factors limit the number of patients that a single therapist can accommodate, resulting in a shortage of available personal therapists or trainers. \\
In the last decade, amid the Fourth Industrial Revolution (Industry 4.0), there has been rapid technological growth and transformation driven by increased interconnectivity and the advent of smart automation. Industries and healthcare have been particularly impacted, now relying on advanced technologies such as big data, cloud computing, deep learning, the Internet of Things (IoT), mobile devices, and robotics. Smart healthcare (s-health) has emerged as a paradigm, leveraging intelligent models, smart sensors, cloud services, and communication networks to offer context-aware healthcare services. These technological advancements provide significant support to medical personnel by automating monitoring tasks and enabling the storage of patients' medical histories in the cloud. With s-health and robotics systems, therapists can remotely track patients' training progress, allowing them to manage a larger number of patients concurrently. \\
Researchers are striving to automate teaching and monitoring processes by proposing virtual coaches capable of suggesting exercises, monitoring execution, and sending results to doctors~\cite{MUKHERJEE2022102231,Aly_Abbasimoshaei_Kern_2022,10.1145/3056540.3076191}. While a computer, screen, and camera suffice to implement a virtual coach, humanoid robots offer a better solution. Interacting with a dynamic and expressive robot is more engaging than merely viewing a computer screen. A humanoid robot can demonstrate exercises, making movements easier for the elderly to understand. \\
In this paper, we introduce a robotic coach, Dr.VCoach, designed to guide elders in their daily training sessions to maintain physical fitness and age actively. Dr.VCoach can verbally interact with users, manage training schedules for multiple users, demonstrate exercises, monitor exercise execution, and evaluate user performance. We present the design and implementation of the complete robotic coach system and evaluate it on nine elder subjects in their homes. Each subject completed a System Usability Scale (SUS) survey. Additionally, we created a video action recognition dataset tailored for training models to recognize gentle gymnastic exercises and augmented it using a novel synthetic video generator. \\
\color{black}Therefore, the contributions of this paper are as follows:
\begin{enumerate}
    \item Definition of a new set of exercises to promote active aging.
    \item Creation of a novel dataset for action prediction augmented in simulation using domain randomization, along with the release of a synthetic video creation tool.
    \item Design and deployment of an affordable robotic coach tested on patients in a real-life scenario.
\end{enumerate}
In this paper, we focus on the implementation of a robotic coach and its interaction with elderly patients. The integration of a smart healthcare cloud database to store patient progress for remote evaluation by therapists was not implemented. We have decided to leave this aspect for future work.\color{black} \\
This work is part of a European project called DrVCoach, which aims to develop a robotic coach for monitoring and motivating seniors to perform a daily gentle physical exercise routine to maintain physical fitness as they age\footnote{\url{https://drvcoach.unica.it/}}. \\
The remainder of this paper is organized as follows: Section~\ref{sec:relWork} reviews the state-of-the-art on video data augmentation, and virtual and robotic coach architectures. Section~\ref{sec:back} provides a detailed explanation of our robotic coach design and the data augmentation system. Sections~\ref{sec:dataGen} and \ref{sec:Arch} present the implementation details of the synthetic video generator and the robotic coach system, respectively. Section~\ref{sec:sysAtWork} analyzes test results on the data generator and the robotic coach. Finally, Section~\ref{sec:conc} draws conclusions and future directions where we are headed.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related Work}
\label{sec:relWork}
In this section, we present the progress achieved by researchers thus far in developing systems designed to assist patients in their daily physical training routines and rehabilitation sessions. The objective of these systems is to automate either part or the entirety of the training or rehabilitation session, ranging from machine-human interaction to the monitoring and evaluation of exercises. Our analysis of the state-of-the-art does not solely focus on complete virtual coaching systems; instead, we also delve into the subproblem of video monitoring systems. We have divided our study into two parts: the first subsection focuses on virtual coaching systems and assistive robotics, while the second subsection delves deeper into existing video action recognition models and related data augmentation techniques.  

\subsection{Virtual coaching and assistive robotics}

Physiotherapy, rehabilitation, and exercises promoting active aging must be regularly performed by elderly patients. Due to their advanced age, they may not always be able to undergo daily training sessions at care center facilities. In such cases, home-based training becomes the preferred solution. However, since therapists cannot always be present at the patients' homes, automated social and assistive technology becomes invaluable. The automation of monitoring, in particular, has been extensively studied by researchers, and several solutions are now available~\cite{lee2019learning, lee2020towards}. Various types of sensors are available for automated monitoring in the healthcare domain, including biosignal sensors integrated within wearable devices, air quality sensors, acoustic sensors, cameras (RGB, infrared, and depth), among others~\cite{cauli2023sensor}. \\
Cameras, in particular, are powerful tools capable of capturing body features and motion. Supported by machine learning models, RGB cameras can precisely track the location of bodies and objects, monitor their skeletal pose, and detect their motion. External cameras offer the advantage of being non-invasive compared to wearable sensors. For example, Lee et al.~\cite{lee2019learning} presented a model for exercise recognition and evaluation for post-stroke rehabilitation, utilizing threshold models with binary classifications. The model input was data captured by a Kinect sensor (Depth-RGB camera). In a subsequent work~\cite{lee2020towards}, the same authors presented an interactive approach that combines machine learning and rule-based models to automatically assess a patient’s rehabilitation exercise and customize personalized corrective feedback. \\
Encouraging and stimulating the patient during the training session is essential. Virtual and augmented reality can be appropriate technologies for realizing virtual coaching systems. Mostajeran et al.~\cite{mostajeran2020augmented} investigated the acceptance of an AR coaching system for balance training, which can be performed at home. The results suggest that older adults find the system encouraging and stimulating. However, the usability of the AR system showed a significant negative correlation with participants’ age. Elderly patients may find it more difficult to empathize with a virtual avatar compared to a physical one, such as a robot. Several researchers have conducted studies to analyze the level of acceptance of socially assistive robots by elderly patients in rehabilitation and active aging training scenarios~\cite{beer2012domesticated, esposito2016supporting, winkle2018social, vcaic2020robotic}. These studies examined the interaction between socially assistive robots and older patients~\cite{beer2012domesticated, winkle2018social}, integration of ICT and robotic technologies to assist nurse practitioners~\cite{esposito2016supporting}, and elderly people’s social perception of human versus robotic coaches in the context of an active and healthy aging program~\cite{vcaic2020robotic}. All authors reported a positive reception by patients for assistive robots. \\
Rehabilitation and assistive robotics encompass a range of topics, including exercise training robots, smart prostheses and orthoses, monitoring for rehabilitation, and robotic smart home technologies. In their work~\cite{van2016rehabilitation}, the authors present a comprehensive survey on rehabilitation robotics for persons seeking to recover physical, social, communication, or cognitive function, and assist persons who have a chronic disability in accomplishing activities of daily living. \\
Initial studies on assistive robotics for rehabilitation and physical training used mobile robots to monitor, assist, encourage, and socially interact with patients~\cite{tapus2008user, gockley2006encouraging, mataric2007socially}. While a mobile robot’s physical embodiment positively influences human task-related behavior, an anthropomorphic appearance would enhance interaction with human subjects, conveying a higher level of empathy. \\
Initially, humanoid socially assistive robots were adopted to assist patients in physical exercises in seated scenarios with limited motion to their upper body. Fasola et al.~\cite{fasola2013socially} proposed a socially assistive robot designed to engage elderly users in physical exercise. During this study, older adults showed a preference for the physically embodied robot coach over a virtual coach. Similarly, Swift et al.~\cite{swift2015effects} studied an autonomous Socially Assistive Robot (SAR) coach that investigates the effect on individuals post-stroke in a seated reaching task. \\
With advancements in robotics, recent studies tend to use full-body humanoid robots for assistance in rehabilitation and physical exercises. In their work~\cite{tanguy2016computational}, Tanguy et al. designed and implemented a humanoid robot coach capable of demonstrating rehabilitation exercises to patients, observing a patient's exercises, providing feedback to improve performance, and offering encouragement. The authors used imitation learning techniques based on Gaussian Mixture Models, making the robot easily programmable by medical experts without specific robotics knowledge. More recently, Lee et al.~\cite{lee2024enabling} used the Nao robot to coach, monitor, and correct post-stroke patients during their rehabilitation exercises. They studied the acceptance of the robotic coach by both patients and therapists, before and after testing the system. The model used for monitoring and evaluating the exercises performed by the patients was based on skeletal extraction and pose detection from video extracted from a Kinect sensor. \\
Our study is aligned with the work of Lee et al.~\cite{lee2024enabling}, but we focus our attention on implementing a robotic coach to promote active aging instead of focusing solely on post-stroke rehabilitation. To our knowledge, our work is the first full implementation of a robotic coach using a state-of-the-art model based on Vision Transformers (ViT) to monitor physical exercises performed by elderly patients in realistic scenarios. Compared to similar recent studies adopting D-RGB cameras~\cite{fasola2013socially, tanguy2016computational, lee2024enabling}, our system uses affordable RGB webcams to monitor and analyze the patients' training sessions. This solution was made possible thanks to the generation of a synthetic augmented dataset for training.\\
\color{black}Table~\ref{tab:SotA} summarizes a comparison of the main features of our approach against state-of-the-art systems. \color{black}

\begin{sidewaystable}
  \centering
  \resizebox{\textwidth}{!}{\begin{tabular}{@{}l|cccccc@{}}
    \toprule
    \multicolumn{5}{c}{\textbf{Virtual and Robotic Coach Systems}} \\
    \midrule
    System & Case study & Coach platform & Sensors & Action Monitoring Model & Full body motion analysis & Code available\\
    \midrule
    Gockley and Matari~\cite{gockley2006encouraging} (2006) & Stroke rehabilitation & Mobile robot (Pioneer 2-DX) & Inertial measurement units & No real-time monitoring & No (upper-limb) & No \\
    Mataric et al.~\cite{mataric2007socially} (2007) & Stroke rehabilitation & Mobile robot (Pioneer 2-DX) & Inertial measurement units & Threshold on inertial sensor raw data & No (upper-limb) & No \\
    Fasola et al.~\cite{fasola2013socially} (2013) & Elders physical training & Bandit (Humanoid upperbody on a Pioneer 2-DX) & RGB camera & Image segmentation and rule-based models & No (upper-limb) & No \\
    Swift et al.~\cite{swift2015effects} (2015) & Stroke rehabilitation & Bandit (Humanoid upperbody on a Pioneer 2-DX) & RGB, Kinect, and IMU & No real-time monitoring & No (upper-limb) & No \\
    Tanguy et al.~\cite{tanguy2016computational} (2016) & Rehabilitation healthcare & Humanoid robot (Poppy) & Kinect (RGB-D) & Imitation learning on Gaussian Mixture Models & Yes & No \\
    Lee et al.~\cite{lee2019learning} (2019) & Stroke rehabilitation & Computer screen & Kinect (RGB-D) & Threshold models with binary classifications & Yes & No \\
    Lee et al.~\cite{lee2020towards} (2020) & Stroke rehabilitation & Humanoid robot (Nao) and tablet & Kinect (RGB-D) & Machine learning and rule-based models & No (upper-limb) & No \\
    Mostajeran et al.~\cite{mostajeran2020augmented} (2020) & Balance Training in an Aging Population & Meta 2 head-mounted display (HMD) & Kinect (RGB-D) & No real-time monitoring & Yes & No \\
    Lee et al.~\cite{lee2024enabling} (2024) & Stroke rehabilitation & Humanoid robot (Nao) and tablet & Kinect (RGB-D) & Skeletal extraction and pose detection & Yes & No \\
    \midrule
    \textbf{Dr.Vcoach (our solution)} & Elders physical training & Humanoid robot (Nao) and laptop & RGB camera & Timesformer (Vision Transformers) & Yes & Yes \\
    \bottomrule
  \end{tabular}}
  \caption{Comparison of our solution with other state-of-the-art systems.}
  \label{tab:SotA}
\end{sidewaystable}


\subsection{Video action recognition and data augmentation}
\label{sec:videoActRec}
\begin{figure*}[t]
  \centering
   \includegraphics[width=1.0\linewidth]{Images/robotic_coach_final_system.png}
   \caption{System architecture.}
   \label{fig:system}
\end{figure*}
Over the past decade, CNNs have become the de facto standard for image recognition~\cite{ji20123d, simonyan2014two, yue2015beyond}. Nevertheless, in the last three years, models based on ViT architecture~\cite{vaswani2017attention} are quickly replacing CNNs~\cite{dosovitskiy2020image, bertasius2021space, arnab2021vivit, jaegle2021perceiver}. For a comprehensive review of video transformers, please refer to the work of Selva et al.~\cite{selva2023video}. \\
While models designed for image analysis can be used for video analysis, it is essential to consider the temporal dimension inherent in videos. For this reason, state-of-the-art models for video analysis are designed to capture temporal information. These models include Optical flow-based methods~\cite{simonyan2014two}, 3D CNNs~\cite{ji20123d}, Recurrent Neural Networks (RNN)\cite{yue2015beyond, lee2017desire}, and ViT with space-time attention\cite{bertasius2021space}. \\
CNNs and ViT models require a substantial amount of data for successful training. Several data augmentation techniques for image analysis have been introduced and well-presented in recent review papers by Shorten et al. and Khalifa et al.~\cite{shorten2019survey, khalifa2021comprehensive}. These surveys provide a comprehensive overview of image data augmentation, covering basic image manipulations such as geometrical and color space transformations, kernel filters, noise injection, mixing images, and random erasing. They also delve into more recent DL approaches, including feature space augmentation, adversarial training, GAN-based techniques, and Neural Style Transfer.\\
A different approach involves the generation of synthetic images using simulators capable of emulating the appearance and physics of the real world. Game engines are often employed for generating new images due to their powerful graphic and physics engines~\cite{tremblay2018deep}. Game engines permit the creation of synthetic images with high variability in the depicted scene, such as object positions, background, illumination, and camera positions. The concept of producing a synthetic dataset with high variability in image appearance is the core idea behind the Domain Randomization (DR) paradigm introduced by Tobin et al.\cite{tobin2017domain}. In DR, the simulated scene parameters need to be highly randomized to generate images that fully cover the data distribution to be learned. When dealing with videos, it is essential to consider the correlation between time and space, not only in the design of learning models but also in the generation of training datasets. In simulations, the physical interaction between objects (e.g., rigid bodies interaction and gravity), their motions, and the animation of subjects in the scene become crucial aspects for the generation of synthetic videos\cite{cauli2022survey}. \\
While several large RGB video datasets for generic action recognition tasks already exist~\cite{marszalek09, Kuehne11, soomro2012ucf101, smaira2020short, rodriguez2008action, KarpathyCVPR14, singh2010muhavi, li2013anomaly, weinland2006free}, for more specific action recognition tasks, additional data may need to be collected. One solution is fine-tuning the DL model on these large generic video datasets. A different approach to address the shortage of data is the use of data augmentation techniques specific to videos. The recent survey titled "Survey on videos data augmentation for deep learning models"~\cite{cauli2022survey} provides a deep analysis of video data augmentation through simulation. \\
In contrast with the mentioned approaches, in this paper, we introduce a new synthetic video generator specifically designed for action recognition. While some existing solutions address object detection in static images, such as the Unreal Engine 4 plugin 'NVIDIA Deep learning Dataset Synthesizer (NDDS)'\cite{to2018ndds}, and others offer tools for generating synthetic videos for action recognition, such as ElderSim, a platform for synthetic data generation focusing on human action recognition within house interiors\cite{Hwang2021}, none of these options provides the level of flexibility and randomization required for our augmented action recognition dataset.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Background}
\label{sec:back}
The work described in this paper focuses on the implementation of a robotic coach, named 'Dr.VCoach', designed to assist elderly individuals during their daily physical training, as depicted in Figure~\ref{fig:system}. The capabilities of the robot include:
\begin{enumerate}
    \item Understanding verbal commands from the user through a conversational module.
    \item Defining the sequence of exercises and presenting them to the user with verbal descriptions and self-performance via a control module.
    \item Monitoring the user's performance during exercises using RGB cameras, identifying errors, and offering corrections through a video action recognition module.
\end{enumerate}
While various datasets for video action recognition are available, none fully encompass the movements and labels required for this work. Consequently, we generated a new dataset by capturing RGB videos of individuals performing gentle gymnastic exercises in a laboratory setting. This core dataset underwent augmentation in simulation using a synthetic image generator implemented in Unity. Simulated avatars, animated with the original recorded data, were subjected to randomization in motion, background, lighting, and camera position, resulting in a significantly expanded dataset. This versatile tool serves a dual purpose: it can augment preexisting datasets, enriching them with diverse examples, or create entirely new datasets tailored to specific research needs. \\
To train and test our system, we selected, in consultation with a professional personal trainer, 11 gentle gymnastic exercises. Gentle gymnastics represents a form of physical activity that involves slow and progressive movements designed to mobilize the entire body. These exercises are suitable for individuals with varying levels of training, including sedentary office workers, older adults, and athletes. Regular participation in gentle gymnastics offers several advantages, including weight management, posture enhancement, and muscle toning. For older adults, a daily routine of gentle gymnastic exercises can contribute to maintaining better physical fitness. See Section~\ref{sec:dataGen} for a description of the exercises. \\
Initially, we collected a dataset of 1647 videos of 15 subjects performing the 11 exercises in front of a camera. Subsequently, we utilized our synthetic video generator to produce a dataset of 5000 videos depicting a human avatar executing one of the 11 exercises based on the collected video animations. The final novel augmented dataset comprises 6647 videos, obtained by combining the 5000 videos generated with the synthetic video generator with the 1647 videos collected in the lab. \\
The augmented dataset was utilized to train a state-of-the-art action recognition model (Timesformer~\cite{bertasius2021space}). This model underwent testing on a dataset of 239 videos featuring three subjects performing the 11 exercises outdoors. Additionally, the trained model was integrated into the video action recognition module of the final robotic coach system. \\
The system was implemented on Zora~\cite{Zora}, a Nao~\cite{NAO} robot with a software layer designed to facilitate its use by individuals without expertise in information and communication technology (ICT). The Nao robot is commonly employed in studies involving assistive robotics, with a focus on social interactions and affectivity. In medical and physical intervention contexts, the Nao robot is primarily utilized as a motivator or demonstrator. \textcolor{black}{The Nao robot offers several advantages as a robotic coach system. First, its humanoid body makes it highly suitable for demonstrating physical exercises to patients. Second, its human-like appearance and appealing design enhance social acceptability, improving interactions with patients. Additionally, the Nao robot is more affordable than larger and more complex humanoid robots on the market, making it a practical choice for healthcare facilities and private users. Furthermore, Nao robots are widely used in academic and medical settings, fostering a large and active user community for support and collaboration.} \\
The Aldebaran Choregraphe Suite~\cite{choregraphe} was utilized to program the 11 exercises on the robot. For the conversational module, we employed a question-answering model based on ChatGPT function calling. To monitor the actions of the subjects, we utilized the Timesformer action recognition model trained on the augmented dataset. The final robotic coach system was capable of initiating a training session with a human subject, proposing and demonstrating the training routine for the session, and monitoring the correct execution of the exercises by the subject (recognizing the action performed and indicating when the wrong exercise was being executed). We conducted tests with 9 subjects and administered a survey to gather feedback on their experience interacting with the robotic coach.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Dataset Generation}
\label{sec:dataGen}
To address the common scarcity of data for video action recognition, we developed a video generator capable of producing synthetic videos featuring a subject performing specific actions. Our generator is implemented using the Unity~\cite{Unity} game engine along with the Perception~\cite{UnityPP} and Synthetic Humans~\cite{UnitySH} packages. Unity is distinguished by its user-friendly interface, comprehensive and practical C\# scripting API, cross-platform building options, a large online asset store, and a free-to-use plan for projects without revenue. Moreover, Unity offers several plugins specifically developed for researchers in Computer Vision, Artificial Intelligence, and Robotics. The Unity Perception Package is a toolkit for generating synthetic datasets for computer vision, offering a set of predefined and customizable scene randomizers, automatic labeling tools, and a C\# library to customize all the parameters of the data generator. Synthetic Humans, on the other hand, is a plugin used to procedurally generate and place human avatars in a virtual scene. \\
The videos generated by our tool feature a virtual humanoid avatar placed in a highly randomized scene, with the avatar performing one action randomly selected from a pool of preselected actions. The pool of actions used in our tests was derived from real videos using the pose landmark detection pre-trained model from MediaPipe Solutions~\cite{Mediapipe}, a suite of libraries, models, and tools for applying AI and ML to vision, text, and audio analysis. We created scripts to extract the 3D body pose from videos of a human subject performing gentle gymnastic exercises, which we later used to generate Unity animation clips for our generator. \\
\textcolor{black}{Specifically, our MediaPipe script extracts the skeletal pose of the subject for each video frame, representing it as joint positions and orientations in the camera's reference frame using a variant of BlazePose~\cite{bazarevsky2020blazepose}. We then developed a C\# script that first transforms the joint positions and orientations from the camera's reference frame to Unity's scene reference frame. The transformed pose is subsequently used to generate avatar animations within Unity.}

\begin{figure}[t]
  \centering
   \includegraphics[width=1.0\linewidth]{Images/GeneratorExamples.png}
   \caption{Frames extracted from the videos generated by our Synthetic Video Generator.}
   \label{fig:generated}
\end{figure}

Figure~\ref{fig:generated} illustrates several frames rendered by our synthetic video generator. These scenes depict a human avatar performing an exercise, set against various indoor or outdoor environments. The background is created using a skybox with a randomly selected HDRI texture, playing a crucial role in the scene's global illumination. This is complemented by the inclusion of a directional light with adjustable color and intensity. The virtual camera consistently faces the avatar, but we introduce randomization in various aspects, including its distance from the avatar, vertical and horizontal translations, longitudinal axis rotation, and 3D spatial positioning. Additionally, the avatar itself is procedurally generated using the Synthetic Humans Unity package, which can create human avatars with randomized features such as age, gender, ethnicity, height, weight, and clothing, derived from available asset pools. \\
For each generated video, the synthetic video generator provides the following outputs:
\begin{enumerate}
    \item A .png image for each frame of the video. The resolution of the images is an adjustable parameter of the generator.
    \item A .csv file for each frame that includes information on various labels automatically calculated by the generator. This information encompasses the name of the action, 3D camera pose, 3D global pose of each joint of the avatar, 2D pose of each joint in camera space, and avatar metadata (age, ethnicity, etc.).
\end{enumerate} 
The synthetic video generator and the MediaPipe scripts for extracting skeletal animations from videos and converting them into Unity animation clips are accessible online through the provided link in the footnote\footnote{\url{https://github.com/nigno17/Synthetic-video-generator}}.

\subsection{The Dataset}
\begin{figure}[t]
  \centering
   \includegraphics[width=1.0\linewidth]{Images/exercisesblurred.png}
   \caption{The 11 exercises selected for our studies.}
   \label{fig:exercises}
\end{figure}

We first defined a suitable set of gentle gymnastic exercises to incorporate into the daily training routines of older individuals. To assist in this process, we enlisted the expertise of a professional personal trainer with 13 years of experience. Through collaboration with the personal trainer, we finalized a set of 11 exercises:
\begin{enumerate}
    \item Backward arm rotations
    \item Forward arm rotations
    \item Lateral bends
    \item Jumping jacks
    \item Frontal bends
    \item Hips rotations
    \item Lunges
    \item Squats
    \item Torso rotations
    \item Bent torso rotations
    \item Seated torso rotations
\end{enumerate}
Figure~\ref{fig:exercises} depicts a visual representation of each of the 11 exercises.
\begin{figure}[t]
  \centering
   \includegraphics[width=1.0\linewidth]{Images/Collected.png}
   \caption{Frames extracted from the collected videos. Upper row: frames from the training/validation set. Bottom row: frames from the test set.}
   \label{fig:collected}
\end{figure}

\color{black}To support our study, we initiated the collection of a small real-life video dataset in our laboratory, which we subsequently augmented by generating a larger synthetic dataset using our Synthetic Video Generator. We enlisted 23 subjects among students (14) and professors (9) from the Department of Mathematics and Computer Science at the University of Cagliari to participate in the collection of two distinct sets of videos. The initial set was recorded indoors utilizing the Intel RealSense D455 camera~\cite{RealSense}, while the subsequent set employed the onboard camera of the Xiaomi Poco3 cellphone. Each subject executed each of the 11 exercises multiple times, resulting in a video captured for each repetition. The first twenty subjects underwent recording in a controlled environment within our laboratory. The position of the RealSense camera remained fixed in front of the subjects and remained unchanged throughout the entire recording session. Subsequently, this dataset was partitioned to form a training set comprising recordings from the first 15 subjects and a validation set containing recordings from the remaining five subjects. Refer to the upper row of Figure~\ref{fig:collected} for exemplar frames extracted from the training and validation set videos. The recordings of the final three subjects were designated to compose our test set. These videos were captured outdoors, within the courtyard of our department, under varying light conditions, across multiple locations, and with diverse camera positions. This dataset aimed to replicate a more realistic setting. \color{black}The bottom row of Figure 3 showcases exemplar frames extracted from this dataset.\\
To summarize, we divided our data into three datasets:
\begin{enumerate}
    \item \textbf{Training set:} This dataset, used for training, contains 1647 videos. Each video corresponds to one repetition of one of the 11 exercises executed by one of the first 15 students recorded in the lab. Each exercise was repeated roughly 10 times by each student.
    \item \textbf{Validation set:} This dataset, used for validation, contains 539 videos. Each video corresponds to one repetition of one of the 11 exercises executed by the remaining five students recorded in the lab. Each exercise was repeated roughly 10 times by each student.
    \item \textbf{Test set:} This dataset, used for testing, contains 329 videos. Each video corresponds to one repetition of one of the 11 exercises executed by one of the three students recorded outdoors. Each exercise was repeated roughly 10 times by each student.
\end{enumerate}
Training, Validation, and Test collected datasets are accessible online through the provided link in the footnote\footnote{\url{https://drive.google.com/drive/folders/1GzkfOD9byPzOhIPMceiBB6DRdCHRX4Ud?usp=drive_link}}.\\
To train the Video Action Recognition module of our robotic coach, we augmented the collected training set generating a synthetic video set of data. We extracted the body motions from all the videos in the collected training set using the Mediapipe pose landmark detection model. We then imported those body motions as Unity animation clips into our synthetic video generator via a C\# script. We generated a dataset of 5000 videos displaying a human avatar performing one of the 11 exercises based on the imported animations. Each video was generated by randomizing avatar appearance (age, ethnicity, height, gender, body mass, and clothes), background images, scene illumination (color, intensity, and direction), animation speed, and camera position and orientation. Figure~\ref{fig:generated} displays some example of generated frames.\\
The generated data was used to create a final "Augmented training set" that contains 6647 videos. It combines the 5000 videos generated with the synthetic video generator with the 1647 training videos collected in the lab.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Architecture of the System}
\label{sec:Arch}
\color{black}In this section, we will analyze the final robotic coach architecture and the implementation of each of its modules in more detail (see Fig~\ref{fig:det_system}). The following subsections describe the three main modules of the system one by one.
\color{black}
\begin{figure*}[t]
  \centering
   \includegraphics[width=1.0\linewidth]{Images/DetailedSystemArch.png}
   \caption{\color{black}Flow diagram of interactions between modules.\color{black}}
   \label{fig:det_system}
\end{figure*}

\subsection{Video Action Recognition Module}
The role of the Video Action Recognition module is to analyze video clips received from a camera placed next to the robot, showing elderly users performing the exercises. The module can recognize which of the 11 actions has been performed by the elderly person and determine if it is the requested exercise or not. The core of the module can be implemented using a Deep Learning video action recognition model. \\
Following the evaluation of our augmented dataset on two cutting-edge models for video action recognition (one based on Convolutional Neural Networks (CNNs), known as I3D~\cite{carreira2017quo}, and the other utilizing Vision Transformer (ViT) architecture~\cite{bertasius2021space}, referred to as Timesformer~\cite{bertasius2021space}), we decided to implement the Video Action Recognition module using the Timesformer model. ViT architectures are rapidly becoming the de facto standard for video and image processing and classification. The Timesformer architecture is an extension of the ViT model tailored for video action recognition. The attention model in Timesformer not only encompasses the spatial domain but also extends its influence over the temporal domain. The input to the Timesformer model consists of a sequence of RGB frames, with each frame divided into N non-overlapping patches. The self-attention mechanism, known as Divided Space-Time Attention, initially calculates the temporal attention between a patch and all corresponding spatial patches in other frames. Subsequently, it computes the spatial attention for the resulting temporal encoding with respect to all other patches within the analyzed frame. \\
\textcolor{black}{More specifically, the Video Action Recognition Module receives the pre-cropped video clip of the next action to analyze from the Control Module. The clip is preprocessed, and each frame is divided into patches that are linearly transformed into embedding vectors. These embedded patches are then analyzed by the Timesformer model using the Divided Space-Time Attention mechanism. For each video clip, the Timesformer produces a probability distribution across 11 possible actions. These probabilities are then sent back to the Control Module for further analysis.}

\subsection{Conversational module}
The Conversational module serves as an interface to manage verbal interactions between the elderly user utilizing the system and the robotic coach. It should be capable of engaging in conversation with the user, understanding specific verbal commands, and providing vocal updates on the status of the training session. \\ 
We have implemented our Conversational module using the ChatGPT API. More specifically, we leverage the API's innovative “function calling” capability. This feature enables new ChatGPT models to identify specific questions or commands from users and respond with a structured call request instead of standard text replies. By incorporating function calling, chatbots gain the ability to interact with other systems, allowing the GPT models to handle inquiries that require the invocation of an external function. Put simply, function calling offers an additional means of instructing AI models on how to engage with the external environment. \\
We designed the Conversational Module as a chatbot using the python ChatGPT API. We used the function calling capability to define the following function calls:
\begin{enumerate}
    \item \textbf{Function:} "start\_session". \textbf{Description:} "Start the daily training section for user". \textbf{Effect:} starting the Control Module for a specific user.
    \item \textbf{Function:} "stop\_session". \textbf{Description:} "Stop the training section". \textbf{Effect:} stopping the Control Module.
    \item \textbf{Function:} "start\_exercise". \textbf{Description:} "Start monitoring the next exercise". \textbf{Effect:} the Control Module start the monitoring phase for the next exercise.
    \item \textbf{Function:} "repeat\_exercise". \textbf{Description:} "Repeat monitoring the last exercise". \textbf{Effect:} the Control Module start the monitoring phase for the past exercise again.
    \item \textbf{Function:} "show\_exercise". \textbf{Description:} "Show me the exercise again". \textbf{Effect:} showing again the exercise via video and performed by the robot.
\end{enumerate}
To enable voice control of the Conversational Module and hear its responses, we utilized the capabilities of OpenAI's speech-to-text (Whisper) and text-to-speech (TTS) models. We developed a submodule connected with the Conversational Module that is capable of:
\begin{enumerate}
    \item Receiving all responses from the Conversational Module in real-time as strings of text.
    \item Generating audio tracks for each received string via the TTS model and playing them.
    \item Listening to the user for verbal commands or conversational responses via the laptop microphone.
    \item Converting audio tracks received from the microphone into strings of text to be passed to the Conversational Module using the Whisper model.
\end{enumerate}

\subsection{Control module}
\begin{figure*}[t]
  \centering
   \includegraphics[width=0.9\linewidth]{Images/DrVCoachProgramW.png}
   \caption{Control Module in action: A) Login screen with the "Add New User" window open. B) Screen displaying the exercises scheduled for the day. C) Monitoring and recording interface. D) Results display screen.}
   \label{fig:controlM}
\end{figure*}

The Control module is the core of our robotic coach and synchronizes all the other modules. Specifically, the functions of the Control Module are: 
\begin{enumerate}
    \item Initialize a training session receiving a command via the Conversational Module.
    \item Select the sequence of exercises for the daily training session based on the schedule defined by a therapist.
    \item Show the exercises to the user (with verbal descriptions) both via a video on a screen and performed by the robot through the Control Module.
    \item Monitor the execution of each exercise via the Video Action Recognition Module.
    \item Show the evaluation of the training session to the user/patient.
\end{enumerate}
Before integrating all the blocks of our system, we decided to implement a prototype of the Control Module that runs on a laptop and does not interact with the robot and the Conversational Module. The prototype is controlled via keyboard and mouse. Figure~\ref{fig:controlM} shows four different screenshots of the prototype running.
The program can store info and data of multiple patients. At first the prototype asks the patient to select an existing account or create a new one (figure~\ref{fig:controlM}-A). After logging in, a screen with the daily training list of exercises is shown. In this screen the patient can play a sample video and read a verbal explanation of each exercise (figure~\ref{fig:controlM}-B). When the patient is ready, the monitoring process for each exercise starts with the recording screen. In this modality the patient must copy the exercise shown by a video on the screen for several repetitions. Other than the sample video, the real-time video of the patients captured from the webcam and the name of the exercise that has been executed are shown on screen (figure~\ref{fig:controlM}-C). During the monitoring the videos captured from the camera for each repetition are stored and analyzed by the Video Action Recognition Module (the Timesformer model trained on our augmented dataset). After the execution of each exercise the program shows a screen with the evaluation of the execution (figure~\ref{fig:controlM}-D). In the summary page, for each repetition is possible to see: 
\begin{enumerate}
    \item The recorded video.
    \item The ordered list of the prediction confidence of our action recognition model for each exercise.
    \item An icon showing the quality of the execution: 
    \begin{enumerate}
        \item Green circle – good execution (the tested exercise is predicted as the more probable)
        \item Yellow circle – average execution (the tested exercise is predicted as the second or third more probable)
        \item Red circle – bad execution (the tested exercise is out of the top three predictions)
    \end{enumerate}
\end{enumerate}
Finished the training session, the program stores all the video and evaluation results in a separated folder for each registered patient. The working code of the Control Module prototype is available online at the provided link in the footnote\footnote{\url{https://github.com/nigno17/DrVCoach}}. \\
Before implementing the final system, we generated robot animations for the exercises on the Zora robot using the Softbank Choregraphe Suite. However, due to the Zora robot's limited motion capabilities, certain exercises could not be faithfully animated. Specifically, the Jumping Jacks exercise could not be implemented as the Zora robot is unable to perform jumping motions.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{System at work}
\label{sec:sysAtWork}
We decided to conduct two different tests to evaluate the proposed robotic coach system. First, we tested the validity of the augmented dataset generated using our synthetic video generator. This step was essential before using the augmented dataset to train the Timesformer model in the Video Action Recognition module of the final system. In the second test, we integrated all the system modules and tested the resulting final version of the robotic coach on nine healthy adult subjects aged over 50. After the test, we asked the subjects to complete a System Usability Scale (SUS) survey and analyzed their responses.

\subsection{Dataset validation}
\begin{figure*}[t]
  \centering
   \includegraphics[width=1.0\linewidth]{Images/confusionTimesTest.png}
   \caption{Confusion Matrices of the Timesformer model trained on the collected and augmented dataset. The model trained on the augmented dataset has an high accuracy (the matrix is almost diagonal), presenting some confusion only between the backward and forward arm rotations, two exercises visually very similar.}
   \label{fig:confusion}
\end{figure*}

We decided to evaluate our datasets on two cutting-edge models: one based on Convolutional Neural Networks (CNNs), known as I3D , and the other utilizing Vision Transformer (ViT) architecture, referred to as Timesformer. \textcolor{black}{The experiments were conducted on a laptop featuring an AMD Ryzen 9 5900HX CPU and an NVIDIA GeForce RTX 3080 Laptop GPU. The models, which were pretrained on ImageNet, were trained for 15 epochs in each experiment. Video frames were processed by resizing the smaller dimension to 256 pixels and applying a central crop to produce 224x224 images. After cropping, ImageNet normalization was applied to standardize the frames. The training process for both networks employed the stochastic gradient descent (SGD) optimizer, configured with a learning rate of 0.005, a momentum of 0.9, and a weight decay of 0.0001.}\\
The Two-Stream Inflated 3D ConvNet (I3D) is a video action recognition model based on 3D CNNs. The Timesformer architecture is an extension of the ViT model tailored for video action recognition.\\
The two models have been trained on both our collected and augmented dataset. The I3D obtained an accuracy on our test set of 19\% trained on the collected and 83\% trained on the augmented. The Timesformer had a better accuracy when trained on the collected dataset, 54\%, and an accuracy in line with I3D when trained on the augmented dataset, 82\%. In general, the Timesformer showed a better generalization (better results trained on the small and less varied collected dataset), faster training time during finetuning, and good results trained on the varied augmented dataset (over 80\% accuracy, confusion matrix depicted in figure~\ref{fig:confusion}). Moreover, ViT architectures are rapidly becoming the de facto standard for video and image processing and classification. For these reasons, we decided to implement the Dr. VCoach "Video Action Recognition Module" using the Timesformer model trained on our augmented dataset.

\subsection{Final system validation}
% \begin{figure}[t]
%   \centering
%    \includegraphics[width=0.9\linewidth]{Images/GenderDistribution.png}
%    \caption{Gender distribution within subjects.}
%    \label{fig:genderD}
% \end{figure}
The final setup of the Dr. VCoach system consists of the Zora robot and a laptop running all the modules, placed in front of the user/patient. The patient interacts via voice with the Conversational Module's chatbot to control the system. In this way, the system can provide more information about the exercises to the user, both through videos and the robot's embodied animation. \\
To perform the final test and validate the robotic coach, all the components presented in section~\ref{sec:Arch} were integrated together. First, all the functions of the Conversational Module were connected to the Control Module prototype. This allowed the execution of connected commands on the Control Module when a patient gives one of the specified commands via function calling (for example, when the patient asks to start the daily training session, the Control Module program is launched). Subsequently, the robot was integrated into the system. Whenever the Control Module displays an exercise video to the patient, a signal is sent to the robot to start the corresponding animation. However, for the Jumping Jack exercise, which cannot be performed by the robot, only the video is played. \\
We selected nine healthy adults over 50 years old to validate the final system. A unique ID was assigned to each subject to maintain anonymity. The subjects' ages range from 51 to 86 years old, with an equal distribution between men (5 subjects) and women (4 subjects). 
% (see Figure\ref{fig:genderD}). 
Table~\ref{tab:sub} presents a list of all subjects with their IDs, ages, and genders.
\begin{table}
  \centering
  \begin{tabular}{@{}l|c|c@{}}
    \toprule
    \multicolumn{3}{c}{\textbf{Subjects list}} \\
    \midrule
    Subject id & Age & Gender \\
    \midrule
    Sub1 & 51 & Man \\
    Sub2 & 56 & Woman \\
    Sub3 & 86 & Man \\
    Sub4 & 54 & Man \\
    Sub5 & 82 & Woman \\
    Sub6 & 77 & Man \\
    Sub7 & 69 & Woman \\
    Sub8 & 54 & Man \\
    Sub9 & 57 & Woman \\
    \bottomrule
  \end{tabular}
  \caption{List of the selected subjects for the test.}
  \label{tab:sub}
\end{table}

\begin{figure*}[t]
  \centering
   \includegraphics[width=1.0\linewidth]{Images/sessXexe.png}
   \caption{Number of total training sessions on each exercise for all the subjects.}
   \label{fig:sessXexe}
\end{figure*}
After an initial explanation of how the system works, each subject used the system for three training sessions. During each session, the robotic coach presented the subject with three different exercises randomly selected from the pool of 11 exercises defined previously. The subject then performed each exercise for 5 repetitions, after which the Control Module displayed the evaluation screen. Each exercise could be repeated only twice during the three training sessions. Consequently, each subject performed a minimum of 5 and a maximum of 9 different exercises during the test. Figure~\ref{fig:sessXexe} illustrates the distribution of training sessions for each exercise throughout the entire test phase. On average, each exercise was presented an equal number of times across all sessions, with the exception of lunges, which unfortunately appeared less frequently than the others. All the subjects signed an informed consent form before commencing the training sessions. The form contains information regarding the purpose, process, potential benefits, and risks of the study, as well as details on data collection procedures, time commitment, voluntary participation, and the right to withdraw (without prejudice to care). Additionally, the form provides assurance of confidentiality, stating that research records identifying participants will be kept confidential to the extent permitted by applicable laws and regulations.

\begin{figure*}[t]
  \centering
   \includegraphics[width=1.0\linewidth]{Images/accXexe.png}
   \caption{Top-1 and Top-3 accuracy obtained by the subjects for each exercise during their training sessions.}
   \label{fig:accXexe}
\end{figure*}
The system stores the classification predictions and the captured videos of all the sessions, with each subject having their own folder. To evaluate the effectiveness of the training coach, we analyzed the top-1 and top-3 accuracy of the Action Video Recognition module. Figure~\ref{fig:accXexe} illustrates the accuracy of the system across all training sessions and grouped by exercise. Overall, the system's accuracy is slightly lower than the results obtained in our previous test set, with the majority of exercises achieving a top-1 accuracy of over 70\% and top-3 accuracy of over 90\%. The challenge of the trained model in distinguishing between forward and backward arm rotations is also evident in this final test. Specifically, the top-1 accuracy for these two exercises is low (45\% and 68\%, respectively), while their top-3 accuracy is close to 100\% (95\% and 94\%). These results are noteworthy for two reasons:
\begin{enumerate}
    \item The age of the subjects in this study is higher than that of the subjects enrolled in collecting our previous test set (who were in their 20s), potentially resulting in poorer exercise performance.
    \item The training sessions were recorded in participants' homes, in an uncontrolled environment. This led to high variance in illumination conditions, distracting elements in the environment (such as other individuals not performing the exercises), and instances of body parts being cropped (refer to Figure~\ref{fig:testFrames} for examples).
\end{enumerate}

\begin{figure*}[t]
  \centering
   \includegraphics[width=1.0\linewidth]{Images/accXsub.png}
   \caption{Top-1 and Top-3 accuracy obtained by each subject during his/her training sessions.}
   \label{fig:accXsub}
\end{figure*}
Another important factor in evaluating the effectiveness of the system is its ability to recognize when a specific exercise is performed poorly. In such cases, the system should not classify the correct exercise as the most probable one. To gain insight into this aspect, we analyzed the top-1 and top-3 accuracy achieved by each subject during all training sessions (results are depicted in figure~\ref{fig:accXsub}). We anticipated a decrease in accuracy with increasing age of the subjects, as older individuals are expected to perform the exercises less effectively compared to younger ones. Indeed, the two subjects over 80 years old achieved significantly lower accuracy than the other subjects (top-1 of 33\% and 44\%), followed closely by subject 6 (77 years old, top-1 accuracy of 66\%).

\begin{figure*}[t]
  \centering
   \includegraphics[width=1.0\linewidth]{Images/SUS.png}
   \caption{System Usability Scale (SUS) survey's questions with the average answer rating across the subjects.}
   \label{fig:SUS}
\end{figure*}
\begin{figure*}[t]
  \centering
   \includegraphics[width=1.0\linewidth]{Images/trainQuest.png}
   \caption{List of survey's questions on the training session with the average answer rating across the subjects.}
   \label{fig:trainQuest}
\end{figure*}
After the training sessions, we asked the subjects to complete a survey regarding their experience. The survey was divided into four sections:
\begin{enumerate}
    \item Background: Subjects were asked to provide their personal information, including name, date of birth, gender, and email address.
    \item System Usability Scale (SUS): This section consisted of a set of standard questions regarding the usability of the system. Each question was rated on a scale from 1 (strongly disagree) to 5 (strongly agree). Figure~\ref{fig:SUS} displays the list of questions and the average ratings obtained across all subjects.
    \item DrVCoach results analytics: This section contained questions regarding the quality of the training sessions. Each question was rated on a scale from 1 (low) to 5 (high). Figure~\ref{fig:trainQuest} presents the list of questions and the average ratings obtained across all subjects.
    \item Open questions: Subjects were asked to identify the strengths and weaknesses of the system, as well as suggest any additional features to be included.
\end{enumerate}  
The survey results indicate a positive response from the subjects for both the SUS questions and those regarding the quality of the training sessions. Subjects found the system easy to use and well-integrated; however, they expressed some uncertainty about using it frequently in their everyday lives.

\begin{figure}[t]
  \centering
   \includegraphics[width=1.0\linewidth]{Images/TestFrames.png}
   \caption{Frames extracted from the videos captured during the validation test of the system.}
   \label{fig:testFrames}
\end{figure}
When asked about the strengths of the system, most of the subjects highlighted its ability to help them maintain their physical fitness ('it helps older people maintain their physical fitness', 'perfect for home training', 'it helps me move my body'), as well as the ease of use of the system ('the system presents the exercises clearly', 'it is easy to learn and very useful', 'the exercise explanations are clear enough'). Regarding the weaknesses of the system, some subjects did not identify any, while others noted the need to familiarize themselves with the program's functionalities before use ('I need to familiarize myself with the system's functionalities before using it', 'the system lacks a user manual or tutorial'), and the absence of a warm-up session ('I believe it could include a warm-up session at the beginning'). \\
Only three subjects mentioned additional features they would like to see: 'the system should include a warm-up phase before starting the training session', 'I would appreciate having more exercises', and 'I would like an audio explanation of each exercise in addition to the practical examples'.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Conclusion}
\label{sec:conc}
\color{black}
In this paper, we introduced a robotic coach capable of guiding elderly patients through their daily physical exercise routines to maintain an active lifestyle and age healthily. The robotic coach can verbally interact with patients, initiate training sessions, describe and demonstrate exercises to be performed, monitor and evaluate exercise execution using an external camera, and display the subjects' performances on a screen. Additionally, we provided a collected dataset of human subjects performing exercises in front of the camera and developed a tool to augment this dataset by generating randomized synthetic videos. The entire system was tested on nine elderly subjects in their homes, followed by the administration of a SUS survey to each subject. \\
\color{black}
Thanks to our generative data augmentation architecture and the video action recognition module based on Vision Transformer, the robotic coach successfully recognized and evaluated the execution of exercises performed by the subjects during testing. The system demonstrated a high level of robustness in uncontrolled environments where training sessions were recorded, including variations in illumination conditions, distracting elements in the environment, and instances of body parts being cropped. \\
Upon analyzing the subjects' responses to the survey, we found a high level of acceptance of the robotic coach for guiding and motivating them during training. When asked about the system's strengths, subjects highlighted its usefulness and ease of use. The only drawbacks mentioned were the time required to become familiar with the system and the absence of a warming-up stage before starting the training session. The positive reception from the nine elderly subjects during testing is encouraging for the broader adoption of robotic coaches and assistive robots in healthcare scenarios such as physical training and rehabilitation.\\
\textcolor{black}{We plan to extend our work in the future to address some of the current limitations and explore additional avenues for enhancement. The first step will involve improving the action recognition module by upgrading the system to handle multiple patients on screen and recognizing actions that involve human-human and human-object interactions. To achieve these enhancements, our Unity-based synthetic generator must be updated to generate videos featuring multiple actors, human interactions, and the inclusion of handheld and interactive objects. \\
The second step will focus on integrating smart healthcare features into the system. Specifically, we plan to add a cloud database to store patient data, training programs, and progress. Additionally, we intend to develop a mobile app connected to the database, enabling both patients and therapists to interact with the system. These features will be crucial for therapists, allowing them to remotely monitor patient progress, update training programs, and provide real-time feedback. \\
Finally, we aim to test the completed system in a real-world scenario, with patients using the system over an extended period (e.g., one month). During this final test, therapists will select appropriate training programs for each patient and closely monitor their progress.}

\backmatter

\bmhead{Supplementary information}

If your article has accompanying supplementary file/s please state so here. 

Authors reporting data from electrophoretic gels and blots should supply the full unprocessed scans for key as part of their Supplementary information. This may be requested by the editorial team/s if it is missing.

Please refer to Journal-level guidance for any specific requirements.

\bmhead{Acknowledgements}
This research was funded by the European Union's Horizon 2020 Marie Skłodowska-Curie Actions Individual Fellowships under the Grant No. 101031646.

\section*{Declarations}

\subsection{Funding}
This research was funded by the European Union's Horizon 2020 Marie Skłodowska-Curie Actions Individual Fellowships under the Grant No. 101031646.

\subsection{Competing Interests}
%The authors declare no conflicts of interest

\subsection{Ethics approval and consent to participate} 
Informed consent was obtained from all subjects involved in the study.

\subsection{Consent for publication} 
Written informed consent has been obtained from the patients to publish this paper.

\subsection{Data availability}
Training, Validation, and Test collected datasets are accessible online through the following link: \url{https://drive.google.com/drive/folders/1GzkfOD9byPzOhIPMceiBB6DRdCHRX4Ud?usp=drive_link}.

\subsection{Code availability}
The synthetic video generator and the MediaPipe scripts for extracting skeletal animations from videos and converting them into Unity animation clips are accessible online through the following link: \url{https://github.com/nigno17/Synthetic-video-generator}. \\
The working code of the Control Module prototype is available online at the following link: \url{https://github.com/nigno17/DrVCoach}.


\subsection{Materials availability}
Not applicable

\subsection{Author contribution}
hidden for blind rule
%Conceptualization, N. Cauli and D. Reforgiato Recupero; methodology, N. Cauli and D. Reforgiato Recupero; software, N. Cauli; investigation, N. Cauli and D. Reforgiato Recupero; data curation, N. Cauli; writing---original draft preparation, N. Cauli; writing---review and editing, D. Reforgiato Recupero; supervision, D. Reforgiato Recupero; All authors have read and agreed to the published version of the manuscript.



\bibliography{main}% common bib file
%% if required, the content of .bbl file can be included here once bbl is generated
%%\input sn-article.bbl


\end{document}
