diff options
Diffstat (limited to 'analysis/days.R')
| -rw-r--r-- | analysis/days.R | 184 |
1 files changed, 91 insertions, 93 deletions
diff --git a/analysis/days.R b/analysis/days.R index 2155819..4997ffd 100644 --- a/analysis/days.R +++ b/analysis/days.R @@ -30,12 +30,13 @@ nwireless=2 nsimkey=4 nsimulations=nseed*nwakeupfor*nwireless*nsimkey # Must be 3200 -## Load data +## Load data and prepare the data data=suppressMessages(read_csv("../CCGRID2022.csv"))%>%distinct() # Note that in the data experiment wireless=="lora",seed==1,wakeupfor==60,simkey=="baseline" is present 2 times in the CSV file data_seed=data%>%group_by(simkey,wireless,wakeupfor,seed)%>%summarize(energy=sum(energy),coverage=sum(nDataRcv))%>% mutate(efficiency=energy/coverage)%>% ungroup() +## F1_Score of multiclass vectors F1_Score2=function(truth, pred){ result=sapply(c("baseline","extended","hint","hintandextended"),function(c){ cur_truth=truth[truth==c] @@ -48,6 +49,7 @@ F1_Score2=function(truth, pred){ do.call("cbind",result) } +## Train models generate_accuracy_for=function(ignore_hint=FALSE,seed_max=200,attempts_max=2,wrl="lora",wuf=180) { attempts=seq(1,attempts_max) results=sapply(attempts,function(attempt){ @@ -94,98 +96,66 @@ generate_accuracy_for=function(ignore_hint=FALSE,seed_max=200,attempts_max=2,wrl } -generate_accuracy_energy = function(wireless,wakeupfor,steps=10, accuracy=10,ignore_hint=TRUE){ +generate_accuracy_energy = function(wireless,wakeupfor,steps=1, accuracy=10,ignore_hint=TRUE){ + ## Setup variables npolicies=4 data_seed_for_energy=data_seed%>%ungroup() if(ignore_hint){npolicies=npolicies-1;data_seed_for_energy=data_seed%>%filter(simkey!="hint")} data_seed_for_energy=data_seed_for_energy%>%filter(wireless==!!wireless,wakeupfor==!!wakeupfor) + ## Generate inputs result=tibble() result_energy=tibble() for(i in seq(1,160,by=steps)){ # We stop at 80% of the data (this way test set is at least 20%) + print(paste("Step",i)) acc=generate_accuracy_for(ignore_hint=ignore_hint,seed=i,attempts_max=accuracy,wrl=wireless,wuf=wakeupfor) - result_energy=rbind(result_energy,data_seed_for_energy%>%filter(seed<=i)%>%summarize(energy=sum(energy),seed_max=i,days=i*npolicies,months=days/30,setup=paste0(!!wireless," ",!!wakeupfor,"s"),wireless=!!wireless,wakeupfor=!!wakeupfor)) + result_energy=rbind(result_energy,data_seed_for_energy%>%filter(seed<=i)%>%summarize(energy=sum(energy),seed_max=i,days=i*npolicies,setup=paste0(!!wireless," ",!!wakeupfor,"s"),wireless=!!wireless,wakeupfor=!!wakeupfor)) result=rbind(result,acc) } - list(accuracy=result%>%mutate(days=seed_max*npolicies,months=days/30), # Since 3 policies (since ignore_hint=TRUE) + list(accuracy=result%>%mutate(days=seed_max*npolicies), # Since 3 policies (since ignore_hint=TRUE) energy=result_energy) } -result_energy_policy=rbind(data_seed, - data_seed%>%mutate(seed=seed+200), - data_seed%>%mutate(seed=seed+400), - data_seed%>%mutate(seed=seed+600)) # Almost same as if each experiment run 4 times more seed seed -result_energy_policy=result_energy_policy%>%group_by(wireless,wakeupfor,simkey)%>%mutate(energy=cumsum(energy),setup=paste0(wireless," ",wakeupfor,"s"),days=seed,months=days/30) -# Generate accuracy for each wireless and uptime +########## Generate accuracy, energy, F1-Score and coverage data ########## if(F){ # Toggle to train lora60=generate_accuracy_energy("lora",60) lora180=generate_accuracy_energy("lora",180) nbiot60=generate_accuracy_energy("nbiot",60) nbiot180=generate_accuracy_energy("nbiot",180) accuracy=rbind(lora60$accuracy,lora180$accuracy,nbiot60$accuracy,nbiot180$accuracy) + coverage=data_seed%>%filter(simkey!="hint")%>%group_by(wireless,wakeupfor,seed)%>%summarize(coverage=sum(coverage))%>%mutate(days=seed*3)%>%filter(days %in% !!lora60$energy$days)%>%select(-seed) energy=rbind(lora60$energy, lora180$energy, nbiot60$energy, - nbiot180$energy) -# energy$setup=factor(energy$setup,levels=c("lora 60s","nbiot 60s","lora 180s", "nbiot 180s"),ordered=TRUE) + nbiot180$energy)%>%left_join(coverage,by=c("wireless","wakeupfor","days")) } -## Summarize -result_summary=accuracy%>%group_by(wireless,wakeupfor,months,model)%>% - summarize( - mean_accuracy=mean(accuracy),sd_accuracy=sd(accuracy),min_accuracy=min(accuracy),max_accuracy=max(accuracy), - mean_f1_baseline=mean(f1_baseline),sd_f1_baseline=sd(f1_baseline),min_f1_baseline=min(f1_baseline),max_f1_baseline=max(f1_baseline), - mean_f1_hint=mean(f1_hint),sd_f1_hint=sd(f1_hint),min_f1_hint=min(f1_hint),max_f1_hint=max(f1_hint), - mean_f1_extended=mean(f1_extended),sd_f1_extended=sd(f1_extended),min_f1_extended=min(f1_extended),max_f1_extended=max(f1_extended), - mean_f1_hintandextended=mean(f1_hintandextended),sd_f1_hintandextended=sd(f1_hintandextended),min_f1_hintandextended=min(f1_hintandextended),max_f1_hintandextended=max(f1_hintandextended)) - -## Result max -metrics_peak=result_summary%>%group_by(wireless,wakeupfor,model)%>% - summarize(max_accuracy=max(mean_accuracy)) - - - - - - -result_energy_policy=result_energy_policy%>%filter(days %in% !!energy$days) -csv_table=result_energy_policy%>% - full_join(energy,by=c("days","wireless","wakeupfor"),suffix=c("","_training")) -csv_table=csv_table%>%group_by(wireless,wakeupfor)%>%summarize(delta=energy-energy_training,simkey=simkey,days=days,months=months) - - -write("wireless,wakeupfor,policy,slope,intercept","figures/delta_energy.csv") -csv_table%>%group_by(wireless,wakeupfor,simkey)%>%group_walk(function(data,grp){ - grp=as.list(grp) - reg=lm(delta/1e3 ~ months,data) - slope=round(as.numeric(reg$coefficients["months"]),digits=1) - intercept=round(as.numeric(reg$coefficients[1]),digits=1) - print(paste0("Wireless=",grp$wireless," Wakeupfor=",grp$wakeupfor," Policy=",grp$simkey," Slope=",slope," Intercept=",intercept)) - write(paste(grp$wireless,grp$wakeupfor,grp$simkey,slope,intercept,sep=","),"figures/delta_energy.csv",append=T) -}) - - -ggplot(csv_table,aes(months,delta/1e3,color=simkey,shape=simkey))+ - geom_line(size=1.2)+geom_point(size=3)+ylab("Delta in energy (kJ)")+xlab("Training months")+ - facet_wrap(~wireless+wakeupfor,scale="free")+ - scale_color_viridis(discrete=TRUE,option="H",end=0.95) - -stopifnot(1) - -ggplot(data=energy,aes(months,energy/1e6,group=setup,fill=setup))+ - geom_bar(stat="identity",position="dodge")+ - scale_fill_viridis(discrete=TRUE,option="D")+ - labs(fill="Nodes wireless technology and uptime")+theme(legend.position=c(0.2,0.75))+ - scale_x_continuous(breaks = seq(0, 15, by = 1))+ - xlab("Number of months")+ylab("Energy consumption (MJ)")+ - geom_point(data=result_energy_policy%>%filter(simkey=="hintandextended"),aes(months,energy/1e6,group=setup,color=simkey),size=1)+facet_wrap(~wireless+wakeupfor) -ggsave("figures/months_energy.pdf",width=8.5,height=4) -ggplot(data=result_summary%>%mutate(model=ifelse(model=="knn","KNN","DT")),aes(linetype=model))+ - geom_line(aes(months,mean_f1_baseline,color="Baseline"),size=1.2)+ - geom_line(aes(months,mean_f1_extended,color="Extended"),size=1.2)+ - geom_line(aes(months,mean_f1_hintandextended,color="Hintandextended"),size=1.2)+labs(color="Classes",linetype="Model")+ +########## Build learning curve (Accuracy+F1_Scores) ########## +learning_curves=accuracy%>%group_by(wireless,wakeupfor,days,model)%>% + summarize( + ## Accuracy + mean_accuracy=mean(accuracy),sd_accuracy=sd(accuracy), + min_accuracy=min(accuracy),max_accuracy=max(accuracy), + ## F1-Score Baseline + mean_f1_baseline=mean(f1_baseline),sd_f1_baseline=sd(f1_baseline), + min_f1_baseline=min(f1_baseline),max_f1_baseline=max(f1_baseline), + ## F1-Score Hint + mean_f1_hint=mean(f1_hint),sd_f1_hint=sd(f1_hint),min_f1_hint=min(f1_hint), + max_f1_hint=max(f1_hint), + ## F1-Score Extended + mean_f1_extended=mean(f1_extended),sd_f1_extended=sd(f1_extended), + min_f1_extended=min(f1_extended),max_f1_extended=max(f1_extended), + ## F1-Score HintAndExtended + mean_f1_hintandextended=mean(f1_hintandextended),sd_f1_hintandextended=sd(f1_hintandextended), + min_f1_hintandextended=min(f1_hintandextended),max_f1_hintandextended=max(f1_hintandextended)) + + +ggplot(data=learning_curves%>%mutate(model=ifelse(model=="knn","KNN","DT")),aes(linetype=model))+ + geom_line(aes(days,mean_f1_baseline,color="Baseline"),size=1.2)+ + geom_line(aes(days,mean_f1_extended,color="Extended"),size=1.2)+ + geom_line(aes(days,mean_f1_hintandextended,color="Hintandextended"),size=1.2)+labs(color="Classes",linetype="Model")+ facet_wrap(~wireless+wakeupfor)+ scale_x_continuous(breaks = seq(0, 15, by = 1))+ scale_y_continuous(breaks = seq(0, 1, by = 0.1))+ @@ -196,14 +166,14 @@ ggplot(data=result_summary%>%mutate(model=ifelse(model=="knn","KNN","DT")),aes(l legend.spacing=unit(-0.2,"cm"), legend.box.margin=margin(1,1,1,1), legend.box.background = element_rect(fill = "white", color = "black",size=0.8))+ - xlab("Training months")+ylab("Classes F1-Score")+ + xlab("Training days")+ylab("Classes F1-Score")+ scale_color_viridis(discrete=TRUE,end=0.7) -ggsave("figures/months_f1-score.pdf",width=8.5,height=6) +ggsave("figures/days_f1-score.pdf",width=8.5,height=6) ## Plot Merge Accuracy -ggplot(data=result_summary%>%mutate(model=ifelse(model=="knn","KNN","DT")),aes(months,mean_accuracy))+ - geom_line(aes(linetype=model),size=1.2)+xlab("Training months")+ylab("Model accuracy")+labs(linetype="Models")+ +ggplot(data=learning_curves%>%mutate(model=ifelse(model=="knn","KNN","DT")),aes(days,mean_accuracy))+ + geom_line(aes(linetype=model),size=1.2)+xlab("Training days")+ylab("Model accuracy")+labs(linetype="Models")+ scale_x_continuous(breaks = seq(0, 15, by = 1))+ scale_y_continuous(breaks = seq(0, 1, by = 0.1))+ facet_wrap(~wireless+wakeupfor)+ @@ -212,29 +182,57 @@ ggplot(data=result_summary%>%mutate(model=ifelse(model=="knn","KNN","DT")),aes(m legend.position = c(0.38,0.68), legend.background = element_rect(fill = "white", color = "black",size=0.8))+ scale_color_viridis(discrete=TRUE,end=0.7) -ggsave("figures/months_accuracy.pdf",width=8.5,height=6) -stopifnot(1) - -## Plot accuracy + F1-Score -sapply(c("knn","tree"),function(grp){ - data=result_summary%>%filter(model==grp) - plot=ggplot(data,aes(months,mean_accuracy))+ - geom_ribbon(aes(ymin=mean_accuracy-sd_accuracy,ymax=mean_accuracy+sd_accuracy),alpha=0.2,color=NA)+ - geom_line(size=1.1)+geom_point(size=3)+ - geom_point(data=data%>%drop_na(mean_f1_baseline),aes(months,mean_f1_baseline,color="baseline"))+geom_line(data=data%>%drop_na(mean_f1_baseline),aes(months,mean_f1_baseline,color="baseline"))+ - geom_point(data=data%>%drop_na(mean_f1_extended),aes(months,mean_f1_extended,color="extended"))+geom_line(data=data%>%drop_na(mean_f1_extended),aes(months,mean_f1_extended,color="extended"))+ - geom_point(data=data%>%drop_na(mean_f1_hintandextended),aes(months,mean_f1_hintandextended,color="hintandextended"))+geom_line(data=data%>%drop_na(mean_f1_hintandextended),aes(months,mean_f1_hintandextended,color="hintandextended")) - - if(any(!is.na(data$mean_f1_hint))){ - plot=plot+geom_point(data=data%>%drop_na(mean_f1_hint),aes(months,mean_f1_hint,color="hint"))+geom_line(data=data%>%drop_na(mean_f1_hint),aes(months,mean_f1_hint,color="hint")) - } +ggsave("figures/days_accuracy.pdf",width=8.5,height=6) + + +########## Energy and delta with raw policies ########## +## First we extended the number of seed to cover the entire duration of the training +data_seed_energy=rbind(data_seed, + data_seed%>%mutate(seed=seed+200), + data_seed%>%mutate(seed=seed+400), + data_seed%>%mutate(seed=seed+600)) # Almost same as if each experiment run 4 times more seed seed +## Compute the cumulative energy of each policies in each configuration accross the 4*200 days +data_seed_energy=data_seed_energy%>%group_by(wireless,wakeupfor,simkey)%>%mutate(energy=cumsum(energy),setup=paste0(wireless," ",wakeupfor,"s"),days=seed) +## Now filter the data +data_seed_energy=data_seed_energy%>%filter(days %in% !!energy$days) +## Compute the delta data +energy_coverage_delta=data_seed_energy%>% + full_join(energy,by=c("days","wireless","wakeupfor"),suffix=c("","_training")) +energy_coverage_delta=energy_coverage_delta%>%group_by(wireless,wakeupfor)%>%summarize(delta_energy=energy-energy_training,simkey=simkey,days=days,delta_coverage=coverage-coverage_training) + + + - plot=plot+xlab("Number of months")+ylab(paste("Mean",grp,"accuracy"))+ggtitle(paste(grp,"accuracy"))+ -# geom_hline(data=metrics_peak%>%filter(model==grp),aes(yintercept=max_accuracy),color="red",size=1)+ -# geom_text(data=metrics_peak%>%filter(model==grp),x=0,aes(y=max_accuracy,label = round(max_accuracy,digits=1),vjust=-1),color="red")+ - facet_wrap(~wireless+wakeupfor)+scale_x_continuous(breaks = seq(0, max(result_summary$months), by = 1))+ylim(c(NA,1))+labs(color="F1 Score")+ - theme(legend.position="top") - ggsave(paste0("figures/months_",grp,".pdf"),width=9,height=8) - print(plot) +write("wireless,wakeupfor,policy,slope,intercept,delta_coverage","figures/delta_energy_coverage.csv") +energy_coverage_delta%>%group_by(wireless,wakeupfor,simkey)%>%group_walk(function(data,grp){ + grp=as.list(grp) + reg=lm(delta_energy/1e3 ~ days,data) + slope=round(as.numeric(reg$coefficients["days"]),digits=1) + intercept=round(as.numeric(reg$coefficients[1]),digits=1) + mean_delta_coverage=round(mean(data$delta_coverage),digits=1) + print(paste0("Wireless=",grp$wireless," Wakeupfor=",grp$wakeupfor," Policy=",grp$simkey," Slope=",slope," Intercept=",intercept," Delta Coverage=",mean_delta_coverage)) + write(paste(grp$wireless,grp$wakeupfor,grp$simkey,slope,intercept,mean_delta_coverage,sep=","),"figures/delta_energy_coverage.csv",append=T) }) +ggplot(energy_coverage_delta,aes(days,delta_energy/1e3,color=simkey,shape=simkey))+ + geom_line(size=1.2)+ylab("Delta in energy (kJ)")+xlab("Training days")+ + facet_wrap(~wireless+wakeupfor,scale="free")+ + scale_color_viridis(discrete=TRUE,option="H",end=0.95) +ggsave("figures/delta_energy_training.pdf") + +ggplot(energy_coverage_delta,aes(days,delta_coverage,color=simkey))+ + geom_line(size=1.2)+ylab("Delta in coverage")+xlab("Training days")+ + facet_wrap(~wireless+wakeupfor,scale="free")+ + scale_color_viridis(discrete=TRUE,end=0.9) +ggsave("figures/delta_coverage_training.pdf") + +ggplot(data=energy%>%filter(((days/3) %% 6) == 0),aes(days,energy/1e6,group=setup,fill=setup))+ + geom_bar(stat="identity",position="dodge")+ + scale_fill_viridis(discrete=TRUE,option="D")+ + labs(fill="Nodes wireless technology and uptime")+theme(legend.position=c(0.2,0.75))+ + scale_x_continuous(breaks = seq(0, max(energy$days), by = 15))+ + xlab("Number of days")+ylab("Energy consumption (MJ)") +ggsave("figures/days_energy.pdf",width=8.5,height=4) + + + |