Chapter 6 Non Linear Regression

6.0.1 LOESS (Local Regression)

Creating a data set

set.seed(5366)
x=runif(1000,0,2*pi)
y=sin(x)+0.1*rnorm(1000)
mydata=data.frame(x,y)
plot(y~x,data=mydata)

trainsample=function(dataset,trainprop){
  return(sample(nrow(dataset),
                round(nrow(dataset)*trainprop,0)))
}

train=trainsample(mydata,0.7)

Loess

model=loess(y~x,data=mydata[train,],
            control=loess.control(surface="direct"))
plot(model)
plot(model$x,model$y)

plot(model$x,model$fitted)
lines(sort(model$x),sin(sort(model$x)),col='red')

Predicting values on test set

yhat=predict(model,newdata=mydata[-train,])
plot(mydata$y[-train],yhat)

cor(mydata$y[-train],yhat)

## [1] 0.9903859

Effect of Span/Bandwidth

span = 0.75 (default)

model=loess(y~x,data=mydata[train,],
            control=loess.control(surface="direct"),
            span=0.75)
plot(model$x,model$fitted)

temp=function(s){
  model=loess(y~x,data=mydata[train,],
              control=loess.control(surface="direct"),
              span=s)
  plot(model$x,model$fitted)
}

Try plotting the loess models using the temp function created. Use the following values

temp(0.75)

temp(0.01)

temp(0.05)

temp(0.1)

temp(0.2)

temp(0.3)

temp(0.4)

temp(0.5)

temp(0.6)

temp(0.7)

temp(0.75)

temp(0.8)

temp(0.9)

temp(1)

temp(1.1)

temp(1.3)

temp(1.4)

temp(1.5)

temp(2.0)

temp(2.5)

temp(3.0)

temp(10)

temp(100)

temp(1000)

plot(x,y)

l.model=lm(y~x,data=mydata[train,])
plot(mydata[train,])
abline(l.model,col='red')

6.0.2 Weighted k-Nearest Neighbors

library(kknn)
fit.sine = train.kknn(y~x,data=mydata)
yhat=(kknn(y~x,train=mydata[train,],test=mydata[-train,],
           k=11,kernel="optimal",distance=2))$fitted.values
plot(mydata$y[-train],yhat)

cor(mydata$y[-train],yhat)

## [1] 0.98947

plot(mydata$x[-train],yhat)
lines(sort(mydata$x[-train]),
      sin(sort(mydata$x[-train])),
      col="red")

6.0.3 Support Vector Regression

library(e1071)
model=svm(y~x,data=mydata[train,])
yhat=predict(model,newdata=mydata[-train,])
plot(mydata$y[-train],yhat)

cor(mydata$y[-train],yhat)

## [1] 0.9904471

plot(mydata$x[-train],yhat)
lines(sort(mydata$x[-train]),
      sin(sort(mydata$x[-train])),
      col="red")

6.0.4 Neural Networks

library(nnet)
model=nnet(y~x,data=mydata[train,],size=10,linout=TRUE)

## # weights:  31
## initial  value 466.507980 
## iter  10 value 151.208253
## iter  20 value 80.529881
## iter  30 value 46.578187
## iter  40 value 12.647240
## iter  50 value 7.738789
## iter  60 value 7.653897
## iter  70 value 7.649360
## iter  80 value 7.646350
## iter  90 value 7.642754
## iter 100 value 7.641623
## final  value 7.641623 
## stopped after 100 iterations

yhat=predict(model,newdata=mydata[-train,])
plot(mydata$y[-train],yhat)

cor(mydata$y[-train],yhat)

##           [,1]
## [1,] 0.9905847

plot(mydata$x[-train],yhat)
lines(sort(mydata$x[-train]),
      sin(sort(mydata$x[-train])),
      col="red")

6.0.5 Random Forest

library(randomForest)

model=randomForest(y~x,data=mydata[train,])
yhat=predict(model,newdata=mydata[-train,])
plot(mydata$y[-train],yhat)

cor(mydata$y[-train],yhat)

## [1] 0.9870691

plot(mydata$x[-train],yhat)
lines(sort(mydata$x[-train]),
      sin(sort(mydata$x[-train])),
      col="red")