---
title: "Lab 8 Solutions"
output: pdf_document
---


## Linear separable dataset.

```{r}
#####Generate dataset###

set.seed(300)
x.pos <- matrix(rnorm(100*2,mean=0), nrow = 100, ncol = 2)
set.seed(300)
x.neg <- matrix(rnorm(100*2,mean=3), nrow = 100, ncol = 2)
y <- c(rep(1,100),rep(-1,100))
x <- rbind(x.pos,x.neg)
dat <- data.frame(x = x, y = as.factor(y))


plot(x, col = ifelse(y>0,1,2))
legend("topleft",c("Positive","Negative"),col=seq(2),pch=1,text.col=seq(2))

#####Training set and test set#####
set.seed(400) 
train <-  sample(200, 200*0.7)
dat.train <-  dat[train,]
x.test <- x[-train,]
y.test <- y[-train]


```

#### 1)	Install and load the library `e1071`.
```{r}
#install.packages("e1071")
library(e1071)
```

#### 2)	Build a linear SVM model with `cost = 1`.
```{r}
svmfit.train.C1 <- svm(y ~ .,
                       data = dat.train, 
                       kernel = "linear", 
                       cost = 1,
                       scale = FALSE)
```

#### 3)	Report how many support vectors are from each class respectively.
```{r}
summary(svmfit.train.C1)
```

There are 11 support vectors, 6 from the class of `y = -1` and 5 from the class of`y = 1`.


#### 4)	Plot the svm model and check how many support vectors are on the wrong side of the boundary and how many data points are very close to the margin.
```{r}
plot(svmfit.train.C1, dat.train)
```

There are 4 points on the wrong side of the boundary. 

There are quite a few very close to the margin, at least 5 data points on the pink side and at least 2 on the cyan side. 


#### 5)	Predict the class label of `y` on the test set and estimate the test error rate.
```{r}
y.pred.C1 <- predict(svmfit.train.C1, newdata = x.test)
mean(y.pred.C1 != y.test)
```

#### 6)	Now try a smaller `cost = 0.01` and a larger `cost = 1e5` and repeat step 2)-5). 
```{r}

#######cost = 0.01########
svmfit.train.C001 <- svm(y ~ .,
                         data = dat.train,
                         kernel = "linear", 
                         cost = 0.01,
                         scale = FALSE)
summary(svmfit.train.C001)
plot(svmfit.train.C001,dat.train)
y.pred.C001 <- predict(svmfit.train.C001, newdata = x.test)
mean(y.pred.C001 != y.test)

#######cost = 1e5########
svmfit.train.C1e5 <- svm(y ~ ., 
                         data = dat.train, 
                         kernel = "linear", 
                         cost = 1e5,
                         scale = FALSE)
summary(svmfit.train.C1e5)
plot(svmfit.train.C1e5,dat.train)
y.pred.C1e5 <- predict(svmfit.train.C1e5, newdata = x.test)
mean(y.pred.C1e5 != y.test)

```

#### 7)	Use `tune()` function to select the best model (tune the parameter `C` or `cost`). Set seed to be 1.
```{r}
set.seed(1)
tune.out <- tune(svm, 
                 y ~ ., 
                 data = dat.train, 
                 kernel = "linear", 
                 ranges = list(cost = c(0.001,0.01,0.1,1,5,10,100,1000,10000,1e5)))
summary(tune.out)
```


## Linearly inseparable dataset.
```{r}
#####Generate dataset###

set.seed(300)
x.pos1 <- matrix(rnorm(30*2,mean=0), nrow=30, ncol=2)
x.pos2 <- matrix(rnorm(30*2,mean=3), nrow=30, ncol=2)
set.seed(300)
x.neg1 <- matrix(c(rnorm(30,mean=0)+3,rnorm(30,mean=0)),nrow=30,ncol=2)
x.neg2<- matrix(c(rnorm(30,mean=3)-3,rnorm(30,mean=3)),nrow=30,ncol=2)
y <- c(rep(1,60),rep(-1,60))
x <- rbind(x.pos1, x.pos2, x.neg1, x.neg2)
dat2 <- data.frame(x = x, y = as.factor(y))


plot(x, 
     col = ifelse(y > 0, 1, 2), 
     pch = ifelse(y > 0, 1, 2))
legend("topleft", 
       c("Positive","Negative"), 
       col = seq(2), 
       pch = 1:2, 
       text.col = seq(2))

#####Training set and test set#####
set.seed(400)
train <- sample(120, 120*0.7)  
#There are 120 data points in total and we take 70% to be training set
dat2.train <- dat2[train,]
x.test <- x[-train,]
y.test <- y[-train]
```

#### 8)	Build an SVM model with a radial kernel, `gamma = 1` and `cost = 1`. In this model, a) see the summary; 2) plot the svm and 3) estimate the test error rate.

```{r}
svmfit.radial.G1C1 <- svm(y ~ ., data = dat2.train, 
                          kernel = "radial", 
                          gamma = 1, cost = 1,
                          scale = F)
summary(svmfit.radial.G1C1)
plot(svmfit.radial.G1C1, data = dat2.train)
y.pred.radial.G1C1 <- predict(svmfit.radial.G1C1, newdata = x.test)
mean(y.pred.radial.G1C1 != y.test)
```

#### 9)	Build an SVM model with a radial kernel, `gamma = 1` and `cost = 1e5`. In this model, a) see the summary; 2) plot the svm and 3) estimate the test error rate.
```{r}
svmfit.radial.G1C1e5 <- svm(y ~ .,data = dat2.train, 
                            kernel = "radial", 
                            gamma = 1,cost = 1e5,
                            scale = F)
summary(svmfit.radial.G1C1e5)
plot(svmfit.radial.G1C1e5,data = dat2.train)
y.pred.radial.G1C1e5 <- predict(svmfit.radial.G1C1e5, newdata = x.test)
mean(y.pred.radial.G1C1e5 != y.test)

```

#### 10)	Choose the best choice of `gamma` and `cost` for an SVM with a radial kernel. 
```{r}
set.seed(1)
tune.out.radial <- tune(svm, y ~ ., 
                        data = dat2.train, kernel = "radial",
                        ranges = list(cost = c(0.1,1,10,100,1000),
                                      gamma = c(0.5,1,2,3,4)))

summary(tune.out.radial)
```

#### 11)	Use the best model to estimate the test error rate.
```{r}
y.pred.radial.best <- predict(tune.out.radial$best.model, newdata = x.test)
mean(y.pred.radial.best != y.test)

```


