First commit
This commit is contained in:
Dib, Gerges
110
.gitignore
vendored
Normal file
110
.gitignore
vendored
Normal file
@@ -0,0 +1,110 @@
|
||||
|
||||
### Python template
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
MANIFEST
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
.hypothesis/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
.static_storage/
|
||||
.media/
|
||||
local_settings.py
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# celery beat schedule file
|
||||
celerybeat-schedule
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# Environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
|
||||
*.pkl
|
||||
*-solved.ipynb
|
||||
|
||||
97
Exercise1/Data/ex1data1.txt
Executable file
97
Exercise1/Data/ex1data1.txt
Executable file
@@ -0,0 +1,97 @@
|
||||
6.1101,17.592
|
||||
5.5277,9.1302
|
||||
8.5186,13.662
|
||||
7.0032,11.854
|
||||
5.8598,6.8233
|
||||
8.3829,11.886
|
||||
7.4764,4.3483
|
||||
8.5781,12
|
||||
6.4862,6.5987
|
||||
5.0546,3.8166
|
||||
5.7107,3.2522
|
||||
14.164,15.505
|
||||
5.734,3.1551
|
||||
8.4084,7.2258
|
||||
5.6407,0.71618
|
||||
5.3794,3.5129
|
||||
6.3654,5.3048
|
||||
5.1301,0.56077
|
||||
6.4296,3.6518
|
||||
7.0708,5.3893
|
||||
6.1891,3.1386
|
||||
20.27,21.767
|
||||
5.4901,4.263
|
||||
6.3261,5.1875
|
||||
5.5649,3.0825
|
||||
18.945,22.638
|
||||
12.828,13.501
|
||||
10.957,7.0467
|
||||
13.176,14.692
|
||||
22.203,24.147
|
||||
5.2524,-1.22
|
||||
6.5894,5.9966
|
||||
9.2482,12.134
|
||||
5.8918,1.8495
|
||||
8.2111,6.5426
|
||||
7.9334,4.5623
|
||||
8.0959,4.1164
|
||||
5.6063,3.3928
|
||||
12.836,10.117
|
||||
6.3534,5.4974
|
||||
5.4069,0.55657
|
||||
6.8825,3.9115
|
||||
11.708,5.3854
|
||||
5.7737,2.4406
|
||||
7.8247,6.7318
|
||||
7.0931,1.0463
|
||||
5.0702,5.1337
|
||||
5.8014,1.844
|
||||
11.7,8.0043
|
||||
5.5416,1.0179
|
||||
7.5402,6.7504
|
||||
5.3077,1.8396
|
||||
7.4239,4.2885
|
||||
7.6031,4.9981
|
||||
6.3328,1.4233
|
||||
6.3589,-1.4211
|
||||
6.2742,2.4756
|
||||
5.6397,4.6042
|
||||
9.3102,3.9624
|
||||
9.4536,5.4141
|
||||
8.8254,5.1694
|
||||
5.1793,-0.74279
|
||||
21.279,17.929
|
||||
14.908,12.054
|
||||
18.959,17.054
|
||||
7.2182,4.8852
|
||||
8.2951,5.7442
|
||||
10.236,7.7754
|
||||
5.4994,1.0173
|
||||
20.341,20.992
|
||||
10.136,6.6799
|
||||
7.3345,4.0259
|
||||
6.0062,1.2784
|
||||
7.2259,3.3411
|
||||
5.0269,-2.6807
|
||||
6.5479,0.29678
|
||||
7.5386,3.8845
|
||||
5.0365,5.7014
|
||||
10.274,6.7526
|
||||
5.1077,2.0576
|
||||
5.7292,0.47953
|
||||
5.1884,0.20421
|
||||
6.3557,0.67861
|
||||
9.7687,7.5435
|
||||
6.5159,5.3436
|
||||
8.5172,4.2415
|
||||
9.1802,6.7981
|
||||
6.002,0.92695
|
||||
5.5204,0.152
|
||||
5.0594,2.8214
|
||||
5.7077,1.8451
|
||||
7.6366,4.2959
|
||||
5.8707,7.2029
|
||||
5.3054,1.9869
|
||||
8.2934,0.14454
|
||||
13.394,9.0551
|
||||
5.4369,0.61705
|
||||
47
Exercise1/Data/ex1data2.txt
Executable file
47
Exercise1/Data/ex1data2.txt
Executable file
@@ -0,0 +1,47 @@
|
||||
2104,3,399900
|
||||
1600,3,329900
|
||||
2400,3,369000
|
||||
1416,2,232000
|
||||
3000,4,539900
|
||||
1985,4,299900
|
||||
1534,3,314900
|
||||
1427,3,198999
|
||||
1380,3,212000
|
||||
1494,3,242500
|
||||
1940,4,239999
|
||||
2000,3,347000
|
||||
1890,3,329999
|
||||
4478,5,699900
|
||||
1268,3,259900
|
||||
2300,4,449900
|
||||
1320,2,299900
|
||||
1236,3,199900
|
||||
2609,4,499998
|
||||
3031,4,599000
|
||||
1767,3,252900
|
||||
1888,2,255000
|
||||
1604,3,242900
|
||||
1962,4,259900
|
||||
3890,3,573900
|
||||
1100,3,249900
|
||||
1458,3,464500
|
||||
2526,3,469000
|
||||
2200,3,475000
|
||||
2637,3,299900
|
||||
1839,2,349900
|
||||
1000,1,169900
|
||||
2040,4,314900
|
||||
3137,3,579900
|
||||
1811,4,285900
|
||||
1437,3,249900
|
||||
1239,3,229900
|
||||
2132,4,345000
|
||||
4215,4,549000
|
||||
2162,4,287000
|
||||
1664,2,368500
|
||||
2238,3,329900
|
||||
2567,4,314000
|
||||
1200,3,299000
|
||||
852,2,179900
|
||||
1852,4,299900
|
||||
1203,3,239500
|
||||
BIN
Exercise1/Figures/cost_function.png
Executable file
BIN
Exercise1/Figures/cost_function.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 120 KiB |
BIN
Exercise1/Figures/dataset1.png
Executable file
BIN
Exercise1/Figures/dataset1.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 12 KiB |
BIN
Exercise1/Figures/learning_rate.png
Executable file
BIN
Exercise1/Figures/learning_rate.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 8.5 KiB |
BIN
Exercise1/Figures/regression_result.png
Executable file
BIN
Exercise1/Figures/regression_result.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 17 KiB |
1307
Exercise1/exercise1.ipynb
Executable file
1307
Exercise1/exercise1.ipynb
Executable file
File diff suppressed because it is too large
Load Diff
48
Exercise1/utils.py
Executable file
48
Exercise1/utils.py
Executable file
@@ -0,0 +1,48 @@
|
||||
import numpy as np
|
||||
import sys
|
||||
sys.path.append('..')
|
||||
|
||||
from submission import SubmissionBase
|
||||
|
||||
|
||||
class Grader(SubmissionBase):
|
||||
X1 = np.column_stack((np.ones(20), np.exp(1) + np.exp(2) * np.linspace(0.1, 2, 20)))
|
||||
Y1 = X1[:, 1] + np.sin(X1[:, 0]) + np.cos(X1[:, 1])
|
||||
X2 = np.column_stack((X1, X1[:, 1]**0.5, X1[:, 1]**0.25))
|
||||
Y2 = np.power(Y1, 0.5) + Y1
|
||||
|
||||
def __init__(self):
|
||||
part_names = ['Warm up exercise',
|
||||
'Computing Cost (for one variable)',
|
||||
'Gradient Descent (for one variable)',
|
||||
'Feature Normalization',
|
||||
'Computing Cost (for multiple variables)',
|
||||
'Gradient Descent (for multiple variables)',
|
||||
'Normal Equations']
|
||||
super().__init__('linear-regression', part_names)
|
||||
|
||||
def __iter__(self):
|
||||
for part_id in range(1, 8):
|
||||
try:
|
||||
func = self.functions[part_id]
|
||||
|
||||
# Each part has different expected arguments/different function
|
||||
if part_id == 1:
|
||||
res = func()
|
||||
elif part_id == 2:
|
||||
res = func(self.X1, self.Y1, np.array([0.5, -0.5]))
|
||||
elif part_id == 3:
|
||||
res = func(self.X1, self.Y1, np.array([0.5, -0.5]), 0.01, 10)
|
||||
elif part_id == 4:
|
||||
res = func(self.X2[:, 1:4])
|
||||
elif part_id == 5:
|
||||
res = func(self.X2, self.Y2, np.array([0.1, 0.2, 0.3, 0.4]))
|
||||
elif part_id == 6:
|
||||
res = func(self.X2, self.Y2, np.array([-0.1, -0.2, -0.3, -0.4]), 0.01, 10)
|
||||
elif part_id == 7:
|
||||
res = func(self.X2, self.Y2)
|
||||
else:
|
||||
raise KeyError
|
||||
yield part_id, res
|
||||
except KeyError:
|
||||
yield part_id, 0
|
||||
100
Exercise2/Data/ex2data1.txt
Executable file
100
Exercise2/Data/ex2data1.txt
Executable file
@@ -0,0 +1,100 @@
|
||||
34.62365962451697,78.0246928153624,0
|
||||
30.28671076822607,43.89499752400101,0
|
||||
35.84740876993872,72.90219802708364,0
|
||||
60.18259938620976,86.30855209546826,1
|
||||
79.0327360507101,75.3443764369103,1
|
||||
45.08327747668339,56.3163717815305,0
|
||||
61.10666453684766,96.51142588489624,1
|
||||
75.02474556738889,46.55401354116538,1
|
||||
76.09878670226257,87.42056971926803,1
|
||||
84.43281996120035,43.53339331072109,1
|
||||
95.86155507093572,38.22527805795094,0
|
||||
75.01365838958247,30.60326323428011,0
|
||||
82.30705337399482,76.48196330235604,1
|
||||
69.36458875970939,97.71869196188608,1
|
||||
39.53833914367223,76.03681085115882,0
|
||||
53.9710521485623,89.20735013750205,1
|
||||
69.07014406283025,52.74046973016765,1
|
||||
67.94685547711617,46.67857410673128,0
|
||||
70.66150955499435,92.92713789364831,1
|
||||
76.97878372747498,47.57596364975532,1
|
||||
67.37202754570876,42.83843832029179,0
|
||||
89.67677575072079,65.79936592745237,1
|
||||
50.534788289883,48.85581152764205,0
|
||||
34.21206097786789,44.20952859866288,0
|
||||
77.9240914545704,68.9723599933059,1
|
||||
62.27101367004632,69.95445795447587,1
|
||||
80.1901807509566,44.82162893218353,1
|
||||
93.114388797442,38.80067033713209,0
|
||||
61.83020602312595,50.25610789244621,0
|
||||
38.78580379679423,64.99568095539578,0
|
||||
61.379289447425,72.80788731317097,1
|
||||
85.40451939411645,57.05198397627122,1
|
||||
52.10797973193984,63.12762376881715,0
|
||||
52.04540476831827,69.43286012045222,1
|
||||
40.23689373545111,71.16774802184875,0
|
||||
54.63510555424817,52.21388588061123,0
|
||||
33.91550010906887,98.86943574220611,0
|
||||
64.17698887494485,80.90806058670817,1
|
||||
74.78925295941542,41.57341522824434,0
|
||||
34.1836400264419,75.2377203360134,0
|
||||
83.90239366249155,56.30804621605327,1
|
||||
51.54772026906181,46.85629026349976,0
|
||||
94.44336776917852,65.56892160559052,1
|
||||
82.36875375713919,40.61825515970618,0
|
||||
51.04775177128865,45.82270145776001,0
|
||||
62.22267576120188,52.06099194836679,0
|
||||
77.19303492601364,70.45820000180959,1
|
||||
97.77159928000232,86.7278223300282,1
|
||||
62.07306379667647,96.76882412413983,1
|
||||
91.56497449807442,88.69629254546599,1
|
||||
79.94481794066932,74.16311935043758,1
|
||||
99.2725269292572,60.99903099844988,1
|
||||
90.54671411399852,43.39060180650027,1
|
||||
34.52451385320009,60.39634245837173,0
|
||||
50.2864961189907,49.80453881323059,0
|
||||
49.58667721632031,59.80895099453265,0
|
||||
97.64563396007767,68.86157272420604,1
|
||||
32.57720016809309,95.59854761387875,0
|
||||
74.24869136721598,69.82457122657193,1
|
||||
71.79646205863379,78.45356224515052,1
|
||||
75.3956114656803,85.75993667331619,1
|
||||
35.28611281526193,47.02051394723416,0
|
||||
56.25381749711624,39.26147251058019,0
|
||||
30.05882244669796,49.59297386723685,0
|
||||
44.66826172480893,66.45008614558913,0
|
||||
66.56089447242954,41.09209807936973,0
|
||||
40.45755098375164,97.53518548909936,1
|
||||
49.07256321908844,51.88321182073966,0
|
||||
80.27957401466998,92.11606081344084,1
|
||||
66.74671856944039,60.99139402740988,1
|
||||
32.72283304060323,43.30717306430063,0
|
||||
64.0393204150601,78.03168802018232,1
|
||||
72.34649422579923,96.22759296761404,1
|
||||
60.45788573918959,73.09499809758037,1
|
||||
58.84095621726802,75.85844831279042,1
|
||||
99.82785779692128,72.36925193383885,1
|
||||
47.26426910848174,88.47586499559782,1
|
||||
50.45815980285988,75.80985952982456,1
|
||||
60.45555629271532,42.50840943572217,0
|
||||
82.22666157785568,42.71987853716458,0
|
||||
88.9138964166533,69.80378889835472,1
|
||||
94.83450672430196,45.69430680250754,1
|
||||
67.31925746917527,66.58935317747915,1
|
||||
57.23870631569862,59.51428198012956,1
|
||||
80.36675600171273,90.96014789746954,1
|
||||
68.46852178591112,85.59430710452014,1
|
||||
42.0754545384731,78.84478600148043,0
|
||||
75.47770200533905,90.42453899753964,1
|
||||
78.63542434898018,96.64742716885644,1
|
||||
52.34800398794107,60.76950525602592,0
|
||||
94.09433112516793,77.15910509073893,1
|
||||
90.44855097096364,87.50879176484702,1
|
||||
55.48216114069585,35.57070347228866,0
|
||||
74.49269241843041,84.84513684930135,1
|
||||
89.84580670720979,45.35828361091658,1
|
||||
83.48916274498238,48.38028579728175,1
|
||||
42.2617008099817,87.10385094025457,1
|
||||
99.31500880510394,68.77540947206617,1
|
||||
55.34001756003703,64.9319380069486,1
|
||||
74.77589300092767,89.52981289513276,1
|
||||
118
Exercise2/Data/ex2data2.txt
Executable file
118
Exercise2/Data/ex2data2.txt
Executable file
@@ -0,0 +1,118 @@
|
||||
0.051267,0.69956,1
|
||||
-0.092742,0.68494,1
|
||||
-0.21371,0.69225,1
|
||||
-0.375,0.50219,1
|
||||
-0.51325,0.46564,1
|
||||
-0.52477,0.2098,1
|
||||
-0.39804,0.034357,1
|
||||
-0.30588,-0.19225,1
|
||||
0.016705,-0.40424,1
|
||||
0.13191,-0.51389,1
|
||||
0.38537,-0.56506,1
|
||||
0.52938,-0.5212,1
|
||||
0.63882,-0.24342,1
|
||||
0.73675,-0.18494,1
|
||||
0.54666,0.48757,1
|
||||
0.322,0.5826,1
|
||||
0.16647,0.53874,1
|
||||
-0.046659,0.81652,1
|
||||
-0.17339,0.69956,1
|
||||
-0.47869,0.63377,1
|
||||
-0.60541,0.59722,1
|
||||
-0.62846,0.33406,1
|
||||
-0.59389,0.005117,1
|
||||
-0.42108,-0.27266,1
|
||||
-0.11578,-0.39693,1
|
||||
0.20104,-0.60161,1
|
||||
0.46601,-0.53582,1
|
||||
0.67339,-0.53582,1
|
||||
-0.13882,0.54605,1
|
||||
-0.29435,0.77997,1
|
||||
-0.26555,0.96272,1
|
||||
-0.16187,0.8019,1
|
||||
-0.17339,0.64839,1
|
||||
-0.28283,0.47295,1
|
||||
-0.36348,0.31213,1
|
||||
-0.30012,0.027047,1
|
||||
-0.23675,-0.21418,1
|
||||
-0.06394,-0.18494,1
|
||||
0.062788,-0.16301,1
|
||||
0.22984,-0.41155,1
|
||||
0.2932,-0.2288,1
|
||||
0.48329,-0.18494,1
|
||||
0.64459,-0.14108,1
|
||||
0.46025,0.012427,1
|
||||
0.6273,0.15863,1
|
||||
0.57546,0.26827,1
|
||||
0.72523,0.44371,1
|
||||
0.22408,0.52412,1
|
||||
0.44297,0.67032,1
|
||||
0.322,0.69225,1
|
||||
0.13767,0.57529,1
|
||||
-0.0063364,0.39985,1
|
||||
-0.092742,0.55336,1
|
||||
-0.20795,0.35599,1
|
||||
-0.20795,0.17325,1
|
||||
-0.43836,0.21711,1
|
||||
-0.21947,-0.016813,1
|
||||
-0.13882,-0.27266,1
|
||||
0.18376,0.93348,0
|
||||
0.22408,0.77997,0
|
||||
0.29896,0.61915,0
|
||||
0.50634,0.75804,0
|
||||
0.61578,0.7288,0
|
||||
0.60426,0.59722,0
|
||||
0.76555,0.50219,0
|
||||
0.92684,0.3633,0
|
||||
0.82316,0.27558,0
|
||||
0.96141,0.085526,0
|
||||
0.93836,0.012427,0
|
||||
0.86348,-0.082602,0
|
||||
0.89804,-0.20687,0
|
||||
0.85196,-0.36769,0
|
||||
0.82892,-0.5212,0
|
||||
0.79435,-0.55775,0
|
||||
0.59274,-0.7405,0
|
||||
0.51786,-0.5943,0
|
||||
0.46601,-0.41886,0
|
||||
0.35081,-0.57968,0
|
||||
0.28744,-0.76974,0
|
||||
0.085829,-0.75512,0
|
||||
0.14919,-0.57968,0
|
||||
-0.13306,-0.4481,0
|
||||
-0.40956,-0.41155,0
|
||||
-0.39228,-0.25804,0
|
||||
-0.74366,-0.25804,0
|
||||
-0.69758,0.041667,0
|
||||
-0.75518,0.2902,0
|
||||
-0.69758,0.68494,0
|
||||
-0.4038,0.70687,0
|
||||
-0.38076,0.91886,0
|
||||
-0.50749,0.90424,0
|
||||
-0.54781,0.70687,0
|
||||
0.10311,0.77997,0
|
||||
0.057028,0.91886,0
|
||||
-0.10426,0.99196,0
|
||||
-0.081221,1.1089,0
|
||||
0.28744,1.087,0
|
||||
0.39689,0.82383,0
|
||||
0.63882,0.88962,0
|
||||
0.82316,0.66301,0
|
||||
0.67339,0.64108,0
|
||||
1.0709,0.10015,0
|
||||
-0.046659,-0.57968,0
|
||||
-0.23675,-0.63816,0
|
||||
-0.15035,-0.36769,0
|
||||
-0.49021,-0.3019,0
|
||||
-0.46717,-0.13377,0
|
||||
-0.28859,-0.060673,0
|
||||
-0.61118,-0.067982,0
|
||||
-0.66302,-0.21418,0
|
||||
-0.59965,-0.41886,0
|
||||
-0.72638,-0.082602,0
|
||||
-0.83007,0.31213,0
|
||||
-0.72062,0.53874,0
|
||||
-0.59389,0.49488,0
|
||||
-0.48445,0.99927,0
|
||||
-0.0063364,0.99927,0
|
||||
0.63265,-0.030612,0
|
||||
BIN
Exercise2/Figures/decision_boundary1.png
Executable file
BIN
Exercise2/Figures/decision_boundary1.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 20 KiB |
BIN
Exercise2/Figures/decision_boundary2.png
Executable file
BIN
Exercise2/Figures/decision_boundary2.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 21 KiB |
BIN
Exercise2/Figures/decision_boundary3.png
Executable file
BIN
Exercise2/Figures/decision_boundary3.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 23 KiB |
BIN
Exercise2/Figures/decision_boundary4.png
Executable file
BIN
Exercise2/Figures/decision_boundary4.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 21 KiB |
965
Exercise2/exercise2.ipynb
Executable file
965
Exercise2/exercise2.ipynb
Executable file
@@ -0,0 +1,965 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Programming Exercise 2: Logistic Regression\n",
|
||||
"\n",
|
||||
"## Introduction\n",
|
||||
"\n",
|
||||
"In this exercise, you will implement logistic regression and apply it to two different datasets. Before starting on the programming exercise, we strongly recommend watching the video lectures and completing the review questions for the associated topics.\n",
|
||||
"\n",
|
||||
"All the information you need for solving this assignment is in this notebook, and all the code you will be implementing will take place within this notebook. The assignment can be promptly submitted to the coursera grader directly from this notebook (code and instructions are included below).\n",
|
||||
"\n",
|
||||
"Before we begin with the exercises, we need to import all libraries required for this programming exercise. Throughout the course, we will be using [`numpy`](http://www.numpy.org/) for all arrays and matrix operations, and [`matplotlib`](https://matplotlib.org/) for plotting. In this assignment, we will also use [`scipy`](https://docs.scipy.org/doc/scipy/reference/), which contains scientific and numerical computation functions and tools. \n",
|
||||
"\n",
|
||||
"You can find instructions on how to install required libraries in the README file in the [github repository](https://github.com/dibgerge/ml-coursera-python-assignments)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# used for manipulating directory paths\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Scientific and vector computation for python\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"# Plotting library\n",
|
||||
"from matplotlib import pyplot\n",
|
||||
"\n",
|
||||
"# Optimization module in scipy\n",
|
||||
"from scipy import optimize\n",
|
||||
"\n",
|
||||
"# library written for this exercise providing additional functions for assignment submission, and others\n",
|
||||
"import utils\n",
|
||||
"\n",
|
||||
"# define the submission/grader object for this exercise\n",
|
||||
"grader = utils.Grader()\n",
|
||||
"\n",
|
||||
"# tells matplotlib to embed plots within the notebook\n",
|
||||
"%matplotlib inline"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Submission and Grading\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"After completing each part of the assignment, be sure to submit your solutions to the grader. The following is a breakdown of how each part of this exercise is scored.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"| Section | Part | Submission function | Points \n",
|
||||
"| :- |:- | :- | :-:\n",
|
||||
"| 1 | [Sigmoid Function](#section1) | [`sigmoid`](#sigmoid) | 5 \n",
|
||||
"| 2 | [Compute cost for logistic regression](#section2) | [`costFunction`](#costFunction) | 30 \n",
|
||||
"| 3 | [Gradient for logistic regression](#section2) | [`costFunction`](#costFunction) | 30 \n",
|
||||
"| 4 | [Predict Function](#section4) | [`predict`](#predict) | 5 \n",
|
||||
"| 5 | [Compute cost for regularized LR](#section5) | [`costFunctionReg`](#costFunctionReg) | 15 \n",
|
||||
"| 6 | [Gradient for regularized LR](#section5) | [`costFunctionReg`](#costFunctionReg) | 15 \n",
|
||||
"| | Total Points | | 100 \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"You are allowed to submit your solutions multiple times, and we will take only the highest score into consideration.\n",
|
||||
"\n",
|
||||
"<div class=\"alert alert-block alert-warning\">\n",
|
||||
"At the end of each section in this notebook, we have a cell which contains code for submitting the solutions thus far to the grader. Execute the cell to see your score up to the current section. For all your work to be submitted properly, you must execute those cells at least once. They must also be re-executed everytime the submitted function is updated.\n",
|
||||
"</div>\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1 Logistic Regression\n",
|
||||
"\n",
|
||||
"In this part of the exercise, you will build a logistic regression model to predict whether a student gets admitted into a university. Suppose that you are the administrator of a university department and\n",
|
||||
"you want to determine each applicant’s chance of admission based on their results on two exams. You have historical data from previous applicants that you can use as a training set for logistic regression. For each training example, you have the applicant’s scores on two exams and the admissions\n",
|
||||
"decision. Your task is to build a classification model that estimates an applicant’s probability of admission based the scores from those two exams. \n",
|
||||
"\n",
|
||||
"The following cell will load the data and corresponding labels:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Load data\n",
|
||||
"# The first two columns contains the exam scores and the third column\n",
|
||||
"# contains the label.\n",
|
||||
"data = np.loadtxt(os.path.join('Data', 'ex2data1.txt'), delimiter=',')\n",
|
||||
"X, y = data[:, 0:2], data[:, 2]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 1.1 Visualizing the data\n",
|
||||
"\n",
|
||||
"Before starting to implement any learning algorithm, it is always good to visualize the data if possible. We display the data on a 2-dimensional plot by calling the function `plotData`. You will now complete the code in `plotData` so that it displays a figure where the axes are the two exam scores, and the positive and negative examples are shown with different markers.\n",
|
||||
"\n",
|
||||
"To help you get more familiar with plotting, we have left `plotData` empty so you can try to implement it yourself. However, this is an optional (ungraded) exercise. We also provide our implementation below so you can\n",
|
||||
"copy it or refer to it. If you choose to copy our example, make sure you learn\n",
|
||||
"what each of its commands is doing by consulting the `matplotlib` and `numpy` documentation.\n",
|
||||
"\n",
|
||||
"```python\n",
|
||||
"# Find Indices of Positive and Negative Examples\n",
|
||||
"pos = y == 1\n",
|
||||
"neg = y == 0\n",
|
||||
"\n",
|
||||
"# Plot Examples\n",
|
||||
"pyplot.plot(X[pos, 0], X[pos, 1], 'k*', lw=2, ms=10)\n",
|
||||
"pyplot.plot(X[neg, 0], X[neg, 1], 'ko', mfc='y', ms=8, mec='k', mew=1)\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def plotData(X, y):\n",
|
||||
" \"\"\"\n",
|
||||
" Plots the data points X and y into a new figure. Plots the data \n",
|
||||
" points with * for the positive examples and o for the negative examples.\n",
|
||||
" \n",
|
||||
" Parameters\n",
|
||||
" ----------\n",
|
||||
" X : array_like\n",
|
||||
" An Mx2 matrix representing the dataset. \n",
|
||||
" \n",
|
||||
" y : array_like\n",
|
||||
" Label values for the dataset. A vector of size (M, ).\n",
|
||||
" \n",
|
||||
" Instructions\n",
|
||||
" ------------\n",
|
||||
" Plot the positive and negative examples on a 2D plot, using the\n",
|
||||
" option 'k*' for the positive examples and 'ko' for the negative examples. \n",
|
||||
" \"\"\"\n",
|
||||
" # Create New Figure\n",
|
||||
" fig = pyplot.figure()\n",
|
||||
"\n",
|
||||
" # ====================== YOUR CODE HERE ======================\n",
|
||||
"\n",
|
||||
" \n",
|
||||
" # ============================================================"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Now, we call the implemented function to display the loaded data:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"plotData(X, y)\n",
|
||||
"# add axes labels\n",
|
||||
"pyplot.xlabel('Exam 1 score')\n",
|
||||
"pyplot.ylabel('Exam 2 score')\n",
|
||||
"pyplot.legend(['Admitted', 'Not admitted'])\n",
|
||||
"pass"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a id=\"section1\"></a>\n",
|
||||
"### 1.2 Implementation\n",
|
||||
"\n",
|
||||
"#### 1.2.1 Warmup exercise: sigmoid function\n",
|
||||
"\n",
|
||||
"Before you start with the actual cost function, recall that the logistic regression hypothesis is defined as:\n",
|
||||
"\n",
|
||||
"$$ h_\\theta(x) = g(\\theta^T x)$$\n",
|
||||
"\n",
|
||||
"where function $g$ is the sigmoid function. The sigmoid function is defined as: \n",
|
||||
"\n",
|
||||
"$$g(z) = \\frac{1}{1+e^{-z}}$$.\n",
|
||||
"\n",
|
||||
"Your first step is to implement this function `sigmoid` so it can be\n",
|
||||
"called by the rest of your program. When you are finished, try testing a few\n",
|
||||
"values by calling `sigmoid(x)` in a new cell. For large positive values of `x`, the sigmoid should be close to 1, while for large negative values, the sigmoid should be close to 0. Evaluating `sigmoid(0)` should give you exactly 0.5. Your code should also work with vectors and matrices. **For a matrix, your function should perform the sigmoid function on every element.**\n",
|
||||
"<a id=\"sigmoid\"></a>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def sigmoid(z):\n",
|
||||
" \"\"\"\n",
|
||||
" Compute sigmoid function given the input z.\n",
|
||||
" \n",
|
||||
" Parameters\n",
|
||||
" ----------\n",
|
||||
" z : array_like\n",
|
||||
" The input to the sigmoid function. This can be a 1-D vector \n",
|
||||
" or a 2-D matrix. \n",
|
||||
" \n",
|
||||
" Returns\n",
|
||||
" -------\n",
|
||||
" g : array_like\n",
|
||||
" The computed sigmoid function. g has the same shape as z, since\n",
|
||||
" the sigmoid is computed element-wise on z.\n",
|
||||
" \n",
|
||||
" Instructions\n",
|
||||
" ------------\n",
|
||||
" Compute the sigmoid of each value of z (z can be a matrix, vector or scalar).\n",
|
||||
" \"\"\"\n",
|
||||
" # convert input to a numpy array\n",
|
||||
" z = np.array(z)\n",
|
||||
" \n",
|
||||
" # You need to return the following variables correctly \n",
|
||||
" g = np.zeros(z.shape)\n",
|
||||
"\n",
|
||||
" # ====================== YOUR CODE HERE ======================\n",
|
||||
"\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" # =============================================================\n",
|
||||
" return g"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The following cell evaluates the sigmoid function at `z=0`. You should get a value of 0.5. You can also try different values for `z` to experiment with the sigmoid function."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Test the implementation of sigmoid function here\n",
|
||||
"z = 0\n",
|
||||
"g = sigmoid(z)\n",
|
||||
"\n",
|
||||
"print('g(', z, ') = ', g)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"After completing a part of the exercise, you can submit your solutions for grading by first adding the function you modified to the submission object, and then sending your function to Coursera for grading. \n",
|
||||
"\n",
|
||||
"The submission script will prompt you for your login e-mail and submission token. You can obtain a submission token from the web page for the assignment. You are allowed to submit your solutions multiple times, and we will take only the highest score into consideration.\n",
|
||||
"\n",
|
||||
"Execute the following cell to grade your solution to the first part of this exercise.\n",
|
||||
"\n",
|
||||
"*You should now submit your solutions.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# appends the implemented function in part 1 to the grader object\n",
|
||||
"grader[1] = sigmoid\n",
|
||||
"\n",
|
||||
"# send the added functions to coursera grader for getting a grade on this part\n",
|
||||
"grader.grade()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a id=\"section2\"></a>\n",
|
||||
"#### 1.2.2 Cost function and gradient\n",
|
||||
"\n",
|
||||
"Now you will implement the cost function and gradient for logistic regression. Before proceeding we add the intercept term to X. "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Setup the data matrix appropriately, and add ones for the intercept term\n",
|
||||
"m, n = X.shape\n",
|
||||
"\n",
|
||||
"# Add intercept term to X\n",
|
||||
"X = np.concatenate([np.ones((m, 1)), X], axis=1)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Now, complete the code for the function `costFunction` to return the cost and gradient. Recall that the cost function in logistic regression is\n",
|
||||
"\n",
|
||||
"$$ J(\\theta) = \\frac{1}{m} \\sum_{i=1}^{m} \\left[ -y^{(i)} \\log\\left(h_\\theta\\left( x^{(i)} \\right) \\right) - \\left( 1 - y^{(i)}\\right) \\log \\left( 1 - h_\\theta\\left( x^{(i)} \\right) \\right) \\right]$$\n",
|
||||
"\n",
|
||||
"and the gradient of the cost is a vector of the same length as $\\theta$ where the $j^{th}$\n",
|
||||
"element (for $j = 0, 1, \\cdots , n$) is defined as follows:\n",
|
||||
"\n",
|
||||
"$$ \\frac{\\partial J(\\theta)}{\\partial \\theta_j} = \\frac{1}{m} \\sum_{i=1}^m \\left( h_\\theta \\left( x^{(i)} \\right) - y^{(i)} \\right) x_j^{(i)} $$\n",
|
||||
"\n",
|
||||
"Note that while this gradient looks identical to the linear regression gradient, the formula is actually different because linear and logistic regression have different definitions of $h_\\theta(x)$.\n",
|
||||
"<a id=\"costFunction\"></a>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def costFunction(theta, X, y):\n",
|
||||
" \"\"\"\n",
|
||||
" Compute cost and gradient for logistic regression. \n",
|
||||
" \n",
|
||||
" Parameters\n",
|
||||
" ----------\n",
|
||||
" theta : array_like\n",
|
||||
" The parameters for logistic regression. This a vector\n",
|
||||
" of shape (n+1, ).\n",
|
||||
" \n",
|
||||
" X : array_like\n",
|
||||
" The input dataset of shape (m x n+1) where m is the total number\n",
|
||||
" of data points and n is the number of features. We assume the \n",
|
||||
" intercept has already been added to the input.\n",
|
||||
" \n",
|
||||
" y : arra_like\n",
|
||||
" Labels for the input. This is a vector of shape (m, ).\n",
|
||||
" \n",
|
||||
" Returns\n",
|
||||
" -------\n",
|
||||
" J : float\n",
|
||||
" The computed value for the cost function. \n",
|
||||
" \n",
|
||||
" grad : array_like\n",
|
||||
" A vector of shape (n+1, ) which is the gradient of the cost\n",
|
||||
" function with respect to theta, at the current values of theta.\n",
|
||||
" \n",
|
||||
" Instructions\n",
|
||||
" ------------\n",
|
||||
" Compute the cost of a particular choice of theta. You should set J to \n",
|
||||
" the cost. Compute the partial derivatives and set grad to the partial\n",
|
||||
" derivatives of the cost w.r.t. each parameter in theta.\n",
|
||||
" \"\"\"\n",
|
||||
" # Initialize some useful values\n",
|
||||
" m = y.size # number of training examples\n",
|
||||
"\n",
|
||||
" # You need to return the following variables correctly \n",
|
||||
" J = 0\n",
|
||||
" grad = np.zeros(theta.shape)\n",
|
||||
"\n",
|
||||
" # ====================== YOUR CODE HERE ======================\n",
|
||||
"\n",
|
||||
" \n",
|
||||
" \n",
|
||||
" # =============================================================\n",
|
||||
" return J, grad"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Once you are done call your `costFunction` using two test cases for $\\theta$ by executing the next cell."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Initialize fitting parameters\n",
|
||||
"initial_theta = np.zeros(n+1)\n",
|
||||
"\n",
|
||||
"cost, grad = costFunction(initial_theta, X, y)\n",
|
||||
"\n",
|
||||
"print('Cost at initial theta (zeros): {:.3f}'.format(cost))\n",
|
||||
"print('Expected cost (approx): 0.693\\n')\n",
|
||||
"\n",
|
||||
"print('Gradient at initial theta (zeros):')\n",
|
||||
"print('\\t[{:.4f}, {:.4f}, {:.4f}]'.format(*grad))\n",
|
||||
"print('Expected gradients (approx):\\n\\t[-0.1000, -12.0092, -11.2628]\\n')\n",
|
||||
"\n",
|
||||
"# Compute and display cost and gradient with non-zero theta\n",
|
||||
"test_theta = np.array([-24, 0.2, 0.2])\n",
|
||||
"cost, grad = costFunction(test_theta, X, y)\n",
|
||||
"\n",
|
||||
"print('Cost at test theta: {:.3f}'.format(cost))\n",
|
||||
"print('Expected cost (approx): 0.218\\n')\n",
|
||||
"\n",
|
||||
"print('Gradient at test theta:')\n",
|
||||
"print('\\t[{:.3f}, {:.3f}, {:.3f}]'.format(*grad))\n",
|
||||
"print('Expected gradients (approx):\\n\\t[0.043, 2.566, 2.647]')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"*You should now submit your solutions.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"grader[2] = costFunction\n",
|
||||
"grader[3] = costFunction\n",
|
||||
"grader.grade()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### 1.2.3 Learning parameters using `scipy.optimize`\n",
|
||||
"\n",
|
||||
"In the previous assignment, you found the optimal parameters of a linear regression model by implementing gradient descent. You wrote a cost function and calculated its gradient, then took a gradient descent step accordingly. This time, instead of taking gradient descent steps, you will use the [`scipy.optimize` module](https://docs.scipy.org/doc/scipy/reference/optimize.html). SciPy is a numerical computing library for `python`. It provides an optimization module for root finding and minimization. As of `scipy 1.0`, the function `scipy.optimize.minimize` is the method to use for optimization problems(both constrained and unconstrained).\n",
|
||||
"\n",
|
||||
"For logistic regression, you want to optimize the cost function $J(\\theta)$ with parameters $\\theta$.\n",
|
||||
"Concretely, you are going to use `optimize.minimize` to find the best parameters $\\theta$ for the logistic regression cost function, given a fixed dataset (of X and y values). You will pass to `optimize.minimize` the following inputs:\n",
|
||||
"- `costFunction`: A cost function that, when given the training set and a particular $\\theta$, computes the logistic regression cost and gradient with respect to $\\theta$ for the dataset (X, y). It is important to note that we only pass the name of the function without the parenthesis. This indicates that we are only providing a reference to this function, and not evaluating the result from this function.\n",
|
||||
"- `initial_theta`: The initial values of the parameters we are trying to optimize.\n",
|
||||
"- `(X, y)`: These are additional arguments to the cost function.\n",
|
||||
"- `jac`: Indication if the cost function returns the Jacobian (gradient) along with cost value. (True)\n",
|
||||
"- `method`: Optimization method/algorithm to use\n",
|
||||
"- `options`: Additional options which might be specific to the specific optimization method. In the following, we only tell the algorithm the maximum number of iterations before it terminates.\n",
|
||||
"\n",
|
||||
"If you have completed the `costFunction` correctly, `optimize.minimize` will converge on the right optimization parameters and return the final values of the cost and $\\theta$ in a class object. Notice that by using `optimize.minimize`, you did not have to write any loops yourself, or set a learning rate like you did for gradient descent. This is all done by `optimize.minimize`: you only needed to provide a function calculating the cost and the gradient.\n",
|
||||
"\n",
|
||||
"In the following, we already have code written to call `optimize.minimize` with the correct arguments."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# set options for optimize.minimize\n",
|
||||
"options= {'maxiter': 400}\n",
|
||||
"\n",
|
||||
"# see documention for scipy's optimize.minimize for description about\n",
|
||||
"# the different parameters\n",
|
||||
"# The function returns an object `OptimizeResult`\n",
|
||||
"# We use truncated Newton algorithm for optimization which is \n",
|
||||
"# equivalent to MATLAB's fminunc\n",
|
||||
"# See https://stackoverflow.com/questions/18801002/fminunc-alternate-in-numpy\n",
|
||||
"res = optimize.minimize(costFunction,\n",
|
||||
" initial_theta,\n",
|
||||
" (X, y),\n",
|
||||
" jac=True,\n",
|
||||
" method='TNC',\n",
|
||||
" options=options)\n",
|
||||
"\n",
|
||||
"# the fun property of `OptimizeResult` object returns\n",
|
||||
"# the value of costFunction at optimized theta\n",
|
||||
"cost = res.fun\n",
|
||||
"\n",
|
||||
"# the optimized theta is in the x property\n",
|
||||
"theta = res.x\n",
|
||||
"\n",
|
||||
"# Print theta to screen\n",
|
||||
"print('Cost at theta found by optimize.minimize: {:.3f}'.format(cost))\n",
|
||||
"print('Expected cost (approx): 0.203\\n');\n",
|
||||
"\n",
|
||||
"print('theta:')\n",
|
||||
"print('\\t[{:.3f}, {:.3f}, {:.3f}]'.format(*theta))\n",
|
||||
"print('Expected theta (approx):\\n\\t[-25.161, 0.206, 0.201]')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Once `optimize.minimize` completes, we want to use the final value for $\\theta$ to visualize the decision boundary on the training data as shown in the figure below. \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"To do so, we have written a function `plotDecisionBoundary` for plotting the decision boundary on top of training data. You do not need to write any code for plotting the decision boundary, but we also encourage you to look at the code in `plotDecisionBoundary` to see how to plot such a boundary using the $\\theta$ values. You can find this function in the `utils.py` file which comes with this assignment."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Plot Boundary\n",
|
||||
"utils.plotDecisionBoundary(plotData, theta, X, y)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a id=\"section4\"></a>\n",
|
||||
"#### 1.2.4 Evaluating logistic regression\n",
|
||||
"\n",
|
||||
"After learning the parameters, you can use the model to predict whether a particular student will be admitted. For a student with an Exam 1 score of 45 and an Exam 2 score of 85, you should expect to see an admission\n",
|
||||
"probability of 0.776. Another way to evaluate the quality of the parameters we have found is to see how well the learned model predicts on our training set. In this part, your task is to complete the code in function `predict`. The predict function will produce “1” or “0” predictions given a dataset and a learned parameter vector $\\theta$. \n",
|
||||
"<a id=\"predict\"></a>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def predict(theta, X):\n",
|
||||
" \"\"\"\n",
|
||||
" Predict whether the label is 0 or 1 using learned logistic regression.\n",
|
||||
" Computes the predictions for X using a threshold at 0.5 \n",
|
||||
" (i.e., if sigmoid(theta.T*x) >= 0.5, predict 1)\n",
|
||||
" \n",
|
||||
" Parameters\n",
|
||||
" ----------\n",
|
||||
" theta : array_like\n",
|
||||
" Parameters for logistic regression. A vecotor of shape (n+1, ).\n",
|
||||
" \n",
|
||||
" X : array_like\n",
|
||||
" The data to use for computing predictions. The rows is the number \n",
|
||||
" of points to compute predictions, and columns is the number of\n",
|
||||
" features.\n",
|
||||
"\n",
|
||||
" Returns\n",
|
||||
" -------\n",
|
||||
" p : array_like\n",
|
||||
" Predictions and 0 or 1 for each row in X. \n",
|
||||
" \n",
|
||||
" Instructions\n",
|
||||
" ------------\n",
|
||||
" Complete the following code to make predictions using your learned \n",
|
||||
" logistic regression parameters.You should set p to a vector of 0's and 1's \n",
|
||||
" \"\"\"\n",
|
||||
" m = X.shape[0] # Number of training examples\n",
|
||||
"\n",
|
||||
" # You need to return the following variables correctly\n",
|
||||
" p = np.zeros(m)\n",
|
||||
"\n",
|
||||
" # ====================== YOUR CODE HERE ======================\n",
|
||||
"\n",
|
||||
" \n",
|
||||
" \n",
|
||||
" # ============================================================\n",
|
||||
" return p"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"After you have completed the code in `predict`, we proceed to report the training accuracy of your classifier by computing the percentage of examples it got correct."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Predict probability for a student with score 45 on exam 1 \n",
|
||||
"# and score 85 on exam 2 \n",
|
||||
"prob = sigmoid(np.dot([1, 45, 85], theta))\n",
|
||||
"print('For a student with scores 45 and 85,'\n",
|
||||
" 'we predict an admission probability of {:.3f}'.format(prob))\n",
|
||||
"print('Expected value: 0.775 +/- 0.002\\n')\n",
|
||||
"\n",
|
||||
"# Compute accuracy on our training set\n",
|
||||
"p = predict(theta, X)\n",
|
||||
"print('Train Accuracy: {:.2f} %'.format(np.mean(p == y) * 100))\n",
|
||||
"print('Expected accuracy (approx): 89.00 %')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"*You should now submit your solutions.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"grader[4] = predict\n",
|
||||
"grader.grade()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2 Regularized logistic regression\n",
|
||||
"\n",
|
||||
"In this part of the exercise, you will implement regularized logistic regression to predict whether microchips from a fabrication plant passes quality assurance (QA). During QA, each microchip goes through various tests to ensure it is functioning correctly.\n",
|
||||
"Suppose you are the product manager of the factory and you have the test results for some microchips on two different tests. From these two tests, you would like to determine whether the microchips should be accepted or rejected. To help you make the decision, you have a dataset of test results on past microchips, from which you can build a logistic regression model.\n",
|
||||
"\n",
|
||||
"First, we load the data from a CSV file:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Load Data\n",
|
||||
"# The first two columns contains the X values and the third column\n",
|
||||
"# contains the label (y).\n",
|
||||
"data = np.loadtxt(os.path.join('Data', 'ex2data2.txt'), delimiter=',')\n",
|
||||
"X = data[:, :2]\n",
|
||||
"y = data[:, 2]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 2.1 Visualize the data\n",
|
||||
"\n",
|
||||
"Similar to the previous parts of this exercise, `plotData` is used to generate a figure, where the axes are the two test scores, and the positive (y = 1, accepted) and negative (y = 0, rejected) examples are shown with\n",
|
||||
"different markers."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"plotData(X, y)\n",
|
||||
"# Labels and Legend\n",
|
||||
"pyplot.xlabel('Microchip Test 1')\n",
|
||||
"pyplot.ylabel('Microchip Test 2')\n",
|
||||
"\n",
|
||||
"# Specified in plot order\n",
|
||||
"pyplot.legend(['y = 1', 'y = 0'], loc='upper right')\n",
|
||||
"pass"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"The above figure shows that our dataset cannot be separated into positive and negative examples by a straight-line through the plot. Therefore, a straight-forward application of logistic regression will not perform well on this dataset since logistic regression will only be able to find a linear decision boundary.\n",
|
||||
"\n",
|
||||
"### 2.2 Feature mapping\n",
|
||||
"\n",
|
||||
"One way to fit the data better is to create more features from each data point. In the function `mapFeature` defined in the file `utils.py`, we will map the features into all polynomial terms of $x_1$ and $x_2$ up to the sixth power.\n",
|
||||
"\n",
|
||||
"$$ \\text{mapFeature}(x) = \\begin{bmatrix} 1 & x_1 & x_2 & x_1^2 & x_1 x_2 & x_2^2 & x_1^3 & \\dots & x_1 x_2^5 & x_2^6 \\end{bmatrix}^T $$\n",
|
||||
"\n",
|
||||
"As a result of this mapping, our vector of two features (the scores on two QA tests) has been transformed into a 28-dimensional vector. A logistic regression classifier trained on this higher-dimension feature vector will have a more complex decision boundary and will appear nonlinear when drawn in our 2-dimensional plot.\n",
|
||||
"While the feature mapping allows us to build a more expressive classifier, it also more susceptible to overfitting. In the next parts of the exercise, you will implement regularized logistic regression to fit the data and also see for yourself how regularization can help combat the overfitting problem.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Note that mapFeature also adds a column of ones for us, so the intercept\n",
|
||||
"# term is handled\n",
|
||||
"X = utils.mapFeature(X[:, 0], X[:, 1])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a id=\"section5\"></a>\n",
|
||||
"### 2.3 Cost function and gradient\n",
|
||||
"\n",
|
||||
"Now you will implement code to compute the cost function and gradient for regularized logistic regression. Complete the code for the function `costFunctionReg` below to return the cost and gradient.\n",
|
||||
"\n",
|
||||
"Recall that the regularized cost function in logistic regression is\n",
|
||||
"\n",
|
||||
"$$ J(\\theta) = \\frac{1}{m} \\sum_{i=1}^m \\left[ -y^{(i)}\\log \\left( h_\\theta \\left(x^{(i)} \\right) \\right) - \\left( 1 - y^{(i)} \\right) \\log \\left( 1 - h_\\theta \\left( x^{(i)} \\right) \\right) \\right] + \\frac{\\lambda}{2m} \\sum_{j=1}^n \\theta_j^2 $$\n",
|
||||
"\n",
|
||||
"Note that you should not regularize the parameters $\\theta_0$. The gradient of the cost function is a vector where the $j^{th}$ element is defined as follows:\n",
|
||||
"\n",
|
||||
"$$ \\frac{\\partial J(\\theta)}{\\partial \\theta_0} = \\frac{1}{m} \\sum_{i=1}^m \\left( h_\\theta \\left(x^{(i)}\\right) - y^{(i)} \\right) x_j^{(i)} \\qquad \\text{for } j =0 $$\n",
|
||||
"\n",
|
||||
"$$ \\frac{\\partial J(\\theta)}{\\partial \\theta_j} = \\left( \\frac{1}{m} \\sum_{i=1}^m \\left( h_\\theta \\left(x^{(i)}\\right) - y^{(i)} \\right) x_j^{(i)} \\right) + \\frac{\\lambda}{m}\\theta_j \\qquad \\text{for } j \\ge 1 $$\n",
|
||||
"<a id=\"costFunctionReg\"></a>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def costFunctionReg(theta, X, y, lambda_):\n",
|
||||
" \"\"\"\n",
|
||||
" Compute cost and gradient for logistic regression with regularization.\n",
|
||||
" \n",
|
||||
" Parameters\n",
|
||||
" ----------\n",
|
||||
" theta : array_like\n",
|
||||
" Logistic regression parameters. A vector with shape (n, ). n is \n",
|
||||
" the number of features including any intercept. If we have mapped\n",
|
||||
" our initial features into polynomial features, then n is the total \n",
|
||||
" number of polynomial features. \n",
|
||||
" \n",
|
||||
" X : array_like\n",
|
||||
" The data set with shape (m x n). m is the number of examples, and\n",
|
||||
" n is the number of features (after feature mapping).\n",
|
||||
" \n",
|
||||
" y : array_like\n",
|
||||
" The data labels. A vector with shape (m, ).\n",
|
||||
" \n",
|
||||
" lambda_ : float\n",
|
||||
" The regularization parameter. \n",
|
||||
" \n",
|
||||
" Returns\n",
|
||||
" -------\n",
|
||||
" J : float\n",
|
||||
" The computed value for the regularized cost function. \n",
|
||||
" \n",
|
||||
" grad : array_like\n",
|
||||
" A vector of shape (n, ) which is the gradient of the cost\n",
|
||||
" function with respect to theta, at the current values of theta.\n",
|
||||
" \n",
|
||||
" Instructions\n",
|
||||
" ------------\n",
|
||||
" Compute the cost `J` of a particular choice of theta.\n",
|
||||
" Compute the partial derivatives and set `grad` to the partial\n",
|
||||
" derivatives of the cost w.r.t. each parameter in theta.\n",
|
||||
" \"\"\"\n",
|
||||
" # Initialize some useful values\n",
|
||||
" m = y.size # number of training examples\n",
|
||||
"\n",
|
||||
" # You need to return the following variables correctly \n",
|
||||
" J = 0\n",
|
||||
" grad = np.zeros(theta.shape)\n",
|
||||
"\n",
|
||||
" # ===================== YOUR CODE HERE ======================\n",
|
||||
"\n",
|
||||
" \n",
|
||||
" \n",
|
||||
" # =============================================================\n",
|
||||
" return J, grad"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Once you are done with the `costFunctionReg`, we call it below using the initial value of $\\theta$ (initialized to all zeros), and also another test case where $\\theta$ is all ones."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Initialize fitting parameters\n",
|
||||
"initial_theta = np.zeros(X.shape[1])\n",
|
||||
"\n",
|
||||
"# Set regularization parameter lambda to 1\n",
|
||||
"# DO NOT use `lambda` as a variable name in python\n",
|
||||
"# because it is a python keyword\n",
|
||||
"lambda_ = 1\n",
|
||||
"\n",
|
||||
"# Compute and display initial cost and gradient for regularized logistic\n",
|
||||
"# regression\n",
|
||||
"cost, grad = costFunctionReg(initial_theta, X, y, lambda_)\n",
|
||||
"\n",
|
||||
"print('Cost at initial theta (zeros): {:.3f}'.format(cost))\n",
|
||||
"print('Expected cost (approx) : 0.693\\n')\n",
|
||||
"\n",
|
||||
"print('Gradient at initial theta (zeros) - first five values only:')\n",
|
||||
"print('\\t[{:.4f}, {:.4f}, {:.4f}, {:.4f}, {:.4f}]'.format(*grad[:5]))\n",
|
||||
"print('Expected gradients (approx) - first five values only:')\n",
|
||||
"print('\\t[0.0085, 0.0188, 0.0001, 0.0503, 0.0115]\\n')\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Compute and display cost and gradient\n",
|
||||
"# with all-ones theta and lambda = 10\n",
|
||||
"test_theta = np.ones(X.shape[1])\n",
|
||||
"cost, grad = costFunctionReg(test_theta, X, y, 10)\n",
|
||||
"\n",
|
||||
"print('------------------------------\\n')\n",
|
||||
"print('Cost at test theta : {:.2f}'.format(cost))\n",
|
||||
"print('Expected cost (approx): 3.16\\n')\n",
|
||||
"\n",
|
||||
"print('Gradient at initial theta (zeros) - first five values only:')\n",
|
||||
"print('\\t[{:.4f}, {:.4f}, {:.4f}, {:.4f}, {:.4f}]'.format(*grad[:5]))\n",
|
||||
"print('Expected gradients (approx) - first five values only:')\n",
|
||||
"print('\\t[0.3460, 0.1614, 0.1948, 0.2269, 0.0922]')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"*You should now submit your solutions.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"grader[5] = costFunctionReg\n",
|
||||
"grader[6] = costFunctionReg\n",
|
||||
"grader.grade()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### 2.3.1 Learning parameters using `scipy.optimize.minimize`\n",
|
||||
"\n",
|
||||
"Similar to the previous parts, you will use `optimize.minimize` to learn the optimal parameters $\\theta$. If you have completed the cost and gradient for regularized logistic regression (`costFunctionReg`) correctly, you should be able to step through the next part of to learn the parameters $\\theta$ using `optimize.minimize`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 2.4 Plotting the decision boundary\n",
|
||||
"\n",
|
||||
"To help you visualize the model learned by this classifier, we have provided the function `plotDecisionBoundary` which plots the (non-linear) decision boundary that separates the positive and negative examples. In `plotDecisionBoundary`, we plot the non-linear decision boundary by computing the classifier’s predictions on an evenly spaced grid and then and draw a contour plot where the predictions change from y = 0 to y = 1. "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 2.5 Optional (ungraded) exercises\n",
|
||||
"\n",
|
||||
"In this part of the exercise, you will get to try out different regularization parameters for the dataset to understand how regularization prevents overfitting.\n",
|
||||
"\n",
|
||||
"Notice the changes in the decision boundary as you vary $\\lambda$. With a small\n",
|
||||
"$\\lambda$, you should find that the classifier gets almost every training example correct, but draws a very complicated boundary, thus overfitting the data. See the following figures for the decision boundaries you should get for different values of $\\lambda$. \n",
|
||||
"\n",
|
||||
"<table>\n",
|
||||
" <tr>\n",
|
||||
" <td style=\"text-align:center\">\n",
|
||||
" No regularization (overfitting)<img src=\"Figures/decision_boundary3.png\">\n",
|
||||
" </td> \n",
|
||||
" <td style=\"text-align:center\">\n",
|
||||
" Decision boundary with regularization\n",
|
||||
" <img src=\"Figures/decision_boundary2.png\">\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align:center\">\n",
|
||||
" Decision boundary with too much regularization\n",
|
||||
" <img src=\"Figures/decision_boundary4.png\">\n",
|
||||
" </td> \n",
|
||||
" <tr>\n",
|
||||
"</table>\n",
|
||||
"\n",
|
||||
"This is not a good decision boundary: for example, it predicts that a point at $x = (−0.25, 1.5)$ is accepted $(y = 1)$, which seems to be an incorrect decision given the training set.\n",
|
||||
"With a larger $\\lambda$, you should see a plot that shows an simpler decision boundary which still separates the positives and negatives fairly well. However, if $\\lambda$ is set to too high a value, you will not get a good fit and the decision boundary will not follow the data so well, thus underfitting the data."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Initialize fitting parameters\n",
|
||||
"initial_theta = np.zeros(X.shape[1])\n",
|
||||
"\n",
|
||||
"# Set regularization parameter lambda to 1 (you should vary this)\n",
|
||||
"lambda_ = 1\n",
|
||||
"\n",
|
||||
"# set options for optimize.minimize\n",
|
||||
"options= {'maxiter': 100}\n",
|
||||
"\n",
|
||||
"res = optimize.minimize(costFunctionReg,\n",
|
||||
" initial_theta,\n",
|
||||
" (X, y, lambda_),\n",
|
||||
" jac=True,\n",
|
||||
" method='TNC',\n",
|
||||
" options=options)\n",
|
||||
"\n",
|
||||
"# the fun property of OptimizeResult object returns\n",
|
||||
"# the value of costFunction at optimized theta\n",
|
||||
"cost = res.fun\n",
|
||||
"\n",
|
||||
"# the optimized theta is in the x property of the result\n",
|
||||
"theta = res.x\n",
|
||||
"\n",
|
||||
"utils.plotDecisionBoundary(plotData, theta, X, y)\n",
|
||||
"pyplot.xlabel('Microchip Test 1')\n",
|
||||
"pyplot.ylabel('Microchip Test 2')\n",
|
||||
"pyplot.legend(['y = 1', 'y = 0'])\n",
|
||||
"pyplot.grid(False)\n",
|
||||
"pyplot.title('lambda = %0.2f' % lambda_)\n",
|
||||
"\n",
|
||||
"# Compute accuracy on our training set\n",
|
||||
"p = predict(theta, X)\n",
|
||||
"\n",
|
||||
"print('Train Accuracy: %.1f %%' % (np.mean(p == y) * 100))\n",
|
||||
"print('Expected accuracy (with lambda = 1): 83.1 % (approx)\\n')\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"*You do not need to submit any solutions for these optional (ungraded) exercises.*"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
147
Exercise2/utils.py
Executable file
147
Exercise2/utils.py
Executable file
@@ -0,0 +1,147 @@
|
||||
import sys
|
||||
import numpy as np
|
||||
from matplotlib import pyplot
|
||||
|
||||
sys.path.append('..')
|
||||
from submission import SubmissionBase
|
||||
|
||||
|
||||
def mapFeature(X1, X2, degree=6):
|
||||
"""
|
||||
Maps the two input features to quadratic features used in the regularization exercise.
|
||||
|
||||
Returns a new feature array with more features, comprising of
|
||||
X1, X2, X1.^2, X2.^2, X1*X2, X1*X2.^2, etc..
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X1 : array_like
|
||||
A vector of shape (m, 1), containing one feature for all examples.
|
||||
|
||||
X2 : array_like
|
||||
A vector of shape (m, 1), containing a second feature for all examples.
|
||||
Inputs X1, X2 must be the same size.
|
||||
|
||||
degree: int, optional
|
||||
The polynomial degree.
|
||||
|
||||
Returns
|
||||
-------
|
||||
: array_like
|
||||
A matrix of of m rows, and columns depend on the degree of polynomial.
|
||||
"""
|
||||
if X1.ndim > 0:
|
||||
out = [np.ones(X1.shape[0])]
|
||||
else:
|
||||
out = [np.ones(1)]
|
||||
|
||||
for i in range(1, degree + 1):
|
||||
for j in range(i + 1):
|
||||
out.append((X1 ** (i - j)) * (X2 ** j))
|
||||
|
||||
if X1.ndim > 0:
|
||||
return np.stack(out, axis=1)
|
||||
else:
|
||||
return np.array(out)
|
||||
|
||||
|
||||
def plotDecisionBoundary(plotData, theta, X, y):
|
||||
"""
|
||||
Plots the data points X and y into a new figure with the decision boundary defined by theta.
|
||||
Plots the data points with * for the positive examples and o for the negative examples.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
plotData : func
|
||||
A function reference for plotting the X, y data.
|
||||
|
||||
theta : array_like
|
||||
Parameters for logistic regression. A vector of shape (n+1, ).
|
||||
|
||||
X : array_like
|
||||
The input dataset. X is assumed to be a either:
|
||||
1) Mx3 matrix, where the first column is an all ones column for the intercept.
|
||||
2) MxN, N>3 matrix, where the first column is all ones.
|
||||
|
||||
y : array_like
|
||||
Vector of data labels of shape (m, ).
|
||||
"""
|
||||
# make sure theta is a numpy array
|
||||
theta = np.array(theta)
|
||||
|
||||
# Plot Data (remember first column in X is the intercept)
|
||||
plotData(X[:, 1:3], y)
|
||||
|
||||
if X.shape[1] <= 3:
|
||||
# Only need 2 points to define a line, so choose two endpoints
|
||||
plot_x = np.array([np.min(X[:, 1]) - 2, np.max(X[:, 1]) + 2])
|
||||
|
||||
# Calculate the decision boundary line
|
||||
plot_y = (-1. / theta[2]) * (theta[1] * plot_x + theta[0])
|
||||
|
||||
# Plot, and adjust axes for better viewing
|
||||
pyplot.plot(plot_x, plot_y)
|
||||
|
||||
# Legend, specific for the exercise
|
||||
pyplot.legend(['Admitted', 'Not admitted', 'Decision Boundary'])
|
||||
pyplot.xlim([30, 100])
|
||||
pyplot.ylim([30, 100])
|
||||
else:
|
||||
# Here is the grid range
|
||||
u = np.linspace(-1, 1.5, 50)
|
||||
v = np.linspace(-1, 1.5, 50)
|
||||
|
||||
z = np.zeros((u.size, v.size))
|
||||
# Evaluate z = theta*x over the grid
|
||||
for i, ui in enumerate(u):
|
||||
for j, vj in enumerate(v):
|
||||
z[i, j] = np.dot(mapFeature(ui, vj), theta)
|
||||
|
||||
z = z.T # important to transpose z before calling contour
|
||||
# print(z)
|
||||
|
||||
# Plot z = 0
|
||||
pyplot.contour(u, v, z, levels=[0], linewidths=2, colors='g')
|
||||
pyplot.contourf(u, v, z, levels=[np.min(z), 0, np.max(z)], cmap='Greens', alpha=0.4)
|
||||
|
||||
|
||||
class Grader(SubmissionBase):
|
||||
X = np.stack([np.ones(20),
|
||||
np.exp(1) * np.sin(np.arange(1, 21)),
|
||||
np.exp(0.5) * np.cos(np.arange(1, 21))], axis=1)
|
||||
|
||||
y = (np.sin(X[:, 0] + X[:, 1]) > 0).astype(float)
|
||||
|
||||
def __init__(self):
|
||||
part_names = ['Sigmoid Function',
|
||||
'Logistic Regression Cost',
|
||||
'Logistic Regression Gradient',
|
||||
'Predict',
|
||||
'Regularized Logistic Regression Cost',
|
||||
'Regularized Logistic Regression Gradient']
|
||||
super().__init__('logistic-regression', part_names)
|
||||
|
||||
def __iter__(self):
|
||||
for part_id in range(1, 7):
|
||||
try:
|
||||
func = self.functions[part_id]
|
||||
|
||||
# Each part has different expected arguments/different function
|
||||
if part_id == 1:
|
||||
res = func(self.X)
|
||||
elif part_id == 2:
|
||||
res = func(np.array([0.25, 0.5, -0.5]), self.X, self.y)
|
||||
elif part_id == 3:
|
||||
J, grad = func(np.array([0.25, 0.5, -0.5]), self.X, self.y)
|
||||
res = grad
|
||||
elif part_id == 4:
|
||||
res = func(np.array([0.25, 0.5, -0.5]), self.X)
|
||||
elif part_id == 5:
|
||||
res = func(np.array([0.25, 0.5, -0.5]), self.X, self.y, 0.1)
|
||||
elif part_id == 6:
|
||||
res = func(np.array([0.25, 0.5, -0.5]), self.X, self.y, 0.1)[1]
|
||||
else:
|
||||
raise KeyError
|
||||
yield part_id, res
|
||||
except KeyError:
|
||||
yield part_id, 0
|
||||
BIN
Exercise3/Data/ex3data1.mat
Executable file
BIN
Exercise3/Data/ex3data1.mat
Executable file
Binary file not shown.
BIN
Exercise3/Data/ex3weights.mat
Executable file
BIN
Exercise3/Data/ex3weights.mat
Executable file
Binary file not shown.
BIN
Exercise3/Figures/neuralnetwork.png
Executable file
BIN
Exercise3/Figures/neuralnetwork.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 54 KiB |
923
Exercise3/exercise3.ipynb
Executable file
923
Exercise3/exercise3.ipynb
Executable file
@@ -0,0 +1,923 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Programming Exercise 3\n",
|
||||
"# Multi-class Classification and Neural Networks\n",
|
||||
"\n",
|
||||
"## Introduction\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"In this exercise, you will implement one-vs-all logistic regression and neural networks to recognize handwritten digits. Before starting the programming exercise, we strongly recommend watching the video lectures and completing the review questions for the associated topics. \n",
|
||||
"\n",
|
||||
"All the information you need for solving this assignment is in this notebook, and all the code you will be implementing will take place within this notebook. The assignment can be promptly submitted to the coursera grader directly from this notebook (code and instructions are included below).\n",
|
||||
"\n",
|
||||
"Before we begin with the exercises, we need to import all libraries required for this programming exercise. Throughout the course, we will be using [`numpy`](http://www.numpy.org/) for all arrays and matrix operations, [`matplotlib`](https://matplotlib.org/) for plotting, and [`scipy`](https://docs.scipy.org/doc/scipy/reference/) for scientific and numerical computation functions and tools. You can find instructions on how to install required libraries in the README file in the [github repository](https://github.com/dibgerge/ml-coursera-python-assignments)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# used for manipulating directory paths\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Scientific and vector computation for python\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"# Plotting library\n",
|
||||
"from matplotlib import pyplot\n",
|
||||
"\n",
|
||||
"# Optimization module in scipy\n",
|
||||
"from scipy import optimize\n",
|
||||
"\n",
|
||||
"# will be used to load MATLAB mat datafile format\n",
|
||||
"from scipy.io import loadmat\n",
|
||||
"\n",
|
||||
"# library written for this exercise providing additional functions for assignment submission, and others\n",
|
||||
"import utils\n",
|
||||
"\n",
|
||||
"# define the submission/grader object for this exercise\n",
|
||||
"grader = utils.Grader()\n",
|
||||
"\n",
|
||||
"# tells matplotlib to embed plots within the notebook\n",
|
||||
"%matplotlib inline"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Submission and Grading\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"After completing each part of the assignment, be sure to submit your solutions to the grader. The following is a breakdown of how each part of this exercise is scored.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"| Section | Part | Submission function | Points \n",
|
||||
"| :- |:- | :- | :-: \n",
|
||||
"| 1 | [Regularized Logistic Regression](#section1) | [`lrCostFunction`](#lrCostFunction) | 30 \n",
|
||||
"| 2 | [One-vs-all classifier training](#section2) | [`oneVsAll`](#oneVsAll) | 20 \n",
|
||||
"| 3 | [One-vs-all classifier prediction](#section3) | [`predictOneVsAll`](#predictOneVsAll) | 20 \n",
|
||||
"| 4 | [Neural Network Prediction Function](#section4) | [`predict`](#predict) | 30\n",
|
||||
"| | Total Points | | 100 \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"You are allowed to submit your solutions multiple times, and we will take only the highest score into consideration.\n",
|
||||
"\n",
|
||||
"<div class=\"alert alert-block alert-warning\">\n",
|
||||
"At the end of each section in this notebook, we have a cell which contains code for submitting the solutions thus far to the grader. Execute the cell to see your score up to the current section. For all your work to be submitted properly, you must execute those cells at least once. They must also be re-executed everytime the submitted function is updated.\n",
|
||||
"</div>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1 Multi-class Classification\n",
|
||||
"\n",
|
||||
"For this exercise, you will use logistic regression and neural networks to recognize handwritten digits (from 0 to 9). Automated handwritten digit recognition is widely used today - from recognizing zip codes (postal codes)\n",
|
||||
"on mail envelopes to recognizing amounts written on bank checks. This exercise will show you how the methods you have learned can be used for this classification task.\n",
|
||||
"\n",
|
||||
"In the first part of the exercise, you will extend your previous implementation of logistic regression and apply it to one-vs-all classification.\n",
|
||||
"\n",
|
||||
"### 1.1 Dataset\n",
|
||||
"\n",
|
||||
"You are given a data set in `ex3data1.mat` that contains 5000 training examples of handwritten digits (This is a subset of the [MNIST](http://yann.lecun.com/exdb/mnist) handwritten digit dataset). The `.mat` format means that that the data has been saved in a native Octave/MATLAB matrix format, instead of a text (ASCII) format like a csv-file. We use the `.mat` format here because this is the dataset provided in the MATLAB version of this assignment. Fortunately, python provides mechanisms to load MATLAB native format using the `loadmat` function within the `scipy.io` module. This function returns a python dictionary with keys containing the variable names within the `.mat` file. \n",
|
||||
"\n",
|
||||
"There are 5000 training examples in `ex3data1.mat`, where each training example is a 20 pixel by 20 pixel grayscale image of the digit. Each pixel is represented by a floating point number indicating the grayscale intensity at that location. The 20 by 20 grid of pixels is “unrolled” into a 400-dimensional vector. Each of these training examples becomes a single row in our data matrix `X`. This gives us a 5000 by 400 matrix `X` where every row is a training example for a handwritten digit image.\n",
|
||||
"\n",
|
||||
"$$ X = \\begin{bmatrix} - \\: (x^{(1)})^T \\: - \\\\ -\\: (x^{(2)})^T \\:- \\\\ \\vdots \\\\ - \\: (x^{(m)})^T \\:- \\end{bmatrix} $$\n",
|
||||
"\n",
|
||||
"The second part of the training set is a 5000-dimensional vector `y` that contains labels for the training set. \n",
|
||||
"We start the exercise by first loading the dataset. Execute the cell below, you do not need to write any code here."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# 20x20 Input Images of Digits\n",
|
||||
"input_layer_size = 400\n",
|
||||
"\n",
|
||||
"# 10 labels, from 1 to 10 (note that we have mapped \"0\" to label 10)\n",
|
||||
"num_labels = 10\n",
|
||||
"\n",
|
||||
"# training data stored in arrays X, y\n",
|
||||
"data = loadmat(os.path.join('Data', 'ex3data1.mat'))\n",
|
||||
"X, y = data['X'], data['y'].ravel()\n",
|
||||
"\n",
|
||||
"# set the zero digit to 0, rather than its mapped 10 in this dataset\n",
|
||||
"# This is an artifact due to the fact that this dataset was used in \n",
|
||||
"# MATLAB where there is no index 0\n",
|
||||
"y[y == 10] = 0\n",
|
||||
"\n",
|
||||
"m = y.size"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 1.2 Visualizing the data\n",
|
||||
"\n",
|
||||
"You will begin by visualizing a subset of the training set. In the following cell, the code randomly selects selects 100 rows from `X` and passes those rows to the `displayData` function. This function maps each row to a 20 pixel by 20 pixel grayscale image and displays the images together. We have provided the `displayData` function in the file `utils.py`. You are encouraged to examine the code to see how it works. Run the following cell to visualize the data."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Randomly select 100 data points to display\n",
|
||||
"rand_indices = np.random.choice(m, 100, replace=False)\n",
|
||||
"sel = X[rand_indices, :]\n",
|
||||
"\n",
|
||||
"utils.displayData(sel)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"source": [
|
||||
"### 1.3 Vectorizing Logistic Regression\n",
|
||||
"\n",
|
||||
"You will be using multiple one-vs-all logistic regression models to build a multi-class classifier. Since there are 10 classes, you will need to train 10 separate logistic regression classifiers. To make this training efficient, it is important to ensure that your code is well vectorized. In this section, you will implement a vectorized version of logistic regression that does not employ any `for` loops. You can use your code in the previous exercise as a starting point for this exercise. \n",
|
||||
"\n",
|
||||
"To test your vectorized logistic regression, we will use custom data as defined in the following cell."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# test values for the parameters theta\n",
|
||||
"theta_t = np.array([-2, -1, 1, 2], dtype=float)\n",
|
||||
"\n",
|
||||
"# test values for the inputs\n",
|
||||
"X_t = np.concatenate([np.ones((5, 1)), np.arange(1, 16).reshape(5, 3, order='F')/10.0], axis=1)\n",
|
||||
"\n",
|
||||
"# test values for the labels\n",
|
||||
"y_t = np.array([1, 0, 1, 0, 1])\n",
|
||||
"\n",
|
||||
"# test value for the regularization parameter\n",
|
||||
"lambda_t = 3"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a id=\"section1\"></a>\n",
|
||||
"#### 1.3.1 Vectorizing the cost function \n",
|
||||
"\n",
|
||||
"We will begin by writing a vectorized version of the cost function. Recall that in (unregularized) logistic regression, the cost function is\n",
|
||||
"\n",
|
||||
"$$ J(\\theta) = \\frac{1}{m} \\sum_{i=1}^m \\left[ -y^{(i)} \\log \\left( h_\\theta\\left( x^{(i)} \\right) \\right) - \\left(1 - y^{(i)} \\right) \\log \\left(1 - h_\\theta \\left( x^{(i)} \\right) \\right) \\right] $$\n",
|
||||
"\n",
|
||||
"To compute each element in the summation, we have to compute $h_\\theta(x^{(i)})$ for every example $i$, where $h_\\theta(x^{(i)}) = g(\\theta^T x^{(i)})$ and $g(z) = \\frac{1}{1+e^{-z}}$ is the sigmoid function. It turns out that we can compute this quickly for all our examples by using matrix multiplication. Let us define $X$ and $\\theta$ as\n",
|
||||
"\n",
|
||||
"$$ X = \\begin{bmatrix} - \\left( x^{(1)} \\right)^T - \\\\ - \\left( x^{(2)} \\right)^T - \\\\ \\vdots \\\\ - \\left( x^{(m)} \\right)^T - \\end{bmatrix} \\qquad \\text{and} \\qquad \\theta = \\begin{bmatrix} \\theta_0 \\\\ \\theta_1 \\\\ \\vdots \\\\ \\theta_n \\end{bmatrix} $$\n",
|
||||
"\n",
|
||||
"Then, by computing the matrix product $X\\theta$, we have: \n",
|
||||
"\n",
|
||||
"$$ X\\theta = \\begin{bmatrix} - \\left( x^{(1)} \\right)^T\\theta - \\\\ - \\left( x^{(2)} \\right)^T\\theta - \\\\ \\vdots \\\\ - \\left( x^{(m)} \\right)^T\\theta - \\end{bmatrix} = \\begin{bmatrix} - \\theta^T x^{(1)} - \\\\ - \\theta^T x^{(2)} - \\\\ \\vdots \\\\ - \\theta^T x^{(m)} - \\end{bmatrix} $$\n",
|
||||
"\n",
|
||||
"In the last equality, we used the fact that $a^Tb = b^Ta$ if $a$ and $b$ are vectors. This allows us to compute the products $\\theta^T x^{(i)}$ for all our examples $i$ in one line of code.\n",
|
||||
"\n",
|
||||
"#### 1.3.2 Vectorizing the gradient\n",
|
||||
"\n",
|
||||
"Recall that the gradient of the (unregularized) logistic regression cost is a vector where the $j^{th}$ element is defined as\n",
|
||||
"\n",
|
||||
"$$ \\frac{\\partial J }{\\partial \\theta_j} = \\frac{1}{m} \\sum_{i=1}^m \\left( \\left( h_\\theta\\left(x^{(i)}\\right) - y^{(i)} \\right)x_j^{(i)} \\right) $$\n",
|
||||
"\n",
|
||||
"To vectorize this operation over the dataset, we start by writing out all the partial derivatives explicitly for all $\\theta_j$,\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{align*}\n",
|
||||
"\\begin{bmatrix} \n",
|
||||
"\\frac{\\partial J}{\\partial \\theta_0} \\\\\n",
|
||||
"\\frac{\\partial J}{\\partial \\theta_1} \\\\\n",
|
||||
"\\frac{\\partial J}{\\partial \\theta_2} \\\\\n",
|
||||
"\\vdots \\\\\n",
|
||||
"\\frac{\\partial J}{\\partial \\theta_n}\n",
|
||||
"\\end{bmatrix} = &\n",
|
||||
"\\frac{1}{m} \\begin{bmatrix}\n",
|
||||
"\\sum_{i=1}^m \\left( \\left(h_\\theta\\left(x^{(i)}\\right) - y^{(i)} \\right)x_0^{(i)}\\right) \\\\\n",
|
||||
"\\sum_{i=1}^m \\left( \\left(h_\\theta\\left(x^{(i)}\\right) - y^{(i)} \\right)x_1^{(i)}\\right) \\\\\n",
|
||||
"\\sum_{i=1}^m \\left( \\left(h_\\theta\\left(x^{(i)}\\right) - y^{(i)} \\right)x_2^{(i)}\\right) \\\\\n",
|
||||
"\\vdots \\\\\n",
|
||||
"\\sum_{i=1}^m \\left( \\left(h_\\theta\\left(x^{(i)}\\right) - y^{(i)} \\right)x_n^{(i)}\\right) \\\\\n",
|
||||
"\\end{bmatrix} \\\\\n",
|
||||
"= & \\frac{1}{m} \\sum_{i=1}^m \\left( \\left(h_\\theta\\left(x^{(i)}\\right) - y^{(i)} \\right)x^{(i)}\\right) \\\\\n",
|
||||
"= & \\frac{1}{m} X^T \\left( h_\\theta(x) - y\\right)\n",
|
||||
"\\end{align*}\n",
|
||||
"$$\n",
|
||||
"\n",
|
||||
"where\n",
|
||||
"\n",
|
||||
"$$ h_\\theta(x) - y = \n",
|
||||
"\\begin{bmatrix}\n",
|
||||
"h_\\theta\\left(x^{(1)}\\right) - y^{(1)} \\\\\n",
|
||||
"h_\\theta\\left(x^{(2)}\\right) - y^{(2)} \\\\\n",
|
||||
"\\vdots \\\\\n",
|
||||
"h_\\theta\\left(x^{(m)}\\right) - y^{(m)} \n",
|
||||
"\\end{bmatrix} $$\n",
|
||||
"\n",
|
||||
"Note that $x^{(i)}$ is a vector, while $h_\\theta\\left(x^{(i)}\\right) - y^{(i)}$ is a scalar (single number).\n",
|
||||
"To understand the last step of the derivation, let $\\beta_i = (h_\\theta\\left(x^{(m)}\\right) - y^{(m)})$ and\n",
|
||||
"observe that:\n",
|
||||
"\n",
|
||||
"$$ \\sum_i \\beta_ix^{(i)} = \\begin{bmatrix} \n",
|
||||
"| & | & & | \\\\\n",
|
||||
"x^{(1)} & x^{(2)} & \\cdots & x^{(m)} \\\\\n",
|
||||
"| & | & & | \n",
|
||||
"\\end{bmatrix}\n",
|
||||
"\\begin{bmatrix}\n",
|
||||
"\\beta_1 \\\\\n",
|
||||
"\\beta_2 \\\\\n",
|
||||
"\\vdots \\\\\n",
|
||||
"\\beta_m\n",
|
||||
"\\end{bmatrix} = x^T \\beta\n",
|
||||
"$$\n",
|
||||
"\n",
|
||||
"where the values $\\beta_i = \\left( h_\\theta(x^{(i)} - y^{(i)} \\right)$.\n",
|
||||
"\n",
|
||||
"The expression above allows us to compute all the partial derivatives\n",
|
||||
"without any loops. If you are comfortable with linear algebra, we encourage you to work through the matrix multiplications above to convince yourself that the vectorized version does the same computations. \n",
|
||||
"\n",
|
||||
"Your job is to write the unregularized cost function `lrCostFunction` which returns both the cost function $J(\\theta)$ and its gradient $\\frac{\\partial J}{\\partial \\theta}$. Your implementation should use the strategy we presented above to calculate $\\theta^T x^{(i)}$. You should also use a vectorized approach for the rest of the cost function. A fully vectorized version of `lrCostFunction` should not contain any loops.\n",
|
||||
"\n",
|
||||
"<div class=\"alert alert-box alert-warning\">\n",
|
||||
"**Debugging Tip:** Vectorizing code can sometimes be tricky. One common strategy for debugging is to print out the sizes of the matrices you are working with using the `shape` property of `numpy` arrays. For example, given a data matrix $X$ of size $100 \\times 20$ (100 examples, 20 features) and $\\theta$, a vector with size $20$, you can observe that `np.dot(X, theta)` is a valid multiplication operation, while `np.dot(theta, X)` is not. Furthermore, if you have a non-vectorized version of your code, you can compare the output of your vectorized code and non-vectorized code to make sure that they produce the same outputs.\n",
|
||||
"</div>\n",
|
||||
"<a id=\"lrCostFunction\"></a>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def lrCostFunction(theta, X, y, lambda_):\n",
|
||||
" \"\"\"\n",
|
||||
" Computes the cost of using theta as the parameter for regularized\n",
|
||||
" logistic regression and the gradient of the cost w.r.t. to the parameters.\n",
|
||||
" \n",
|
||||
" Parameters\n",
|
||||
" ----------\n",
|
||||
" theta : array_like\n",
|
||||
" Logistic regression parameters. A vector with shape (n, ). n is \n",
|
||||
" the number of features including any intercept. \n",
|
||||
" \n",
|
||||
" X : array_like\n",
|
||||
" The data set with shape (m x n). m is the number of examples, and\n",
|
||||
" n is the number of features (including intercept).\n",
|
||||
" \n",
|
||||
" y : array_like\n",
|
||||
" The data labels. A vector with shape (m, ).\n",
|
||||
" \n",
|
||||
" lambda_ : float\n",
|
||||
" The regularization parameter. \n",
|
||||
" \n",
|
||||
" Returns\n",
|
||||
" -------\n",
|
||||
" J : float\n",
|
||||
" The computed value for the regularized cost function. \n",
|
||||
" \n",
|
||||
" grad : array_like\n",
|
||||
" A vector of shape (n, ) which is the gradient of the cost\n",
|
||||
" function with respect to theta, at the current values of theta.\n",
|
||||
" \n",
|
||||
" Instructions\n",
|
||||
" ------------\n",
|
||||
" Compute the cost of a particular choice of theta. You should set J to the cost.\n",
|
||||
" Compute the partial derivatives and set grad to the partial\n",
|
||||
" derivatives of the cost w.r.t. each parameter in theta\n",
|
||||
" \n",
|
||||
" Hint 1\n",
|
||||
" ------\n",
|
||||
" The computation of the cost function and gradients can be efficiently\n",
|
||||
" vectorized. For example, consider the computation\n",
|
||||
" \n",
|
||||
" sigmoid(X * theta)\n",
|
||||
" \n",
|
||||
" Each row of the resulting matrix will contain the value of the prediction\n",
|
||||
" for that example. You can make use of this to vectorize the cost function\n",
|
||||
" and gradient computations. \n",
|
||||
" \n",
|
||||
" Hint 2\n",
|
||||
" ------\n",
|
||||
" When computing the gradient of the regularized cost function, there are\n",
|
||||
" many possible vectorized solutions, but one solution looks like:\n",
|
||||
" \n",
|
||||
" grad = (unregularized gradient for logistic regression)\n",
|
||||
" temp = theta \n",
|
||||
" temp[0] = 0 # because we don't add anything for j = 0\n",
|
||||
" grad = grad + YOUR_CODE_HERE (using the temp variable)\n",
|
||||
" \n",
|
||||
" Hint 3\n",
|
||||
" ------\n",
|
||||
" We have provided the implementatation of the sigmoid function within \n",
|
||||
" the file `utils.py`. At the start of the notebook, we imported this file\n",
|
||||
" as a module. Thus to access the sigmoid function within that file, you can\n",
|
||||
" do the following: `utils.sigmoid(z)`.\n",
|
||||
" \n",
|
||||
" \"\"\"\n",
|
||||
" #Initialize some useful values\n",
|
||||
" m = y.size\n",
|
||||
" \n",
|
||||
" # convert labels to ints if their type is bool\n",
|
||||
" if y.dtype == bool:\n",
|
||||
" y = y.astype(int)\n",
|
||||
" \n",
|
||||
" # You need to return the following variables correctly\n",
|
||||
" J = 0\n",
|
||||
" grad = np.zeros(theta.shape)\n",
|
||||
" \n",
|
||||
" # ====================== YOUR CODE HERE ======================\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" \n",
|
||||
" # =============================================================\n",
|
||||
" return J, grad"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### 1.3.3 Vectorizing regularized logistic regression\n",
|
||||
"\n",
|
||||
"After you have implemented vectorization for logistic regression, you will now\n",
|
||||
"add regularization to the cost function. Recall that for regularized logistic\n",
|
||||
"regression, the cost function is defined as\n",
|
||||
"\n",
|
||||
"$$ J(\\theta) = \\frac{1}{m} \\sum_{i=1}^m \\left[ -y^{(i)} \\log \\left(h_\\theta\\left(x^{(i)} \\right)\\right) - \\left( 1 - y^{(i)} \\right) \\log\\left(1 - h_\\theta \\left(x^{(i)} \\right) \\right) \\right] + \\frac{\\lambda}{2m} \\sum_{j=1}^n \\theta_j^2 $$\n",
|
||||
"\n",
|
||||
"Note that you should not be regularizing $\\theta_0$ which is used for the bias term.\n",
|
||||
"Correspondingly, the partial derivative of regularized logistic regression cost for $\\theta_j$ is defined as\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{align*}\n",
|
||||
"& \\frac{\\partial J(\\theta)}{\\partial \\theta_0} = \\frac{1}{m} \\sum_{i=1}^m \\left( h_\\theta\\left( x^{(i)} \\right) - y^{(i)} \\right) x_j^{(i)} & \\text{for } j = 0 \\\\\n",
|
||||
"& \\frac{\\partial J(\\theta)}{\\partial \\theta_0} = \\left( \\frac{1}{m} \\sum_{i=1}^m \\left( h_\\theta\\left( x^{(i)} \\right) - y^{(i)} \\right) x_j^{(i)} \\right) + \\frac{\\lambda}{m} \\theta_j & \\text{for } j \\ge 1\n",
|
||||
"\\end{align*}\n",
|
||||
"$$\n",
|
||||
"\n",
|
||||
"Now modify your code in lrCostFunction in the [**previous cell**](#lrCostFunction) to account for regularization. Once again, you should not put any loops into your code.\n",
|
||||
"\n",
|
||||
"<div class=\"alert alert-box alert-warning\">\n",
|
||||
"**python/numpy Tip:** When implementing the vectorization for regularized logistic regression, you might often want to only sum and update certain elements of $\\theta$. In `numpy`, you can index into the matrices to access and update only certain elements. For example, A[:, 3:5]\n",
|
||||
"= B[:, 1:3] will replaces the columns with index 3 to 5 of A with the columns with index 1 to 3 from B. To select columns (or rows) until the end of the matrix, you can leave the right hand side of the colon blank. For example, A[:, 2:] will only return elements from the $3^{rd}$ to last columns of $A$. If you leave the left hand size of the colon blank, you will select elements from the beginning of the matrix. For example, A[:, :2] selects the first two columns, and is equivalent to A[:, 0:2]. In addition, you can use negative indices to index arrays from the end. Thus, A[:, :-1] selects all columns of A except the last column, and A[:, -5:] selects the $5^{th}$ column from the end to the last column. Thus, you could use this together with the sum and power ($^{**}$) operations to compute the sum of only the elements you are interested in (e.g., `np.sum(z[1:]**2)`). In the starter code, `lrCostFunction`, we have also provided hints on yet another possible method computing the regularized gradient.\n",
|
||||
"</div>\n",
|
||||
"\n",
|
||||
"Once you finished your implementation, you can call the function `lrCostFunction` to test your solution using the following cell:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"J, grad = lrCostFunction(theta_t, X_t, y_t, lambda_t)\n",
|
||||
"\n",
|
||||
"print('Cost : {:.6f}'.format(J))\n",
|
||||
"print('Expected cost: 2.534819')\n",
|
||||
"print('-----------------------')\n",
|
||||
"print('Gradients:')\n",
|
||||
"print(' [{:.6f}, {:.6f}, {:.6f}, {:.6f}]'.format(*grad))\n",
|
||||
"print('Expected gradients:')\n",
|
||||
"print(' [0.146561, -0.548558, 0.724722, 1.398003]');"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"After completing a part of the exercise, you can submit your solutions for grading by first adding the function you modified to the submission object, and then sending your function to Coursera for grading. \n",
|
||||
"\n",
|
||||
"The submission script will prompt you for your login e-mail and submission token. You can obtain a submission token from the web page for the assignment. You are allowed to submit your solutions multiple times, and we will take only the highest score into consideration.\n",
|
||||
"\n",
|
||||
"*Execute the following cell to grade your solution to the first part of this exercise.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# appends the implemented function in part 1 to the grader object\n",
|
||||
"grader[1] = lrCostFunction\n",
|
||||
"\n",
|
||||
"# send the added functions to coursera grader for getting a grade on this part\n",
|
||||
"grader.grade()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a id=\"section2\"></a>\n",
|
||||
"### 1.4 One-vs-all Classification\n",
|
||||
"\n",
|
||||
"In this part of the exercise, you will implement one-vs-all classification by training multiple regularized logistic regression classifiers, one for each of the $K$ classes in our dataset. In the handwritten digits dataset, $K = 10$, but your code should work for any value of $K$. \n",
|
||||
"\n",
|
||||
"You should now complete the code for the function `oneVsAll` below, to train one classifier for each class. In particular, your code should return all the classifier parameters in a matrix $\\theta \\in \\mathbb{R}^{K \\times (N +1)}$, where each row of $\\theta$ corresponds to the learned logistic regression parameters for one class. You can do this with a “for”-loop from $0$ to $K-1$, training each classifier independently.\n",
|
||||
"\n",
|
||||
"Note that the `y` argument to this function is a vector of labels from 0 to 9. When training the classifier for class $k \\in \\{0, ..., K-1\\}$, you will want a K-dimensional vector of labels $y$, where $y_j \\in 0, 1$ indicates whether the $j^{th}$ training instance belongs to class $k$ $(y_j = 1)$, or if it belongs to a different\n",
|
||||
"class $(y_j = 0)$. You may find logical arrays helpful for this task. \n",
|
||||
"\n",
|
||||
"Furthermore, you will be using scipy's `optimize.minimize` for this exercise. \n",
|
||||
"<a id=\"oneVsAll\"></a>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def oneVsAll(X, y, num_labels, lambda_):\n",
|
||||
" \"\"\"\n",
|
||||
" Trains num_labels logistic regression classifiers and returns\n",
|
||||
" each of these classifiers in a matrix all_theta, where the i-th\n",
|
||||
" row of all_theta corresponds to the classifier for label i.\n",
|
||||
" \n",
|
||||
" Parameters\n",
|
||||
" ----------\n",
|
||||
" X : array_like\n",
|
||||
" The input dataset of shape (m x n). m is the number of \n",
|
||||
" data points, and n is the number of features. Note that we \n",
|
||||
" do not assume that the intercept term (or bias) is in X, however\n",
|
||||
" we provide the code below to add the bias term to X. \n",
|
||||
" \n",
|
||||
" y : array_like\n",
|
||||
" The data labels. A vector of shape (m, ).\n",
|
||||
" \n",
|
||||
" num_labels : int\n",
|
||||
" Number of possible labels.\n",
|
||||
" \n",
|
||||
" lambda_ : float\n",
|
||||
" The logistic regularization parameter.\n",
|
||||
" \n",
|
||||
" Returns\n",
|
||||
" -------\n",
|
||||
" all_theta : array_like\n",
|
||||
" The trained parameters for logistic regression for each class.\n",
|
||||
" This is a matrix of shape (K x n+1) where K is number of classes\n",
|
||||
" (ie. `numlabels`) and n is number of features without the bias.\n",
|
||||
" \n",
|
||||
" Instructions\n",
|
||||
" ------------\n",
|
||||
" You should complete the following code to train `num_labels`\n",
|
||||
" logistic regression classifiers with regularization parameter `lambda_`. \n",
|
||||
" \n",
|
||||
" Hint\n",
|
||||
" ----\n",
|
||||
" You can use y == c to obtain a vector of 1's and 0's that tell you\n",
|
||||
" whether the ground truth is true/false for this class.\n",
|
||||
" \n",
|
||||
" Note\n",
|
||||
" ----\n",
|
||||
" For this assignment, we recommend using `scipy.optimize.minimize(method='CG')`\n",
|
||||
" to optimize the cost function. It is okay to use a for-loop \n",
|
||||
" (`for c in range(num_labels):`) to loop over the different classes.\n",
|
||||
" \n",
|
||||
" Example Code\n",
|
||||
" ------------\n",
|
||||
" \n",
|
||||
" # Set Initial theta\n",
|
||||
" initial_theta = np.zeros(n + 1)\n",
|
||||
" \n",
|
||||
" # Set options for minimize\n",
|
||||
" options = {'maxiter', 50}\n",
|
||||
" \n",
|
||||
" # Run minimize to obtain the optimal theta. This function will \n",
|
||||
" # return a class object where theta is in `res.x` and cost in `res.fun`\n",
|
||||
" res = optimize.minimize(lrCostFunction, \n",
|
||||
" initial_theta, \n",
|
||||
" (X, (y == c), lambda_), \n",
|
||||
" jac=True, \n",
|
||||
" method='TNC')\n",
|
||||
" options=options) \n",
|
||||
" \"\"\"\n",
|
||||
" # Some useful variables\n",
|
||||
" m, n = X.shape\n",
|
||||
" \n",
|
||||
" # You need to return the following variables correctly \n",
|
||||
" all_theta = np.zeros((num_labels, n + 1))\n",
|
||||
"\n",
|
||||
" # Add ones to the X data matrix\n",
|
||||
" X = np.concatenate([np.ones((m, 1)), X], axis=1)\n",
|
||||
"\n",
|
||||
" # ====================== YOUR CODE HERE ======================\n",
|
||||
" \n",
|
||||
"\n",
|
||||
"\n",
|
||||
" # ============================================================\n",
|
||||
" return all_theta"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"After you have completed the code for `oneVsAll`, the following cell will use your implementation to train a multi-class classifier. "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"lambda_ = 0.1\n",
|
||||
"all_theta = oneVsAll(X, y, num_labels, lambda_)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"*You should now submit your solutions.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"grader[2] = oneVsAll\n",
|
||||
"grader.grade()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a id=\"section3\"></a>\n",
|
||||
"#### 1.4.1 One-vs-all Prediction\n",
|
||||
"\n",
|
||||
"After training your one-vs-all classifier, you can now use it to predict the digit contained in a given image. For each input, you should compute the “probability” that it belongs to each class using the trained logistic regression classifiers. Your one-vs-all prediction function will pick the class for which the corresponding logistic regression classifier outputs the highest probability and return the class label (0, 1, ..., K-1) as the prediction for the input example. You should now complete the code in the function `predictOneVsAll` to use the one-vs-all classifier for making predictions. \n",
|
||||
"<a id=\"predictOneVsAll\"></a>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def predictOneVsAll(all_theta, X):\n",
|
||||
" \"\"\"\n",
|
||||
" Return a vector of predictions for each example in the matrix X. \n",
|
||||
" Note that X contains the examples in rows. all_theta is a matrix where\n",
|
||||
" the i-th row is a trained logistic regression theta vector for the \n",
|
||||
" i-th class. You should set p to a vector of values from 0..K-1 \n",
|
||||
" (e.g., p = [0, 2, 0, 1] predicts classes 0, 2, 0, 1 for 4 examples) .\n",
|
||||
" \n",
|
||||
" Parameters\n",
|
||||
" ----------\n",
|
||||
" all_theta : array_like\n",
|
||||
" The trained parameters for logistic regression for each class.\n",
|
||||
" This is a matrix of shape (K x n+1) where K is number of classes\n",
|
||||
" and n is number of features without the bias.\n",
|
||||
" \n",
|
||||
" X : array_like\n",
|
||||
" Data points to predict their labels. This is a matrix of shape \n",
|
||||
" (m x n) where m is number of data points to predict, and n is number \n",
|
||||
" of features without the bias term. Note we add the bias term for X in \n",
|
||||
" this function. \n",
|
||||
" \n",
|
||||
" Returns\n",
|
||||
" -------\n",
|
||||
" p : array_like\n",
|
||||
" The predictions for each data point in X. This is a vector of shape (m, ).\n",
|
||||
" \n",
|
||||
" Instructions\n",
|
||||
" ------------\n",
|
||||
" Complete the following code to make predictions using your learned logistic\n",
|
||||
" regression parameters (one-vs-all). You should set p to a vector of predictions\n",
|
||||
" (from 0 to num_labels-1).\n",
|
||||
" \n",
|
||||
" Hint\n",
|
||||
" ----\n",
|
||||
" This code can be done all vectorized using the numpy argmax function.\n",
|
||||
" In particular, the argmax function returns the index of the max element,\n",
|
||||
" for more information see '?np.argmax' or search online. If your examples\n",
|
||||
" are in rows, then, you can use np.argmax(A, axis=1) to obtain the index \n",
|
||||
" of the max for each row.\n",
|
||||
" \"\"\"\n",
|
||||
" m = X.shape[0];\n",
|
||||
" num_labels = all_theta.shape[0]\n",
|
||||
"\n",
|
||||
" # You need to return the following variables correctly \n",
|
||||
" p = np.zeros(m)\n",
|
||||
"\n",
|
||||
" # Add ones to the X data matrix\n",
|
||||
" X = np.concatenate([np.ones((m, 1)), X], axis=1)\n",
|
||||
"\n",
|
||||
" # ====================== YOUR CODE HERE ======================\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" \n",
|
||||
" # ============================================================\n",
|
||||
" return p"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Once you are done, call your `predictOneVsAll` function using the learned value of $\\theta$. You should see that the training set accuracy is about 95.1% (i.e., it classifies 95.1% of the examples in the training set correctly)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"pred = predictOneVsAll(all_theta, X)\n",
|
||||
"print('Training Set Accuracy: {:.2f}%'.format(np.mean(pred == y) * 100))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"*You should now submit your solutions.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"grader[3] = predictOneVsAll\n",
|
||||
"grader.grade()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2 Neural Networks\n",
|
||||
"\n",
|
||||
"In the previous part of this exercise, you implemented multi-class logistic regression to recognize handwritten digits. However, logistic regression cannot form more complex hypotheses as it is only a linear classifier (You could add more features - such as polynomial features - to logistic regression, but that can be very expensive to train).\n",
|
||||
"\n",
|
||||
"In this part of the exercise, you will implement a neural network to recognize handwritten digits using the same training set as before. The neural network will be able to represent complex models that form non-linear hypotheses. For this week, you will be using parameters from a neural network that we have already trained. Your goal is to implement the feedforward propagation algorithm to use our weights for prediction. In next week’s exercise, you will write the backpropagation algorithm for learning the neural network parameters. \n",
|
||||
"\n",
|
||||
"We start by first reloading and visualizing the dataset which contains the MNIST handwritten digits (this is the same as we did in the first part of this exercise, we reload it here to ensure the variables have not been modified). "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# training data stored in arrays X, y\n",
|
||||
"data = loadmat(os.path.join('Data', 'ex3data1.mat'))\n",
|
||||
"X, y = data['X'], data['y'].ravel()\n",
|
||||
"\n",
|
||||
"# set the zero digit to 0, rather than its mapped 10 in this dataset\n",
|
||||
"# This is an artifact due to the fact that this dataset was used in \n",
|
||||
"# MATLAB where there is no index 0\n",
|
||||
"y[y == 10] = 0\n",
|
||||
"\n",
|
||||
"# get number of examples in dataset\n",
|
||||
"m = y.size\n",
|
||||
"\n",
|
||||
"# randomly permute examples, to be used for visualizing one \n",
|
||||
"# picture at a time\n",
|
||||
"indices = np.random.permutation(m)\n",
|
||||
"\n",
|
||||
"# Randomly select 100 data points to display\n",
|
||||
"rand_indices = np.random.choice(m, 100, replace=False)\n",
|
||||
"sel = X[rand_indices, :]\n",
|
||||
"\n",
|
||||
"utils.displayData(sel)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"\n",
|
||||
"### 2.1 Model representation \n",
|
||||
"\n",
|
||||
"Our neural network is shown in the following figure.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"It has 3 layers: an input layer, a hidden layer and an output layer. Recall that our inputs are pixel values of digit images. Since the images are of size 20×20, this gives us 400 input layer units (excluding the extra bias unit which always outputs +1). As before, the training data will be loaded into the variables X and y. \n",
|
||||
"\n",
|
||||
"You have been provided with a set of network parameters ($\\Theta^{(1)}$, $\\Theta^{(2)}$) already trained by us. These are stored in `ex3weights.mat`. The following cell loads those parameters into `Theta1` and `Theta2`. The parameters have dimensions that are sized for a neural network with 25 units in the second layer and 10 output units (corresponding to the 10 digit classes)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Setup the parameters you will use for this exercise\n",
|
||||
"input_layer_size = 400 # 20x20 Input Images of Digits\n",
|
||||
"hidden_layer_size = 25 # 25 hidden units\n",
|
||||
"num_labels = 10 # 10 labels, from 0 to 9\n",
|
||||
"\n",
|
||||
"# Load the .mat file, which returns a dictionary \n",
|
||||
"weights = loadmat(os.path.join('Data', 'ex3weights.mat'))\n",
|
||||
"\n",
|
||||
"# get the model weights from the dictionary\n",
|
||||
"# Theta1 has size 25 x 401\n",
|
||||
"# Theta2 has size 10 x 26\n",
|
||||
"Theta1, Theta2 = weights['Theta1'], weights['Theta2']\n",
|
||||
"\n",
|
||||
"# swap first and last columns of Theta2, due to legacy from MATLAB indexing, \n",
|
||||
"# since the weight file ex3weights.mat was saved based on MATLAB indexing\n",
|
||||
"Theta2 = np.roll(Theta2, 1, axis=0)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a id=\"section4\"></a>\n",
|
||||
"### 2.2 Feedforward Propagation and Prediction\n",
|
||||
"\n",
|
||||
"Now you will implement feedforward propagation for the neural network. You will need to complete the code in the function `predict` to return the neural network’s prediction. You should implement the feedforward computation that computes $h_\\theta(x^{(i)})$ for every example $i$ and returns the associated predictions. Similar to the one-vs-all classification strategy, the prediction from the neural network will be the label that has the largest output $\\left( h_\\theta(x) \\right)_k$.\n",
|
||||
"\n",
|
||||
"<div class=\"alert alert-box alert-warning\">\n",
|
||||
"**Implementation Note:** The matrix $X$ contains the examples in rows. When you complete the code in the function `predict`, you will need to add the column of 1’s to the matrix. The matrices `Theta1` and `Theta2` contain the parameters for each unit in rows. Specifically, the first row of `Theta1` corresponds to the first hidden unit in the second layer. In `numpy`, when you compute $z^{(2)} = \\theta^{(1)}a^{(1)}$, be sure that you index (and if necessary, transpose) $X$ correctly so that you get $a^{(l)}$ as a 1-D vector.\n",
|
||||
"</div>\n",
|
||||
"<a id=\"predict\"></a>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def predict(Theta1, Theta2, X):\n",
|
||||
" \"\"\"\n",
|
||||
" Predict the label of an input given a trained neural network.\n",
|
||||
" \n",
|
||||
" Parameters\n",
|
||||
" ----------\n",
|
||||
" Theta1 : array_like\n",
|
||||
" Weights for the first layer in the neural network.\n",
|
||||
" It has shape (2nd hidden layer size x input size)\n",
|
||||
" \n",
|
||||
" Theta2: array_like\n",
|
||||
" Weights for the second layer in the neural network. \n",
|
||||
" It has shape (output layer size x 2nd hidden layer size)\n",
|
||||
" \n",
|
||||
" X : array_like\n",
|
||||
" The image inputs having shape (number of examples x image dimensions).\n",
|
||||
" \n",
|
||||
" Return \n",
|
||||
" ------\n",
|
||||
" p : array_like\n",
|
||||
" Predictions vector containing the predicted label for each example.\n",
|
||||
" It has a length equal to the number of examples.\n",
|
||||
" \n",
|
||||
" Instructions\n",
|
||||
" ------------\n",
|
||||
" Complete the following code to make predictions using your learned neural\n",
|
||||
" network. You should set p to a vector containing labels \n",
|
||||
" between 0 to (num_labels-1).\n",
|
||||
" \n",
|
||||
" Hint\n",
|
||||
" ----\n",
|
||||
" This code can be done all vectorized using the numpy argmax function.\n",
|
||||
" In particular, the argmax function returns the index of the max element,\n",
|
||||
" for more information see '?np.argmax' or search online. If your examples\n",
|
||||
" are in rows, then, you can use np.argmax(A, axis=1) to obtain the index\n",
|
||||
" of the max for each row.\n",
|
||||
" \n",
|
||||
" Note\n",
|
||||
" ----\n",
|
||||
" Remember, we have supplied the `sigmoid` function in the `utils.py` file. \n",
|
||||
" You can use this function by calling `utils.sigmoid(z)`, where you can \n",
|
||||
" replace `z` by the required input variable to sigmoid.\n",
|
||||
" \"\"\"\n",
|
||||
" # Make sure the input has two dimensions\n",
|
||||
" if X.ndim == 1:\n",
|
||||
" X = X[None] # promote to 2-dimensions\n",
|
||||
" \n",
|
||||
" # useful variables\n",
|
||||
" m = X.shape[0]\n",
|
||||
" num_labels = Theta2.shape[0]\n",
|
||||
"\n",
|
||||
" # You need to return the following variables correctly \n",
|
||||
" p = np.zeros(X.shape[0])\n",
|
||||
"\n",
|
||||
" # ====================== YOUR CODE HERE ======================\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" # =============================================================\n",
|
||||
" return p"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Once you are done, call your predict function using the loaded set of parameters for `Theta1` and `Theta2`. You should see that the accuracy is about 97.5%."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"pred = predict(Theta1, Theta2, X)\n",
|
||||
"print('Training Set Accuracy: {:.1f}%'.format(np.mean(pred == y) * 100))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"After that, we will display images from the training set one at a time, while at the same time printing out the predicted label for the displayed image. \n",
|
||||
"\n",
|
||||
"Run the following cell to display a single image the the neural network's prediction. You can run the cell multiple time to see predictions for different images."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if indices.size > 0:\n",
|
||||
" i, indices = indices[0], indices[1:]\n",
|
||||
" utils.displayData(X[i, :], figsize=(4, 4))\n",
|
||||
" pred = predict(Theta1, Theta2, X[i, :])\n",
|
||||
" print('Neural Network Prediction: {}'.format(*pred))\n",
|
||||
"else:\n",
|
||||
" print('No more images to display!')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"*You should now submit your solutions.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"grader[4] = predict\n",
|
||||
"grader.grade()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
104
Exercise3/utils.py
Executable file
104
Exercise3/utils.py
Executable file
@@ -0,0 +1,104 @@
|
||||
import sys
|
||||
import numpy as np
|
||||
from matplotlib import pyplot
|
||||
|
||||
sys.path.append('..')
|
||||
from submission import SubmissionBase
|
||||
|
||||
|
||||
def displayData(X, example_width=None, figsize=(10, 10)):
|
||||
"""
|
||||
Displays 2D data stored in X in a nice grid.
|
||||
"""
|
||||
# Compute rows, cols
|
||||
if X.ndim == 2:
|
||||
m, n = X.shape
|
||||
elif X.ndim == 1:
|
||||
n = X.size
|
||||
m = 1
|
||||
X = X[None] # Promote to a 2 dimensional array
|
||||
else:
|
||||
raise IndexError('Input X should be 1 or 2 dimensional.')
|
||||
|
||||
example_width = example_width or int(np.round(np.sqrt(n)))
|
||||
example_height = n / example_width
|
||||
|
||||
# Compute number of items to display
|
||||
display_rows = int(np.floor(np.sqrt(m)))
|
||||
display_cols = int(np.ceil(m / display_rows))
|
||||
|
||||
fig, ax_array = pyplot.subplots(display_rows, display_cols, figsize=figsize)
|
||||
fig.subplots_adjust(wspace=0.025, hspace=0.025)
|
||||
|
||||
ax_array = [ax_array] if m == 1 else ax_array.ravel()
|
||||
|
||||
for i, ax in enumerate(ax_array):
|
||||
ax.imshow(X[i].reshape(example_width, example_width, order='F'),
|
||||
cmap='Greys', extent=[0, 1, 0, 1])
|
||||
ax.axis('off')
|
||||
|
||||
|
||||
def sigmoid(z):
|
||||
"""
|
||||
Computes the sigmoid of z.
|
||||
"""
|
||||
return 1.0 / (1.0 + np.exp(-z))
|
||||
|
||||
|
||||
class Grader(SubmissionBase):
|
||||
# Random Test Cases
|
||||
X = np.stack([np.ones(20),
|
||||
np.exp(1) * np.sin(np.arange(1, 21)),
|
||||
np.exp(0.5) * np.cos(np.arange(1, 21))], axis=1)
|
||||
|
||||
y = (np.sin(X[:, 0] + X[:, 1]) > 0).astype(float)
|
||||
|
||||
Xm = np.array([[-1, -1],
|
||||
[-1, -2],
|
||||
[-2, -1],
|
||||
[-2, -2],
|
||||
[1, 1],
|
||||
[1, 2],
|
||||
[2, 1],
|
||||
[2, 2],
|
||||
[-1, 1],
|
||||
[-1, 2],
|
||||
[-2, 1],
|
||||
[-2, 2],
|
||||
[1, -1],
|
||||
[1, -2],
|
||||
[-2, -1],
|
||||
[-2, -2]])
|
||||
ym = np.array([0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3])
|
||||
|
||||
t1 = np.sin(np.reshape(np.arange(1, 25, 2), (4, 3), order='F'))
|
||||
t2 = np.cos(np.reshape(np.arange(1, 41, 2), (4, 5), order='F'))
|
||||
|
||||
def __init__(self):
|
||||
part_names = ['Regularized Logistic Regression',
|
||||
'One-vs-All Classifier Training',
|
||||
'One-vs-All Classifier Prediction',
|
||||
'Neural Network Prediction Function']
|
||||
|
||||
super().__init__('multi-class-classification-and-neural-networks', part_names)
|
||||
|
||||
def __iter__(self):
|
||||
for part_id in range(1, 5):
|
||||
try:
|
||||
func = self.functions[part_id]
|
||||
|
||||
# Each part has different expected arguments/different function
|
||||
if part_id == 1:
|
||||
res = func(np.array([0.25, 0.5, -0.5]), self.X, self.y, 0.1)
|
||||
res = np.hstack(res).tolist()
|
||||
elif part_id == 2:
|
||||
res = func(self.Xm, self.ym, 4, 0.1)
|
||||
elif part_id == 3:
|
||||
res = func(self.t1, self.Xm) + 1
|
||||
elif part_id == 4:
|
||||
res = func(self.t1, self.t2, self.Xm) + 1
|
||||
else:
|
||||
raise KeyError
|
||||
yield part_id, res
|
||||
except KeyError:
|
||||
yield part_id, 0
|
||||
BIN
Exercise4/Data/ex4data1.mat
Executable file
BIN
Exercise4/Data/ex4data1.mat
Executable file
Binary file not shown.
BIN
Exercise4/Data/ex4weights.mat
Executable file
BIN
Exercise4/Data/ex4weights.mat
Executable file
Binary file not shown.
BIN
Exercise4/Figures/ex4-backpropagation.png
Executable file
BIN
Exercise4/Figures/ex4-backpropagation.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 42 KiB |
BIN
Exercise4/Figures/neural_network.png
Executable file
BIN
Exercise4/Figures/neural_network.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 54 KiB |
924
Exercise4/exercise4.ipynb
Executable file
924
Exercise4/exercise4.ipynb
Executable file
@@ -0,0 +1,924 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Programming Exercise 4: Neural Networks Learning\n",
|
||||
"\n",
|
||||
"## Introduction\n",
|
||||
"\n",
|
||||
"In this exercise, you will implement the backpropagation algorithm for neural networks and apply it to the task of hand-written digit recognition. Before starting on the programming exercise, we strongly recommend watching the video lectures and completing the review questions for the associated topics.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"All the information you need for solving this assignment is in this notebook, and all the code you will be implementing will take place within this notebook. The assignment can be promptly submitted to the coursera grader directly from this notebook (code and instructions are included below).\n",
|
||||
"\n",
|
||||
"Before we begin with the exercises, we need to import all libraries required for this programming exercise. Throughout the course, we will be using [`numpy`](http://www.numpy.org/) for all arrays and matrix operations, [`matplotlib`](https://matplotlib.org/) for plotting, and [`scipy`](https://docs.scipy.org/doc/scipy/reference/) for scientific and numerical computation functions and tools. You can find instructions on how to install required libraries in the README file in the [github repository](https://github.com/dibgerge/ml-coursera-python-assignments)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# used for manipulating directory paths\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Scientific and vector computation for python\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"# Plotting library\n",
|
||||
"from matplotlib import pyplot\n",
|
||||
"\n",
|
||||
"# Optimization module in scipy\n",
|
||||
"from scipy import optimize\n",
|
||||
"\n",
|
||||
"# will be used to load MATLAB mat datafile format\n",
|
||||
"from scipy.io import loadmat\n",
|
||||
"\n",
|
||||
"# library written for this exercise providing additional functions for assignment submission, and others\n",
|
||||
"import utils\n",
|
||||
"\n",
|
||||
"# define the submission/grader object for this exercise\n",
|
||||
"grader = utils.Grader()\n",
|
||||
"\n",
|
||||
"# tells matplotlib to embed plots within the notebook\n",
|
||||
"%matplotlib inline"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Submission and Grading\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"After completing each part of the assignment, be sure to submit your solutions to the grader. The following is a breakdown of how each part of this exercise is scored.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"| Section | Part | Submission function | Points \n",
|
||||
"| :- |:- | :- | :-: \n",
|
||||
"| 1 | [Feedforward and Cost Function](#section1) | [`nnCostFunction`](#nnCostFunction) | 30 \n",
|
||||
"| 2 | [Regularized Cost Function](#section2) | [`nnCostFunction`](#nnCostFunction) | 15 \n",
|
||||
"| 3 | [Sigmoid Gradient](#section3) | [`sigmoidGradient`](#sigmoidGradient) | 5 \n",
|
||||
"| 4 | [Neural Net Gradient Function (Backpropagation)](#section4) | [`nnCostFunction`](#nnCostFunction) | 40 \n",
|
||||
"| 5 | [Regularized Gradient](#section5) | [`nnCostFunction`](#nnCostFunction) |10 \n",
|
||||
"| | Total Points | | 100 \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"You are allowed to submit your solutions multiple times, and we will take only the highest score into consideration.\n",
|
||||
"\n",
|
||||
"<div class=\"alert alert-block alert-warning\">\n",
|
||||
"At the end of each section in this notebook, we have a cell which contains code for submitting the solutions thus far to the grader. Execute the cell to see your score up to the current section. For all your work to be submitted properly, you must execute those cells at least once.\n",
|
||||
"</div>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Neural Networks\n",
|
||||
"\n",
|
||||
"In the previous exercise, you implemented feedforward propagation for neural networks and used it to predict handwritten digits with the weights we provided. In this exercise, you will implement the backpropagation algorithm to learn the parameters for the neural network.\n",
|
||||
"\n",
|
||||
"We start the exercise by first loading the dataset. "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# training data stored in arrays X, y\n",
|
||||
"data = loadmat(os.path.join('Data', 'ex4data1.mat'))\n",
|
||||
"X, y = data['X'], data['y'].ravel()\n",
|
||||
"\n",
|
||||
"# set the zero digit to 0, rather than its mapped 10 in this dataset\n",
|
||||
"# This is an artifact due to the fact that this dataset was used in \n",
|
||||
"# MATLAB where there is no index 0\n",
|
||||
"y[y == 10] = 0\n",
|
||||
"\n",
|
||||
"# Number of training examples\n",
|
||||
"m = y.size"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 1.1 Visualizing the data\n",
|
||||
"\n",
|
||||
"You will begin by visualizing a subset of the training set, using the function `displayData`, which is the same function we used in Exercise 3. It is provided in the `utils.py` file for this assignment as well. The dataset is also the same one you used in the previous exercise.\n",
|
||||
"\n",
|
||||
"There are 5000 training examples in `ex4data1.mat`, where each training example is a 20 pixel by 20 pixel grayscale image of the digit. Each pixel is represented by a floating point number indicating the grayscale intensity at that location. The 20 by 20 grid of pixels is “unrolled” into a 400-dimensional vector. Each\n",
|
||||
"of these training examples becomes a single row in our data matrix $X$. This gives us a 5000 by 400 matrix $X$ where every row is a training example for a handwritten digit image.\n",
|
||||
"\n",
|
||||
"$$ X = \\begin{bmatrix} - \\left(x^{(1)} \\right)^T - \\\\\n",
|
||||
"- \\left(x^{(2)} \\right)^T - \\\\\n",
|
||||
"\\vdots \\\\\n",
|
||||
"- \\left(x^{(m)} \\right)^T - \\\\\n",
|
||||
"\\end{bmatrix}\n",
|
||||
"$$\n",
|
||||
"\n",
|
||||
"The second part of the training set is a 5000-dimensional vector `y` that contains labels for the training set. \n",
|
||||
"The following cell randomly selects 100 images from the dataset and plots them."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Randomly select 100 data points to display\n",
|
||||
"rand_indices = np.random.choice(m, 100, replace=False)\n",
|
||||
"sel = X[rand_indices, :]\n",
|
||||
"\n",
|
||||
"utils.displayData(sel)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 1.2 Model representation\n",
|
||||
"\n",
|
||||
"Our neural network is shown in the following figure.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"It has 3 layers - an input layer, a hidden layer and an output layer. Recall that our inputs are pixel values\n",
|
||||
"of digit images. Since the images are of size $20 \\times 20$, this gives us 400 input layer units (not counting the extra bias unit which always outputs +1). The training data was loaded into the variables `X` and `y` above.\n",
|
||||
"\n",
|
||||
"You have been provided with a set of network parameters ($\\Theta^{(1)}, \\Theta^{(2)}$) already trained by us. These are stored in `ex4weights.mat` and will be loaded in the next cell of this notebook into `Theta1` and `Theta2`. The parameters have dimensions that are sized for a neural network with 25 units in the second layer and 10 output units (corresponding to the 10 digit classes)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Setup the parameters you will use for this exercise\n",
|
||||
"input_layer_size = 400 # 20x20 Input Images of Digits\n",
|
||||
"hidden_layer_size = 25 # 25 hidden units\n",
|
||||
"num_labels = 10 # 10 labels, from 0 to 9\n",
|
||||
"\n",
|
||||
"# Load the weights into variables Theta1 and Theta2\n",
|
||||
"weights = loadmat(os.path.join('Data', 'ex4weights.mat'))\n",
|
||||
"\n",
|
||||
"# Theta1 has size 25 x 401\n",
|
||||
"# Theta2 has size 10 x 26\n",
|
||||
"Theta1, Theta2 = weights['Theta1'], weights['Theta2']\n",
|
||||
"\n",
|
||||
"# swap first and last columns of Theta2, due to legacy from MATLAB indexing, \n",
|
||||
"# since the weight file ex3weights.mat was saved based on MATLAB indexing\n",
|
||||
"Theta2 = np.roll(Theta2, 1, axis=0)\n",
|
||||
"\n",
|
||||
"# Unroll parameters \n",
|
||||
"nn_params = np.concatenate([Theta1.ravel(), Theta2.ravel()])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a id=\"section1\"></a>\n",
|
||||
"### 1.3 Feedforward and cost function\n",
|
||||
"\n",
|
||||
"Now you will implement the cost function and gradient for the neural network. First, complete the code for the function `nnCostFunction` in the next cell to return the cost.\n",
|
||||
"\n",
|
||||
"Recall that the cost function for the neural network (without regularization) is:\n",
|
||||
"\n",
|
||||
"$$ J(\\theta) = \\frac{1}{m} \\sum_{i=1}^{m}\\sum_{k=1}^{K} \\left[ - y_k^{(i)} \\log \\left( \\left( h_\\theta \\left( x^{(i)} \\right) \\right)_k \\right) - \\left( 1 - y_k^{(i)} \\right) \\log \\left( 1 - \\left( h_\\theta \\left( x^{(i)} \\right) \\right)_k \\right) \\right]$$\n",
|
||||
"\n",
|
||||
"where $h_\\theta \\left( x^{(i)} \\right)$ is computed as shown in the neural network figure above, and K = 10 is the total number of possible labels. Note that $h_\\theta(x^{(i)})_k = a_k^{(3)}$ is the activation (output\n",
|
||||
"value) of the $k^{th}$ output unit. Also, recall that whereas the original labels (in the variable y) were 0, 1, ..., 9, for the purpose of training a neural network, we need to encode the labels as vectors containing only values 0 or 1, so that\n",
|
||||
"\n",
|
||||
"$$ y = \n",
|
||||
"\\begin{bmatrix} 1 \\\\ 0 \\\\ 0 \\\\\\vdots \\\\ 0 \\end{bmatrix}, \\quad\n",
|
||||
"\\begin{bmatrix} 0 \\\\ 1 \\\\ 0 \\\\ \\vdots \\\\ 0 \\end{bmatrix}, \\quad \\cdots \\quad \\text{or} \\qquad\n",
|
||||
"\\begin{bmatrix} 0 \\\\ 0 \\\\ 0 \\\\ \\vdots \\\\ 1 \\end{bmatrix}.\n",
|
||||
"$$\n",
|
||||
"\n",
|
||||
"For example, if $x^{(i)}$ is an image of the digit 5, then the corresponding $y^{(i)}$ (that you should use with the cost function) should be a 10-dimensional vector with $y_5 = 1$, and the other elements equal to 0.\n",
|
||||
"\n",
|
||||
"You should implement the feedforward computation that computes $h_\\theta(x^{(i)})$ for every example $i$ and sum the cost over all examples. **Your code should also work for a dataset of any size, with any number of labels** (you can assume that there are always at least $K \\ge 3$ labels).\n",
|
||||
"\n",
|
||||
"<div class=\"alert alert-box alert-warning\">\n",
|
||||
"**Implementation Note:** The matrix $X$ contains the examples in rows (i.e., X[i,:] is the i-th training example $x^{(i)}$, expressed as a $n \\times 1$ vector.) When you complete the code in `nnCostFunction`, you will need to add the column of 1’s to the X matrix. The parameters for each unit in the neural network is represented in Theta1 and Theta2 as one row. Specifically, the first row of Theta1 corresponds to the first hidden unit in the second layer. You can use a for-loop over the examples to compute the cost.\n",
|
||||
"</div>\n",
|
||||
"<a id=\"nnCostFunction\"></a>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def nnCostFunction(nn_params,\n",
|
||||
" input_layer_size,\n",
|
||||
" hidden_layer_size,\n",
|
||||
" num_labels,\n",
|
||||
" X, y, lambda_=0.0):\n",
|
||||
" \"\"\"\n",
|
||||
" Implements the neural network cost function and gradient for a two layer neural \n",
|
||||
" network which performs classification. \n",
|
||||
" \n",
|
||||
" Parameters\n",
|
||||
" ----------\n",
|
||||
" nn_params : array_like\n",
|
||||
" The parameters for the neural network which are \"unrolled\" into \n",
|
||||
" a vector. This needs to be converted back into the weight matrices Theta1\n",
|
||||
" and Theta2.\n",
|
||||
" \n",
|
||||
" input_layer_size : int\n",
|
||||
" Number of features for the input layer. \n",
|
||||
" \n",
|
||||
" hidden_layer_size : int\n",
|
||||
" Number of hidden units in the second layer.\n",
|
||||
" \n",
|
||||
" num_labels : int\n",
|
||||
" Total number of labels, or equivalently number of units in output layer. \n",
|
||||
" \n",
|
||||
" X : array_like\n",
|
||||
" Input dataset. A matrix of shape (m x input_layer_size).\n",
|
||||
" \n",
|
||||
" y : array_like\n",
|
||||
" Dataset labels. A vector of shape (m,).\n",
|
||||
" \n",
|
||||
" lambda_ : float, optional\n",
|
||||
" Regularization parameter.\n",
|
||||
" \n",
|
||||
" Returns\n",
|
||||
" -------\n",
|
||||
" J : float\n",
|
||||
" The computed value for the cost function at the current weight values.\n",
|
||||
" \n",
|
||||
" grad : array_like\n",
|
||||
" An \"unrolled\" vector of the partial derivatives of the concatenatation of\n",
|
||||
" neural network weights Theta1 and Theta2.\n",
|
||||
" \n",
|
||||
" Instructions\n",
|
||||
" ------------\n",
|
||||
" You should complete the code by working through the following parts.\n",
|
||||
" \n",
|
||||
" - Part 1: Feedforward the neural network and return the cost in the \n",
|
||||
" variable J. After implementing Part 1, you can verify that your\n",
|
||||
" cost function computation is correct by verifying the cost\n",
|
||||
" computed in the following cell.\n",
|
||||
" \n",
|
||||
" - Part 2: Implement the backpropagation algorithm to compute the gradients\n",
|
||||
" Theta1_grad and Theta2_grad. You should return the partial derivatives of\n",
|
||||
" the cost function with respect to Theta1 and Theta2 in Theta1_grad and\n",
|
||||
" Theta2_grad, respectively. After implementing Part 2, you can check\n",
|
||||
" that your implementation is correct by running checkNNGradients provided\n",
|
||||
" in the utils.py module.\n",
|
||||
" \n",
|
||||
" Note: The vector y passed into the function is a vector of labels\n",
|
||||
" containing values from 0..K-1. You need to map this vector into a \n",
|
||||
" binary vector of 1's and 0's to be used with the neural network\n",
|
||||
" cost function.\n",
|
||||
" \n",
|
||||
" Hint: We recommend implementing backpropagation using a for-loop\n",
|
||||
" over the training examples if you are implementing it for the \n",
|
||||
" first time.\n",
|
||||
" \n",
|
||||
" - Part 3: Implement regularization with the cost function and gradients.\n",
|
||||
" \n",
|
||||
" Hint: You can implement this around the code for\n",
|
||||
" backpropagation. That is, you can compute the gradients for\n",
|
||||
" the regularization separately and then add them to Theta1_grad\n",
|
||||
" and Theta2_grad from Part 2.\n",
|
||||
" \n",
|
||||
" Note \n",
|
||||
" ----\n",
|
||||
" We have provided an implementation for the sigmoid function in the file \n",
|
||||
" `utils.py` accompanying this assignment.\n",
|
||||
" \"\"\"\n",
|
||||
" # Reshape nn_params back into the parameters Theta1 and Theta2, the weight matrices\n",
|
||||
" # for our 2 layer neural network\n",
|
||||
" Theta1 = np.reshape(nn_params[:hidden_layer_size * (input_layer_size + 1)],\n",
|
||||
" (hidden_layer_size, (input_layer_size + 1)))\n",
|
||||
"\n",
|
||||
" Theta2 = np.reshape(nn_params[(hidden_layer_size * (input_layer_size + 1)):],\n",
|
||||
" (num_labels, (hidden_layer_size + 1)))\n",
|
||||
"\n",
|
||||
" # Setup some useful variables\n",
|
||||
" m = y.size\n",
|
||||
" \n",
|
||||
" # You need to return the following variables correctly \n",
|
||||
" J = 0\n",
|
||||
" Theta1_grad = np.zeros(Theta1.shape)\n",
|
||||
" Theta2_grad = np.zeros(Theta2.shape)\n",
|
||||
"\n",
|
||||
" # ====================== YOUR CODE HERE ======================\n",
|
||||
"\n",
|
||||
" \n",
|
||||
" \n",
|
||||
" # ================================================================\n",
|
||||
" # Unroll gradients\n",
|
||||
" # grad = np.concatenate([Theta1_grad.ravel(order=order), Theta2_grad.ravel(order=order)])\n",
|
||||
" grad = np.concatenate([Theta1_grad.ravel(), Theta2_grad.ravel()])\n",
|
||||
"\n",
|
||||
" return J, grad"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<div class=\"alert alert-box alert-warning\">\n",
|
||||
"Use the following links to go back to the different parts of this exercise that require to modify the function `nnCostFunction`.<br>\n",
|
||||
"\n",
|
||||
"Back to:\n",
|
||||
"- [Feedforward and cost function](#section1)\n",
|
||||
"- [Regularized cost](#section2)\n",
|
||||
"- [Neural Network Gradient (Backpropagation)](#section4)\n",
|
||||
"- [Regularized Gradient](#section5)\n",
|
||||
"</div>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Once you are done, call your `nnCostFunction` using the loaded set of parameters for `Theta1` and `Theta2`. You should see that the cost is about 0.287629."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"lambda_ = 0\n",
|
||||
"J, _ = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,\n",
|
||||
" num_labels, X, y, lambda_)\n",
|
||||
"print('Cost at parameters (loaded from ex4weights): %.6f ' % J)\n",
|
||||
"print('The cost should be about : 0.287629.')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"*You should now submit your solutions.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"grader = utils.Grader()\n",
|
||||
"grader[1] = nnCostFunction\n",
|
||||
"grader.grade()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a id=\"section2\"></a>\n",
|
||||
"### 1.4 Regularized cost function\n",
|
||||
"\n",
|
||||
"The cost function for neural networks with regularization is given by:\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"$$ J(\\theta) = \\frac{1}{m} \\sum_{i=1}^{m}\\sum_{k=1}^{K} \\left[ - y_k^{(i)} \\log \\left( \\left( h_\\theta \\left( x^{(i)} \\right) \\right)_k \\right) - \\left( 1 - y_k^{(i)} \\right) \\log \\left( 1 - \\left( h_\\theta \\left( x^{(i)} \\right) \\right)_k \\right) \\right] + \\frac{\\lambda}{2 m} \\left[ \\sum_{j=1}^{25} \\sum_{k=1}^{400} \\left( \\Theta_{j,k}^{(1)} \\right)^2 + \\sum_{j=1}^{10} \\sum_{k=1}^{25} \\left( \\Theta_{j,k}^{(2)} \\right)^2 \\right] $$\n",
|
||||
"\n",
|
||||
"You can assume that the neural network will only have 3 layers - an input layer, a hidden layer and an output layer. However, your code should work for any number of input units, hidden units and outputs units. While we\n",
|
||||
"have explicitly listed the indices above for $\\Theta^{(1)}$ and $\\Theta^{(2)}$ for clarity, do note that your code should in general work with $\\Theta^{(1)}$ and $\\Theta^{(2)}$ of any size. Note that you should not be regularizing the terms that correspond to the bias. For the matrices `Theta1` and `Theta2`, this corresponds to the first column of each matrix. You should now add regularization to your cost function. Notice that you can first compute the unregularized cost function $J$ using your existing `nnCostFunction` and then later add the cost for the regularization terms.\n",
|
||||
"\n",
|
||||
"[Click here to go back to `nnCostFunction` for editing.](#nnCostFunction)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Once you are done, the next cell will call your `nnCostFunction` using the loaded set of parameters for `Theta1` and `Theta2`, and $\\lambda = 1$. You should see that the cost is about 0.383770."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Weight regularization parameter (we set this to 1 here).\n",
|
||||
"lambda_ = 1\n",
|
||||
"J, _ = nnCostFunction(nn_params, input_layer_size, hidden_layer_size,\n",
|
||||
" num_labels, X, y, lambda_)\n",
|
||||
"\n",
|
||||
"print('Cost at parameters (loaded from ex4weights): %.6f' % J)\n",
|
||||
"print('This value should be about : 0.383770.')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"*You should now submit your solutions.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"grader[2] = nnCostFunction\n",
|
||||
"grader.grade()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2 Backpropagation\n",
|
||||
"\n",
|
||||
"In this part of the exercise, you will implement the backpropagation algorithm to compute the gradient for the neural network cost function. You will need to update the function `nnCostFunction` so that it returns an appropriate value for `grad`. Once you have computed the gradient, you will be able to train the neural network by minimizing the cost function $J(\\theta)$ using an advanced optimizer such as `scipy`'s `optimize.minimize`.\n",
|
||||
"You will first implement the backpropagation algorithm to compute the gradients for the parameters for the (unregularized) neural network. After you have verified that your gradient computation for the unregularized case is correct, you will implement the gradient for the regularized neural network."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a id=\"section3\"></a>\n",
|
||||
"### 2.1 Sigmoid Gradient\n",
|
||||
"\n",
|
||||
"To help you get started with this part of the exercise, you will first implement\n",
|
||||
"the sigmoid gradient function. The gradient for the sigmoid function can be\n",
|
||||
"computed as\n",
|
||||
"\n",
|
||||
"$$ g'(z) = \\frac{d}{dz} g(z) = g(z)\\left(1-g(z)\\right) $$\n",
|
||||
"\n",
|
||||
"where\n",
|
||||
"\n",
|
||||
"$$ \\text{sigmoid}(z) = g(z) = \\frac{1}{1 + e^{-z}} $$\n",
|
||||
"\n",
|
||||
"Now complete the implementation of `sigmoidGradient` in the next cell.\n",
|
||||
"<a id=\"sigmoidGradient\"></a>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def sigmoidGradient(z):\n",
|
||||
" \"\"\"\n",
|
||||
" Computes the gradient of the sigmoid function evaluated at z. \n",
|
||||
" This should work regardless if z is a matrix or a vector. \n",
|
||||
" In particular, if z is a vector or matrix, you should return\n",
|
||||
" the gradient for each element.\n",
|
||||
" \n",
|
||||
" Parameters\n",
|
||||
" ----------\n",
|
||||
" z : array_like\n",
|
||||
" A vector or matrix as input to the sigmoid function. \n",
|
||||
" \n",
|
||||
" Returns\n",
|
||||
" --------\n",
|
||||
" g : array_like\n",
|
||||
" Gradient of the sigmoid function. Has the same shape as z. \n",
|
||||
" \n",
|
||||
" Instructions\n",
|
||||
" ------------\n",
|
||||
" Compute the gradient of the sigmoid function evaluated at\n",
|
||||
" each value of z (z can be a matrix, vector or scalar).\n",
|
||||
" \n",
|
||||
" Note\n",
|
||||
" ----\n",
|
||||
" We have provided an implementation of the sigmoid function \n",
|
||||
" in `utils.py` file accompanying this assignment.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" g = np.zeros(z.shape)\n",
|
||||
"\n",
|
||||
" # ====================== YOUR CODE HERE ======================\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" # =============================================================\n",
|
||||
" return g"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"When you are done, the following cell call `sigmoidGradient` on a given vector `z`. Try testing a few values by calling `sigmoidGradient(z)`. For large values (both positive and negative) of z, the gradient should be close to 0. When $z = 0$, the gradient should be exactly 0.25. Your code should also work with vectors and matrices. For a matrix, your function should perform the sigmoid gradient function on every element."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"z = np.array([-1, -0.5, 0, 0.5, 1])\n",
|
||||
"g = sigmoidGradient(z)\n",
|
||||
"print('Sigmoid gradient evaluated at [-1 -0.5 0 0.5 1]:\\n ')\n",
|
||||
"print(g)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"*You should now submit your solutions.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"grader[3] = sigmoidGradient\n",
|
||||
"grader.grade()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 2.2 Random Initialization\n",
|
||||
"\n",
|
||||
"When training neural networks, it is important to randomly initialize the parameters for symmetry breaking. One effective strategy for random initialization is to randomly select values for $\\Theta^{(l)}$ uniformly in the range $[-\\epsilon_{init}, \\epsilon_{init}]$. You should use $\\epsilon_{init} = 0.12$. This range of values ensures that the parameters are kept small and makes the learning more efficient.\n",
|
||||
"\n",
|
||||
"<div class=\"alert alert-box alert-warning\">\n",
|
||||
"One effective strategy for choosing $\\epsilon_{init}$ is to base it on the number of units in the network. A good choice of $\\epsilon_{init}$ is $\\epsilon_{init} = \\frac{\\sqrt{6}}{\\sqrt{L_{in} + L_{out}}}$ where $L_{in} = s_l$ and $L_{out} = s_{l+1}$ are the number of units in the layers adjacent to $\\Theta^{l}$.\n",
|
||||
"</div>\n",
|
||||
"\n",
|
||||
"Your job is to complete the function `randInitializeWeights` to initialize the weights for $\\Theta$. Modify the function by filling in the following code:\n",
|
||||
"\n",
|
||||
"```python\n",
|
||||
"# Randomly initialize the weights to small values\n",
|
||||
"W = np.random.rand(L_out, 1 + L_in) * 2 * epsilon_init - epsilon_init\n",
|
||||
"```\n",
|
||||
"Note that we give the function an argument for $\\epsilon$ with default value `epsilon_init = 0.12`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def randInitializeWeights(L_in, L_out, epsilon_init=0.12):\n",
|
||||
" \"\"\"\n",
|
||||
" Randomly initialize the weights of a layer in a neural network.\n",
|
||||
" \n",
|
||||
" Parameters\n",
|
||||
" ----------\n",
|
||||
" L_in : int\n",
|
||||
" Number of incomming connections.\n",
|
||||
" \n",
|
||||
" L_out : int\n",
|
||||
" Number of outgoing connections. \n",
|
||||
" \n",
|
||||
" epsilon_init : float, optional\n",
|
||||
" Range of values which the weight can take from a uniform \n",
|
||||
" distribution.\n",
|
||||
" \n",
|
||||
" Returns\n",
|
||||
" -------\n",
|
||||
" W : array_like\n",
|
||||
" The weight initialiatized to random values. Note that W should\n",
|
||||
" be set to a matrix of size(L_out, 1 + L_in) as\n",
|
||||
" the first column of W handles the \"bias\" terms.\n",
|
||||
" \n",
|
||||
" Instructions\n",
|
||||
" ------------\n",
|
||||
" Initialize W randomly so that we break the symmetry while training\n",
|
||||
" the neural network. Note that the first column of W corresponds \n",
|
||||
" to the parameters for the bias unit.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" # You need to return the following variables correctly \n",
|
||||
" W = np.zeros((L_out, 1 + L_in))\n",
|
||||
"\n",
|
||||
" # ====================== YOUR CODE HERE ======================\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" # ============================================================\n",
|
||||
" return W"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"*You do not need to submit any code for this part of the exercise.*\n",
|
||||
"\n",
|
||||
"Execute the following cell to initialize the weights for the 2 layers in the neural network using the `randInitializeWeights` function."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print('Initializing Neural Network Parameters ...')\n",
|
||||
"\n",
|
||||
"initial_Theta1 = randInitializeWeights(input_layer_size, hidden_layer_size)\n",
|
||||
"initial_Theta2 = randInitializeWeights(hidden_layer_size, num_labels)\n",
|
||||
"\n",
|
||||
"# Unroll parameters\n",
|
||||
"initial_nn_params = np.concatenate([initial_Theta1.ravel(), initial_Theta2.ravel()], axis=0)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a id=\"section4\"></a>\n",
|
||||
"### 2.4 Backpropagation\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Now, you will implement the backpropagation algorithm. Recall that the intuition behind the backpropagation algorithm is as follows. Given a training example $(x^{(t)}, y^{(t)})$, we will first run a “forward pass” to compute all the activations throughout the network, including the output value of the hypothesis $h_\\theta(x)$. Then, for each node $j$ in layer $l$, we would like to compute an “error term” $\\delta_j^{(l)}$ that measures how much that node was “responsible” for any errors in our output.\n",
|
||||
"\n",
|
||||
"For an output node, we can directly measure the difference between the network’s activation and the true target value, and use that to define $\\delta_j^{(3)}$ (since layer 3 is the output layer). For the hidden units, you will compute $\\delta_j^{(l)}$ based on a weighted average of the error terms of the nodes in layer $(l+1)$. In detail, here is the backpropagation algorithm (also depicted in the figure above). You should implement steps 1 to 4 in a loop that processes one example at a time. Concretely, you should implement a for-loop `for t in range(m)` and place steps 1-4 below inside the for-loop, with the $t^{th}$ iteration performing the calculation on the $t^{th}$ training example $(x^{(t)}, y^{(t)})$. Step 5 will divide the accumulated gradients by $m$ to obtain the gradients for the neural network cost function.\n",
|
||||
"\n",
|
||||
"1. Set the input layer’s values $(a^{(1)})$ to the $t^{th }$training example $x^{(t)}$. Perform a feedforward pass, computing the activations $(z^{(2)}, a^{(2)}, z^{(3)}, a^{(3)})$ for layers 2 and 3. Note that you need to add a `+1` term to ensure that the vectors of activations for layers $a^{(1)}$ and $a^{(2)}$ also include the bias unit. In `numpy`, if a 1 is a column matrix, adding one corresponds to `a_1 = np.concatenate([np.ones((m, 1)), a_1], axis=1)`.\n",
|
||||
"\n",
|
||||
"1. For each output unit $k$ in layer 3 (the output layer), set \n",
|
||||
"$$\\delta_k^{(3)} = \\left(a_k^{(3)} - y_k \\right)$$\n",
|
||||
"where $y_k \\in \\{0, 1\\}$ indicates whether the current training example belongs to class $k$ $(y_k = 1)$, or if it belongs to a different class $(y_k = 0)$. You may find logical arrays helpful for this task (explained in the previous programming exercise).\n",
|
||||
"\n",
|
||||
"1. For the hidden layer $l = 2$, set \n",
|
||||
"$$ \\delta^{(2)} = \\left( \\Theta^{(2)} \\right)^T \\delta^{(3)} * g'\\left(z^{(2)} \\right)$$\n",
|
||||
"Note that the symbol $*$ performs element wise multiplication in `numpy`.\n",
|
||||
"\n",
|
||||
"1. Accumulate the gradient from this example using the following formula. Note that you should skip or remove $\\delta_0^{(2)}$. In `numpy`, removing $\\delta_0^{(2)}$ corresponds to `delta_2 = delta_2[1:]`.\n",
|
||||
"\n",
|
||||
"1. Obtain the (unregularized) gradient for the neural network cost function by dividing the accumulated gradients by $\\frac{1}{m}$:\n",
|
||||
"$$ \\frac{\\partial}{\\partial \\Theta_{ij}^{(l)}} J(\\Theta) = D_{ij}^{(l)} = \\frac{1}{m} \\Delta_{ij}^{(l)}$$\n",
|
||||
"\n",
|
||||
"<div class=\"alert alert-box alert-warning\">\n",
|
||||
"**Python/Numpy tip**: You should implement the backpropagation algorithm only after you have successfully completed the feedforward and cost functions. While implementing the backpropagation alogrithm, it is often useful to use the `shape` function to print out the shapes of the variables you are working with if you run into dimension mismatch errors.\n",
|
||||
"</div>\n",
|
||||
"\n",
|
||||
"[Click here to go back and update the function `nnCostFunction` with the backpropagation algorithm](#nnCostFunction)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"After you have implemented the backpropagation algorithm, we will proceed to run gradient checking on your implementation. The gradient check will allow you to increase your confidence that your code is\n",
|
||||
"computing the gradients correctly.\n",
|
||||
"\n",
|
||||
"### 2.4 Gradient checking \n",
|
||||
"\n",
|
||||
"In your neural network, you are minimizing the cost function $J(\\Theta)$. To perform gradient checking on your parameters, you can imagine “unrolling” the parameters $\\Theta^{(1)}$, $\\Theta^{(2)}$ into a long vector $\\theta$. By doing so, you can think of the cost function being $J(\\Theta)$ instead and use the following gradient checking procedure.\n",
|
||||
"\n",
|
||||
"Suppose you have a function $f_i(\\theta)$ that purportedly computes $\\frac{\\partial}{\\partial \\theta_i} J(\\theta)$; you’d like to check if $f_i$ is outputting correct derivative values.\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\text{Let } \\theta^{(i+)} = \\theta + \\begin{bmatrix} 0 \\\\ 0 \\\\ \\vdots \\\\ \\epsilon \\\\ \\vdots \\\\ 0 \\end{bmatrix}\n",
|
||||
"\\quad \\text{and} \\quad \\theta^{(i-)} = \\theta - \\begin{bmatrix} 0 \\\\ 0 \\\\ \\vdots \\\\ \\epsilon \\\\ \\vdots \\\\ 0 \\end{bmatrix}\n",
|
||||
"$$\n",
|
||||
"\n",
|
||||
"So, $\\theta^{(i+)}$ is the same as $\\theta$, except its $i^{th}$ element has been incremented by $\\epsilon$. Similarly, $\\theta^{(i−)}$ is the corresponding vector with the $i^{th}$ element decreased by $\\epsilon$. You can now numerically verify $f_i(\\theta)$’s correctness by checking, for each $i$, that:\n",
|
||||
"\n",
|
||||
"$$ f_i\\left( \\theta \\right) \\approx \\frac{J\\left( \\theta^{(i+)}\\right) - J\\left( \\theta^{(i-)} \\right)}{2\\epsilon} $$\n",
|
||||
"\n",
|
||||
"The degree to which these two values should approximate each other will depend on the details of $J$. But assuming $\\epsilon = 10^{-4}$, you’ll usually find that the left- and right-hand sides of the above will agree to at least 4 significant digits (and often many more).\n",
|
||||
"\n",
|
||||
"We have implemented the function to compute the numerical gradient for you in `computeNumericalGradient` (within the file `utils.py`). While you are not required to modify the file, we highly encourage you to take a look at the code to understand how it works.\n",
|
||||
"\n",
|
||||
"In the next cell we will run the provided function `checkNNGradients` which will create a small neural network and dataset that will be used for checking your gradients. If your backpropagation implementation is correct,\n",
|
||||
"you should see a relative difference that is less than 1e-9.\n",
|
||||
"\n",
|
||||
"<div class=\"alert alert-box alert-success\">\n",
|
||||
"**Practical Tip**: When performing gradient checking, it is much more efficient to use a small neural network with a relatively small number of input units and hidden units, thus having a relatively small number\n",
|
||||
"of parameters. Each dimension of $\\theta$ requires two evaluations of the cost function and this can be expensive. In the function `checkNNGradients`, our code creates a small random model and dataset which is used with `computeNumericalGradient` for gradient checking. Furthermore, after you are confident that your gradient computations are correct, you should turn off gradient checking before running your learning algorithm.\n",
|
||||
"</div>\n",
|
||||
"\n",
|
||||
"<div class=\"alert alert-box alert-success\">\n",
|
||||
"**Practical Tip:** Gradient checking works for any function where you are computing the cost and the gradient. Concretely, you can use the same `computeNumericalGradient` function to check if your gradient implementations for the other exercises are correct too (e.g., logistic regression’s cost function).\n",
|
||||
"</div>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"utils.checkNNGradients(nnCostFunction)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"*Once your cost function passes the gradient check for the (unregularized) neural network cost function, you should submit the neural network gradient function (backpropagation).*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"grader[4] = nnCostFunction\n",
|
||||
"grader.grade()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a id=\"section5\"></a>\n",
|
||||
"### 2.5 Regularized Neural Network\n",
|
||||
"\n",
|
||||
"After you have successfully implemented the backpropagation algorithm, you will add regularization to the gradient. To account for regularization, it turns out that you can add this as an additional term *after* computing the gradients using backpropagation.\n",
|
||||
"\n",
|
||||
"Specifically, after you have computed $\\Delta_{ij}^{(l)}$ using backpropagation, you should add regularization using\n",
|
||||
"\n",
|
||||
"$$ \\begin{align} \n",
|
||||
"& \\frac{\\partial}{\\partial \\Theta_{ij}^{(l)}} J(\\Theta) = D_{ij}^{(l)} = \\frac{1}{m} \\Delta_{ij}^{(l)} & \\qquad \\text{for } j = 0 \\\\\n",
|
||||
"& \\frac{\\partial}{\\partial \\Theta_{ij}^{(l)}} J(\\Theta) = D_{ij}^{(l)} = \\frac{1}{m} \\Delta_{ij}^{(l)} + \\frac{\\lambda}{m} \\Theta_{ij}^{(l)} & \\qquad \\text{for } j \\ge 1\n",
|
||||
"\\end{align}\n",
|
||||
"$$\n",
|
||||
"\n",
|
||||
"Note that you should *not* be regularizing the first column of $\\Theta^{(l)}$ which is used for the bias term. Furthermore, in the parameters $\\Theta_{ij}^{(l)}$, $i$ is indexed starting from 1, and $j$ is indexed starting from 0. Thus, \n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\Theta^{(l)} = \\begin{bmatrix}\n",
|
||||
"\\Theta_{1,0}^{(i)} & \\Theta_{1,1}^{(l)} & \\cdots \\\\\n",
|
||||
"\\Theta_{2,0}^{(i)} & \\Theta_{2,1}^{(l)} & \\cdots \\\\\n",
|
||||
"\\vdots & ~ & \\ddots\n",
|
||||
"\\end{bmatrix}\n",
|
||||
"$$\n",
|
||||
"\n",
|
||||
"[Now modify your code that computes grad in `nnCostFunction` to account for regularization.](#nnCostFunction)\n",
|
||||
"\n",
|
||||
"After you are done, the following cell runs gradient checking on your implementation. If your code is correct, you should expect to see a relative difference that is less than 1e-9."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Check gradients by running checkNNGradients\n",
|
||||
"lambda_ = 3\n",
|
||||
"utils.checkNNGradients(nnCostFunction, lambda_)\n",
|
||||
"\n",
|
||||
"# Also output the costFunction debugging values\n",
|
||||
"debug_J, _ = nnCostFunction(nn_params, input_layer_size,\n",
|
||||
" hidden_layer_size, num_labels, X, y, lambda_)\n",
|
||||
"\n",
|
||||
"print('\\n\\nCost at (fixed) debugging parameters (w/ lambda = %f): %f ' % (lambda_, debug_J))\n",
|
||||
"print('(for lambda = 3, this value should be about 0.576051)')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"grader[5] = nnCostFunction\n",
|
||||
"grader.grade()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 2.6 Learning parameters using `scipy.optimize.minimize`\n",
|
||||
"\n",
|
||||
"After you have successfully implemented the neural network cost function\n",
|
||||
"and gradient computation, the next step we will use `scipy`'s minimization to learn a good set parameters."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# After you have completed the assignment, change the maxiter to a larger\n",
|
||||
"# value to see how more training helps.\n",
|
||||
"options= {'maxiter': 100}\n",
|
||||
"\n",
|
||||
"# You should also try different values of lambda\n",
|
||||
"lambda_ = 1\n",
|
||||
"\n",
|
||||
"# Create \"short hand\" for the cost function to be minimized\n",
|
||||
"costFunction = lambda p: nnCostFunction(p, input_layer_size,\n",
|
||||
" hidden_layer_size,\n",
|
||||
" num_labels, X, y, lambda_)\n",
|
||||
"\n",
|
||||
"# Now, costFunction is a function that takes in only one argument\n",
|
||||
"# (the neural network parameters)\n",
|
||||
"res = optimize.minimize(costFunction,\n",
|
||||
" initial_nn_params,\n",
|
||||
" jac=True,\n",
|
||||
" method='TNC',\n",
|
||||
" options=options)\n",
|
||||
"\n",
|
||||
"# get the solution of the optimization\n",
|
||||
"nn_params = res.x\n",
|
||||
" \n",
|
||||
"# Obtain Theta1 and Theta2 back from nn_params\n",
|
||||
"Theta1 = np.reshape(nn_params[:hidden_layer_size * (input_layer_size + 1)],\n",
|
||||
" (hidden_layer_size, (input_layer_size + 1)))\n",
|
||||
"\n",
|
||||
"Theta2 = np.reshape(nn_params[(hidden_layer_size * (input_layer_size + 1)):],\n",
|
||||
" (num_labels, (hidden_layer_size + 1)))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"After the training completes, we will proceed to report the training accuracy of your classifier by computing the percentage of examples it got correct. If your implementation is correct, you should see a reported\n",
|
||||
"training accuracy of about 95.3% (this may vary by about 1% due to the random initialization). It is possible to get higher training accuracies by training the neural network for more iterations. We encourage you to try\n",
|
||||
"training the neural network for more iterations (e.g., set `maxiter` to 400) and also vary the regularization parameter $\\lambda$. With the right learning settings, it is possible to get the neural network to perfectly fit the training set."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"pred = utils.predict(Theta1, Theta2, X)\n",
|
||||
"print('Training Set Accuracy: %f' % (np.mean(pred == y) * 100))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 3 Visualizing the Hidden Layer\n",
|
||||
"\n",
|
||||
"One way to understand what your neural network is learning is to visualize what the representations captured by the hidden units. Informally, given a particular hidden unit, one way to visualize what it computes is to find an input $x$ that will cause it to activate (that is, to have an activation value \n",
|
||||
"($a_i^{(l)}$) close to 1). For the neural network you trained, notice that the $i^{th}$ row of $\\Theta^{(1)}$ is a 401-dimensional vector that represents the parameter for the $i^{th}$ hidden unit. If we discard the bias term, we get a 400 dimensional vector that represents the weights from each input pixel to the hidden unit.\n",
|
||||
"\n",
|
||||
"Thus, one way to visualize the “representation” captured by the hidden unit is to reshape this 400 dimensional vector into a 20 × 20 image and display it (It turns out that this is equivalent to finding the input that gives the highest activation for the hidden unit, given a “norm” constraint on the input (i.e., $||x||_2 \\le 1$)). \n",
|
||||
"\n",
|
||||
"The next cell does this by using the `displayData` function and it will show you an image with 25 units,\n",
|
||||
"each corresponding to one hidden unit in the network. In your trained network, you should find that the hidden units corresponds roughly to detectors that look for strokes and other patterns in the input."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"utils.displayData(Theta1[:, 1:])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 3.1 Optional (ungraded) exercise\n",
|
||||
"\n",
|
||||
"In this part of the exercise, you will get to try out different learning settings for the neural network to see how the performance of the neural network varies with the regularization parameter $\\lambda$ and number of training steps (the `maxiter` option when using `scipy.optimize.minimize`). Neural networks are very powerful models that can form highly complex decision boundaries. Without regularization, it is possible for a neural network to “overfit” a training set so that it obtains close to 100% accuracy on the training set but does not as well on new examples that it has not seen before. You can set the regularization $\\lambda$ to a smaller value and the `maxiter` parameter to a higher number of iterations to see this for youself."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
226
Exercise4/utils.py
Executable file
226
Exercise4/utils.py
Executable file
@@ -0,0 +1,226 @@
|
||||
import sys
|
||||
import numpy as np
|
||||
from matplotlib import pyplot
|
||||
|
||||
sys.path.append('..')
|
||||
from submission import SubmissionBase
|
||||
|
||||
|
||||
def displayData(X, example_width=None, figsize=(10, 10)):
|
||||
"""
|
||||
Displays 2D data stored in X in a nice grid.
|
||||
"""
|
||||
# Compute rows, cols
|
||||
if X.ndim == 2:
|
||||
m, n = X.shape
|
||||
elif X.ndim == 1:
|
||||
n = X.size
|
||||
m = 1
|
||||
X = X[None] # Promote to a 2 dimensional array
|
||||
else:
|
||||
raise IndexError('Input X should be 1 or 2 dimensional.')
|
||||
|
||||
example_width = example_width or int(np.round(np.sqrt(n)))
|
||||
example_height = n / example_width
|
||||
|
||||
# Compute number of items to display
|
||||
display_rows = int(np.floor(np.sqrt(m)))
|
||||
display_cols = int(np.ceil(m / display_rows))
|
||||
|
||||
fig, ax_array = pyplot.subplots(display_rows, display_cols, figsize=figsize)
|
||||
fig.subplots_adjust(wspace=0.025, hspace=0.025)
|
||||
|
||||
ax_array = [ax_array] if m == 1 else ax_array.ravel()
|
||||
|
||||
for i, ax in enumerate(ax_array):
|
||||
# Display Image
|
||||
h = ax.imshow(X[i].reshape(example_width, example_width, order='F'),
|
||||
cmap='Greys', extent=[0, 1, 0, 1])
|
||||
ax.axis('off')
|
||||
|
||||
|
||||
def predict(Theta1, Theta2, X):
|
||||
"""
|
||||
Predict the label of an input given a trained neural network
|
||||
Outputs the predicted label of X given the trained weights of a neural
|
||||
network(Theta1, Theta2)
|
||||
"""
|
||||
# Useful values
|
||||
m = X.shape[0]
|
||||
num_labels = Theta2.shape[0]
|
||||
|
||||
# You need to return the following variables correctly
|
||||
p = np.zeros(m)
|
||||
h1 = sigmoid(np.dot(np.concatenate([np.ones((m, 1)), X], axis=1), Theta1.T))
|
||||
h2 = sigmoid(np.dot(np.concatenate([np.ones((m, 1)), h1], axis=1), Theta2.T))
|
||||
p = np.argmax(h2, axis=1)
|
||||
return p
|
||||
|
||||
|
||||
def debugInitializeWeights(fan_out, fan_in):
|
||||
"""
|
||||
Initialize the weights of a layer with fan_in incoming connections and fan_out outgoings
|
||||
connections using a fixed strategy. This will help you later in debugging.
|
||||
|
||||
Note that W should be set a matrix of size (1+fan_in, fan_out) as the first row of W handles
|
||||
the "bias" terms.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
fan_out : int
|
||||
The number of outgoing connections.
|
||||
|
||||
fan_in : int
|
||||
The number of incoming connections.
|
||||
|
||||
Returns
|
||||
-------
|
||||
W : array_like (1+fan_in, fan_out)
|
||||
The initialized weights array given the dimensions.
|
||||
"""
|
||||
# Initialize W using "sin". This ensures that W is always of the same values and will be
|
||||
# useful for debugging
|
||||
W = np.sin(np.arange(1, 1 + (1+fan_in)*fan_out))/10.0
|
||||
W = W.reshape(fan_out, 1+fan_in, order='F')
|
||||
return W
|
||||
|
||||
|
||||
def computeNumericalGradient(J, theta, e=1e-4):
|
||||
"""
|
||||
Computes the gradient using "finite differences" and gives us a numerical estimate of the
|
||||
gradient.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
J : func
|
||||
The cost function which will be used to estimate its numerical gradient.
|
||||
|
||||
theta : array_like
|
||||
The one dimensional unrolled network parameters. The numerical gradient is computed at
|
||||
those given parameters.
|
||||
|
||||
e : float (optional)
|
||||
The value to use for epsilon for computing the finite difference.
|
||||
|
||||
Notes
|
||||
-----
|
||||
The following code implements numerical gradient checking, and
|
||||
returns the numerical gradient. It sets `numgrad[i]` to (a numerical
|
||||
approximation of) the partial derivative of J with respect to the
|
||||
i-th input argument, evaluated at theta. (i.e., `numgrad[i]` should
|
||||
be the (approximately) the partial derivative of J with respect
|
||||
to theta[i].)
|
||||
"""
|
||||
numgrad = np.zeros(theta.shape)
|
||||
perturb = np.diag(e * np.ones(theta.shape))
|
||||
for i in range(theta.size):
|
||||
loss1, _ = J(theta - perturb[:, i])
|
||||
loss2, _ = J(theta + perturb[:, i])
|
||||
numgrad[i] = (loss2 - loss1)/(2*e)
|
||||
return numgrad
|
||||
|
||||
|
||||
def checkNNGradients(nnCostFunction, lambda_=0):
|
||||
"""
|
||||
Creates a small neural network to check the backpropagation gradients. It will output the
|
||||
analytical gradients produced by your backprop code and the numerical gradients
|
||||
(computed using computeNumericalGradient). These two gradient computations should result in
|
||||
very similar values.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
nnCostFunction : func
|
||||
A reference to the cost function implemented by the student.
|
||||
|
||||
lambda_ : float (optional)
|
||||
The regularization parameter value.
|
||||
"""
|
||||
input_layer_size = 3
|
||||
hidden_layer_size = 5
|
||||
num_labels = 3
|
||||
m = 5
|
||||
|
||||
# We generate some 'random' test data
|
||||
Theta1 = debugInitializeWeights(hidden_layer_size, input_layer_size)
|
||||
Theta2 = debugInitializeWeights(num_labels, hidden_layer_size)
|
||||
|
||||
# Reusing debugInitializeWeights to generate X
|
||||
X = debugInitializeWeights(m, input_layer_size - 1)
|
||||
y = np.arange(1, 1+m) % num_labels
|
||||
# print(y)
|
||||
# Unroll parameters
|
||||
nn_params = np.concatenate([Theta1.ravel(), Theta2.ravel()])
|
||||
|
||||
# short hand for cost function
|
||||
costFunc = lambda p: nnCostFunction(p, input_layer_size, hidden_layer_size,
|
||||
num_labels, X, y, lambda_)
|
||||
cost, grad = costFunc(nn_params)
|
||||
numgrad = computeNumericalGradient(costFunc, nn_params)
|
||||
|
||||
# Visually examine the two gradient computations.The two columns you get should be very similar.
|
||||
print(np.stack([numgrad, grad], axis=1))
|
||||
print('The above two columns you get should be very similar.')
|
||||
print('(Left-Your Numerical Gradient, Right-Analytical Gradient)\n')
|
||||
|
||||
# Evaluate the norm of the difference between two the solutions. If you have a correct
|
||||
# implementation, and assuming you used e = 0.0001 in computeNumericalGradient, then diff
|
||||
# should be less than 1e-9.
|
||||
diff = np.linalg.norm(numgrad - grad)/np.linalg.norm(numgrad + grad)
|
||||
|
||||
print('If your backpropagation implementation is correct, then \n'
|
||||
'the relative difference will be small (less than 1e-9). \n'
|
||||
'Relative Difference: %g' % diff)
|
||||
|
||||
|
||||
def sigmoid(z):
|
||||
"""
|
||||
Computes the sigmoid of z.
|
||||
"""
|
||||
return 1.0 / (1.0 + np.exp(-z))
|
||||
|
||||
|
||||
class Grader(SubmissionBase):
|
||||
X = np.reshape(3 * np.sin(np.arange(1, 31)), (3, 10), order='F')
|
||||
Xm = np.reshape(np.sin(np.arange(1, 33)), (16, 2), order='F') / 5
|
||||
ym = np.arange(1, 17) % 4
|
||||
t1 = np.sin(np.reshape(np.arange(1, 25, 2), (4, 3), order='F'))
|
||||
t2 = np.cos(np.reshape(np.arange(1, 41, 2), (4, 5), order='F'))
|
||||
t = np.concatenate([t1.ravel(), t2.ravel()], axis=0)
|
||||
|
||||
def __init__(self):
|
||||
part_names = ['Feedforward and Cost Function',
|
||||
'Regularized Cost Function',
|
||||
'Sigmoid Gradient',
|
||||
'Neural Network Gradient (Backpropagation)',
|
||||
'Regularized Gradient']
|
||||
super().__init__('neural-network-learning', part_names)
|
||||
|
||||
def __iter__(self):
|
||||
for part_id in range(1, 6):
|
||||
try:
|
||||
func = self.functions[part_id]
|
||||
|
||||
# Each part has different expected arguments/different function
|
||||
if part_id == 1:
|
||||
res = func(self.t, 2, 4, 4, self.Xm, self.ym, 0)[0]
|
||||
elif part_id == 2:
|
||||
res = func(self.t, 2, 4, 4, self.Xm, self.ym, 1.5)
|
||||
elif part_id == 3:
|
||||
res = func(self.X, )
|
||||
elif part_id == 4:
|
||||
J, grad = func(self.t, 2, 4, 4, self.Xm, self.ym, 0)
|
||||
grad1 = np.reshape(grad[:12], (4, 3))
|
||||
grad2 = np.reshape(grad[12:], (4, 5))
|
||||
grad = np.concatenate([grad1.ravel('F'), grad2.ravel('F')])
|
||||
res = np.hstack([J, grad]).tolist()
|
||||
elif part_id == 5:
|
||||
J, grad = func(self.t, 2, 4, 4, self.Xm, self.ym, 1.5)
|
||||
grad1 = np.reshape(grad[:12], (4, 3))
|
||||
grad2 = np.reshape(grad[12:], (4, 5))
|
||||
grad = np.concatenate([grad1.ravel('F'), grad2.ravel('F')])
|
||||
res = np.hstack([J, grad]).tolist()
|
||||
else:
|
||||
raise KeyError
|
||||
yield part_id, res
|
||||
except KeyError:
|
||||
yield part_id, 0
|
||||
BIN
Exercise5/Data/ex4data1.mat
Executable file
BIN
Exercise5/Data/ex4data1.mat
Executable file
Binary file not shown.
BIN
Exercise5/Data/ex4weights.mat
Executable file
BIN
Exercise5/Data/ex4weights.mat
Executable file
Binary file not shown.
BIN
Exercise5/Data/ex5data1.mat
Executable file
BIN
Exercise5/Data/ex5data1.mat
Executable file
Binary file not shown.
BIN
Exercise5/Figures/cross_validation.png
Normal file
BIN
Exercise5/Figures/cross_validation.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
BIN
Exercise5/Figures/learning_curve.png
Executable file
BIN
Exercise5/Figures/learning_curve.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 20 KiB |
BIN
Exercise5/Figures/learning_curve_random.png
Normal file
BIN
Exercise5/Figures/learning_curve_random.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 19 KiB |
BIN
Exercise5/Figures/linear_fit.png
Executable file
BIN
Exercise5/Figures/linear_fit.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
BIN
Exercise5/Figures/polynomial_learning_curve.png
Normal file
BIN
Exercise5/Figures/polynomial_learning_curve.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 20 KiB |
BIN
Exercise5/Figures/polynomial_learning_curve_reg_1.png
Normal file
BIN
Exercise5/Figures/polynomial_learning_curve_reg_1.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 19 KiB |
BIN
Exercise5/Figures/polynomial_regression.png
Normal file
BIN
Exercise5/Figures/polynomial_regression.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 23 KiB |
BIN
Exercise5/Figures/polynomial_regression_reg_1.png
Normal file
BIN
Exercise5/Figures/polynomial_regression_reg_1.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 21 KiB |
BIN
Exercise5/Figures/polynomial_regression_reg_100.png
Normal file
BIN
Exercise5/Figures/polynomial_regression_reg_100.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 21 KiB |
927
Exercise5/exercise5.ipynb
Executable file
927
Exercise5/exercise5.ipynb
Executable file
@@ -0,0 +1,927 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Programming Exercise 5:\n",
|
||||
"# Regularized Linear Regression and Bias vs Variance\n",
|
||||
"\n",
|
||||
"## Introduction\n",
|
||||
"\n",
|
||||
"In this exercise, you will implement regularized linear regression and use it to study models with different bias-variance properties. Before starting on the programming exercise, we strongly recommend watching the video lectures and completing the review questions for the associated topics.\n",
|
||||
"\n",
|
||||
"All the information you need for solving this assignment is in this notebook, and all the code you will be implementing will take place within this notebook. The assignment can be promptly submitted to the coursera grader directly from this notebook (code and instructions are included below).\n",
|
||||
"\n",
|
||||
"Before we begin with the exercises, we need to import all libraries required for this programming exercise. Throughout the course, we will be using [`numpy`](http://www.numpy.org/) for all arrays and matrix operations, [`matplotlib`](https://matplotlib.org/) for plotting, and [`scipy`](https://docs.scipy.org/doc/scipy/reference/) for scientific and numerical computation functions and tools. You can find instructions on how to install required libraries in the README file in the [github repository](https://github.com/dibgerge/ml-coursera-python-assignments)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# used for manipulating directory paths\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Scientific and vector computation for python\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"# Plotting library\n",
|
||||
"from matplotlib import pyplot\n",
|
||||
"\n",
|
||||
"# Optimization module in scipy\n",
|
||||
"from scipy import optimize\n",
|
||||
"\n",
|
||||
"# will be used to load MATLAB mat datafile format\n",
|
||||
"from scipy.io import loadmat\n",
|
||||
"\n",
|
||||
"# library written for this exercise providing additional functions for assignment submission, and others\n",
|
||||
"import utils\n",
|
||||
"\n",
|
||||
"# define the submission/grader object for this exercise\n",
|
||||
"grader = utils.Grader()\n",
|
||||
"\n",
|
||||
"# tells matplotlib to embed plots within the notebook\n",
|
||||
"%matplotlib inline"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Submission and Grading\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"After completing each part of the assignment, be sure to submit your solutions to the grader. The following is a breakdown of how each part of this exercise is scored.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"| Section | Part | Submitted Function | Points |\n",
|
||||
"| :- |:- |:- | :-: |\n",
|
||||
"| 1 | [Regularized Linear Regression Cost Function](#section1) | [`linearRegCostFunction`](#linearRegCostFunction) | 25 |\n",
|
||||
"| 2 | [Regularized Linear Regression Gradient](#section2) | [`linearRegCostFunction`](#linearRegCostFunction) |25 |\n",
|
||||
"| 3 | [Learning Curve](#section3) | [`learningCurve`](#func2) | 20 |\n",
|
||||
"| 4 | [Polynomial Feature Mapping](#section4) | [`polyFeatures`](#polyFeatures) | 10 |\n",
|
||||
"| 5 | [Cross Validation Curve](#section5) | [`validationCurve`](#validationCurve) | 20 |\n",
|
||||
"| | Total Points | |100 |\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"You are allowed to submit your solutions multiple times, and we will take only the highest score into consideration.\n",
|
||||
"\n",
|
||||
"<div class=\"alert alert-block alert-warning\">\n",
|
||||
"At the end of each section in this notebook, we have a cell which contains code for submitting the solutions thus far to the grader. Execute the cell to see your score up to the current section. For all your work to be submitted properly, you must execute those cells at least once.\n",
|
||||
"</div>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a id=\"section1\"></a>\n",
|
||||
"## 1 Regularized Linear Regression\n",
|
||||
"\n",
|
||||
"In the first half of the exercise, you will implement regularized linear regression to predict the amount of water flowing out of a dam using the change of water level in a reservoir. In the next half, you will go through some diagnostics of debugging learning algorithms and examine the effects of bias v.s.\n",
|
||||
"variance. \n",
|
||||
"\n",
|
||||
"### 1.1 Visualizing the dataset\n",
|
||||
"\n",
|
||||
"We will begin by visualizing the dataset containing historical records on the change in the water level, $x$, and the amount of water flowing out of the dam, $y$. This dataset is divided into three parts:\n",
|
||||
"\n",
|
||||
"- A **training** set that your model will learn on: `X`, `y`\n",
|
||||
"- A **cross validation** set for determining the regularization parameter: `Xval`, `yval`\n",
|
||||
"- A **test** set for evaluating performance. These are “unseen” examples which your model did not see during training: `Xtest`, `ytest`\n",
|
||||
"\n",
|
||||
"Run the next cell to plot the training data. In the following parts, you will implement linear regression and use that to fit a straight line to the data and plot learning curves. Following that, you will implement polynomial regression to find a better fit to the data."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Load from ex5data1.mat, where all variables will be store in a dictionary\n",
|
||||
"data = loadmat(os.path.join('Data', 'ex5data1.mat'))\n",
|
||||
"\n",
|
||||
"# Extract train, test, validation data from dictionary\n",
|
||||
"# and also convert y's form 2-D matrix (MATLAB format) to a numpy vector\n",
|
||||
"X, y = data['X'], data['y'][:, 0]\n",
|
||||
"Xtest, ytest = data['Xtest'], data['ytest'][:, 0]\n",
|
||||
"Xval, yval = data['Xval'], data['yval'][:, 0]\n",
|
||||
"\n",
|
||||
"# m = Number of examples\n",
|
||||
"m = y.size\n",
|
||||
"\n",
|
||||
"# Plot training data\n",
|
||||
"pyplot.plot(X, y, 'ro', ms=10, mec='k', mew=1)\n",
|
||||
"pyplot.xlabel('Change in water level (x)')\n",
|
||||
"pyplot.ylabel('Water flowing out of the dam (y)');"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 1.2 Regularized linear regression cost function\n",
|
||||
"\n",
|
||||
"Recall that regularized linear regression has the following cost function:\n",
|
||||
"\n",
|
||||
"$$ J(\\theta) = \\frac{1}{2m} \\left( \\sum_{i=1}^m \\left( h_\\theta\\left( x^{(i)} \\right) - y^{(i)} \\right)^2 \\right) + \\frac{\\lambda}{2m} \\left( \\sum_{j=1}^n \\theta_j^2 \\right)$$\n",
|
||||
"\n",
|
||||
"where $\\lambda$ is a regularization parameter which controls the degree of regularization (thus, help preventing overfitting). The regularization term puts a penalty on the overall cost J. As the magnitudes of the model parameters $\\theta_j$ increase, the penalty increases as well. Note that you should not regularize\n",
|
||||
"the $\\theta_0$ term.\n",
|
||||
"\n",
|
||||
"You should now complete the code in the function `linearRegCostFunction` in the next cell. Your task is to calculate the regularized linear regression cost function. If possible, try to vectorize your code and avoid writing loops.\n",
|
||||
"<a id=\"linearRegCostFunction\"></a>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def linearRegCostFunction(X, y, theta, lambda_=0.0):\n",
|
||||
" \"\"\"\n",
|
||||
" Compute cost and gradient for regularized linear regression \n",
|
||||
" with multiple variables. Computes the cost of using theta as\n",
|
||||
" the parameter for linear regression to fit the data points in X and y. \n",
|
||||
" \n",
|
||||
" Parameters\n",
|
||||
" ----------\n",
|
||||
" X : array_like\n",
|
||||
" The dataset. Matrix with shape (m x n + 1) where m is the \n",
|
||||
" total number of examples, and n is the number of features \n",
|
||||
" before adding the bias term.\n",
|
||||
" \n",
|
||||
" y : array_like\n",
|
||||
" The functions values at each datapoint. A vector of\n",
|
||||
" shape (m, ).\n",
|
||||
" \n",
|
||||
" theta : array_like\n",
|
||||
" The parameters for linear regression. A vector of shape (n+1,).\n",
|
||||
" \n",
|
||||
" lambda_ : float, optional\n",
|
||||
" The regularization parameter.\n",
|
||||
" \n",
|
||||
" Returns\n",
|
||||
" -------\n",
|
||||
" J : float\n",
|
||||
" The computed cost function. \n",
|
||||
" \n",
|
||||
" grad : array_like\n",
|
||||
" The value of the cost function gradient w.r.t theta. \n",
|
||||
" A vector of shape (n+1, ).\n",
|
||||
" \n",
|
||||
" Instructions\n",
|
||||
" ------------\n",
|
||||
" Compute the cost and gradient of regularized linear regression for\n",
|
||||
" a particular choice of theta.\n",
|
||||
" You should set J to the cost and grad to the gradient.\n",
|
||||
" \"\"\"\n",
|
||||
" # Initialize some useful values\n",
|
||||
" m = y.size # number of training examples\n",
|
||||
"\n",
|
||||
" # You need to return the following variables correctly \n",
|
||||
" J = 0\n",
|
||||
" grad = np.zeros(theta.shape)\n",
|
||||
"\n",
|
||||
" # ====================== YOUR CODE HERE ======================\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" # ============================================================\n",
|
||||
" return J, grad"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"When you are finished, the next cell will run your cost function using `theta` initialized at `[1, 1]`. You should expect to see an output of 303.993."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"theta = np.array([1, 1])\n",
|
||||
"J, _ = linearRegCostFunction(np.concatenate([np.ones((m, 1)), X], axis=1), y, theta, 1)\n",
|
||||
"\n",
|
||||
"print('Cost at theta = [1, 1]:\\t %f ' % J)\n",
|
||||
"print('This value should be about 303.993192)\\n' % J)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"After completing a part of the exercise, you can submit your solutions for grading by first adding the function you modified to the submission object, and then sending your function to Coursera for grading. \n",
|
||||
"\n",
|
||||
"The submission script will prompt you for your login e-mail and submission token. You can obtain a submission token from the web page for the assignment. You are allowed to submit your solutions multiple times, and we will take only the highest score into consideration.\n",
|
||||
"\n",
|
||||
"*Execute the following cell to grade your solution to the first part of this exercise.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"grader[1] = linearRegCostFunction\n",
|
||||
"grader.grade()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a id=\"section2\"></a>\n",
|
||||
"### 1.3 Regularized linear regression gradient\n",
|
||||
"\n",
|
||||
"Correspondingly, the partial derivative of the cost function for regularized linear regression is defined as:\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{align}\n",
|
||||
"& \\frac{\\partial J(\\theta)}{\\partial \\theta_0} = \\frac{1}{m} \\sum_{i=1}^m \\left( h_\\theta \\left(x^{(i)} \\right) - y^{(i)} \\right) x_j^{(i)} & \\qquad \\text{for } j = 0 \\\\\n",
|
||||
"& \\frac{\\partial J(\\theta)}{\\partial \\theta_j} = \\left( \\frac{1}{m} \\sum_{i=1}^m \\left( h_\\theta \\left( x^{(i)} \\right) - y^{(i)} \\right) x_j^{(i)} \\right) + \\frac{\\lambda}{m} \\theta_j & \\qquad \\text{for } j \\ge 1\n",
|
||||
"\\end{align}\n",
|
||||
"$$\n",
|
||||
"\n",
|
||||
"In the function [`linearRegCostFunction`](#linearRegCostFunction) above, add code to calculate the gradient, returning it in the variable `grad`. <font color='red'><b>Do not forget to re-execute the cell containing this function to update the function's definition.</b></font>\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"When you are finished, use the next cell to run your gradient function using theta initialized at `[1, 1]`. You should expect to see a gradient of `[-15.30, 598.250]`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"theta = np.array([1, 1])\n",
|
||||
"J, grad = linearRegCostFunction(np.concatenate([np.ones((m, 1)), X], axis=1), y, theta, 1)\n",
|
||||
"\n",
|
||||
"print('Gradient at theta = [1, 1]: [{:.6f}, {:.6f}] '.format(*grad))\n",
|
||||
"print(' (this value should be about [-15.303016, 598.250744])\\n')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"*You should now submit your solutions.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"grader[2] = linearRegCostFunction\n",
|
||||
"grader.grade()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Fitting linear regression\n",
|
||||
"\n",
|
||||
"Once your cost function and gradient are working correctly, the next cell will run the code in `trainLinearReg` (found in the module `utils.py`) to compute the optimal values of $\\theta$. This training function uses `scipy`'s optimization module to minimize the cost function.\n",
|
||||
"\n",
|
||||
"In this part, we set regularization parameter $\\lambda$ to zero. Because our current implementation of linear regression is trying to fit a 2-dimensional $\\theta$, regularization will not be incredibly helpful for a $\\theta$ of such low dimension. In the later parts of the exercise, you will be using polynomial regression with regularization.\n",
|
||||
"\n",
|
||||
"Finally, the code in the next cell should also plot the best fit line, which should look like the figure below. \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"The best fit line tells us that the model is not a good fit to the data because the data has a non-linear pattern. While visualizing the best fit as shown is one possible way to debug your learning algorithm, it is not always easy to visualize the data and model. In the next section, you will implement a function to generate learning curves that can help you debug your learning algorithm even if it is not easy to visualize the\n",
|
||||
"data."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# add a columns of ones for the y-intercept\n",
|
||||
"X_aug = np.concatenate([np.ones((m, 1)), X], axis=1)\n",
|
||||
"theta = utils.trainLinearReg(linearRegCostFunction, X_aug, y, lambda_=0)\n",
|
||||
"\n",
|
||||
"# Plot fit over the data\n",
|
||||
"pyplot.plot(X, y, 'ro', ms=10, mec='k', mew=1.5)\n",
|
||||
"pyplot.xlabel('Change in water level (x)')\n",
|
||||
"pyplot.ylabel('Water flowing out of the dam (y)')\n",
|
||||
"pyplot.plot(X, np.dot(X_aug, theta), '--', lw=2);"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a id=\"section3\"></a>\n",
|
||||
"## 2 Bias-variance\n",
|
||||
"\n",
|
||||
"An important concept in machine learning is the bias-variance tradeoff. Models with high bias are not complex enough for the data and tend to underfit, while models with high variance overfit to the training data.\n",
|
||||
"\n",
|
||||
"In this part of the exercise, you will plot training and test errors on a learning curve to diagnose bias-variance problems.\n",
|
||||
"\n",
|
||||
"### 2.1 Learning Curves\n",
|
||||
"\n",
|
||||
"You will now implement code to generate the learning curves that will be useful in debugging learning algorithms. Recall that a learning curve plots training and cross validation error as a function of training set size. Your job is to fill in the function `learningCurve` in the next cell, so that it returns a vector of errors for the training set and cross validation set.\n",
|
||||
"\n",
|
||||
"To plot the learning curve, we need a training and cross validation set error for different training set sizes. To obtain different training set sizes, you should use different subsets of the original training set `X`. Specifically, for a training set size of $i$, you should use the first $i$ examples (i.e., `X[:i, :]`\n",
|
||||
"and `y[:i]`).\n",
|
||||
"\n",
|
||||
"You can use the `trainLinearReg` function (by calling `utils.trainLinearReg(...)`) to find the $\\theta$ parameters. Note that the `lambda_` is passed as a parameter to the `learningCurve` function.\n",
|
||||
"After learning the $\\theta$ parameters, you should compute the error on the training and cross validation sets. Recall that the training error for a dataset is defined as\n",
|
||||
"\n",
|
||||
"$$ J_{\\text{train}} = \\frac{1}{2m} \\left[ \\sum_{i=1}^m \\left(h_\\theta \\left( x^{(i)} \\right) - y^{(i)} \\right)^2 \\right] $$\n",
|
||||
"\n",
|
||||
"In particular, note that the training error does not include the regularization term. One way to compute the training error is to use your existing cost function and set $\\lambda$ to 0 only when using it to compute the training error and cross validation error. When you are computing the training set error, make sure you compute it on the training subset (i.e., `X[:n,:]` and `y[:n]`) instead of the entire training set. However, for the cross validation error, you should compute it over the entire cross validation set. You should store\n",
|
||||
"the computed errors in the vectors error train and error val.\n",
|
||||
"\n",
|
||||
"<a id=\"func2\"></a>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def learningCurve(X, y, Xval, yval, lambda_=0):\n",
|
||||
" \"\"\"\n",
|
||||
" Generates the train and cross validation set errors needed to plot a learning curve\n",
|
||||
" returns the train and cross validation set errors for a learning curve. \n",
|
||||
" \n",
|
||||
" In this function, you will compute the train and test errors for\n",
|
||||
" dataset sizes from 1 up to m. In practice, when working with larger\n",
|
||||
" datasets, you might want to do this in larger intervals.\n",
|
||||
" \n",
|
||||
" Parameters\n",
|
||||
" ----------\n",
|
||||
" X : array_like\n",
|
||||
" The training dataset. Matrix with shape (m x n + 1) where m is the \n",
|
||||
" total number of examples, and n is the number of features \n",
|
||||
" before adding the bias term.\n",
|
||||
" \n",
|
||||
" y : array_like\n",
|
||||
" The functions values at each training datapoint. A vector of\n",
|
||||
" shape (m, ).\n",
|
||||
" \n",
|
||||
" Xval : array_like\n",
|
||||
" The validation dataset. Matrix with shape (m_val x n + 1) where m is the \n",
|
||||
" total number of examples, and n is the number of features \n",
|
||||
" before adding the bias term.\n",
|
||||
" \n",
|
||||
" yval : array_like\n",
|
||||
" The functions values at each validation datapoint. A vector of\n",
|
||||
" shape (m_val, ).\n",
|
||||
" \n",
|
||||
" lambda_ : float, optional\n",
|
||||
" The regularization parameter.\n",
|
||||
" \n",
|
||||
" Returns\n",
|
||||
" -------\n",
|
||||
" error_train : array_like\n",
|
||||
" A vector of shape m. error_train[i] contains the training error for\n",
|
||||
" i examples.\n",
|
||||
" error_val : array_like\n",
|
||||
" A vecotr of shape m. error_val[i] contains the validation error for\n",
|
||||
" i training examples.\n",
|
||||
" \n",
|
||||
" Instructions\n",
|
||||
" ------------\n",
|
||||
" Fill in this function to return training errors in error_train and the\n",
|
||||
" cross validation errors in error_val. i.e., error_train[i] and \n",
|
||||
" error_val[i] should give you the errors obtained after training on i examples.\n",
|
||||
" \n",
|
||||
" Notes\n",
|
||||
" -----\n",
|
||||
" - You should evaluate the training error on the first i training\n",
|
||||
" examples (i.e., X[:i, :] and y[:i]).\n",
|
||||
" \n",
|
||||
" For the cross-validation error, you should instead evaluate on\n",
|
||||
" the _entire_ cross validation set (Xval and yval).\n",
|
||||
" \n",
|
||||
" - If you are using your cost function (linearRegCostFunction) to compute\n",
|
||||
" the training and cross validation error, you should call the function with\n",
|
||||
" the lambda argument set to 0. Do note that you will still need to use\n",
|
||||
" lambda when running the training to obtain the theta parameters.\n",
|
||||
" \n",
|
||||
" Hint\n",
|
||||
" ----\n",
|
||||
" You can loop over the examples with the following:\n",
|
||||
" \n",
|
||||
" for i in range(1, m+1):\n",
|
||||
" # Compute train/cross validation errors using training examples \n",
|
||||
" # X[:i, :] and y[:i], storing the result in \n",
|
||||
" # error_train[i-1] and error_val[i-1]\n",
|
||||
" .... \n",
|
||||
" \"\"\"\n",
|
||||
" # Number of training examples\n",
|
||||
" m = y.size\n",
|
||||
"\n",
|
||||
" # You need to return these values correctly\n",
|
||||
" error_train = np.zeros(m)\n",
|
||||
" error_val = np.zeros(m)\n",
|
||||
"\n",
|
||||
" # ====================== YOUR CODE HERE ======================\n",
|
||||
" \n",
|
||||
"\n",
|
||||
" \n",
|
||||
" # =============================================================\n",
|
||||
" return error_train, error_val"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"When you are finished implementing the function `learningCurve`, executing the next cell prints the learning curves and produce a plot similar to the figure below. \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"In the learning curve figure, you can observe that both the train error and cross validation error are high when the number of training examples is increased. This reflects a high bias problem in the model - the linear regression model is too simple and is unable to fit our dataset well. In the next section, you will implement polynomial regression to fit a better model for this dataset."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"X_aug = np.concatenate([np.ones((m, 1)), X], axis=1)\n",
|
||||
"Xval_aug = np.concatenate([np.ones((yval.size, 1)), Xval], axis=1)\n",
|
||||
"error_train, error_val = learningCurve(X_aug, y, Xval_aug, yval, lambda_=0)\n",
|
||||
"\n",
|
||||
"pyplot.plot(np.arange(1, m+1), error_train, np.arange(1, m+1), error_val, lw=2)\n",
|
||||
"pyplot.title('Learning curve for linear regression')\n",
|
||||
"pyplot.legend(['Train', 'Cross Validation'])\n",
|
||||
"pyplot.xlabel('Number of training examples')\n",
|
||||
"pyplot.ylabel('Error')\n",
|
||||
"pyplot.axis([0, 13, 0, 150])\n",
|
||||
"\n",
|
||||
"print('# Training Examples\\tTrain Error\\tCross Validation Error')\n",
|
||||
"for i in range(m):\n",
|
||||
" print(' \\t%d\\t\\t%f\\t%f' % (i+1, error_train[i], error_val[i]))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"*You should now submit your solutions.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"grader[3] = learningCurve\n",
|
||||
"grader.grade()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a id=\"section4\"></a>\n",
|
||||
"\n",
|
||||
"## 3 Polynomial regression\n",
|
||||
"\n",
|
||||
"The problem with our linear model was that it was too simple for the data\n",
|
||||
"and resulted in underfitting (high bias). In this part of the exercise, you will address this problem by adding more features. For polynomial regression, our hypothesis has the form:\n",
|
||||
"\n",
|
||||
"$$\n",
|
||||
"\\begin{align}\n",
|
||||
"h_\\theta(x) &= \\theta_0 + \\theta_1 \\times (\\text{waterLevel}) + \\theta_2 \\times (\\text{waterLevel})^2 + \\cdots + \\theta_p \\times (\\text{waterLevel})^p \\\\\n",
|
||||
"& = \\theta_0 + \\theta_1 x_1 + \\theta_2 x_2 + \\cdots + \\theta_p x_p\n",
|
||||
"\\end{align}\n",
|
||||
"$$\n",
|
||||
"\n",
|
||||
"Notice that by defining $x_1 = (\\text{waterLevel})$, $x_2 = (\\text{waterLevel})^2$ , $\\cdots$, $x_p =\n",
|
||||
"(\\text{waterLevel})^p$, we obtain a linear regression model where the features are the various powers of the original value (waterLevel).\n",
|
||||
"\n",
|
||||
"Now, you will add more features using the higher powers of the existing feature $x$ in the dataset. Your task in this part is to complete the code in the function `polyFeatures` in the next cell. The function should map the original training set $X$ of size $m \\times 1$ into its higher powers. Specifically, when a training set $X$ of size $m \\times 1$ is passed into the function, the function should return a $m \\times p$ matrix `X_poly`, where column 1 holds the original values of X, column 2 holds the values of $X^2$, column 3 holds the values of $X^3$, and so on. Note that you don’t have to account for the zero-eth power in this function.\n",
|
||||
"\n",
|
||||
"<a id=\"polyFeatures\"></a>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def polyFeatures(X, p):\n",
|
||||
" \"\"\"\n",
|
||||
" Maps X (1D vector) into the p-th power.\n",
|
||||
" \n",
|
||||
" Parameters\n",
|
||||
" ----------\n",
|
||||
" X : array_like\n",
|
||||
" A data vector of size m, where m is the number of examples.\n",
|
||||
" \n",
|
||||
" p : int\n",
|
||||
" The polynomial power to map the features. \n",
|
||||
" \n",
|
||||
" Returns \n",
|
||||
" -------\n",
|
||||
" X_poly : array_like\n",
|
||||
" A matrix of shape (m x p) where p is the polynomial \n",
|
||||
" power and m is the number of examples. That is:\n",
|
||||
" \n",
|
||||
" X_poly[i, :] = [X[i], X[i]**2, X[i]**3 ... X[i]**p]\n",
|
||||
" \n",
|
||||
" Instructions\n",
|
||||
" ------------\n",
|
||||
" Given a vector X, return a matrix X_poly where the p-th column of\n",
|
||||
" X contains the values of X to the p-th power.\n",
|
||||
" \"\"\"\n",
|
||||
" # You need to return the following variables correctly.\n",
|
||||
" X_poly = np.zeros((X.shape[0], p))\n",
|
||||
"\n",
|
||||
" # ====================== YOUR CODE HERE ======================\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" # ============================================================\n",
|
||||
" return X_poly"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Now you have a function that will map features to a higher dimension. The next cell will apply it to the training set, the test set, and the cross validation set."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"p = 8\n",
|
||||
"\n",
|
||||
"# Map X onto Polynomial Features and Normalize\n",
|
||||
"X_poly = polyFeatures(X, p)\n",
|
||||
"X_poly, mu, sigma = utils.featureNormalize(X_poly)\n",
|
||||
"X_poly = np.concatenate([np.ones((m, 1)), X_poly], axis=1)\n",
|
||||
"\n",
|
||||
"# Map X_poly_test and normalize (using mu and sigma)\n",
|
||||
"X_poly_test = polyFeatures(Xtest, p)\n",
|
||||
"X_poly_test -= mu\n",
|
||||
"X_poly_test /= sigma\n",
|
||||
"X_poly_test = np.concatenate([np.ones((ytest.size, 1)), X_poly_test], axis=1)\n",
|
||||
"\n",
|
||||
"# Map X_poly_val and normalize (using mu and sigma)\n",
|
||||
"X_poly_val = polyFeatures(Xval, p)\n",
|
||||
"X_poly_val -= mu\n",
|
||||
"X_poly_val /= sigma\n",
|
||||
"X_poly_val = np.concatenate([np.ones((yval.size, 1)), X_poly_val], axis=1)\n",
|
||||
"\n",
|
||||
"print('Normalized Training Example 1:')\n",
|
||||
"X_poly[0, :]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"*You should now submit your solutions.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"grader[4] = polyFeatures\n",
|
||||
"grader.grade()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 3.1 Learning Polynomial Regression\n",
|
||||
"\n",
|
||||
"After you have completed the function `polyFeatures`, we will proceed to train polynomial regression using your linear regression cost function.\n",
|
||||
"\n",
|
||||
"Keep in mind that even though we have polynomial terms in our feature vector, we are still solving a linear regression optimization problem. The polynomial terms have simply turned into features that we can use for linear regression. We are using the same cost function and gradient that you wrote for the earlier part of this exercise.\n",
|
||||
"\n",
|
||||
"For this part of the exercise, you will be using a polynomial of degree 8. It turns out that if we run the training directly on the projected data, will not work well as the features would be badly scaled (e.g., an example with $x = 40$ will now have a feature $x_8 = 40^8 = 6.5 \\times 10^{12}$). Therefore, you will\n",
|
||||
"need to use feature normalization.\n",
|
||||
"\n",
|
||||
"Before learning the parameters $\\theta$ for the polynomial regression, we first call `featureNormalize` and normalize the features of the training set, storing the mu, sigma parameters separately. We have already implemented this function for you (in `utils.py` module) and it is the same function from the first exercise.\n",
|
||||
"\n",
|
||||
"After learning the parameters $\\theta$, you should see two plots generated for polynomial regression with $\\lambda = 0$, which should be similar to the ones here:\n",
|
||||
"\n",
|
||||
"<table>\n",
|
||||
" <tr>\n",
|
||||
" <td><img src=\"Figures/polynomial_regression.png\"></td>\n",
|
||||
" <td><img src=\"Figures/polynomial_learning_curve.png\"></td>\n",
|
||||
" </tr>\n",
|
||||
"</table>\n",
|
||||
"\n",
|
||||
"You should see that the polynomial fit is able to follow the datapoints very well, thus, obtaining a low training error. The figure on the right shows that the training error essentially stays zero for all numbers of training samples. However, the polynomial fit is very complex and even drops off at the extremes. This is an indicator that the polynomial regression model is overfitting the training data and will not generalize well.\n",
|
||||
"\n",
|
||||
"To better understand the problems with the unregularized ($\\lambda = 0$) model, you can see that the learning curve shows the same effect where the training error is low, but the cross validation error is high. There is a gap between the training and cross validation errors, indicating a high variance problem."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"lambda_ = 100\n",
|
||||
"theta = utils.trainLinearReg(linearRegCostFunction, X_poly, y,\n",
|
||||
" lambda_=lambda_, maxiter=55)\n",
|
||||
"\n",
|
||||
"# Plot training data and fit\n",
|
||||
"pyplot.plot(X, y, 'ro', ms=10, mew=1.5, mec='k')\n",
|
||||
"\n",
|
||||
"utils.plotFit(polyFeatures, np.min(X), np.max(X), mu, sigma, theta, p)\n",
|
||||
"\n",
|
||||
"pyplot.xlabel('Change in water level (x)')\n",
|
||||
"pyplot.ylabel('Water flowing out of the dam (y)')\n",
|
||||
"pyplot.title('Polynomial Regression Fit (lambda = %f)' % lambda_)\n",
|
||||
"pyplot.ylim([-20, 50])\n",
|
||||
"\n",
|
||||
"pyplot.figure()\n",
|
||||
"error_train, error_val = learningCurve(X_poly, y, X_poly_val, yval, lambda_)\n",
|
||||
"pyplot.plot(np.arange(1, 1+m), error_train, np.arange(1, 1+m), error_val)\n",
|
||||
"\n",
|
||||
"pyplot.title('Polynomial Regression Learning Curve (lambda = %f)' % lambda_)\n",
|
||||
"pyplot.xlabel('Number of training examples')\n",
|
||||
"pyplot.ylabel('Error')\n",
|
||||
"pyplot.axis([0, 13, 0, 100])\n",
|
||||
"pyplot.legend(['Train', 'Cross Validation'])\n",
|
||||
"\n",
|
||||
"print('Polynomial Regression (lambda = %f)\\n' % lambda_)\n",
|
||||
"print('# Training Examples\\tTrain Error\\tCross Validation Error')\n",
|
||||
"for i in range(m):\n",
|
||||
" print(' \\t%d\\t\\t%f\\t%f' % (i+1, error_train[i], error_val[i]))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"One way to combat the overfitting (high-variance) problem is to add regularization to the model. In the next section, you will get to try different $\\lambda$ parameters to see how regularization can lead to a better model.\n",
|
||||
"\n",
|
||||
"### 3.2 Optional (ungraded) exercise: Adjusting the regularization parameter\n",
|
||||
"\n",
|
||||
"In this section, you will get to observe how the regularization parameter affects the bias-variance of regularized polynomial regression. You should now modify the the lambda parameter and try $\\lambda = 1, 100$. For each of these values, the script should generate a polynomial fit to the data and also a learning curve.\n",
|
||||
"\n",
|
||||
"For $\\lambda = 1$, the generated plots should look like the the figure below. You should see a polynomial fit that follows the data trend well (left) and a learning curve (right) showing that both the cross validation and training error converge to a relatively low value. This shows the $\\lambda = 1$ regularized polynomial regression model does not have the high-bias or high-variance problems. In effect, it achieves a good trade-off between bias and variance.\n",
|
||||
"\n",
|
||||
"<table>\n",
|
||||
" <tr>\n",
|
||||
" <td><img src=\"Figures/polynomial_regression_reg_1.png\"></td>\n",
|
||||
" <td><img src=\"Figures/polynomial_learning_curve_reg_1.png\"></td>\n",
|
||||
" </tr>\n",
|
||||
"</table>\n",
|
||||
"\n",
|
||||
"For $\\lambda = 100$, you should see a polynomial fit (figure below) that does not follow the data well. In this case, there is too much regularization and the model is unable to fit the training data.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"*You do not need to submit any solutions for this optional (ungraded) exercise.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"<a id=\"section5\"></a>\n",
|
||||
"### 3.3 Selecting $\\lambda$ using a cross validation set\n",
|
||||
"\n",
|
||||
"From the previous parts of the exercise, you observed that the value of $\\lambda$ can significantly affect the results of regularized polynomial regression on the training and cross validation set. In particular, a model without regularization ($\\lambda = 0$) fits the training set well, but does not generalize. Conversely, a model with too much regularization ($\\lambda = 100$) does not fit the training set and testing set well. A good choice of $\\lambda$ (e.g., $\\lambda = 1$) can provide a good fit to the data.\n",
|
||||
"\n",
|
||||
"In this section, you will implement an automated method to select the $\\lambda$ parameter. Concretely, you will use a cross validation set to evaluate how good each $\\lambda$ value is. After selecting the best $\\lambda$ value using the cross validation set, we can then evaluate the model on the test set to estimate\n",
|
||||
"how well the model will perform on actual unseen data. \n",
|
||||
"\n",
|
||||
"Your task is to complete the code in the function `validationCurve`. Specifically, you should should use the `utils.trainLinearReg` function to train the model using different values of $\\lambda$ and compute the training error and cross validation error. You should try $\\lambda$ in the following range: {0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10}.\n",
|
||||
"<a id=\"validationCurve\"></a>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def validationCurve(X, y, Xval, yval):\n",
|
||||
" \"\"\"\n",
|
||||
" Generate the train and validation errors needed to plot a validation\n",
|
||||
" curve that we can use to select lambda_.\n",
|
||||
" \n",
|
||||
" Parameters\n",
|
||||
" ----------\n",
|
||||
" X : array_like\n",
|
||||
" The training dataset. Matrix with shape (m x n) where m is the \n",
|
||||
" total number of training examples, and n is the number of features \n",
|
||||
" including any polynomial features.\n",
|
||||
" \n",
|
||||
" y : array_like\n",
|
||||
" The functions values at each training datapoint. A vector of\n",
|
||||
" shape (m, ).\n",
|
||||
" \n",
|
||||
" Xval : array_like\n",
|
||||
" The validation dataset. Matrix with shape (m_val x n) where m is the \n",
|
||||
" total number of validation examples, and n is the number of features \n",
|
||||
" including any polynomial features.\n",
|
||||
" \n",
|
||||
" yval : array_like\n",
|
||||
" The functions values at each validation datapoint. A vector of\n",
|
||||
" shape (m_val, ).\n",
|
||||
" \n",
|
||||
" Returns\n",
|
||||
" -------\n",
|
||||
" lambda_vec : list\n",
|
||||
" The values of the regularization parameters which were used in \n",
|
||||
" cross validation.\n",
|
||||
" \n",
|
||||
" error_train : list\n",
|
||||
" The training error computed at each value for the regularization\n",
|
||||
" parameter.\n",
|
||||
" \n",
|
||||
" error_val : list\n",
|
||||
" The validation error computed at each value for the regularization\n",
|
||||
" parameter.\n",
|
||||
" \n",
|
||||
" Instructions\n",
|
||||
" ------------\n",
|
||||
" Fill in this function to return training errors in `error_train` and\n",
|
||||
" the validation errors in `error_val`. The vector `lambda_vec` contains\n",
|
||||
" the different lambda parameters to use for each calculation of the\n",
|
||||
" errors, i.e, `error_train[i]`, and `error_val[i]` should give you the\n",
|
||||
" errors obtained after training with `lambda_ = lambda_vec[i]`.\n",
|
||||
"\n",
|
||||
" Note\n",
|
||||
" ----\n",
|
||||
" You can loop over lambda_vec with the following:\n",
|
||||
" \n",
|
||||
" for i in range(len(lambda_vec))\n",
|
||||
" lambda = lambda_vec[i]\n",
|
||||
" # Compute train / val errors when training linear \n",
|
||||
" # regression with regularization parameter lambda_\n",
|
||||
" # You should store the result in error_train[i]\n",
|
||||
" # and error_val[i]\n",
|
||||
" ....\n",
|
||||
" \"\"\"\n",
|
||||
" # Selected values of lambda (you should not change this)\n",
|
||||
" lambda_vec = [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]\n",
|
||||
"\n",
|
||||
" # You need to return these variables correctly.\n",
|
||||
" error_train = np.zeros(len(lambda_vec))\n",
|
||||
" error_val = np.zeros(len(lambda_vec))\n",
|
||||
"\n",
|
||||
" # ====================== YOUR CODE HERE ======================\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" # ============================================================\n",
|
||||
" return lambda_vec, error_train, error_val"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"After you have completed the code, the next cell will run your function and plot a cross validation curve of error v.s. $\\lambda$ that allows you select which $\\lambda$ parameter to use. You should see a plot similar to the figure below. \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"In this figure, we can see that the best value of $\\lambda$ is around 3. Due to randomness\n",
|
||||
"in the training and validation splits of the dataset, the cross validation error can sometimes be lower than the training error."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"lambda_vec, error_train, error_val = validationCurve(X_poly, y, X_poly_val, yval)\n",
|
||||
"\n",
|
||||
"pyplot.plot(lambda_vec, error_train, '-o', lambda_vec, error_val, '-o', lw=2)\n",
|
||||
"pyplot.legend(['Train', 'Cross Validation'])\n",
|
||||
"pyplot.xlabel('lambda')\n",
|
||||
"pyplot.ylabel('Error')\n",
|
||||
"\n",
|
||||
"print('lambda\\t\\tTrain Error\\tValidation Error')\n",
|
||||
"for i in range(len(lambda_vec)):\n",
|
||||
" print(' %f\\t%f\\t%f' % (lambda_vec[i], error_train[i], error_val[i]))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"*You should now submit your solutions.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"grader[5] = validationCurve\n",
|
||||
"grader.grade()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 3.4 Optional (ungraded) exercise: Computing test set error\n",
|
||||
"\n",
|
||||
"In the previous part of the exercise, you implemented code to compute the cross validation error for various values of the regularization parameter $\\lambda$. However, to get a better indication of the model’s performance in the real world, it is important to evaluate the “final” model on a test set that was not used in any part of training (that is, it was neither used to select the $\\lambda$ parameters, nor to learn the model parameters $\\theta$). For this optional (ungraded) exercise, you should compute the test error using the best value of $\\lambda$ you found. In our cross validation, we obtained a test error of 3.8599 for $\\lambda = 3$.\n",
|
||||
"\n",
|
||||
"*You do not need to submit any solutions for this optional (ungraded) exercise.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### 3.5 Optional (ungraded) exercise: Plotting learning curves with randomly selected examples\n",
|
||||
"\n",
|
||||
"In practice, especially for small training sets, when you plot learning curves to debug your algorithms, it is often helpful to average across multiple sets of randomly selected examples to determine the training error and cross validation error.\n",
|
||||
"\n",
|
||||
"Concretely, to determine the training error and cross validation error for $i$ examples, you should first randomly select $i$ examples from the training set and $i$ examples from the cross validation set. You will then learn the parameters $\\theta$ using the randomly chosen training set and evaluate the parameters $\\theta$ on the randomly chosen training set and cross validation set. The above steps should then be repeated multiple times (say 50) and the averaged error should be used to determine the training error and cross validation error for $i$ examples.\n",
|
||||
"\n",
|
||||
"For this optional (ungraded) exercise, you should implement the above strategy for computing the learning curves. For reference, the figure below shows the learning curve we obtained for polynomial regression with $\\lambda = 0.01$. Your figure may differ slightly due to the random selection of examples.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"*You do not need to submit any solutions for this optional (ungraded) exercise.*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.4"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
164
Exercise5/utils.py
Executable file
164
Exercise5/utils.py
Executable file
@@ -0,0 +1,164 @@
|
||||
import sys
|
||||
import numpy as np
|
||||
from scipy import optimize
|
||||
from matplotlib import pyplot
|
||||
|
||||
sys.path.append('..')
|
||||
from submission import SubmissionBase
|
||||
|
||||
|
||||
def trainLinearReg(linearRegCostFunction, X, y, lambda_=0.0, maxiter=200):
|
||||
"""
|
||||
Trains linear regression using scipy's optimize.minimize.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : array_like
|
||||
The dataset with shape (m x n+1). The bias term is assumed to be concatenated.
|
||||
|
||||
y : array_like
|
||||
Function values at each datapoint. A vector of shape (m,).
|
||||
|
||||
lambda_ : float, optional
|
||||
The regularization parameter.
|
||||
|
||||
maxiter : int, optional
|
||||
Maximum number of iteration for the optimization algorithm.
|
||||
|
||||
Returns
|
||||
-------
|
||||
theta : array_like
|
||||
The parameters for linear regression. This is a vector of shape (n+1,).
|
||||
"""
|
||||
# Initialize Theta
|
||||
initial_theta = np.zeros(X.shape[1])
|
||||
|
||||
# Create "short hand" for the cost function to be minimized
|
||||
costFunction = lambda t: linearRegCostFunction(X, y, t, lambda_)
|
||||
|
||||
# Now, costFunction is a function that takes in only one argument
|
||||
options = {'maxiter': maxiter}
|
||||
|
||||
# Minimize using scipy
|
||||
res = optimize.minimize(costFunction, initial_theta, jac=True, method='TNC', options=options)
|
||||
return res.x
|
||||
|
||||
|
||||
def featureNormalize(X):
|
||||
"""
|
||||
Normalizes the features in X returns a normalized version of X where the mean value of each
|
||||
feature is 0 and the standard deviation is 1. This is often a good preprocessing step to do when
|
||||
working with learning algorithms.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : array_like
|
||||
An dataset which is a (m x n) matrix, where m is the number of examples,
|
||||
and n is the number of dimensions for each example.
|
||||
|
||||
Returns
|
||||
-------
|
||||
X_norm : array_like
|
||||
The normalized input dataset.
|
||||
|
||||
mu : array_like
|
||||
A vector of size n corresponding to the mean for each dimension across all examples.
|
||||
|
||||
sigma : array_like
|
||||
A vector of size n corresponding to the standard deviations for each dimension across
|
||||
all examples.
|
||||
"""
|
||||
mu = np.mean(X, axis=0)
|
||||
X_norm = X - mu
|
||||
|
||||
sigma = np.std(X_norm, axis=0, ddof=1)
|
||||
X_norm /= sigma
|
||||
return X_norm, mu, sigma
|
||||
|
||||
|
||||
def plotFit(polyFeatures, min_x, max_x, mu, sigma, theta, p):
|
||||
"""
|
||||
Plots a learned polynomial regression fit over an existing figure.
|
||||
Also works with linear regression.
|
||||
Plots the learned polynomial fit with power p and feature normalization (mu, sigma).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
polyFeatures : func
|
||||
A function which generators polynomial features from a single feature.
|
||||
|
||||
min_x : float
|
||||
The minimum value for the feature.
|
||||
|
||||
max_x : float
|
||||
The maximum value for the feature.
|
||||
|
||||
mu : float
|
||||
The mean feature value over the training dataset.
|
||||
|
||||
sigma : float
|
||||
The feature standard deviation of the training dataset.
|
||||
|
||||
theta : array_like
|
||||
The parameters for the trained polynomial linear regression.
|
||||
|
||||
p : int
|
||||
The polynomial order.
|
||||
"""
|
||||
# We plot a range slightly bigger than the min and max values to get
|
||||
# an idea of how the fit will vary outside the range of the data points
|
||||
x = np.arange(min_x - 15, max_x + 25, 0.05).reshape(-1, 1)
|
||||
|
||||
# Map the X values
|
||||
X_poly = polyFeatures(x, p)
|
||||
X_poly -= mu
|
||||
X_poly /= sigma
|
||||
|
||||
# Add ones
|
||||
X_poly = np.concatenate([np.ones((x.shape[0], 1)), X_poly], axis=1)
|
||||
|
||||
# Plot
|
||||
pyplot.plot(x, np.dot(X_poly, theta), '--', lw=2)
|
||||
|
||||
|
||||
class Grader(SubmissionBase):
|
||||
# Random test cases
|
||||
X = np.vstack([np.ones(10),
|
||||
np.sin(np.arange(1, 15, 1.5)),
|
||||
np.cos(np.arange(1, 15, 1.5))]).T
|
||||
y = np.sin(np.arange(1, 31, 3))
|
||||
Xval = np.vstack([np.ones(10),
|
||||
np.sin(np.arange(0, 14, 1.5)),
|
||||
np.cos(np.arange(0, 14, 1.5))]).T
|
||||
yval = np.sin(np.arange(1, 11))
|
||||
|
||||
def __init__(self):
|
||||
part_names = ['Regularized Linear Regression Cost Function',
|
||||
'Regularized Linear Regression Gradient',
|
||||
'Learning Curve',
|
||||
'Polynomial Feature Mapping',
|
||||
'Validation Curve']
|
||||
super().__init__('regularized-linear-regression-and-bias-variance', part_names)
|
||||
|
||||
def __iter__(self):
|
||||
for part_id in range(1, 6):
|
||||
try:
|
||||
func = self.functions[part_id]
|
||||
# Each part has different expected arguments/different function
|
||||
if part_id == 1:
|
||||
res = func(self.X, self.y, np.array([0.1, 0.2, 0.3]), 0.5)
|
||||
elif part_id == 2:
|
||||
theta = np.array([0.1, 0.2, 0.3])
|
||||
res = func(self.X, self.y, theta, 0.5)[1]
|
||||
elif part_id == 3:
|
||||
res = np.hstack(func(self.X, self.y, self.Xval, self.yval, 1)).tolist()
|
||||
elif part_id == 4:
|
||||
res = func(self.X[1, :].reshape(-1, 1), 8)
|
||||
elif part_id == 5:
|
||||
res = np.hstack(func(self.X, self.y, self.Xval, self.yval)).tolist()
|
||||
else:
|
||||
raise KeyError
|
||||
except KeyError:
|
||||
yield part_id, 0
|
||||
yield part_id, res
|
||||
|
||||
10
Exercise6/Data/emailSample1.txt
Executable file
10
Exercise6/Data/emailSample1.txt
Executable file
@@ -0,0 +1,10 @@
|
||||
> Anyone knows how much it costs to host a web portal ?
|
||||
>
|
||||
Well, it depends on how many visitors you're expecting.
|
||||
This can be anywhere from less than 10 bucks a month to a couple of $100.
|
||||
You should checkout http://www.rackspace.com/ or perhaps Amazon EC2
|
||||
if youre running something big..
|
||||
|
||||
To unsubscribe yourself from this mailing list, send an email to:
|
||||
groupname-unsubscribe@egroups.com
|
||||
|
||||
34
Exercise6/Data/emailSample2.txt
Executable file
34
Exercise6/Data/emailSample2.txt
Executable file
@@ -0,0 +1,34 @@
|
||||
Folks,
|
||||
|
||||
my first time posting - have a bit of Unix experience, but am new to Linux.
|
||||
|
||||
|
||||
Just got a new PC at home - Dell box with Windows XP. Added a second hard disk
|
||||
for Linux. Partitioned the disk and have installed Suse 7.2 from CD, which went
|
||||
fine except it didn't pick up my monitor.
|
||||
|
||||
I have a Dell branded E151FPp 15" LCD flat panel monitor and a nVidia GeForce4
|
||||
Ti4200 video card, both of which are probably too new to feature in Suse's default
|
||||
set. I downloaded a driver from the nVidia website and installed it using RPM.
|
||||
Then I ran Sax2 (as was recommended in some postings I found on the net), but
|
||||
it still doesn't feature my video card in the available list. What next?
|
||||
|
||||
Another problem. I have a Dell branded keyboard and if I hit Caps-Lock twice,
|
||||
the whole machine crashes (in Linux, not Windows) - even the on/off switch is
|
||||
inactive, leaving me to reach for the power cable instead.
|
||||
|
||||
If anyone can help me in any way with these probs., I'd be really grateful -
|
||||
I've searched the 'net but have run out of ideas.
|
||||
|
||||
Or should I be going for a different version of Linux such as RedHat? Opinions
|
||||
welcome.
|
||||
|
||||
Thanks a lot,
|
||||
Peter
|
||||
|
||||
--
|
||||
Irish Linux Users' Group: ilug@linux.ie
|
||||
http://www.linux.ie/mailman/listinfo/ilug for (un)subscription information.
|
||||
List maintainer: listmaster@linux.ie
|
||||
|
||||
|
||||
BIN
Exercise6/Data/ex6data1.mat
Executable file
BIN
Exercise6/Data/ex6data1.mat
Executable file
Binary file not shown.
BIN
Exercise6/Data/ex6data2.mat
Executable file
BIN
Exercise6/Data/ex6data2.mat
Executable file
Binary file not shown.
BIN
Exercise6/Data/ex6data3.mat
Executable file
BIN
Exercise6/Data/ex6data3.mat
Executable file
Binary file not shown.
42
Exercise6/Data/spamSample1.txt
Executable file
42
Exercise6/Data/spamSample1.txt
Executable file
@@ -0,0 +1,42 @@
|
||||
Do You Want To Make $1000 Or More Per Week?
|
||||
|
||||
|
||||
|
||||
If you are a motivated and qualified individual - I
|
||||
will personally demonstrate to you a system that will
|
||||
make you $1,000 per week or more! This is NOT mlm.
|
||||
|
||||
|
||||
|
||||
Call our 24 hour pre-recorded number to get the
|
||||
details.
|
||||
|
||||
|
||||
|
||||
000-456-789
|
||||
|
||||
|
||||
|
||||
I need people who want to make serious money. Make
|
||||
the call and get the facts.
|
||||
|
||||
Invest 2 minutes in yourself now!
|
||||
|
||||
|
||||
|
||||
000-456-789
|
||||
|
||||
|
||||
|
||||
Looking forward to your call and I will introduce you
|
||||
to people like yourself who
|
||||
are currently making $10,000 plus per week!
|
||||
|
||||
|
||||
|
||||
000-456-789
|
||||
|
||||
|
||||
|
||||
3484lJGv6-241lEaN9080lRmS6-271WxHo7524qiyT5-438rjUv5615hQcf0-662eiDB9057dMtVl72
|
||||
|
||||
8
Exercise6/Data/spamSample2.txt
Executable file
8
Exercise6/Data/spamSample2.txt
Executable file
@@ -0,0 +1,8 @@
|
||||
Best Buy Viagra Generic Online
|
||||
|
||||
Viagra 100mg x 60 Pills $125, Free Pills & Reorder Discount, Top Selling 100% Quality & Satisfaction guaranteed!
|
||||
|
||||
We accept VISA, Master & E-Check Payments, 90000+ Satisfied Customers!
|
||||
http://medphysitcstech.ru
|
||||
|
||||
|
||||
BIN
Exercise6/Data/spamTest.mat
Executable file
BIN
Exercise6/Data/spamTest.mat
Executable file
Binary file not shown.
BIN
Exercise6/Data/spamTrain.mat
Executable file
BIN
Exercise6/Data/spamTrain.mat
Executable file
Binary file not shown.
1899
Exercise6/Data/vocab.txt
Executable file
1899
Exercise6/Data/vocab.txt
Executable file
File diff suppressed because it is too large
Load Diff
BIN
Exercise6/Figures/dataset1.png
Executable file
BIN
Exercise6/Figures/dataset1.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 7.8 KiB |
BIN
Exercise6/Figures/dataset2.png
Normal file
BIN
Exercise6/Figures/dataset2.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 79 KiB |
BIN
Exercise6/Figures/dataset3.png
Normal file
BIN
Exercise6/Figures/dataset3.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 20 KiB |
BIN
Exercise6/Figures/email.png
Normal file
BIN
Exercise6/Figures/email.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 88 KiB |
BIN
Exercise6/Figures/email_cleaned.png
Normal file
BIN
Exercise6/Figures/email_cleaned.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 54 KiB |
BIN
Exercise6/Figures/svm_c1.png
Normal file
BIN
Exercise6/Figures/svm_c1.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 13 KiB |
BIN
Exercise6/Figures/svm_c100.png
Normal file
BIN
Exercise6/Figures/svm_c100.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 12 KiB |
BIN
Exercise6/Figures/svm_dataset2.png
Normal file
BIN
Exercise6/Figures/svm_dataset2.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 113 KiB |
BIN
Exercise6/Figures/svm_dataset3_best.png
Normal file
BIN
Exercise6/Figures/svm_dataset3_best.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 39 KiB |
BIN
Exercise6/Figures/svm_predictors.png
Executable file
BIN
Exercise6/Figures/svm_predictors.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 18 KiB |
BIN
Exercise6/Figures/vocab.png
Normal file
BIN
Exercise6/Figures/vocab.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 25 KiB |
BIN
Exercise6/Figures/word_indices.png
Normal file
BIN
Exercise6/Figures/word_indices.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 72 KiB |
1029
Exercise6/exercise6.ipynb
Executable file
1029
Exercise6/exercise6.ipynb
Executable file
File diff suppressed because it is too large
Load Diff
718
Exercise6/utils.py
Executable file
718
Exercise6/utils.py
Executable file
@@ -0,0 +1,718 @@
|
||||
import sys
|
||||
|
||||
sys.path.append('..')
|
||||
from submission import SubmissionBase
|
||||
import numpy as np
|
||||
from scipy.io import loadmat
|
||||
from os.path import join
|
||||
from matplotlib import pyplot
|
||||
|
||||
|
||||
def plotData(X, y, grid=False):
|
||||
"""
|
||||
Plots the data points X and y into a new figure. Uses `+` for positive examples, and `o` for
|
||||
negative examples. `X` is assumed to be a Mx2 matrix
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : numpy ndarray
|
||||
X is assumed to be a Mx2 matrix.
|
||||
|
||||
y : numpy ndarray
|
||||
The data labels.
|
||||
|
||||
grid : bool (Optional)
|
||||
Specify whether or not to show the grid in the plot. It is False by default.
|
||||
|
||||
Notes
|
||||
-----
|
||||
This was slightly modified such that it expects y=1 or y=0.
|
||||
"""
|
||||
# Find Indices of Positive and Negative Examples
|
||||
pos = y == 1
|
||||
neg = y == 0
|
||||
|
||||
# Plot Examples
|
||||
pyplot.plot(X[pos, 0], X[pos, 1], 'X', mew=1, ms=10, mec='k')
|
||||
pyplot.plot(X[neg, 0], X[neg, 1], 'o', mew=1, mfc='y', ms=10, mec='k')
|
||||
pyplot.grid(grid)
|
||||
|
||||
|
||||
def svmTrain(X, Y, C, kernelFunction, tol=1e-3, max_passes=5, args=()):
|
||||
"""
|
||||
Trains an SVM classifier using a simplified version of the SMO algorithm.
|
||||
|
||||
Parameters
|
||||
---------
|
||||
X : numpy ndarray
|
||||
(m x n) Matrix of training examples. Each row is a training example, and the
|
||||
jth column holds the jth feature.
|
||||
|
||||
Y : numpy ndarray
|
||||
(m, ) A vector (1-D numpy array) containing 1 for positive examples and 0 for negative examples.
|
||||
|
||||
C : float
|
||||
The standard SVM regularization parameter.
|
||||
|
||||
kernelFunction : func
|
||||
A function handle which computes the kernel. The function should accept two vectors as
|
||||
inputs, and returns a scalar as output.
|
||||
|
||||
tol : float, optional
|
||||
Tolerance value used for determining equality of floating point numbers.
|
||||
|
||||
max_passes : int, optional
|
||||
Controls the number of iterations over the dataset (without changes to alpha)
|
||||
before the algorithm quits.
|
||||
|
||||
args : tuple
|
||||
Extra arguments required for the kernel function, such as the sigma parameter for a
|
||||
Gaussian kernel.
|
||||
|
||||
Returns
|
||||
-------
|
||||
model :
|
||||
The trained SVM model.
|
||||
|
||||
Notes
|
||||
-----
|
||||
This is a simplified version of the SMO algorithm for training SVMs. In practice, if
|
||||
you want to train an SVM classifier, we recommend using an optimized package such as:
|
||||
|
||||
- LIBSVM (http://www.csie.ntu.edu.tw/~cjlin/libsvm/)
|
||||
- SVMLight (http://svmlight.joachims.org/)
|
||||
- scikit-learn (http://scikit-learn.org/stable/modules/svm.html) which contains python wrappers
|
||||
for the LIBSVM library.
|
||||
"""
|
||||
# make sure data is signed int
|
||||
Y = Y.astype(int)
|
||||
# Dataset size parameters
|
||||
m, n = X.shape
|
||||
|
||||
passes = 0
|
||||
E = np.zeros(m)
|
||||
alphas = np.zeros(m)
|
||||
b = 0
|
||||
|
||||
# Map 0 to -1
|
||||
Y[Y == 0] = -1
|
||||
|
||||
# Pre-compute the Kernel Matrix since our dataset is small
|
||||
# (in practice, optimized SVM packages that handle large datasets
|
||||
# gracefully will **not** do this)
|
||||
|
||||
# We have implemented the optimized vectorized version of the Kernels here so
|
||||
# that the SVM training will run faster
|
||||
if kernelFunction.__name__ == 'linearKernel':
|
||||
# Vectorized computation for the linear kernel
|
||||
# This is equivalent to computing the kernel on every pair of examples
|
||||
K = np.dot(X, X.T)
|
||||
elif kernelFunction.__name__ == 'gaussianKernel':
|
||||
# vectorized RBF Kernel
|
||||
# This is equivalent to computing the kernel on every pair of examples
|
||||
X2 = np.sum(X**2, axis=1)
|
||||
K = X2 + X2[:, None] - 2 * np.dot(X, X.T)
|
||||
|
||||
if len(args) > 0:
|
||||
K /= 2*args[0]**2
|
||||
|
||||
K = np.exp(-K)
|
||||
else:
|
||||
K = np.zeros((m, m))
|
||||
for i in range(m):
|
||||
for j in range(i, m):
|
||||
K[i, j] = kernelFunction(X[i, :], X[j, :])
|
||||
K[j, i] = K[i, j]
|
||||
|
||||
while passes < max_passes:
|
||||
num_changed_alphas = 0
|
||||
for i in range(m):
|
||||
E[i] = b + np.sum(alphas * Y * K[:, i]) - Y[i]
|
||||
|
||||
if (Y[i]*E[i] < -tol and alphas[i] < C) or (Y[i]*E[i] > tol and alphas[i] > 0):
|
||||
# select the alpha_j randomly
|
||||
j = np.random.choice(list(range(i)) + list(range(i+1, m)), size=1)[0]
|
||||
|
||||
E[j] = b + np.sum(alphas * Y * K[:, j]) - Y[j]
|
||||
|
||||
alpha_i_old = alphas[i]
|
||||
alpha_j_old = alphas[j]
|
||||
|
||||
if Y[i] == Y[j]:
|
||||
L = max(0, alphas[j] + alphas[i] - C)
|
||||
H = min(C, alphas[j] + alphas[i])
|
||||
else:
|
||||
L = max(0, alphas[j] - alphas[i])
|
||||
H = min(C, C + alphas[j] - alphas[i])
|
||||
|
||||
if L == H:
|
||||
continue
|
||||
|
||||
eta = 2 * K[i, j] - K[i, i] - K[j, j]
|
||||
|
||||
# objective function positive definite, there will be a minimum along the direction
|
||||
# of linear equality constrain, and eta will be greater than zero
|
||||
# we are actually computing -eta here (so we skip of eta >= 0)
|
||||
if eta >= 0:
|
||||
continue
|
||||
|
||||
alphas[j] -= Y[j] * (E[i] - E[j])/eta
|
||||
alphas[j] = max(L, min(H, alphas[j]))
|
||||
|
||||
if abs(alphas[j] - alpha_j_old) < tol:
|
||||
alphas[j] = alpha_j_old
|
||||
continue
|
||||
alphas[i] += Y[i]*Y[j]*(alpha_j_old - alphas[j])
|
||||
|
||||
b1 = b - E[i] - Y[i]*(alphas[i] - alpha_i_old) * K[i, j] \
|
||||
- Y[j] * (alphas[j] - alpha_j_old) * K[i, j]
|
||||
|
||||
b2 = b - E[j] - Y[i]*(alphas[i] - alpha_i_old) * K[i, j] \
|
||||
- Y[j] * (alphas[j] - alpha_j_old) * K[j, j]
|
||||
|
||||
if 0 < alphas[i] < C:
|
||||
b = b1
|
||||
elif 0 < alphas[j] < C:
|
||||
b = b2
|
||||
else:
|
||||
b = (b1 + b2)/2
|
||||
|
||||
num_changed_alphas += 1
|
||||
if num_changed_alphas == 0:
|
||||
passes += 1
|
||||
else:
|
||||
passes = 0
|
||||
|
||||
idx = alphas > 0
|
||||
model = {'X': X[idx, :],
|
||||
'y': Y[idx],
|
||||
'kernelFunction': kernelFunction,
|
||||
'b': b,
|
||||
'args': args,
|
||||
'alphas': alphas[idx],
|
||||
'w': np.dot(alphas * Y, X)}
|
||||
return model
|
||||
|
||||
|
||||
def svmPredict(model, X):
|
||||
"""
|
||||
Returns a vector of predictions using a trained SVM model.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
model : dict
|
||||
The parameters of the trained svm model, as returned by the function svmTrain
|
||||
|
||||
X : array_like
|
||||
A (m x n) matrix where each example is a row.
|
||||
|
||||
Returns
|
||||
-------
|
||||
pred : array_like
|
||||
A (m,) sized vector of predictions {0, 1} values.
|
||||
"""
|
||||
# check if we are getting a vector. If so, then assume we only need to do predictions
|
||||
# for a single example
|
||||
if X.ndim == 1:
|
||||
X = X[np.newaxis, :]
|
||||
|
||||
m = X.shape[0]
|
||||
p = np.zeros(m)
|
||||
pred = np.zeros(m)
|
||||
|
||||
if model['kernelFunction'].__name__ == 'linearKernel':
|
||||
# we can use the weights and bias directly if working with the linear kernel
|
||||
p = np.dot(X, model['w']) + model['b']
|
||||
elif model['kernelFunction'].__name__ == 'gaussianKernel':
|
||||
# vectorized RBF Kernel
|
||||
# This is equivalent to computing the kernel on every pair of examples
|
||||
X1 = np.sum(X**2, 1)
|
||||
X2 = np.sum(model['X']**2, 1)
|
||||
K = X2 + X1[:, None] - 2 * np.dot(X, model['X'].T)
|
||||
|
||||
if len(model['args']) > 0:
|
||||
K /= 2*model['args'][0]**2
|
||||
|
||||
K = np.exp(-K)
|
||||
p = np.dot(K, model['alphas']*model['y']) + model['b']
|
||||
else:
|
||||
# other non-linear kernel
|
||||
for i in range(m):
|
||||
predictions = 0
|
||||
for j in range(model['X'].shape[0]):
|
||||
predictions += model['alphas'][j] * model['y'][j] \
|
||||
* model['kernelFunction'](X[i, :], model['X'][j, :])
|
||||
p[i] = predictions
|
||||
|
||||
pred[p >= 0] = 1
|
||||
return pred
|
||||
|
||||
|
||||
def linearKernel(x1, x2):
|
||||
"""
|
||||
Returns a linear kernel between x1 and x2.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
x1 : numpy ndarray
|
||||
A 1-D vector.
|
||||
|
||||
x2 : numpy ndarray
|
||||
A 1-D vector of same size as x1.
|
||||
|
||||
Returns
|
||||
-------
|
||||
: float
|
||||
The scalar amplitude.
|
||||
"""
|
||||
return np.dot(x1, x2)
|
||||
|
||||
|
||||
def visualizeBoundaryLinear(X, y, model):
|
||||
"""
|
||||
Plots a linear decision boundary learned by the SVM.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : array_like
|
||||
(m x 2) The training data with two features (to plot in a 2-D plane).
|
||||
|
||||
y : array_like
|
||||
(m, ) The data labels.
|
||||
|
||||
model : dict
|
||||
Dictionary of model variables learned by SVM.
|
||||
"""
|
||||
w, b = model['w'], model['b']
|
||||
xp = np.linspace(min(X[:, 0]), max(X[:, 0]), 100)
|
||||
yp = -(w[0] * xp + b)/w[1]
|
||||
|
||||
plotData(X, y)
|
||||
pyplot.plot(xp, yp, '-b')
|
||||
|
||||
|
||||
def visualizeBoundary(X, y, model):
|
||||
"""
|
||||
Plots a non-linear decision boundary learned by the SVM and overlays the data on it.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : array_like
|
||||
(m x 2) The training data with two features (to plot in a 2-D plane).
|
||||
|
||||
y : array_like
|
||||
(m, ) The data labels.
|
||||
|
||||
model : dict
|
||||
Dictionary of model variables learned by SVM.
|
||||
"""
|
||||
plotData(X, y)
|
||||
|
||||
# make classification predictions over a grid of values
|
||||
x1plot = np.linspace(min(X[:, 0]), max(X[:, 0]), 100)
|
||||
x2plot = np.linspace(min(X[:, 1]), max(X[:, 1]), 100)
|
||||
X1, X2 = np.meshgrid(x1plot, x2plot)
|
||||
|
||||
vals = np.zeros(X1.shape)
|
||||
for i in range(X1.shape[1]):
|
||||
this_X = np.stack((X1[:, i], X2[:, i]), axis=1)
|
||||
vals[:, i] = svmPredict(model, this_X)
|
||||
|
||||
pyplot.contour(X1, X2, vals, colors='y', linewidths=2)
|
||||
pyplot.pcolormesh(X1, X2, vals, cmap='YlGnBu', alpha=0.25, edgecolors='None', lw=0)
|
||||
pyplot.grid(False)
|
||||
|
||||
|
||||
def getVocabList():
|
||||
"""
|
||||
Reads the fixed vocabulary list in vocab.txt and returns a cell array of the words
|
||||
% vocabList = GETVOCABLIST() reads the fixed vocabulary list in vocab.txt
|
||||
% and returns a cell array of the words in vocabList.
|
||||
|
||||
:return:
|
||||
"""
|
||||
vocabList = np.genfromtxt(join('Data', 'vocab.txt'), dtype=object)
|
||||
return list(vocabList[:, 1].astype(str))
|
||||
|
||||
|
||||
class PorterStemmer:
|
||||
"""
|
||||
Porter Stemming Algorithm
|
||||
|
||||
This is the Porter stemming algorithm, ported to Python from the
|
||||
version coded up in ANSI C by the author. It may be be regarded
|
||||
as canonical, in that it follows the algorithm presented in
|
||||
|
||||
Porter, 1980, An algorithm for suffix stripping, Program, Vol. 14,
|
||||
no. 3, pp 130-137,
|
||||
|
||||
only differing from it at the points maked --DEPARTURE-- below.
|
||||
|
||||
See also http://www.tartarus.org/~martin/PorterStemmer
|
||||
|
||||
The algorithm as described in the paper could be exactly replicated
|
||||
by adjusting the points of DEPARTURE, but this is barely necessary,
|
||||
because (a) the points of DEPARTURE are definitely improvements, and
|
||||
(b) no encoding of the Porter stemmer I have seen is anything like
|
||||
as exact as this version, even with the points of DEPARTURE!
|
||||
|
||||
Vivake Gupta (v@nano.com)
|
||||
|
||||
Release 1: January 2001
|
||||
|
||||
Further adjustments by Santiago Bruno (bananabruno@gmail.com)
|
||||
to allow word input not restricted to one word per line, leading
|
||||
to:
|
||||
|
||||
release 2: July 2008
|
||||
"""
|
||||
def __init__(self):
|
||||
"""
|
||||
The main part of the stemming algorithm starts here.
|
||||
b is a buffer holding a word to be stemmed. The letters are in b[k0],
|
||||
b[k0+1] ... ending at b[k]. In fact k0 = 0 in this demo program. k is
|
||||
readjusted downwards as the stemming progresses. Zero termination is
|
||||
not in fact used in the algorithm.
|
||||
|
||||
Note that only lower case sequences are stemmed. Forcing to lower case
|
||||
should be done before stem(...) is called.
|
||||
"""
|
||||
self.b = "" # buffer for word to be stemmed
|
||||
self.k = 0
|
||||
self.k0 = 0
|
||||
self.j = 0 # j is a general offset into the string
|
||||
|
||||
def cons(self, i):
|
||||
"""cons(i) is TRUE <=> b[i] is a consonant."""
|
||||
if self.b[i] in 'aeiou':
|
||||
return 0
|
||||
if self.b[i] == 'y':
|
||||
if i == self.k0:
|
||||
return 1
|
||||
else:
|
||||
return not self.cons(i - 1)
|
||||
return 1
|
||||
|
||||
def m(self):
|
||||
"""
|
||||
m() measures the number of consonant sequences between k0 and j.
|
||||
if c is a consonant sequence and v a vowel sequence, and <..>
|
||||
indicates arbitrary presence,
|
||||
|
||||
<c><v> gives 0
|
||||
<c>vc<v> gives 1
|
||||
<c>vcvc<v> gives 2
|
||||
<c>vcvcvc<v> gives 3
|
||||
....
|
||||
"""
|
||||
n = 0
|
||||
i = self.k0
|
||||
while 1:
|
||||
if i > self.j:
|
||||
return n
|
||||
if not self.cons(i):
|
||||
break
|
||||
i = i + 1
|
||||
i = i + 1
|
||||
while 1:
|
||||
while 1:
|
||||
if i > self.j:
|
||||
return n
|
||||
if self.cons(i):
|
||||
break
|
||||
i = i + 1
|
||||
i = i + 1
|
||||
n = n + 1
|
||||
while 1:
|
||||
if i > self.j:
|
||||
return n
|
||||
if not self.cons(i):
|
||||
break
|
||||
i = i + 1
|
||||
i = i + 1
|
||||
|
||||
def vowelinstem(self):
|
||||
"""vowelinstem() is TRUE <=> k0,...j contains a vowel"""
|
||||
for i in range(self.k0, self.j + 1):
|
||||
if not self.cons(i):
|
||||
return 1
|
||||
return 0
|
||||
|
||||
def doublec(self, j):
|
||||
""" doublec(j) is TRUE <=> j,(j-1) contain a double consonant. """
|
||||
if j < (self.k0 + 1):
|
||||
return 0
|
||||
if self.b[j] != self.b[j-1]:
|
||||
return 0
|
||||
return self.cons(j)
|
||||
|
||||
def cvc(self, i):
|
||||
"""
|
||||
cvc(i) is TRUE <=> i-2,i-1,i has the form consonant - vowel - consonant
|
||||
and also if the second c is not w,x or y. this is used when trying to
|
||||
restore an e at the end of a short e.g.
|
||||
|
||||
cav(e), lov(e), hop(e), crim(e), but
|
||||
snow, box, tray.
|
||||
"""
|
||||
if i < (self.k0 + 2) or not self.cons(i) or self.cons(i-1) or not self.cons(i-2):
|
||||
return 0
|
||||
ch = self.b[i]
|
||||
if ch in 'wxy':
|
||||
return 0
|
||||
return 1
|
||||
|
||||
def ends(self, s):
|
||||
"""ends(s) is TRUE <=> k0,...k ends with the string s."""
|
||||
length = len(s)
|
||||
if s[length - 1] != self.b[self.k]: # tiny speed-up
|
||||
return 0
|
||||
if length > (self.k - self.k0 + 1):
|
||||
return 0
|
||||
if self.b[self.k-length+1:self.k+1] != s:
|
||||
return 0
|
||||
self.j = self.k - length
|
||||
return 1
|
||||
|
||||
def setto(self, s):
|
||||
"""setto(s) sets (j+1),...k to the characters in the string s, readjusting k."""
|
||||
length = len(s)
|
||||
self.b = self.b[:self.j+1] + s + self.b[self.j+length+1:]
|
||||
self.k = self.j + length
|
||||
|
||||
def r(self, s):
|
||||
"""r(s) is used further down."""
|
||||
if self.m() > 0:
|
||||
self.setto(s)
|
||||
|
||||
def step1ab(self):
|
||||
"""step1ab() gets rid of plurals and -ed or -ing. e.g.
|
||||
|
||||
caresses -> caress
|
||||
ponies -> poni
|
||||
ties -> ti
|
||||
caress -> caress
|
||||
cats -> cat
|
||||
|
||||
feed -> feed
|
||||
agreed -> agree
|
||||
disabled -> disable
|
||||
|
||||
matting -> mat
|
||||
mating -> mate
|
||||
meeting -> meet
|
||||
milling -> mill
|
||||
messing -> mess
|
||||
|
||||
meetings -> meet
|
||||
"""
|
||||
if self.b[self.k] == 's':
|
||||
if self.ends("sses"):
|
||||
self.k = self.k - 2
|
||||
elif self.ends("ies"):
|
||||
self.setto("i")
|
||||
elif self.b[self.k - 1] != 's':
|
||||
self.k = self.k - 1
|
||||
if self.ends("eed"):
|
||||
if self.m() > 0:
|
||||
self.k = self.k - 1
|
||||
elif (self.ends("ed") or self.ends("ing")) and self.vowelinstem():
|
||||
self.k = self.j
|
||||
if self.ends("at"):
|
||||
self.setto("ate")
|
||||
elif self.ends("bl"):
|
||||
self.setto("ble")
|
||||
elif self.ends("iz"):
|
||||
self.setto("ize")
|
||||
elif self.doublec(self.k):
|
||||
self.k = self.k - 1
|
||||
ch = self.b[self.k]
|
||||
if ch in 'lsz':
|
||||
self.k += 1
|
||||
elif self.m() == 1 and self.cvc(self.k):
|
||||
self.setto("e")
|
||||
|
||||
def step1c(self):
|
||||
"""step1c() turns terminal y to i when there is another vowel in the stem."""
|
||||
if self.ends("y") and self.vowelinstem():
|
||||
self.b = self.b[:self.k] + 'i' + self.b[self.k+1:]
|
||||
|
||||
def step2(self):
|
||||
"""step2() maps double suffices to single ones.
|
||||
so -ization ( = -ize plus -ation) maps to -ize etc. note that the
|
||||
string before the suffix must give m() > 0.
|
||||
"""
|
||||
if self.b[self.k - 1] == 'a':
|
||||
if self.ends("ational"): self.r("ate")
|
||||
elif self.ends("tional"): self.r("tion")
|
||||
elif self.b[self.k - 1] == 'c':
|
||||
if self.ends("enci"): self.r("ence")
|
||||
elif self.ends("anci"): self.r("ance")
|
||||
elif self.b[self.k - 1] == 'e':
|
||||
if self.ends("izer"): self.r("ize")
|
||||
elif self.b[self.k - 1] == 'l':
|
||||
if self.ends("bli"): self.r("ble") # --DEPARTURE--
|
||||
# To match the published algorithm, replace this phrase with
|
||||
# if self.ends("abli"): self.r("able")
|
||||
elif self.ends("alli"): self.r("al")
|
||||
elif self.ends("entli"): self.r("ent")
|
||||
elif self.ends("eli"): self.r("e")
|
||||
elif self.ends("ousli"): self.r("ous")
|
||||
elif self.b[self.k - 1] == 'o':
|
||||
if self.ends("ization"): self.r("ize")
|
||||
elif self.ends("ation"): self.r("ate")
|
||||
elif self.ends("ator"): self.r("ate")
|
||||
elif self.b[self.k - 1] == 's':
|
||||
if self.ends("alism"): self.r("al")
|
||||
elif self.ends("iveness"): self.r("ive")
|
||||
elif self.ends("fulness"): self.r("ful")
|
||||
elif self.ends("ousness"): self.r("ous")
|
||||
elif self.b[self.k - 1] == 't':
|
||||
if self.ends("aliti"): self.r("al")
|
||||
elif self.ends("iviti"): self.r("ive")
|
||||
elif self.ends("biliti"): self.r("ble")
|
||||
elif self.b[self.k - 1] == 'g': # --DEPARTURE--
|
||||
if self.ends("logi"): self.r("log")
|
||||
# To match the published algorithm, delete this phrase
|
||||
|
||||
def step3(self):
|
||||
"""step3() dels with -ic-, -full, -ness etc. similar strategy to step2."""
|
||||
if self.b[self.k] == 'e':
|
||||
if self.ends("icate"): self.r("ic")
|
||||
elif self.ends("ative"): self.r("")
|
||||
elif self.ends("alize"): self.r("al")
|
||||
elif self.b[self.k] == 'i':
|
||||
if self.ends("iciti"): self.r("ic")
|
||||
elif self.b[self.k] == 'l':
|
||||
if self.ends("ical"): self.r("ic")
|
||||
elif self.ends("ful"): self.r("")
|
||||
elif self.b[self.k] == 's':
|
||||
if self.ends("ness"): self.r("")
|
||||
|
||||
def step4(self):
|
||||
"""step4() takes off -ant, -ence etc., in context <c>vcvc<v>."""
|
||||
if self.b[self.k - 1] == 'a':
|
||||
if self.ends("al"): pass
|
||||
else: return
|
||||
elif self.b[self.k - 1] == 'c':
|
||||
if self.ends("ance"): pass
|
||||
elif self.ends("ence"): pass
|
||||
else: return
|
||||
elif self.b[self.k - 1] == 'e':
|
||||
if self.ends("er"): pass
|
||||
else: return
|
||||
elif self.b[self.k - 1] == 'i':
|
||||
if self.ends("ic"): pass
|
||||
else: return
|
||||
elif self.b[self.k - 1] == 'l':
|
||||
if self.ends("able"): pass
|
||||
elif self.ends("ible"): pass
|
||||
else: return
|
||||
elif self.b[self.k - 1] == 'n':
|
||||
if self.ends("ant"): pass
|
||||
elif self.ends("ement"): pass
|
||||
elif self.ends("ment"): pass
|
||||
elif self.ends("ent"): pass
|
||||
else: return
|
||||
elif self.b[self.k - 1] == 'o':
|
||||
if self.ends("ion") and (self.b[self.j] == 's' or self.b[self.j] == 't'): pass
|
||||
elif self.ends("ou"): pass
|
||||
# takes care of -ous
|
||||
else: return
|
||||
elif self.b[self.k - 1] == 's':
|
||||
if self.ends("ism"): pass
|
||||
else: return
|
||||
elif self.b[self.k - 1] == 't':
|
||||
if self.ends("ate"): pass
|
||||
elif self.ends("iti"): pass
|
||||
else: return
|
||||
elif self.b[self.k - 1] == 'u':
|
||||
if self.ends("ous"): pass
|
||||
else: return
|
||||
elif self.b[self.k - 1] == 'v':
|
||||
if self.ends("ive"): pass
|
||||
else: return
|
||||
elif self.b[self.k - 1] == 'z':
|
||||
if self.ends("ize"): pass
|
||||
else: return
|
||||
else:
|
||||
return
|
||||
if self.m() > 1:
|
||||
self.k = self.j
|
||||
|
||||
def step5(self):
|
||||
"""step5() removes a final -e if m() > 1, and changes -ll to -l if
|
||||
m() > 1.
|
||||
"""
|
||||
self.j = self.k
|
||||
if self.b[self.k] == 'e':
|
||||
a = self.m()
|
||||
if a > 1 or (a == 1 and not self.cvc(self.k-1)):
|
||||
self.k = self.k - 1
|
||||
if self.b[self.k] == 'l' and self.doublec(self.k) and self.m() > 1:
|
||||
self.k = self.k -1
|
||||
|
||||
def stem(self, p, i=0, j=None):
|
||||
"""In stem(p,i,j), p is a char pointer, and the string to be stemmed
|
||||
is from p[i] to p[j] inclusive. Typically i is zero and j is the
|
||||
offset to the last character of a string, (p[j+1] == '\0'). The
|
||||
stemmer adjusts the characters p[i] ... p[j] and returns the new
|
||||
end-point of the string, k. Stemming never increases word length, so
|
||||
i <= k <= j. To turn the stemmer into a module, declare 'stem' as
|
||||
extern, and delete the remainder of this file.
|
||||
"""
|
||||
# copy the parameters into statics
|
||||
self.b = p
|
||||
self.k = j or len(p) - 1
|
||||
self.k0 = i
|
||||
if self.k <= self.k0 + 1:
|
||||
return self.b # --DEPARTURE--
|
||||
|
||||
# With this line, strings of length 1 or 2 don't go through the
|
||||
# stemming process, although no mention is made of this in the
|
||||
# published algorithm. Remove the line to match the published
|
||||
# algorithm.
|
||||
|
||||
self.step1ab()
|
||||
self.step1c()
|
||||
self.step2()
|
||||
self.step3()
|
||||
self.step4()
|
||||
self.step5()
|
||||
return self.b[self.k0:self.k+1]
|
||||
|
||||
|
||||
class Grader(SubmissionBase):
|
||||
# Random Test Cases
|
||||
x1 = np.sin(np.arange(1, 11))
|
||||
x2 = np.cos(np.arange(1, 11))
|
||||
ec = 'the quick brown fox jumped over the lazy dog'
|
||||
wi = np.abs(np.round(x1 * 1863)).astype(int)
|
||||
wi = np.concatenate([wi, wi])
|
||||
|
||||
def __init__(self):
|
||||
part_names = ['Gaussian Kernel',
|
||||
'Parameters (C, sigma) for Dataset 3',
|
||||
'Email Processing',
|
||||
'Email Feature Extraction']
|
||||
super().__init__('support-vector-machines', part_names)
|
||||
|
||||
def __iter__(self):
|
||||
for part_id in range(1, 5):
|
||||
try:
|
||||
func = self.functions[part_id]
|
||||
# Each part has different expected arguments/different function
|
||||
if part_id == 1:
|
||||
res = func(self.x1, self.x2, 2)
|
||||
elif part_id == 2:
|
||||
res = np.hstack(func()).tolist()
|
||||
elif part_id == 3:
|
||||
# add one to be compatible with matlab grader
|
||||
res = [ind+1 for ind in func(self.ec, False)]
|
||||
elif part_id == 4:
|
||||
res = func(self.wi)
|
||||
else:
|
||||
raise KeyError
|
||||
yield part_id, res
|
||||
except KeyError:
|
||||
yield part_id, 0
|
||||
BIN
Exercise7/Data/bird_small.mat
Executable file
BIN
Exercise7/Data/bird_small.mat
Executable file
Binary file not shown.
BIN
Exercise7/Data/bird_small.png
Executable file
BIN
Exercise7/Data/bird_small.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 32 KiB |
BIN
Exercise7/Data/ex7data1.mat
Executable file
BIN
Exercise7/Data/ex7data1.mat
Executable file
Binary file not shown.
BIN
Exercise7/Data/ex7data2.mat
Executable file
BIN
Exercise7/Data/ex7data2.mat
Executable file
Binary file not shown.
BIN
Exercise7/Data/ex7faces.mat
Executable file
BIN
Exercise7/Data/ex7faces.mat
Executable file
Binary file not shown.
BIN
Exercise7/Figures/bird_compression.png
Executable file
BIN
Exercise7/Figures/bird_compression.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 70 KiB |
BIN
Exercise7/Figures/faces.png
Executable file
BIN
Exercise7/Figures/faces.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 174 KiB |
BIN
Exercise7/Figures/faces_original.png
Executable file
BIN
Exercise7/Figures/faces_original.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 154 KiB |
BIN
Exercise7/Figures/faces_reconstructed.png
Executable file
BIN
Exercise7/Figures/faces_reconstructed.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 125 KiB |
BIN
Exercise7/Figures/kmeans_result.png
Executable file
BIN
Exercise7/Figures/kmeans_result.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 28 KiB |
BIN
Exercise7/Figures/pca_components.png
Executable file
BIN
Exercise7/Figures/pca_components.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 7.4 KiB |
BIN
Exercise7/Figures/pca_reconstruction.png
Executable file
BIN
Exercise7/Figures/pca_reconstruction.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 17 KiB |
1194
Exercise7/exercise7.ipynb
Executable file
1194
Exercise7/exercise7.ipynb
Executable file
File diff suppressed because it is too large
Load Diff
236
Exercise7/utils.py
Executable file
236
Exercise7/utils.py
Executable file
@@ -0,0 +1,236 @@
|
||||
import sys
|
||||
import numpy as np
|
||||
from matplotlib import pyplot
|
||||
from matplotlib.animation import FuncAnimation
|
||||
import matplotlib as mpl
|
||||
|
||||
sys.path.append('..')
|
||||
from submission import SubmissionBase
|
||||
|
||||
|
||||
def displayData(X, example_width=None, figsize=(10, 10)):
|
||||
"""
|
||||
Displays 2D data in a nice grid.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : array_like
|
||||
The input data of size (m x n) where m is the number of examples and n is the number of
|
||||
features.
|
||||
|
||||
example_width : int, optional
|
||||
THe width of each 2-D image in pixels. If not provided, the image is assumed to be square,
|
||||
and the width is the floor of the square root of total number of pixels.
|
||||
|
||||
figsize : tuple, optional
|
||||
A 2-element tuple indicating the width and height of figure in inches.
|
||||
"""
|
||||
# Compute rows, cols
|
||||
if X.ndim == 2:
|
||||
m, n = X.shape
|
||||
elif X.ndim == 1:
|
||||
n = X.size
|
||||
m = 1
|
||||
X = X[None] # Promote to a 2 dimensional array
|
||||
else:
|
||||
raise IndexError('Input X should be 1 or 2 dimensional.')
|
||||
|
||||
example_width = example_width or int(np.round(np.sqrt(n)))
|
||||
example_height = int(n / example_width)
|
||||
|
||||
# Compute number of items to display
|
||||
display_rows = int(np.floor(np.sqrt(m)))
|
||||
display_cols = int(np.ceil(m / display_rows))
|
||||
|
||||
fig, ax_array = pyplot.subplots(display_rows, display_cols, figsize=figsize)
|
||||
fig.subplots_adjust(wspace=0.025, hspace=0.025)
|
||||
|
||||
ax_array = [ax_array] if m == 1 else ax_array.ravel()
|
||||
|
||||
for i, ax in enumerate(ax_array):
|
||||
ax.imshow(X[i].reshape(example_height, example_width, order='F'), cmap='gray')
|
||||
ax.axis('off')
|
||||
|
||||
|
||||
def featureNormalize(X):
|
||||
"""
|
||||
Normalizes the features in X returns a normalized version of X where the mean value of each
|
||||
feature is 0 and the standard deviation is 1. This is often a good preprocessing step to do when
|
||||
working with learning algorithms.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : array_like
|
||||
An dataset which is a (m x n) matrix, where m is the number of examples,
|
||||
and n is the number of dimensions for each example.
|
||||
|
||||
Returns
|
||||
-------
|
||||
X_norm : array_like
|
||||
The normalized input dataset.
|
||||
|
||||
mu : array_like
|
||||
A vector of size n corresponding to the mean for each dimension across all examples.
|
||||
|
||||
sigma : array_like
|
||||
A vector of size n corresponding to the standard deviations for each dimension across
|
||||
all examples.
|
||||
"""
|
||||
mu = np.mean(X, axis=0)
|
||||
X_norm = X - mu
|
||||
|
||||
sigma = np.std(X_norm, axis=0, ddof=1)
|
||||
X_norm /= sigma
|
||||
return X_norm, mu, sigma
|
||||
|
||||
|
||||
def plotProgresskMeans(i, X, centroid_history, idx_history):
|
||||
"""
|
||||
A helper function that displays the progress of k-Means as it is running. It is intended for use
|
||||
only with 2D data. It plots data points with colors assigned to each centroid. With the
|
||||
previous centroids, it also plots a line between the previous locations and current locations
|
||||
of the centroids.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
i : int
|
||||
Current iteration number of k-means. Used for matplotlib animation function.
|
||||
|
||||
X : array_like
|
||||
The dataset, which is a matrix (m x n). Note since the plot only supports 2D data, n should
|
||||
be equal to 2.
|
||||
|
||||
centroid_history : list
|
||||
A list of computed centroids for all iteration.
|
||||
|
||||
idx_history : list
|
||||
A list of computed assigned indices for all iterations.
|
||||
"""
|
||||
K = centroid_history[0].shape[0]
|
||||
pyplot.gcf().clf()
|
||||
cmap = pyplot.cm.rainbow
|
||||
norm = mpl.colors.Normalize(vmin=0, vmax=2)
|
||||
|
||||
for k in range(K):
|
||||
current = np.stack([c[k, :] for c in centroid_history[:i+1]], axis=0)
|
||||
pyplot.plot(current[:, 0], current[:, 1],
|
||||
'-Xk',
|
||||
mec='k',
|
||||
lw=2,
|
||||
ms=10,
|
||||
mfc=cmap(norm(k)),
|
||||
mew=2)
|
||||
|
||||
pyplot.scatter(X[:, 0], X[:, 1],
|
||||
c=idx_history[i],
|
||||
cmap=cmap,
|
||||
marker='o',
|
||||
s=8**2,
|
||||
linewidths=1,)
|
||||
pyplot.grid(False)
|
||||
pyplot.title('Iteration number %d' % (i+1))
|
||||
|
||||
|
||||
def runkMeans(X, centroids, findClosestCentroids, computeCentroids,
|
||||
max_iters=10, plot_progress=False):
|
||||
"""
|
||||
Runs the K-means algorithm.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : array_like
|
||||
The data set of size (m, n). Each row of X is a single example of n dimensions. The
|
||||
data set is a total of m examples.
|
||||
|
||||
centroids : array_like
|
||||
Initial centroid location for each clusters. This is a matrix of size (K, n). K is the total
|
||||
number of clusters and n is the dimensions of each data point.
|
||||
|
||||
findClosestCentroids : func
|
||||
A function (implemented by student) reference which computes the cluster assignment for
|
||||
each example.
|
||||
|
||||
computeCentroids : func
|
||||
A function(implemented by student) reference which computes the centroid of each cluster.
|
||||
|
||||
max_iters : int, optional
|
||||
Specifies the total number of interactions of K-Means to execute.
|
||||
|
||||
plot_progress : bool, optional
|
||||
A flag that indicates if the function should also plot its progress as the learning happens.
|
||||
This is set to false by default.
|
||||
|
||||
Returns
|
||||
-------
|
||||
centroids : array_like
|
||||
A (K x n) matrix of the computed (updated) centroids.
|
||||
idx : array_like
|
||||
A vector of size (m,) for cluster assignment for each example in the dataset. Each entry
|
||||
in idx is within the range [0 ... K-1].
|
||||
|
||||
anim : FuncAnimation, optional
|
||||
A matplotlib animation object which can be used to embed a video within the jupyter
|
||||
notebook. This is only returned if `plot_progress` is `True`.
|
||||
"""
|
||||
K = centroids.shape[0]
|
||||
idx = None
|
||||
idx_history = []
|
||||
centroid_history = []
|
||||
|
||||
for i in range(max_iters):
|
||||
idx = findClosestCentroids(X, centroids)
|
||||
|
||||
if plot_progress:
|
||||
idx_history.append(idx)
|
||||
centroid_history.append(centroids)
|
||||
|
||||
centroids = computeCentroids(X, idx, K)
|
||||
|
||||
if plot_progress:
|
||||
fig = pyplot.figure()
|
||||
anim = FuncAnimation(fig, plotProgresskMeans,
|
||||
frames=max_iters,
|
||||
interval=500,
|
||||
repeat_delay=2,
|
||||
fargs=(X, centroid_history, idx_history))
|
||||
return centroids, idx, anim
|
||||
|
||||
return centroids, idx
|
||||
|
||||
|
||||
class Grader(SubmissionBase):
|
||||
# Random Test Cases
|
||||
X = np.sin(np.arange(1, 166)).reshape(15, 11, order='F')
|
||||
Z = np.cos(np.arange(1, 122)).reshape(11, 11, order='F')
|
||||
C = Z[:5, :]
|
||||
idx = np.arange(1, 16) % 3
|
||||
|
||||
def __init__(self):
|
||||
part_names = ['Find Closest Centroids (k-Means)',
|
||||
'Compute Centroid Means (k-Means)',
|
||||
'PCA',
|
||||
'Project Data (PCA)',
|
||||
'Recover Data (PCA)']
|
||||
super().__init__('k-means-clustering-and-pca', part_names)
|
||||
|
||||
def __iter__(self):
|
||||
for part_id in range(1, 6):
|
||||
try:
|
||||
func = self.functions[part_id]
|
||||
# Each part has different expected arguments/different function
|
||||
if part_id == 1:
|
||||
res = 1 + func(self.X, self.C)
|
||||
elif part_id == 2:
|
||||
res = func(self.X, self.idx, 3)
|
||||
elif part_id == 3:
|
||||
U, S = func(self.X)
|
||||
res = np.hstack([U.ravel('F'), np.diag(S).ravel('F')]).tolist()
|
||||
elif part_id == 4:
|
||||
res = func(self.X, self.Z, 5)
|
||||
elif part_id == 5:
|
||||
res = func(self.X[:, :5], self.Z, 5)
|
||||
else:
|
||||
raise KeyError
|
||||
yield part_id, res
|
||||
except KeyError:
|
||||
yield part_id, 0
|
||||
BIN
Exercise8/Data/ex8_movieParams.mat
Executable file
BIN
Exercise8/Data/ex8_movieParams.mat
Executable file
Binary file not shown.
BIN
Exercise8/Data/ex8_movies.mat
Executable file
BIN
Exercise8/Data/ex8_movies.mat
Executable file
Binary file not shown.
BIN
Exercise8/Data/ex8data1.mat
Executable file
BIN
Exercise8/Data/ex8data1.mat
Executable file
Binary file not shown.
BIN
Exercise8/Data/ex8data2.mat
Executable file
BIN
Exercise8/Data/ex8data2.mat
Executable file
Binary file not shown.
1682
Exercise8/Data/movie_ids.txt
Executable file
1682
Exercise8/Data/movie_ids.txt
Executable file
File diff suppressed because it is too large
Load Diff
BIN
Exercise8/Figures/gaussian_fit.png
Executable file
BIN
Exercise8/Figures/gaussian_fit.png
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 46 KiB |
1028
Exercise8/exercise8.ipynb
Executable file
1028
Exercise8/exercise8.ipynb
Executable file
File diff suppressed because it is too large
Load Diff
269
Exercise8/utils.py
Executable file
269
Exercise8/utils.py
Executable file
@@ -0,0 +1,269 @@
|
||||
import numpy as np
|
||||
import sys
|
||||
from os.path import join
|
||||
from matplotlib import pyplot
|
||||
|
||||
sys.path.append('..')
|
||||
from submission import SubmissionBase
|
||||
|
||||
|
||||
def normalizeRatings(Y, R):
|
||||
"""
|
||||
Preprocess data by subtracting mean rating for every movie (every row).
|
||||
|
||||
Parameters
|
||||
----------
|
||||
Y : array_like
|
||||
The user ratings for all movies. A matrix of shape (num_movies x num_users).
|
||||
|
||||
R : array_like
|
||||
Indicator matrix for movies rated by users. A matrix of shape (num_movies x num_users).
|
||||
|
||||
Returns
|
||||
-------
|
||||
Ynorm : array_like
|
||||
A matrix of same shape as Y, after mean normalization.
|
||||
|
||||
Ymean : array_like
|
||||
A vector of shape (num_movies, ) containing the mean rating for each movie.
|
||||
"""
|
||||
m, n = Y.shape
|
||||
Ymean = np.zeros(m)
|
||||
Ynorm = np.zeros(Y.shape)
|
||||
|
||||
for i in range(m):
|
||||
idx = R[i, :] == 1
|
||||
Ymean[i] = np.mean(Y[i, idx])
|
||||
Ynorm[i, idx] = Y[i, idx] - Ymean[i]
|
||||
|
||||
return Ynorm, Ymean
|
||||
|
||||
|
||||
def loadMovieList():
|
||||
"""
|
||||
Reads the fixed movie list in movie_ids.txt and returns a list of movie names.
|
||||
|
||||
Returns
|
||||
-------
|
||||
movieNames : list
|
||||
A list of strings, representing all movie names.
|
||||
"""
|
||||
# Read the fixed movieulary list
|
||||
with open(join('Data', 'movie_ids.txt'), encoding='ISO-8859-1') as fid:
|
||||
movies = fid.readlines()
|
||||
|
||||
movieNames = []
|
||||
for movie in movies:
|
||||
parts = movie.split()
|
||||
movieNames.append(' '.join(parts[1:]).strip())
|
||||
return movieNames
|
||||
|
||||
|
||||
def computeNumericalGradient(J, theta, e=1e-4):
|
||||
"""
|
||||
Computes the gradient using "finite differences" and gives us a numerical estimate of the
|
||||
gradient.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
J : func
|
||||
The cost function which will be used to estimate its numerical gradient.
|
||||
|
||||
theta : array_like
|
||||
The one dimensional unrolled network parameters. The numerical gradient is computed at
|
||||
those given parameters.
|
||||
|
||||
e : float (optional)
|
||||
The value to use for epsilon for computing the finite difference.
|
||||
|
||||
Returns
|
||||
-------
|
||||
numgrad : array_like
|
||||
The numerical gradient with respect to theta. Has same shape as theta.
|
||||
|
||||
Notes
|
||||
-----
|
||||
The following code implements numerical gradient checking, and
|
||||
returns the numerical gradient. It sets `numgrad[i]` to (a numerical
|
||||
approximation of) the partial derivative of J with respect to the
|
||||
i-th input argument, evaluated at theta. (i.e., `numgrad[i]` should
|
||||
be the (approximately) the partial derivative of J with respect
|
||||
to theta[i].)
|
||||
"""
|
||||
numgrad = np.zeros(theta.shape)
|
||||
perturb = np.diag(e * np.ones(theta.shape))
|
||||
for i in range(theta.size):
|
||||
loss1, _ = J(theta - perturb[:, i])
|
||||
loss2, _ = J(theta + perturb[:, i])
|
||||
numgrad[i] = (loss2 - loss1)/(2*e)
|
||||
return numgrad
|
||||
|
||||
|
||||
def checkCostFunction(cofiCostFunc, lambda_=0.):
|
||||
"""
|
||||
Creates a collaborative filtering problem to check your cost function and gradients.
|
||||
It will output the analytical gradients produced by your code and the numerical gradients
|
||||
(computed using computeNumericalGradient). These two gradient computations should result
|
||||
in very similar values.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
cofiCostFunc: func
|
||||
Implementation of the cost function.
|
||||
|
||||
lambda_ : float, optional
|
||||
The regularization parameter.
|
||||
"""
|
||||
# Create small problem
|
||||
X_t = np.random.rand(4, 3)
|
||||
Theta_t = np.random.rand(5, 3)
|
||||
|
||||
# Zap out most entries
|
||||
Y = np.dot(X_t, Theta_t.T)
|
||||
Y[np.random.rand(*Y.shape) > 0.5] = 0
|
||||
R = np.zeros(Y.shape)
|
||||
R[Y != 0] = 1
|
||||
|
||||
# Run Gradient Checking
|
||||
X = np.random.randn(*X_t.shape)
|
||||
Theta = np.random.randn(*Theta_t.shape)
|
||||
num_movies, num_users = Y.shape
|
||||
num_features = Theta_t.shape[1]
|
||||
|
||||
params = np.concatenate([X.ravel(), Theta.ravel()])
|
||||
numgrad = computeNumericalGradient(
|
||||
lambda x: cofiCostFunc(x, Y, R, num_users, num_movies, num_features, lambda_), params)
|
||||
|
||||
cost, grad = cofiCostFunc(params, Y, R, num_users,num_movies, num_features, lambda_)
|
||||
|
||||
print(np.stack([numgrad, grad], axis=1))
|
||||
print('\nThe above two columns you get should be very similar.'
|
||||
'(Left-Your Numerical Gradient, Right-Analytical Gradient)')
|
||||
|
||||
diff = np.linalg.norm(numgrad-grad)/np.linalg.norm(numgrad+grad)
|
||||
print('If your cost function implementation is correct, then '
|
||||
'the relative difference will be small (less than 1e-9).')
|
||||
print('\nRelative Difference: %g' % diff)
|
||||
|
||||
|
||||
def multivariateGaussian(X, mu, Sigma2):
|
||||
"""
|
||||
Computes the probability density function of the multivariate gaussian distribution.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : array_like
|
||||
The dataset of shape (m x n). Where there are m examples of n-dimensions.
|
||||
|
||||
mu : array_like
|
||||
A vector of shape (n,) contains the means for each dimension (feature).
|
||||
|
||||
Sigma2 : array_like
|
||||
Either a vector of shape (n,) containing the variances of independent features
|
||||
(i.e. it is the diagonal of the correlation matrix), or the full
|
||||
correlation matrix of shape (n x n) which can represent dependent features.
|
||||
|
||||
Returns
|
||||
------
|
||||
p : array_like
|
||||
A vector of shape (m,) which contains the computed probabilities at each of the
|
||||
provided examples.
|
||||
"""
|
||||
k = mu.size
|
||||
|
||||
# if sigma is given as a diagonal, compute the matrix
|
||||
if Sigma2.ndim == 1:
|
||||
Sigma2 = np.diag(Sigma2)
|
||||
|
||||
X = X - mu
|
||||
p = (2 * np.pi) ** (- k / 2) * np.linalg.det(Sigma2) ** (-0.5)\
|
||||
* np.exp(-0.5 * np.sum(np.dot(X, np.linalg.pinv(Sigma2)) * X, axis=1))
|
||||
return p
|
||||
|
||||
|
||||
def visualizeFit(X, mu, sigma2):
|
||||
"""
|
||||
Visualize the dataset and its estimated distribution.
|
||||
This visualization shows you the probability density function of the Gaussian distribution.
|
||||
Each example has a location (x1, x2) that depends on its feature values.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
X : array_like
|
||||
The dataset of shape (m x 2). Where there are m examples of 2-dimensions. We need at most
|
||||
2-D features to be able to visualize the distribution.
|
||||
|
||||
mu : array_like
|
||||
A vector of shape (n,) contains the means for each dimension (feature).
|
||||
|
||||
sigma2 : array_like
|
||||
Either a vector of shape (n,) containing the variances of independent features
|
||||
(i.e. it is the diagonal of the correlation matrix), or the full
|
||||
correlation matrix of shape (n x n) which can represent dependent features.
|
||||
"""
|
||||
|
||||
X1, X2 = np.meshgrid(np.arange(0, 35.5, 0.5), np.arange(0, 35.5, 0.5))
|
||||
Z = multivariateGaussian(np.stack([X1.ravel(), X2.ravel()], axis=1), mu, sigma2)
|
||||
Z = Z.reshape(X1.shape)
|
||||
|
||||
pyplot.plot(X[:, 0], X[:, 1], 'bx', mec='b', mew=2, ms=8)
|
||||
|
||||
if np.all(abs(Z) != np.inf):
|
||||
pyplot.contour(X1, X2, Z, levels=10**(np.arange(-20., 1, 3)), zorder=100)
|
||||
|
||||
|
||||
class Grader(SubmissionBase):
|
||||
# Random Test Cases
|
||||
n_u = 3
|
||||
n_m = 4
|
||||
n = 5
|
||||
X = np.sin(np.arange(1, 1 + n_m * n)).reshape(n_m, n, order='F')
|
||||
Theta = np.cos(np.arange(1, 1 + n_u * n)).reshape(n_u, n, order='F')
|
||||
Y = np.sin(np.arange(1, 1 + 2 * n_m * n_u, 2)).reshape(n_m, n_u, order='F')
|
||||
R = Y > 0.5
|
||||
pval = np.concatenate([abs(Y.ravel('F')), [0.001], [1]])
|
||||
Y = Y * R # set 'Y' values to 0 for movies not reviewed
|
||||
|
||||
yval = np.concatenate([R.ravel('F'), [1], [0]])
|
||||
#
|
||||
params = np.concatenate([X.ravel(), Theta.ravel()])
|
||||
|
||||
def __init__(self):
|
||||
part_names = ['Estimate Gaussian Parameters',
|
||||
'Select Threshold',
|
||||
'Collaborative Filtering Cost',
|
||||
'Collaborative Filtering Gradient',
|
||||
'Regularized Cost',
|
||||
'Regularized Gradient']
|
||||
super().__init__('anomaly-detection-and-recommender-systems', part_names)
|
||||
|
||||
def __iter__(self):
|
||||
for part_id in range(1, 7):
|
||||
try:
|
||||
func = self.functions[part_id]
|
||||
|
||||
# Each part has different expected arguments/different function
|
||||
if part_id == 1:
|
||||
res = np.hstack(func(self.X)).tolist()
|
||||
elif part_id == 2:
|
||||
res = np.hstack(func(self.yval, self.pval)).tolist()
|
||||
elif part_id == 3:
|
||||
J, grad = func(self.params, self.Y, self.R, self.n_u, self.n_m, self.n)
|
||||
res = J
|
||||
elif part_id == 4:
|
||||
J, grad = func(self.params, self.Y, self.R, self.n_u, self.n_m, self.n, 0)
|
||||
xgrad = grad[:self.n_m*self.n].reshape(self.n_m, self.n)
|
||||
thetagrad = grad[self.n_m*self.n:].reshape(self.n_u, self.n)
|
||||
res = np.hstack([xgrad.ravel('F'), thetagrad.ravel('F')]).tolist()
|
||||
elif part_id == 5:
|
||||
res, _ = func(self.params, self.Y, self.R, self.n_u, self.n_m, self.n, 1.5)
|
||||
elif part_id == 6:
|
||||
J, grad = func(self.params, self.Y, self.R, self.n_u, self.n_m, self.n, 1.5)
|
||||
xgrad = grad[:self.n_m*self.n].reshape(self.n_m, self.n)
|
||||
thetagrad = grad[self.n_m*self.n:].reshape(self.n_u, self.n)
|
||||
res = np.hstack([xgrad.ravel('F'), thetagrad.ravel('F')]).tolist()
|
||||
else:
|
||||
raise KeyError
|
||||
yield part_id, res
|
||||
except KeyError:
|
||||
yield part_id, 0
|
||||
99
README.md
Executable file
99
README.md
Executable file
@@ -0,0 +1,99 @@
|
||||
# [Coursera Machine Learning MOOC by Andrew Ng](https://www.coursera.org/learn/machine-learning)
|
||||
# Python Programming Assignments
|
||||
|
||||
|
||||
|
||||
This repositry contains the python versions of the programming assignments for the [Machine Learning online class](https://www.coursera.org/learn/machine-learning) taught by Professor Andrew Ng. This is perhaps the most popular introductory online machine learning class. In addition to being popular, it is also one of the best Machine learning classes any interested student can take to get started with machine learning. An unfortunate aspect of this class is that the programming assignments are in MATLAB or OCTAVE, probably because this class was made before python become the go-to language in machine learning.
|
||||
|
||||
The Python machine learning ecosystem has grown exponentially in the past few years, and still gaining momentum. I suspect that many students who want to get started with their machine learning journey would like to start it with Python also. It is for those reasons I have decided to re-write all the programming assignments in Python, so students can get acquainted with its ecosystem from the start of their learning journey.
|
||||
|
||||
These assignments work seamlessly with the class and do not require any of the materials published in the MATLAB assignments. Here are some new and useful features for these sets of assignments:
|
||||
|
||||
- The assignments use [Jupyter Notebook](http://jupyter-notebook-beginner-guide.readthedocs.io/en/latest/what_is_jupyter.html), which provides an intuitive flow easier than the original MATLAB/OCTAVE assignments.
|
||||
- The original assignment instructions have been completely re-written and the parts which used to reference MATLAB/OCTAVE functionality have been changed to reference its `python` counterpart.
|
||||
- The re-written instructions are now embedded within the Jupyter Notebook along with the `python` starter code. For each assignment, all work is done solely within the notebook.
|
||||
- The `python` assignments can be submitted for grading. They were tested to work perfectly well with the original Coursera grader that is currently used to grade the MATLAB/OCTAVE versions of the assignments.
|
||||
- After each part of a given assignment, the Jupyter Notebook contains a cell which prompts the user for submitting the current part of the assignment for grading.
|
||||
|
||||
## Downloading the Assignments
|
||||
|
||||
To get started, you can start by either downloading a zip file of these assignments by clicking on the `Clone or download` button. If you have `git` installed on your system, you can clone this repository using :
|
||||
|
||||
clone
|
||||
|
||||
Each assignment is contained in a separate folder. For example, assignment 1 is contained within the folder `Exercise1`. Each folder contains two files:
|
||||
- The assignment `jupyter` notebook, which has a `.ipynb` extension. All the code which you need to write will be written within this notebook.
|
||||
- A python module `utils.py` which contains some helper functions needed for the assignment. Functions within the `utils` module are called from the python notebook. You do not need to modify or add any code to this file.
|
||||
|
||||
## Requirements
|
||||
|
||||
These assignments has been tested and developed using the following libraries:
|
||||
|
||||
- python==3.6.4
|
||||
- numpy==1.13.3
|
||||
- scipy==1.0.0
|
||||
- matplotlib==2.1.2
|
||||
- jupyter==1.0.0
|
||||
- jupyter-client==5.0.1
|
||||
|
||||
We recommend using at least these versions of the required libraries or later. Python 2 is not supported.
|
||||
|
||||
## Python Installation
|
||||
|
||||
We highly recommend using anaconda for installing python. [Click here](https://www.anaconda.com/download/) to go to Anaconda's download page. Make sure to download Python 3.6 version.
|
||||
If you are on a windows machine:
|
||||
- Open the executable after download is complete and follow instructions.
|
||||
- Once installation is complete, open `Anaconda prompt` from the start menu. This will open a terminal with python enabled.
|
||||
|
||||
If you are on a linux machine:
|
||||
|
||||
- Open a terminal and navigate to the directory where Anaconda was downloaded.
|
||||
- Change the permission to the downloaded file so that it can be executed. So if the downloaded file name is `Anaconda3-5.1.0-Linux-x86_64.sh`, then use the following command:
|
||||
|
||||
`chmod a+x Anaconda3-5.1.0-Linux-x86_64.sh`
|
||||
|
||||
- Now, run the installation script using `./Anaconda3-5.1.0-Linux-x86_64.sh`, and follow installation instructions in the terminal.
|
||||
|
||||
|
||||
Once you have installed python, create a new python environment will all the requirements using the following command:
|
||||
|
||||
conda create -n machine_learning python=3.6 scipy=1 numpy=1.13 matplotlib=2.1 jupyter
|
||||
|
||||
After the new environment is setup, activate it using (windows)
|
||||
|
||||
activate machine_learning
|
||||
|
||||
or if you are on a linux machine
|
||||
|
||||
source activate machine_learning
|
||||
|
||||
Now we have our python environment all set up, we can start working on the assignments. To do so, navigate to the directory where the assignments were installed, and launch the jupyter notebook from the terminal using the command
|
||||
|
||||
jupyter notebook
|
||||
|
||||
This should automatically open a tab in the default browser. To start with assignment 1, open the notebook `./Exercise1/exercise1.ipynb`.
|
||||
|
||||
## Python Tutorials
|
||||
|
||||
If you are new to python and to `jupyter` notebooks, no worries! There is a plethora of tutorials and documentation to get you started. Here are a few links which might be of help:
|
||||
|
||||
- [Python Programming](https://pythonprogramming.net/introduction-to-python-programming/): A turorial with videos about the basics of python.
|
||||
|
||||
- [Numpy and matplotlib tutorial](http://cs231n.github.io/python-numpy-tutorial/): We will be using numpy extensively for matrix and vector operations. This is great tutorial to get you started with using numpy and matplotlib for plotting.
|
||||
|
||||
- [Jupyter notebook](https://medium.com/codingthesmartway-com-blog/getting-started-with-jupyter-notebook-for-python-4e7082bd5d46): Getting started with the jupyter notebook.
|
||||
|
||||
- [Python introduction based on the class's MATLAB tutorial](https://github.com/mstampfer/Coursera-Stanford-ML-Python/blob/master/Coursera%20Stanford%20ML%20Python%20wiki.ipynb): This is the equivalent of class's MATLAB tutorial, in python.
|
||||
|
||||
|
||||
## Caveats and tips
|
||||
|
||||
- In many of the exercises, the regularization parameter $\lambda$ is denoted as the variable name `lambda_`, notice the underscore at the end of the name. This is because `lambda` is a reserved python keyword, and should never be used as a variable name.
|
||||
|
||||
- In `numpy`, the function `dot` is used to perform matrix multiplication. The operation '*' only does element-by-element multiplication (unlike MATLAB). If you are using python version 3.5+, the operator '@' is the new matrix multiplication, and it is equivalent to the `dot` function.
|
||||
|
||||
## Acknowledgements
|
||||
|
||||
- I would like to thank professor Andrew Ng and the crew of the Stanford Machine Learning class on Coursera for such an awesome class.
|
||||
|
||||
- Some of the material used, especially the code for submitting assignments for grading is based on [`mstampfer`'s](https://github.com/mstampfer/Coursera-Stanford-ML-Python) python implementation of the assignments.
|
||||
BIN
machinelearning.jpg
Executable file
BIN
machinelearning.jpg
Executable file
Binary file not shown.
|
After Width: | Height: | Size: 66 KiB |
174
requirements.txt
Executable file
174
requirements.txt
Executable file
@@ -0,0 +1,174 @@
|
||||
alabaster==0.7.10
|
||||
anaconda-client==1.6.3
|
||||
anaconda-navigator==1.6.2
|
||||
anaconda-project==0.6.0
|
||||
asn1crypto==0.22.0
|
||||
astroid==1.4.9
|
||||
astropy==2.0.1
|
||||
Babel==2.4.0
|
||||
backports.shutil-get-terminal-size==1.0.0
|
||||
backports.weakref==1.0rc1
|
||||
beautifulsoup4==4.6.0
|
||||
bitarray==0.8.1
|
||||
blaze==0.10.1
|
||||
bleach==1.5.0
|
||||
bokeh==0.12.13
|
||||
boto==2.46.1
|
||||
Bottleneck==1.2.1
|
||||
cffi==1.10.0
|
||||
chardet==3.0.3
|
||||
click==6.7
|
||||
cloudpickle==0.2.2
|
||||
clyent==1.2.2
|
||||
colorama==0.3.9
|
||||
conda==4.4.8
|
||||
contextlib2==0.5.5
|
||||
coverage==4.4.1
|
||||
cryptography==1.8.1
|
||||
cycler==0.10.0
|
||||
Cython==0.25.2
|
||||
cytoolz==0.8.2
|
||||
dask==0.14.3
|
||||
datashape==0.5.4
|
||||
decorator==4.1.2
|
||||
distributed==1.16.3
|
||||
docutils==0.13.1
|
||||
entrypoints==0.2.2
|
||||
et-xmlfile==1.0.1
|
||||
fastcache==1.0.2
|
||||
Flask==0.12.2
|
||||
Flask-Cors==3.0.2
|
||||
future==0.16.0
|
||||
gevent==1.2.1
|
||||
greenlet==0.4.12
|
||||
h5py==2.7.0
|
||||
HeapDict==1.0.0
|
||||
html5lib==0.9999999
|
||||
idna==2.5
|
||||
imagesize==0.7.1
|
||||
ipdb==0.10.3
|
||||
ipdbplugin==1.4.5
|
||||
ipykernel==4.6.1
|
||||
ipython==6.2.0
|
||||
ipython-genutils==0.2.0
|
||||
ipywidgets==6.0.0
|
||||
isort==4.2.5
|
||||
itsdangerous==0.24
|
||||
jdcal==1.3
|
||||
jedi==0.11.0
|
||||
Jinja2==2.9.6
|
||||
jsonschema==2.6.0
|
||||
jupyter==1.0.0
|
||||
jupyter-client==5.0.1
|
||||
jupyter-console==5.1.0
|
||||
jupyter-contrib-core==0.3.3
|
||||
jupyter-contrib-nbextensions==0.3.3
|
||||
jupyter-core==4.3.0
|
||||
jupyter-highlight-selected-word==0.1.0
|
||||
jupyter-latex-envs==1.4.1
|
||||
jupyter-nbextensions-configurator==0.3.0
|
||||
jupyterthemes==0.17.8
|
||||
Keras==2.0.8
|
||||
lazy-object-proxy==1.2.2
|
||||
lesscpy==0.12.0
|
||||
llvmlite==0.20.0
|
||||
locket==0.2.0
|
||||
lxml==4.1.1
|
||||
Mako==1.0.7
|
||||
Markdown==2.6.9
|
||||
MarkupSafe==0.23
|
||||
matplotlib==2.1.2
|
||||
mistune==0.7.4
|
||||
mpmath==0.19
|
||||
msgpack-python==0.4.8
|
||||
multipledispatch==0.4.9
|
||||
navigator-updater==0.1.0
|
||||
nbconvert==5.3.1
|
||||
nbformat==4.4.0
|
||||
networkx==1.11
|
||||
nltk==3.2.3
|
||||
nose==1.3.7
|
||||
notebook==5.0.0
|
||||
numba==0.35.0
|
||||
numexpr==2.6.2
|
||||
numpy==1.13.1
|
||||
numpydoc==0.6.0
|
||||
odo==0.5.0
|
||||
olefile==0.44
|
||||
openpyxl==2.4.7
|
||||
packaging==16.8
|
||||
pandas==0.20.3
|
||||
pandocfilters==1.4.1
|
||||
paramiko==2.1.2
|
||||
parso==0.1.0
|
||||
partd==0.3.8
|
||||
pathlib2==2.2.1
|
||||
patsy==0.4.1
|
||||
pep8==1.7.0
|
||||
pexpect==4.2.1
|
||||
pickleshare==0.7.4
|
||||
Pillow==5.0.0
|
||||
ply==3.10
|
||||
prompt-toolkit==1.0.15
|
||||
protobuf==3.4.0
|
||||
psutil==5.2.2
|
||||
ptyprocess==0.5.2
|
||||
py==1.4.33
|
||||
pyasn1==0.2.3
|
||||
pycosat==0.6.3
|
||||
pycparser==2.17
|
||||
pycrypto==2.6.1
|
||||
pycurl==7.43.0
|
||||
pyflakes==1.5.0
|
||||
Pygments==2.2.0
|
||||
pygpu==0.6.9
|
||||
pylint==1.6.4
|
||||
pyodbc==4.0.16
|
||||
pyOpenSSL==17.0.0
|
||||
pyparsing==2.2.0
|
||||
pytest==3.0.7
|
||||
python-dateutil==2.6.1
|
||||
pytz==2017.2
|
||||
PyWavelets==0.5.2
|
||||
PyYAML==3.12
|
||||
pyzmq==16.0.2
|
||||
QtAwesome==0.4.4
|
||||
qtconsole==4.3.0
|
||||
QtPy==1.2.1
|
||||
requests==2.14.2
|
||||
rope-py3k==0.9.4.post1
|
||||
scikit-image==0.13.0
|
||||
scikit-learn==0.19.0
|
||||
scipy==0.19.1
|
||||
scons==3.0.0a20170821
|
||||
seaborn==0.7.1
|
||||
simplegeneric==0.8.1
|
||||
singledispatch==3.4.0.3
|
||||
six==1.11.0
|
||||
snowballstemmer==1.2.1
|
||||
sortedcollections==0.5.3
|
||||
sortedcontainers==1.5.7
|
||||
Sphinx==1.5.6
|
||||
spyder==3.1.4
|
||||
SQLAlchemy==1.1.9
|
||||
statsmodels==0.8.0
|
||||
sympy==1.0
|
||||
tables==3.4.2
|
||||
tblib==1.3.2
|
||||
tensorflow==1.3.0
|
||||
tensorflow-tensorboard==0.1.5
|
||||
terminado==0.6
|
||||
testpath==0.3
|
||||
Theano==0.9.0
|
||||
toolz==0.8.2
|
||||
tornado==4.5.1
|
||||
traitlets==4.3.2
|
||||
unicodecsv==0.14.1
|
||||
wcwidth==0.1.7
|
||||
Werkzeug==0.12.2
|
||||
widgetsnbextension==2.0.0
|
||||
wrapt==1.10.10
|
||||
xlrd==1.0.0
|
||||
XlsxWriter==0.9.6
|
||||
xlwt==1.2.0
|
||||
zict==0.1.2
|
||||
105
submission.py
Executable file
105
submission.py
Executable file
@@ -0,0 +1,105 @@
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import urlopen
|
||||
import pickle
|
||||
import json
|
||||
from collections import OrderedDict
|
||||
import numpy as np
|
||||
import os
|
||||
|
||||
|
||||
class SubmissionBase:
|
||||
|
||||
submit_url = 'https://www-origin.coursera.org/api/' \
|
||||
'onDemandProgrammingImmediateFormSubmissions.v1'
|
||||
save_file = 'token.pkl'
|
||||
|
||||
def __init__(self, assignment_slug, part_names):
|
||||
self.assignment_slug = assignment_slug
|
||||
self.part_names = part_names
|
||||
self.login = None
|
||||
self.token = None
|
||||
self.functions = OrderedDict()
|
||||
self.args = dict()
|
||||
|
||||
def grade(self):
|
||||
print('\nSubmitting Solutions | Programming Exercise %s\n' % self.assignment_slug)
|
||||
self.login_prompt()
|
||||
|
||||
# Evaluate the different parts of exercise
|
||||
parts = OrderedDict()
|
||||
for part_id, result in self:
|
||||
parts[str(part_id)] = {'output': sprintf('%0.5f ', result)}
|
||||
result, response = self.request(parts)
|
||||
response = json.loads(response)
|
||||
|
||||
# if an error was returned, print it and stop
|
||||
if 'errorMessage' in response:
|
||||
print(response['errorMessage'])
|
||||
return
|
||||
|
||||
# Print the grading table
|
||||
print('%43s | %9s | %-s' % ('Part Name', 'Score', 'Feedback'))
|
||||
print('%43s | %9s | %-s' % ('---------', '-----', '--------'))
|
||||
for part in parts:
|
||||
part_feedback = response['partFeedbacks'][part]
|
||||
part_evaluation = response['partEvaluations'][part]
|
||||
score = '%d / %3d' % (part_evaluation['score'], part_evaluation['maxScore'])
|
||||
print('%43s | %9s | %-s' % (self.part_names[int(part) - 1], score, part_feedback))
|
||||
evaluation = response['evaluation']
|
||||
total_score = '%d / %d' % (evaluation['score'], evaluation['maxScore'])
|
||||
print(' --------------------------------')
|
||||
print('%43s | %9s | %-s\n' % (' ', total_score, ' '))
|
||||
|
||||
def login_prompt(self):
|
||||
if os.path.isfile(self.save_file):
|
||||
with open(self.save_file, 'rb') as f:
|
||||
login, token = pickle.load(f)
|
||||
reenter = input('Use token from last successful submission (%s)? (Y/n): ' % login)
|
||||
|
||||
if reenter == '' or reenter[0] == 'Y' or reenter[0] == 'y':
|
||||
self.login, self.token = login, token
|
||||
return
|
||||
else:
|
||||
os.remove(self.save_file)
|
||||
|
||||
self.login = input('Login (email address): ')
|
||||
self.token = input('Token: ')
|
||||
|
||||
# Save the entered credentials
|
||||
if not os.path.isfile(self.save_file):
|
||||
with open(self.save_file, 'wb') as f:
|
||||
pickle.dump((self.login, self.token), f)
|
||||
|
||||
def request(self, parts):
|
||||
params = {
|
||||
'assignmentSlug': self.assignment_slug,
|
||||
'secret': self.token,
|
||||
'parts': parts,
|
||||
'submitterEmail': self.login}
|
||||
|
||||
params = urlencode({'jsonBody': json.dumps(params)}).encode("utf-8")
|
||||
f = urlopen(self.submit_url, params)
|
||||
try:
|
||||
return 0, f.read()
|
||||
finally:
|
||||
f.close()
|
||||
|
||||
def __iter__(self):
|
||||
for part_id in self.functions:
|
||||
yield part_id
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
self.functions[key] = value
|
||||
|
||||
|
||||
def sprintf(fmt, arg):
|
||||
""" Emulates (part of) Octave sprintf function. """
|
||||
if isinstance(arg, tuple):
|
||||
# for multiple return values, only use the first one
|
||||
arg = arg[0]
|
||||
|
||||
if isinstance(arg, (np.ndarray, list)):
|
||||
# concatenates all elements, column by column
|
||||
return ' '.join(fmt % e for e in np.asarray(arg).ravel('F'))
|
||||
else:
|
||||
return fmt % arg
|
||||
Reference in New Issue
Block a user