One hidden layer Neural Network Neural Networks deeplearning.ai Overview What is a Neural Network? !" !# %& !$ x w ' = )*! + , - = .(') ℒ(-, %) b !" !# %& !$ x ["] ["] ["] ["] ["] ["] [#] [#] ["] [#] [#] [#] [#] 4 ' = 4 ! + , - = .(' ) ' = 4 - + , - = .(' ) ℒ(- , %) ,["] 4[#] ,[#] Andrew Ng One hidden layer Neural Network Neural Network deeplearning.ai Representation Neural Network Representation !" !# %& !$ Andrew Ng One hidden layer Neural Network Computing a deeplearning.ai Neural Network’s Output Neural Network Representation !" !" !# )!! + + -(') , = %& !# %& ' , !$ !$ ' = )!! + + , = -(') Andrew Ng Neural Network Representation $( $( $) #!$ + & +(!) ' = ./ $) ./ ! ' $* $* ! ! = # $ + & $( ' = +(!) $) ./ $* Andrew Ng Neural Network Representation " " " , ["] ["] " '" )" = +" ! + /" , ' " = 3()" ) !" " " " , ["] ["] " '# )# = +# ! + /# , ' # = 3()# ) ! %& # " " " , ["] ["] " '$ )$ = +$ ! + /$ , ' $ = 3()$ ) !$ " " " , ["] ["] " '( )( = +( ! + /( , ' ( = 3()( ) Andrew Ng Neural Network Representation learning ( " " Given input x: %" ( " , ! " = $ " % + ' " % ./ , " (- " " %- ( = )(! ) " (0 ! , = $ , ( " + ' , ( , = )(! , ) Andrew Ng One hidden layer Neural Network Vectorizing across deeplearning.ai multiple examples Vectorizing across multiple examples ' " = ) " ! + + " !" , " = -(' " ) %& !# ' # = ) # , " + + # ! $ , # = -(' # ) Andrew Ng Vectorizing across multiple examples for i = 1 to m: ! " ($) = ' " (($) + * " + " ($) = ,(! " $ ) ! - ($) = ' - + " ($) + * - + - ($) = ,(! - $ ) Andrew Ng One hidden layer Neural Network Explanation deeplearning.ai for vectorized implementation Justification for vectorized implementation Andrew Ng Recap of vectorizing across multiple examples for i = 1 to m !" ' " ()) = , " !()) + . " !# %& / " ()) = 0(' " ) ) !$ ' # ()) = , # / " ()) + . # / # ()) = 0(' # ) ) 1 = !(") !(#) … !(2) 6 " = , " 1 + . " 7 " = 0(6 " ) # # " # ["] 6 = , 7 + . A = /["](") ["](#) … /["](2) / 7 # = 0(6 # ) Andrew Ng One hidden layer Neural Network Activation functions deeplearning.ai Activation functions !" !# %& !$ Given x: ' " = ) " ! + + " , " = -(' " ) ' # = ) # , " + + # # # , = -(' ) Andrew Ng Pros and cons of activation functions a a x z 1 sigmoid: ! = 1 + &'( a a z z Andrew Ng One hidden layer Neural Network Why do you deeplearning.ai need non-linear activation functions? Activation function %" %. 01 %/ Given x: ! " = $ " % + ' " ( " = )["](! " ) ! . = $ . ( " + ' . ( . = )[.](! . ) Andrew Ng One hidden layer Neural Network Derivatives of deeplearning.ai activation functions Sigmoid activation function a 1 !(#) = 1 + )*+ z Andrew Ng Tanh activation function a !(#) = tanh(#) z Andrew Ng ReLU and Leaky ReLU a a z z ReLU Leaky ReLU Andrew Ng One hidden layer Neural Network Gradient descent for deeplearning.ai neural networks Gradient descent for neural networks Andrew Ng Formulas for computing derivatives Andrew Ng One hidden layer Neural Network Backpropagation deeplearning.ai intuition (Optional) Computing gradients Logistic regression % # ! = #$% + ' ) = *(!) ℒ(), /) ' Andrew Ng Neural network gradients &[$] ' )[$] &["] ![#] = &[#]' + )[#] +[#] = ,(![#]) ![0] = &[0]' + )[0] +[0] = ,(![0]) ℒ(+[0], y) )["] Andrew Ng Summary of gradient descent !'[$] = ([$] − 5 * !"[$] = !'[$]( ) !+[$] = !'[$] !'[)] = " $ ,!'[$] ∗ .[)]′(z ) ) !"[)] = !'[)]3, !+[)] = !'[)] Andrew Ng Summary of gradient descent !"[$] = '[$] − ) !6["] = 7["] − 8 , 1 , !*[$] = !"[$]' + !*["] = !6["]7 $ : 1 !-[$] = !"[$] !-["] = ;<. >?:(!6 " , '5A> = 1, BCC<!A:> = DE?C) : !"[+] = * $ .!"[$] ∗ 0[+]′(z + ) !6[$] = * " %!6["] ∗ 0[$]′(Z $ ) 1 !*[+] = !"[+]5. !*[$] = !6[$]G% : 1 !-[+] = !"[+] !-[$] = ;<. >?:(!6 $ , '5A> = 1, BCC<!A:> = DE?C) : Andrew Ng One hidden layer Neural Network Random Initialization deeplearning.ai What happens if you initialize weights to zero? [!] "# !! [$] !! %& [!] "$ !$ Andrew Ng Random initialization [!] "# !! [$] !! %& [!] "$ !$ Andrew Ng.