Add leaky_relu and softmax functions (TheAlgorithms#558)

Author: Navaneeth-Sharma

src/math/leaky_relu.rs

@@ -0,0 +1,47 @@
+//! # Leaky ReLU Function
+//!
+//! The `leaky_relu` function computes the Leaky Rectified Linear Unit (ReLU) values of a given vector
+//! of f64 numbers with a specified alpha parameter.
+//!
+//! The Leaky ReLU activation function is commonly used in neural networks to introduce a small negative
+//! slope (controlled by the alpha parameter) for the negative input values, preventing neurons from dying
+//! during training.
+//!
+//! ## Formula
+//!
+//! For a given input vector `x` and an alpha parameter `alpha`, the Leaky ReLU function computes the output
+//! `y` as follows:
+//!
+//! `y_i = { x_i if x_i >= 0, alpha * x_i if x_i < 0 }`
+//!
+//! ## Leaky ReLU Function Implementation
+//!
+//! This implementation takes a reference to a vector of f64 values and an alpha parameter, and returns a new
+//! vector with the Leaky ReLU transformation applied to each element. The input vector is not altered.
+//!
+pub fn leaky_relu(vector: &Vec<f64>, alpha: f64) -> Vec<f64> {
+    let mut _vector = vector.to_owned();
+
+    for value in &mut _vector {
+        if value < &mut 0. {
+            *value *= alpha;
+        }
+    }
+
+    _vector
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_leaky_relu() {
+        let test_vector = vec![-10., 2., -3., 4., -5., 10., 0.05];
+        let alpha = 0.01;
+        assert_eq!(
+            leaky_relu(&test_vector, alpha),
+            vec![-0.1, 2.0, -0.03, 4.0, -0.05, 10.0, 0.05]
+        );
+    }
+}

src/math/mod.rs

@@ -24,6 +24,7 @@ mod interest;
 mod interpolation;
 mod karatsuba_multiplication;
 mod lcm_of_n_numbers;
+mod leaky_relu;
 mod linear_sieve;
 mod matrix_ops;
 mod mersenne_primes;
@@ -44,6 +45,7 @@ mod sigmoid;
 mod signum;
 mod simpson_integration;
 mod sine;
+mod softmax;
 mod square_pyramidal_numbers;
 mod square_root;
 mod sum_of_digits;
@@ -83,6 +85,7 @@ pub use self::interest::{compound_interest, simple_interest};
 pub use self::interpolation::{lagrange_polynomial_interpolation, linear_interpolation};
 pub use self::karatsuba_multiplication::multiply;
 pub use self::lcm_of_n_numbers::lcm;
+pub use self::leaky_relu::leaky_relu;
 pub use self::linear_sieve::LinearSieve;
 pub use self::matrix_ops::Matrix;
 pub use self::mersenne_primes::{get_mersenne_primes, is_mersenne_prime};
@@ -103,6 +106,7 @@ pub use self::sigmoid::sigmoid;
 pub use self::signum::signum;
 pub use self::simpson_integration::simpson_integration;
 pub use self::sine::sine;
+pub use self::softmax::softmax;
 pub use self::square_pyramidal_numbers::square_pyramidal_number;
 pub use self::square_root::{fast_inv_sqrt, square_root};
 pub use self::sum_of_digits::{sum_digits_iterative, sum_digits_recursive};

src/math/softmax.rs

@@ -0,0 +1,56 @@
+//! # Softmax Function
+//!
+//! The `softmax` function computes the softmax values of a given array of f32 numbers.
+//!
+//! The softmax operation is often used in machine learning for converting a vector of real numbers into a
+//! probability distribution. It exponentiates each element in the input array, and then normalizes the
+//! results so that they sum to 1.
+//!
+//! ## Formula
+//!
+//! For a given input array `x`, the softmax function computes the output `y` as follows:
+//!
+//! `y_i = e^(x_i) / sum(e^(x_j) for all j)`
+//!
+//! ## Softmax Function Implementation
+//!
+//! This implementation uses the `std::f32::consts::E` constant for the base of the exponential function. and
+//! f32 vectors to compute the values. The function creates a new vector and not altering the input vector.
+//!
+use std::f32::consts::E;
+
+pub fn softmax(array: Vec<f32>) -> Vec<f32> {
+    let mut softmax_array = array.clone();
+
+    for value in &mut softmax_array {
+        *value = E.powf(*value);
+    }
+
+    let sum: f32 = softmax_array.iter().sum();
+
+    for value in &mut softmax_array {
+        *value /= sum;
+    }
+
+    softmax_array
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_softmax() {
+        let test = vec![9.0, 0.5, -3.0, 0.0, 3.0];
+        assert_eq!(
+            softmax(test),
+            vec![
+                0.9971961,
+                0.00020289792,
+                6.126987e-6,
+                0.00012306382,
+                0.0024718025
+            ]
+        );
+    }
+}