/home/runner/work/deep-learning-algo-impls/deep-learning-algo-impls/include/ml/svm.hpp

A class implementing Support Vector Machine with automatic differentiation.

A class implementing Support Vector Machine with automatic differentiationThis class provides functionality for binary classification using Support Vector Machines with automatic gradient computation similar to PyTorch's autograd engine.

Template Parameters

T	The data type for matrix elements (typically float or double)

// Create an SVM with RBF kernel and autograd
SVM<double> svm(KernelType::RBF, 1.0, 0.1);
 
// Fit SVM to training data with automatic gradient computation
Matrix<double> X_train = training_features;
std::vector<int> y_train = training_labels;
svm.fit(X_train, y_train);
 
// Predict on test data
Matrix<double> X_test = test_features;
std::vector<int> predictions = svm.predict(X_test);

#pragma once
 
#include <functional>
#include <vector>
#include "../utils/matrix.hpp"
#include "../utils/autograd.hpp"
 
namespace ml {
    using namespace utils;
 
    enum class KernelType {
        LINEAR, 
        POLYNOMIAL, 
        RBF, 
        SIGMOID 
    };
 
    template<typename T>
    class SVM {
    public:
        SVM(KernelType kernel_type = KernelType::RBF, T C = 1.0, T gamma = 1.0, 
                   int degree = 3, T coef0 = 0.0, T tol = 1e-3, size_t max_iter = 1000, T learning_rate = 0.01);
 
        void fit(const Matrix<T> &X, const std::vector<int> &y);
 
        std::vector<int> predict(const Matrix<T> &X) const;
 
        Matrix<T> predict_proba(const Matrix<T> &X) const;
 
        std::vector<T> decision_function(const Matrix<T> &X) const;
 
        Matrix<T> support_vectors() const;
 
        std::vector<size_t> support() const;
 
        std::vector<T> dual_coef() const;
 
        T intercept() const;
 
        std::vector<T> loss_history() const { return loss_history_; }
 
    private:
        KernelType kernel_type_; 
        T C_; 
        T gamma_; 
        int degree_; 
        T coef0_; 
        T tol_; 
        size_t max_iter_; 
        T learning_rate_; 
 
        Variable<T> weights_; 
        Variable<T> bias_; 
        Matrix<T> support_vectors_; 
        std::vector<size_t> support_indices_; 
        std::vector<T> dual_coef_; 
        std::vector<int> classes_; 
        bool is_fitted_; 
        std::vector<T> loss_history_; 
 
        Variable<T> kernel(const Variable<T> &x1, const Variable<T> &x2) const;
 
        Variable<T> kernel_matrix(const Variable<T> &X1, const Variable<T> &X2) const;
 
        Variable<T> compute_loss(const Variable<T> &X, const std::vector<int> &y) const;
 
        T gradient_step(const Matrix<T> &X, const std::vector<int> &y);
 
        Variable<T> to_variable(const Matrix<T> &matrix, bool requires_grad = false) const;
    };
 
    // Type aliases
    using SVMF = SVM<float>;
    using SVMD = SVM<double>;
 
} // namespace ml