inline double ActivationFunction_ReLU(double x) {
  return std::max(0.0, x);
}
inline double ActivationFunction_Sigmoid(double x) {
  return 1.0 / (1 + exp(-x));
}
inline double ActivationFunction_Tanh(double x) {
  return (tanh(x) + 1.0) / 2;
}

double AnnRun(const double x[2], double* w) {
  double f = ActivationFunction_ReLU(x[0] * w[0] + x[1] * w[1] - w[2]);
  double g = ActivationFunction_ReLU(x[0] * w[3] + x[1] * w[4] - w[5]);
  return ActivationFunction_Sigmoid(f * w[6] + g * w[7] - w[8]);
}

dy/da = x
dy/db = 1

delta_a = delta_y / x
delta_b = delta_y

// BackPropagation.cpp
//

#include <iostream>

void Train(double& a,
           double& b,
           double input,
           double expect_output,
           double learning_rate) {
  double delta_y = expect_output - (input * a + b);
  if (input != 0) {
    a += (delta_y / input) * learning_rate;
  }
  b += delta_y * learning_rate;
}

int main() {
  // 要求的函数是：y = 2 * x + 3
  const double input[4] = {0, 1, 2, 3};
  const double expect_output[4] = {3, 5, 7, 9};

  // 初始化状态是：y = 1 * x + 4
  double a = 1.0;
  double b = 4.0;

  std::cout << "Initial: y = " << a << " * x + " << b << "\n";

  // 两轮就搞定了
  for (int t = 0; t < 2; ++t) {
    for (int i = 0; i < 4; ++i) {
      Train(a, b, input[i], expect_output[i], 1);
    }
  }
  std::cout << "Trained: y = " << a << " * x + " << b << "\n";

  return 0;
}

// TrainXor_RandomSearch.cpp
// UMUTech @ 2018-07-05 23:45:52
// Be aware that I'm only a novice to ANN. My apologies for any wrong info.
//
#include <algorithm>
#include <iostream>
#include <random>

std::default_random_engine random_engine;

void RandomizeW(double* w, size_t size) {
  std::uniform_real_distribution<double> r(0, 1);
  for (size_t i = 0; i < size; ++i) {
    w[i] = r(random_engine);
  }
}

void PrintW(double* w, size_t size) {
  for (size_t i = 0; i < size; ++i) {
    std::cout << i << "\t" << w[i] << "\n";
  }
}

double ActivationFunction(double x) {
  // ReLU
  return std::max(0.0, x);
}

double AnnRun(const double x[2], double* w) {
  // bias 乘了 -1，让结果更好地收敛到 [0, 1]
  double f = ActivationFunction(x[0] * w[0] + x[1] * w[1] - w[2]);
  double g = ActivationFunction(x[0] * w[3] + x[1] * w[4] - w[5]);
  return ActivationFunction(f * w[6] + g * w[7] - w[8]);
}

int main() {
  const double input[4][2] = {{0, 0}, {0, 1}, {1, 0}, {1, 1}};
  const double expect_output[4] = {0, 1, 1, 0};

  double last_error = 1000;

  double w[3 * 3];
  double w_copy[3 * 3];

  std::random_device rd;
  random_engine.seed(rd());

  int train_count = 0;
  for (; last_error > 0.01; ++train_count) {
    if (train_count % 10000 == 0) {
      std::cout << "Randomize\n";
      RandomizeW(w, _countof(w));
    }

    memcpy(w_copy, w, sizeof(w));

    // 随机改变 w
    std::uniform_real_distribution<double> r(-0.5, 0.5);
    for (int i = 0; i < 3 * 3; ++i) {
      w[i] += r(random_engine);
    }

    double error = pow(AnnRun(input[0], w) - expect_output[0], 2.0);
    error += pow(AnnRun(input[1], w) - expect_output[1], 2.0);
    error += pow(AnnRun(input[2], w) - expect_output[2], 2.0);
    error += pow(AnnRun(input[3], w) - expect_output[3], 2.0);

    if (error < last_error) {
      // 错误率更小，保存
      last_error = error;
    } else {
      // 恢复 w
      memcpy(w, w_copy, sizeof(w));
    }
  }

  printf("Finished in %d loops.\n", train_count);

  PrintW(w, _countof(w));

  /* Run the network and see what it predicts. */
  printf("Output for [%1.f, %1.f] is %1.f.\n", input[0][0], input[0][1],
         AnnRun(input[0], w));
  printf("Output for [%1.f, %1.f] is %1.f.\n", input[1][0], input[1][1],
         AnnRun(input[1], w));
  printf("Output for [%1.f, %1.f] is %1.f.\n", input[2][0], input[2][1],
         AnnRun(input[2], w));
  printf("Output for [%1.f, %1.f] is %1.f.\n", input[3][0], input[3][1],
         AnnRun(input[3], w));

  return 0;
}

Randomize
Finished in 344 loops.
0       -1.18943
1       -1.60685
2       -0.848489
3       1.28751
4       1.21697
5       0.532657
6       -2.27322
7       -0.77646
8       -1.57966
Output for [0, 0] is 0.
Output for [0, 1] is 1.
Output for [1, 0] is 1.
Output for [1, 1] is 0.

Randomize
Finished in 444 loops.
0       1.6138
1       1.4345
2       1.33925
3       1.50895
4       1.09461
5       -0.283878
6       -2.37528
7       1.08117
8       0.239626
Output for [0, 0] is 0.
Output for [0, 1] is 1.
Output for [1, 0] is 1.
Output for [1, 1] is 0.

// Convert the name of the video card to a character array and store it.
error = wcstombs_s(&stringLength, m_videoCardDescription, 128, adapterDesc.Description, 128);

sigmoid(x) = 1.0 / (1 + exp(-x))

f(x1, x2) = sigmoid(w1 * g(x1, x2) + w2 * h(x1, x2) + w3)

g(x1, x2) = sigmoid(wg1 * x1 + wg2 * x2 + wg3)
h(x1, x2) = sigmoid(wh1 * x1 + wh2 * x2 + wh3)

wg1 wg2 wg3
wh1 wh2 wh3
w1 w2 w3

-5.734 -6.029 1.777
-3.261 -3.172 4.460
-6.581 5.826 -2.444

g(0, 0) = sigmoid(1.777) ≈ 0.855326
h(0, 0) = sigmoid(4.460) ≈ 0.988570
f(0.855326, 0.988569) = sigmoid(-6.581 * 0.855326 + 5.826 * 0.988570 + -2.444)
  = sigmoid(-2.313491586) ≈ 0.090012 ≈ 0