Skip to content

De-correlate boost random number generators in tests #300

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

De-correlate boost random number generators in tests #300

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
89afcf2
add test demonstrating weird boost RNG issue when sampling from a
jeffdonahue Apr 7, 2014
047e3fa
add analogous test for uniform instead of gaussian
jeffdonahue Apr 7, 2014
9feec8c
cleanup log messages
jeffdonahue Apr 7, 2014
ca1f4c3
Set uncorrelated seeds for boost random number generators in tests
kloudkl Apr 7, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
231 changes: 230 additions & 1 deletion src/caffe/test/test_random_number_generator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// Copyright 2014 BVLC and contributors.

#include <boost/uuid/seed_rng.hpp>
#include <cuda_runtime.h>
#include <cmath>
#include <cstring>
Expand Down Expand Up @@ -34,7 +35,7 @@ class RandomNumberGeneratorTest : public ::testing::Test {
}

Dtype mean_bound(const Dtype std, const size_t sample_size) {
return std/sqrt(static_cast<double>(sample_size));
return 2 * std / sqrt(static_cast<double>(sample_size));
}
};

Expand Down Expand Up @@ -95,4 +96,232 @@ TYPED_TEST(RandomNumberGeneratorTest, TestRngBernoulli) {
}


TYPED_TEST(RandomNumberGeneratorTest, TestRngGaussianTimesBernoulli) {
size_t sample_size = 10000;
SyncedMemory gaussian_data(sample_size * sizeof(TypeParam));
SyncedMemory bernoulli_data(sample_size * sizeof(int));
int seed = time(NULL) * getpid();
Caffe::set_random_seed(seed);
// Sample from 0 mean Gaussian
TypeParam mu = 0;
TypeParam sigma = 1;
caffe_vRngGaussian(sample_size, reinterpret_cast<TypeParam*>(
gaussian_data.mutable_cpu_data()), mu, sigma);
TypeParam true_mean = mu;
TypeParam true_std = sigma;
TypeParam bound = this->mean_bound(true_std, sample_size);
TypeParam empirical_mean = this->sample_mean(
reinterpret_cast<const TypeParam*>(gaussian_data.cpu_data()),
sample_size);
EXPECT_NEAR(empirical_mean, true_mean, bound);
int num_pos = 0;
int num_neg = 0;
int num_zeros = 0;
TypeParam* samples =
static_cast<TypeParam*>(gaussian_data.mutable_cpu_data());
for (int i = 0; i < sample_size; ++i) {
if (samples[i] == TypeParam(0)) {
++num_zeros;
} else if (samples[i] > TypeParam(0)) {
++num_pos;
} else if (samples[i] < TypeParam(0)) {
++num_neg;
}
}
// Check that we have no zeros (possible to generate 0s, but highly
// improbable), and roughly half positives and half negatives (with bound
// computed from a Bernoulli with p = 0.5).
EXPECT_EQ(0, num_zeros);
double p = 0.5;
true_mean = p;
true_std = sqrt(p * (1 - p));
bound = this->mean_bound(true_std, sample_size);
TypeParam expected_num_each_sign = sample_size * p;
LOG(INFO) << "Gaussian: Expected " << expected_num_each_sign << " positives"
<< "; got " << num_pos;
LOG(INFO) << "Gaussian: Expected " << expected_num_each_sign << " negatives"
<< "; got " << num_neg;
EXPECT_NEAR(expected_num_each_sign, num_pos, sample_size * bound);
EXPECT_NEAR(expected_num_each_sign, num_neg, sample_size * bound);
// Sample from Bernoulli with p = 0.3
p = 0.3;
seed = boost::uuids::detail::seed_rng()();
Caffe::set_random_seed(seed);
caffe_vRngBernoulli(sample_size,
reinterpret_cast<int*>(bernoulli_data.mutable_cpu_data()), p);
true_mean = p;
true_std = sqrt(p * (1 - p));
bound = this->mean_bound(true_std, sample_size);
empirical_mean =
this->sample_mean((const int *)bernoulli_data.cpu_data(), sample_size);
LOG(INFO) << "Bernoulli: Expected mean = " << true_mean
<< "; sample mean = " << empirical_mean;
EXPECT_NEAR(empirical_mean, true_mean, bound);
int bernoulli_num_zeros = 0;
int num_ones = 0;
int num_other = 0;
const int* bernoulli_samples =
reinterpret_cast<const int*>(bernoulli_data.cpu_data());
for (int i = 0; i < sample_size; ++i) {
if (bernoulli_samples[i] == 0) {
++bernoulli_num_zeros;
} else if (bernoulli_samples[i] == 1) {
++num_ones;
} else {
++num_other;
}
}
LOG(INFO) << "Bernoulli: zeros: " << bernoulli_num_zeros
<< "; ones: " << num_ones << "; other: " << num_other;
EXPECT_EQ(0, num_other);
EXPECT_NEAR(sample_size * empirical_mean, num_ones, 1);
EXPECT_NEAR(sample_size * (1.0 - empirical_mean), bernoulli_num_zeros, 1);
// Multiply Gaussian by Bernoulli
for (int i = 0; i < sample_size; ++i) {
samples[i] *= bernoulli_samples[i];
}
num_pos = 0;
num_neg = 0;
num_zeros = 0;
for (int i = 0; i < sample_size; ++i) {
if (samples[i] == TypeParam(0)) {
++num_zeros;
} else if (samples[i] > TypeParam(0)) {
++num_pos;
} else if (samples[i] < TypeParam(0)) {
++num_neg;
}
}
// Check that we have as many zeros as Bernoulli, and roughly half positives
// and half negatives (with bound computed from a Bernoulli with p = 0.5).
EXPECT_EQ(bernoulli_num_zeros, num_zeros);
p = 0.5;
true_mean = p;
true_std = sqrt(p * (1 - p));
int sub_sample_size = sample_size - bernoulli_num_zeros;
bound = this->mean_bound(true_std, sub_sample_size);
expected_num_each_sign = sub_sample_size * p;
LOG(INFO) << "Gaussian*Bernoulli: Expected " << expected_num_each_sign
<< " positives; got " << num_pos;
LOG(INFO) << "Gaussian*Bernoulli: Expected " << expected_num_each_sign
<< " negatives; got " << num_neg;
EXPECT_NEAR(expected_num_each_sign, num_pos, sample_size * bound);
EXPECT_NEAR(expected_num_each_sign, num_neg, sample_size * bound);
}


TYPED_TEST(RandomNumberGeneratorTest, TestRngUniformTimesBernoulli) {
size_t sample_size = 10000;
SyncedMemory uniform_data(sample_size * sizeof(TypeParam));
SyncedMemory bernoulli_data(sample_size * sizeof(int));
int seed = time(NULL) * getpid();
Caffe::set_random_seed(seed);
// Sample from Uniform on [-1, 1]
TypeParam a = -1;
TypeParam b = 1;
caffe_vRngUniform(sample_size, reinterpret_cast<TypeParam*>(
uniform_data.mutable_cpu_data()), a, b);
TypeParam true_mean = (a + b) / 2;
TypeParam true_std = (b - a) / sqrt(12);
TypeParam bound = this->mean_bound(true_std, sample_size);
TypeParam empirical_mean = this->sample_mean(
reinterpret_cast<const TypeParam*>(uniform_data.cpu_data()),
sample_size);
EXPECT_NEAR(empirical_mean, true_mean, bound);
int num_pos = 0;
int num_neg = 0;
int num_zeros = 0;
TypeParam* samples =
static_cast<TypeParam*>(uniform_data.mutable_cpu_data());
for (int i = 0; i < sample_size; ++i) {
if (samples[i] == TypeParam(0)) {
++num_zeros;
} else if (samples[i] > TypeParam(0)) {
++num_pos;
} else if (samples[i] < TypeParam(0)) {
++num_neg;
}
}
// Check that we have no zeros (possible to generate 0s, but highly
// improbable), and roughly half positives and half negatives (with bound
// computed from a Bernoulli with p = 0.5).
EXPECT_EQ(0, num_zeros);
double p = 0.5;
true_mean = p;
true_std = sqrt(p * (1 - p));
bound = this->mean_bound(true_std, sample_size);
TypeParam expected_num_each_sign = sample_size * p;
LOG(INFO) << "Uniform: Expected " << expected_num_each_sign << " positives"
<< "; got " << num_pos;
LOG(INFO) << "Uniform: Expected " << expected_num_each_sign << " negatives"
<< "; got " << num_neg;
EXPECT_NEAR(expected_num_each_sign, num_pos, sample_size * bound);
EXPECT_NEAR(expected_num_each_sign, num_neg, sample_size * bound);
// Sample from Bernoulli with p = 0.3
p = 0.3;
seed = boost::uuids::detail::seed_rng()();
Caffe::set_random_seed(seed);
caffe_vRngBernoulli(sample_size,
reinterpret_cast<int*>(bernoulli_data.mutable_cpu_data()), p);
true_mean = p;
true_std = sqrt(p * (1 - p));
bound = this->mean_bound(true_std, sample_size);
empirical_mean =
this->sample_mean((const int *)bernoulli_data.cpu_data(), sample_size);
LOG(INFO) << "Bernoulli: Expected mean = " << true_mean
<< "; sample mean = " << empirical_mean;
EXPECT_NEAR(empirical_mean, true_mean, bound);
int bernoulli_num_zeros = 0;
int num_ones = 0;
int num_other = 0;
const int* bernoulli_samples =
reinterpret_cast<const int*>(bernoulli_data.cpu_data());
for (int i = 0; i < sample_size; ++i) {
if (bernoulli_samples[i] == 0) {
++bernoulli_num_zeros;
} else if (bernoulli_samples[i] == 1) {
++num_ones;
} else {
++num_other;
}
}
LOG(INFO) << "Bernoulli: zeros: " << bernoulli_num_zeros
<< "; ones: " << num_ones << "; other: " << num_other;
EXPECT_EQ(0, num_other);
EXPECT_NEAR(sample_size * empirical_mean, num_ones, 1);
EXPECT_NEAR(sample_size * (1.0 - empirical_mean), bernoulli_num_zeros, 1);
// Multiply Uniform by Bernoulli
for (int i = 0; i < sample_size; ++i) {
samples[i] *= bernoulli_samples[i];
}
num_pos = 0;
num_neg = 0;
num_zeros = 0;
for (int i = 0; i < sample_size; ++i) {
if (samples[i] == TypeParam(0)) {
++num_zeros;
} else if (samples[i] > TypeParam(0)) {
++num_pos;
} else if (samples[i] < TypeParam(0)) {
++num_neg;
}
}
// Check that we have as many zeros as Bernoulli, and roughly half positives
// and half negatives (with bound computed from a Bernoulli with p = 0.5).
EXPECT_EQ(bernoulli_num_zeros, num_zeros);
p = 0.5;
true_mean = p;
true_std = sqrt(p * (1 - p));
int sub_sample_size = sample_size - bernoulli_num_zeros;
bound = this->mean_bound(true_std, sub_sample_size);
expected_num_each_sign = sub_sample_size * p;
LOG(INFO) << "Uniform*Bernoulli: Expected " << expected_num_each_sign
<< " positives; got " << num_pos;
LOG(INFO) << "Uniform*Bernoulli: Expected " << expected_num_each_sign
<< " negatives; got " << num_neg;
EXPECT_NEAR(expected_num_each_sign, num_pos, sample_size * bound);
EXPECT_NEAR(expected_num_each_sign, num_neg, sample_size * bound);
}


} // namespace caffe