Cheetah - SKA - PSS - Prototype Time Domain Search Pipeline
FldoTester.cpp
1 /*
2  * The MIT License (MIT)
3  *
4  * Copyright (c) 2016 The SKA organisation
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to deal
8  * in the Software without restriction, including without limitation the rights
9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10  * copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in all
14  * copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22  * SOFTWARE.
23  */
24 #include "cheetah/fldo/test_utils/FldoTester.h"
25 #include "cheetah/fldo/CommonDefs.h"
26 #include "cheetah/fldo/Types.h"
27 #include "cheetah/generators/GaussianNoise.h"
28 #include "cheetah/generators/GaussianNoiseConfig.h"
29 
30 #include "cheetah/sigproc/SigProcFileStream.h"
31 #include "cheetah/fldo/cuda/CommonDefs.h"
32 #include <boost/filesystem.hpp>
33 #include "panda/test/gtest.h"
34 
35 //#define DEEP_TESTING 1
36 // Oss: to define DEEP_TESTING it can be added in the CMakeList.txt file
37 // (in the cheetah/cheetah/fldo/cuda/test directory) the line:
38 // add_definitions(-DDEEP_TESTING). cmake files need to be re-generated to
39 // let this modify take effect.
40 namespace ska {
41 namespace cheetah {
42 namespace fldo {
43 namespace test {
44 
45 template<class FldoAlgo>
46 typename FldoTesterTraits<FldoAlgo>::Api& FldoTesterTraits<FldoAlgo>::api(PoolType& pool)
47 {
48  if(!_api) {
49  _config.pool(pool);
50  _config.template config<typename FldoAlgo::Config>().active(true);
51  _api.reset(new Api(_config, _handler));
52  }
53  return *_api;
54 }
55 
56 template<class FldoAlgo>
57 typename FldoTesterTraits<FldoAlgo>::FldoHandler& FldoTesterTraits<FldoAlgo>::handler()
58 {
59  return _handler;
60 }
61 
62 template <typename TestTraits>
63 FldoTester<TestTraits>::FldoTester()
64  : BaseT()
65 {
66 }
67 
68 template <typename TestTraits>
69 FldoTester<TestTraits>::~FldoTester()
70 {
71 }
72 
73 template<typename TestTraits>
74 void FldoTester<TestTraits>::SetUp()
75 {
76 }
77 
78 template<typename TestTraits>
79 void FldoTester<TestTraits>::TearDown()
80 {
81 }
82 
84 constexpr int candidate_number = 5 ;
85 
86 void load_candidates(data::Scl &scl_data)
87 {
88  typedef data::Candidate<Cpu, float> CandidateType;
89 
90  //TODO: check if float or double is needed. In this case we need to
91  //update the Scl definition !!
92  //open the file with candidates
93  std::ifstream myfile;
94  std::string line;
95  double period, pdot,dm;
96 #ifdef DEEP_TESTING
97 
98  myfile.exceptions (std::ifstream::failbit | std::ifstream::badbit );
99  try {
100  myfile.open (panda::test::test_file("candidates.txt"), std::fstream::in);
101  std::size_t ident = 0; // candidate identifier number
102  while (getline(myfile, line)) {
103  std::stringstream ss;
104  ss << line;
105  //scan line to get pulsar info
106  ss >> period >> pdot >> dm;
107  PANDA_LOG_DEBUG << "period: " << period << " pdot: " << pdot << " dm: " << dm;
108  CandidateType::MsecTimeType period_val(period * boost::units::si::seconds);
109  CandidateType::Dm dm_val(dm * data::parsecs_per_cube_cm);
110  CandidateType::SecPerSecType pdot_val(pdot);
111  //store values inside a candidate structure
112  CandidateType candidate(period_val, pdot_val, dm_val, ident);
113  ident ++;
114  //add it to the scl vector
115  scl_data.push_back(candidate);
116  }
117  myfile.close();
118  }
119  catch(std::ifstream::failure e) {
120  PANDA_LOG_ERROR << "Exception opening/reading/closing Candidates file:"
121  << e.what();
122  }
123 #else // -> !DEEP_TESTING
124 
125  //double fake_period[] = {5.94685039e-04, 7.88370079e-04, 9.82055118e-04, 1.17574016e-03, 1.12340000e-03 };
126  double fake_period[] = {1.17574016e-03, 5.94685039e-04, 7.88370079e-04, 1.12340000e-03 , 9.82055118e-04 };
127  double fake_pdot[] = {0.0, 0.0, 0.0, 0.0, 0.0};
128  double fake_dm[] = {1.01000000e+01, 1.01000000e+01, 1.01000000e+01, 1.01000000e+01, 1.01000000e+01 };
129  for (int i = 0 ; i < candidate_number ; i++ ) {
130  period = fake_period[i];
131  pdot = fake_pdot[i];
132  dm = fake_dm[i];
133  PANDA_LOG_DEBUG << "period: " << period << " pdot: " << pdot << " dm: " << dm;
134  CandidateType::MsecTimeType period_val(period * boost::units::si::seconds);
135  CandidateType::Dm dm_val(dm * data::parsecs_per_cube_cm);
136  CandidateType::SecPerSecType pdot_val(pdot);
137  //store values inside a candidate structure
138  CandidateType candidate(period_val, pdot_val, dm_val, i);
139  //add it to the scl vector
140  scl_data.push_back(candidate);
141  }
142 
143 #endif //DEEP_TESTING
144  //Print for debug the value inside the data::Scl vector
145  for (data::VectorLike<std::vector<data::Scl::CandidateType>>::ConstIterator it = scl_data.begin() ;
146  it != scl_data.end(); ++it) {
147  PANDA_LOG_DEBUG << " period " << (*it).period()
148  << " pdot " << (*it).pdot()
149  << " dm " << (*it).dm()
150  << " ident " << (*it).ident() ;
151  }
152  //sort the candidates on period basis using lambda function (ascending order)
153  std::sort (scl_data.begin(), scl_data.end(), [](CandidateType first, CandidateType second) {
154  return (first.period() < second.period());});
155 }
156 
157 POOL_ALGORITHM_TYPED_TEST_P(FldoTester, test_empty_data)
158 {
159  typedef typename TypeParam::TimeFrequencyType DataType;
160  TypeParam traits;
161  auto& api = traits.api(pool);
162 
163  std::vector<std::shared_ptr<DataType>> tf_data;
164  data::Scl scl_data;
165  auto job = api(tf_data, scl_data);
166  job->wait();
167 
168  std::shared_ptr<data::Ocld> results = traits.handler().data();
169  ASSERT_NE(nullptr, results.get());
170  ASSERT_EQ(0U, results->size());
171 }
172 
173 //
174 // test_empty_gaussian
175 // This test executes the folding operator on noisy data generated at runtime.
176 // This test adds information to the data about the sampling time and
177 // frequecies.
178 POOL_ALGORITHM_TYPED_TEST_P(FldoTester, test_empty_gaussian_data)
179 {
180  typedef typename TypeParam::TimeFrequencyType DataType;
181  TypeParam traits;
182  auto& api = traits.api(pool);
183 
184  typename DataType::FrequencyType delta( 1.0 * boost::units::si::mega * boost::units::si::hertz);
185  typename DataType::FrequencyType start( 100.0 * boost::units::si::mega * boost::units::si::hertz);
186  typedef data::Candidate<Cpu, float> CandidateType;
187  //fldo::Config fldo_config;
188  generators::GaussianNoiseConfig config;
189  generators::GaussianNoise<typename DataType::DataType> noise(config);
190 
191  // generate some noise
192  std::vector<std::shared_ptr<DataType>> tf_data;
193 
194  for(int i = 0; i < 64 ; ++i) {
195  std::shared_ptr<DataType> time_frequency_data(new DataType(data::DimensionSize<data::Time>(512), data::DimensionSize<data::Frequency>(1024))); // at least one channel
196  time_frequency_data->set_channel_frequencies_const_width(start, delta);
197  boost::units::quantity<boost::units::si::time, double> dt(64 * boost::units::si::micro * boost::units::si::seconds);
198  //set the sample interval for generated data
199  time_frequency_data->sample_interval(dt);
200  noise.next(*time_frequency_data);
201  // appends a new element to the end of the container
202  tf_data.emplace_back(time_frequency_data);
203  }
204  data::Scl scl_data;
205  //load the list of pulsar candidates
206  //
207  load_candidates(scl_data);
208  // init of config variables
209  //fldo_config.phases(fldo::o_phases);
210  //
211  ASSERT_FALSE(traits.handler().executed());
212  auto job = api(tf_data, scl_data);
213  job->wait();
214 
215  ASSERT_TRUE(traits.handler().executed());
216 /*
217  std::shared_ptr<data::Ocld> results = traits.handler().data();
218 
219  // check for all elements no candidates with S/N > 3
220  ASSERT_GT(results->size(), 0);
221  if (0 < results->size() ) {
222  for (int i = 0 ; i < candidate_number ; i++ ) {
223  ASSERT_GE(3. , (*results)[0].sigma());
224  }
225  }
226 */
227 }
228 #ifdef DEEP_TESTING
229 
230 //
231 // test_sig_proc_file
232 // This test executes the folding operator on data read from a file in
233 // sigproc format.
234 // This tests gets the number of samples stored into the file and then calculates the
235 // optimal number of samples taking into account the number of channels,
236 // sub-integration and max rebin value.
237 // It then organizes the data in a number of chunks equal to the number of
238 // sub-integrations and call the folding algorithm.
239 POOL_ALGORITHM_TYPED_TEST_P(FldoTester, test_sigproc_file)
240 {
241  typedef typename TypeParam::TimeFrequencyType DataType;
242  TypeParam traits;
243  auto& api = traits.api(pool);
244 
245  int nsubints = 64;
246  std::ifstream file_stream;
247  std::stringstream ss;
248  ss << "ska.dat";
249  file_stream.exceptions (std::ifstream::failbit | std::ifstream::badbit );
250  try {
251  file_stream.open(panda::test::test_file(ss.str()), std::ios::binary);
252  sigproc::SigProcHeader r;
253  r.read(file_stream);
254  file_stream.close();
255  PANDA_LOG_DEBUG << "number of bits: " << r.number_of_bits();
256  PANDA_LOG_DEBUG << "header size: " << r.size();
257  //PANDA_LOG << "number of bits: " << r.number_of_bits();
258  //PANDA_LOG << "header size: " << r.size();
259  size_t nchannels = r.number_of_channels();
260  size_t file_size = boost::filesystem::file_size(panda::test::test_file(ss.str()));
261  uint64_t nsamples = (file_size - r.size()) * (8/r.number_of_bits());
262  PANDA_LOG_DEBUG << "number of channels: " << nchannels;
263  PANDA_LOG_DEBUG << "number of samples: " << nsamples;
264  //-data::TimeType sampling_time = r.sample_interval();
265  //-double tobs = (nsamples/nchannels) * sampling_time.value();
266  // we want the data to be a multiple also of the fldo::max_prebin value
267  if ((nsamples % (nsubints * nchannels * fldo::cuda::max_prebin)) != 0) {
268  nsamples = (uint64_t)((nsamples/(((float)nsubints) *nchannels* fldo::cuda::max_prebin))) * (nsubints * nchannels * fldo::cuda::max_prebin);
269  //-tobs = (nsamples/nchannels) * sampling_time.value();
270  }
271  std::vector<std::shared_ptr<DataType>> tf_data;
272  sigproc::Config config;
273  //set the data file name
274  config.set_sigproc_files(panda::test::test_file(ss.str()));
275  // configure the chunk data dimension. The number ito specify has to be calculated as:
276  // chunk_number_of_samples = number_of_sample_in_a_chunk/numer_of_channels
277  // because the routine that allocates the chunk dimension (resize()) multiplies the value
278  // of chunk_number_of_samples for the number of channels specified into the header
279  config.set_chunk_samples(nsamples/nsubints/nchannels);
280  sigproc::SigProcFileStream stream(config);
281  panda::DataManager<sigproc::SigProcFileStream> chunk_manager(stream);
282  stream.process();
283  for (int i = 0; i < nsubints; ++i) {
284  PANDA_LOG_DEBUG <<"Loading subint: " << i;
285  std::tuple<std::shared_ptr<DataType>> data = chunk_manager.next();
286  std::shared_ptr<DataType> time_frequency_data = std::get<0>(data);
287  tf_data.emplace_back(time_frequency_data);
288  }
289  PANDA_LOG <<"Loaded data from file! ";
290  data::Scl scl_data;
291  //load the list of pulsar candidates from a file
292  load_candidates(scl_data);
293 
294  auto job = api(tf_data, scl_data);
295  job->wait();
296  std::shared_ptr<data::Ocld> results = api.handler().data();
297 
298  ASSERT_EQ(0U, results->size());
299  }
300  catch(std::ifstream::failure e) {
301  std::stringstream s;
302  s << "Exception accessing data file:"
303  << e.what();
304  PANDA_LOG_ERROR << s.str();
305 
306  }
307 }
308 
309 #endif // DEEP_TESTING
310 
311 // each test defined by ALGORITHM_TYPED_TEST_P must be added to the
312 // test register (each one as an element of the comma seperated list)
313 #ifdef DEEP_TESTING
314 REGISTER_TYPED_TEST_CASE_P(FldoTester, test_empty_data, test_empty_gaussian_data, test_sigproc_file);
315 #else
316 REGISTER_TYPED_TEST_CASE_P(FldoTester, test_empty_data, test_empty_gaussian_data);
317 #endif
318 
319 } // namespace test
320 } // namespace fldo
321 } // namespace cheetah
322 } // namespace ska
Some limits and constants for FLDO.
Definition: Brdz.h:35