Adds optional source-file-key specifier in config file

  - Also adds time remaining estimation to status printout

include/config.hpp

@@ -139,7 +139,11 @@ class Config{
 
         std::vector<DataFileDescriptor> get_source_files() {
             std::vector<DataFileDescriptor> source_files;
-            YAML::Node source_files_nd = root["source-files"];
+            std::string source_file_key = "source-files";
+            if (root["source-file-key"]) {
+                source_file_key = root["source-file-key"].as<std::string>();
+            }
+            YAML::Node source_files_nd = root[source_file_key];
             if(source_files_nd.IsSequence()) {
                 for (const auto& f : source_files_nd) {
                     source_files.push_back(f.as<DataFileDescriptor>());

include/dataset.hpp

@@ -31,6 +31,7 @@
 #ifndef dataset_hpp
 #define dataset_hpp
 #include <iostream>
+#include <sys/time.h>
 #include "value.hpp"
 #include "container.hpp"
 #include "memtrack.hpp"
@@ -56,6 +57,37 @@ namespace fv {
         }
 
         int max_events;
+        timeval start_time;
+
+        void print_status() {
+            size_t m_used = fv_util::getCurrentRSS() / 1024 / 1024;
+            timeval curr_time;
+            gettimeofday(&curr_time, nullptr);
+            float delta_secs = (curr_time.tv_sec - start_time.tv_sec) + (curr_time.tv_usec - start_time.tv_usec) / 1E6f;
+            int current_event = get_current_event();
+            float events_per_second = current_event / delta_secs;
+            int secs_remaining = int((get_events() - current_event) / events_per_second);
+            std::stringstream time_remaining;
+            if (secs_remaining > 3600.) {
+                int hours = secs_remaining / 3600;
+                secs_remaining %= 3600;
+                time_remaining << hours << "H ";
+            }
+            if (secs_remaining > 60) {
+                int minutes = secs_remaining / 60;
+                secs_remaining %= 60;
+                time_remaining << minutes << "M ";
+
+            }
+            if (secs_remaining > 0) {
+                time_remaining << secs_remaining << "S";
+            }
+            std::cout << "\rprocessing event: " << current_event + 1 << "/" << get_events()
+                      << " of file: " << get_current_file().filename
+                      << ", " << m_used <<  "MB used "
+                      << ", " << time_remaining.str() << " est. time remaining"
+                      << std::flush;
+        }
 
     protected:
         std::map<std::string, GenContainer *> containers;
@@ -78,15 +110,11 @@ namespace fv {
         virtual void save_config() = 0;
 
     public:
-        bool next(bool print_status=true) {
-            if(print_status) {
-                size_t m_used = fv_util::getCurrentRSS() / 1024 / 1024;
-                size_t m_peak = fv_util::getPeakRSS() / 1024 / 1024;
-                std::cout << "\rprocessing event: " << get_current_event() + 1 << "/" << get_events()
-                          << ", " << m_used << "/" << m_peak << "MB used/peak"
-                          << " of file: " << get_current_file().filename << std::flush;
-            }
-            if (max_events && get_current_event() + 1 >= max_events) return false;
+        bool next(bool verbose=true) {
+            int current_event = get_current_event();
+            if (current_event == 0) gettimeofday(&start_time, nullptr);
+            if (verbose and (((current_event + 1) % 500) == 0)) print_status();
+            if (max_events && current_event + 1 >= max_events) return false;
             GenValue::reset();
             return load_next();
         }