AstraLog-HPC
Loading...
Searching...
No Matches
BatchWriter.hpp
Go to the documentation of this file.
1
5#pragma once
6
7#include <algorithm>
8#include <string>
9#include <memory>
10#include <utility>
11#include <vector>
12#include <unordered_map>
13#include <unordered_set>
14#include <common/io.hpp>
15
21 public:
27 ConfigReader(const std::string& sensors_filepath,
28 std::string rules_filepath);
29
34 [[nodiscard]] int optimal_batch_size() const;
35
36
37
38 private:
39 std::string sensors_filepath;
40 std::string rules_filepath;
42};
43
53 public:
62 int batch_size,
63 std::string batch_filepath,
64 const std::string& sensors_filepath,
65 const std::string& rules_filepath)
66 : sensor_list_(astralog::loadSensorIds(sensors_filepath)),
69 batch_size_(batch_size),
70 batch_filepath_(std::move(batch_filepath)),
71 configurator_(std::make_unique<ConfigReader>(sensors_filepath, rules_filepath))
72 {}
73
74 ~BatchWriter() = default;
75
85 void receive(const std::string& timestamp, const std::string& sensor_id, const std::string& packet);
86
91 int get_batch_size() const { return batch_size_; }
92
103 int get_num_proc() const {
104 constexpr int NODE_CORES = 48; // g100_usr_prod: 2 sockets x 24 cores x 1 thread/core (SMT off, verified via sinfo)
105 constexpr int CPUS_PER_TASK = 4; // matches the template; <= socket size (24) so a rank's OMP team never crosses a NUMA socket
106 constexpr int MAX_TASKS = NODE_CORES / CPUS_PER_TASK;
107 const int by_sensors = static_cast<int>((num_sensors_ + 3) / 4);
108 return std::clamp(by_sensors, 1, MAX_TASKS);
109 }
110
115 bool is_batch_full() const { return std::cmp_greater_equal(buffer_.size(), batch_size_); }
116
124 bool row_in_progress() const { return !current_row_.slots.empty(); }
125
130 bool has_buffered() const { return !buffer_.empty(); }
131
136 std::string flush();
137
138 private:
143 struct CurrentRow {
144 std::string timestamp;
145 std::unordered_map<std::string, std::string> slots;
146 };
147
149 std::vector<std::string> sensor_list_;
150 std::unordered_set<std::string> sensor_set_;
151 size_t num_sensors_ = 0;
152
153 std::vector<std::string> buffer_;
156 std::string batch_filepath_;
157
158 std::unique_ptr<ConfigReader> configurator_;
159
163 std::string make_error_packet(const std::string& timestamp, const std::string& sensor_id) const;
164
171 void close_row();
172};
Accumulates incoming packets into per-timestamp rows and flushes complete batches to disk.
Definition BatchWriter.hpp:52
int get_batch_size() const
Returns the configured maximum batch size.
Definition BatchWriter.hpp:91
std::string flush()
Writes the buffered rows to a numbered batch file and clears the buffer.
Definition BatchWriter.cpp:58
bool has_buffered() const
Returns true if there are completed rows buffered but not yet flushed.
Definition BatchWriter.hpp:130
int batch_size_
Maximum number of packets (not rows) before a flush is triggered.
Definition BatchWriter.hpp:154
void receive(const std::string &timestamp, const std::string &sensor_id, const std::string &packet)
Adds the packet to the current row; fills the slot with an error packet if packet is empty.
Definition BatchWriter.cpp:25
std::vector< std::string > sensor_list_
Ordered list of all configured sensor IDs.
Definition BatchWriter.hpp:149
std::unordered_set< std::string > sensor_set_
Hash set of sensor IDs for O(1) membership checks.
Definition BatchWriter.hpp:150
std::string make_error_packet(const std::string &timestamp, const std::string &sensor_id) const
Returns a JSON error packet (value "E") for the given timestamp and sensor.
Definition BatchWriter.cpp:8
BatchWriter(int batch_size, std::string batch_filepath, const std::string &sensors_filepath, const std::string &rules_filepath)
Constructs a new BatchWriter.
Definition BatchWriter.hpp:61
std::vector< std::string > buffer_
Completed rows waiting to be flushed to disk.
Definition BatchWriter.hpp:153
bool is_batch_full() const
Returns true if the buffer has reached the batch size limit.
Definition BatchWriter.hpp:115
CurrentRow current_row_
The row currently being filled.
Definition BatchWriter.hpp:148
int get_num_proc() const
Number of MPI tasks to request: ~1 per 4 sensors, at least one.
Definition BatchWriter.hpp:103
size_t num_sensors_
Total number of configured sensors.
Definition BatchWriter.hpp:151
void close_row()
Flushes the current row to buffer_, padding missing sensors with error packets.
Definition BatchWriter.cpp:15
std::string batch_filepath_
Base file path for batch output files.
Definition BatchWriter.hpp:156
~BatchWriter()=default
std::unique_ptr< ConfigReader > configurator_
Computes the default batch size from the configuration.
Definition BatchWriter.hpp:158
bool row_in_progress() const
Returns true when some but not all sensors have reported for the current timestamp.
Definition BatchWriter.hpp:124
int batch_counter_
Counter for generating unique batch file names.
Definition BatchWriter.hpp:155
Parses sensor and rule configuration files to derive the optimal batch size.
Definition BatchWriter.hpp:20
int optimal_batch_size() const
Returns the optimal number of rows per batch based on the sensor count.
Definition BatchWriter.cpp:80
size_t sensors_number
Number of sensors; used to compute the optimal batch size.
Definition BatchWriter.hpp:41
std::string rules_filepath
Path to the rules configuration file.
Definition BatchWriter.hpp:40
std::string sensors_filepath
Path to the sensors configuration file.
Definition BatchWriter.hpp:39
Small shared I/O helpers used across modules.
Definition io.cpp:5
Tracks the state of the row currently being built.
Definition BatchWriter.hpp:143
std::string timestamp
Shared timestamp for all packets in this row.
Definition BatchWriter.hpp:144
std::unordered_map< std::string, std::string > slots
Maps sensor_id to its JSON packet; missing entries become error packets when the row closes.
Definition BatchWriter.hpp:145