AstraLog-HPC
Loading...
Searching...
No Matches
Collector.hpp
Go to the documentation of this file.
1
5#pragma once
6
9#include <common/errors.hpp>
10#include <atomic>
11#include <functional>
12#include <memory>
13#include <string>
14#include <cstdlib>
15#include <iostream>
16#include <thread>
17#include <chrono>
18#include <atomic>
19#include <queue>
20#include <mutex>
21#include <condition_variable>
23
24#include <mqtt/async_client.h>
28static constexpr int N_RETRY_ATTEMPTS = 5;
29
30
35class Connector : public virtual mqtt::callback, public virtual mqtt::iaction_listener {
36 private:
37 int nretry_ = 0;
38 mqtt::async_client& cli_;
39 mqtt::connect_options& connOpts_;
40 std::string topic_;
41 std::function<void(const std::string&)> on_message_;
42 std::function<void(const std::string&)> on_error_;
43
48 void message_arrived(mqtt::const_message_ptr msg) override;
49
53 void reconnect();
54
59 void on_failure(const mqtt::token& tok) override;
60
65 void on_success(const mqtt::token& tok) override;
66
71 void connected(const std::string& cause) override;
72
77 void connection_lost(const std::string& cause) override;
78
79 public:
88 Connector(mqtt::async_client& cli, mqtt::connect_options& connOpts,
89 std::string topic,
90 std::function<void(const std::string&)> on_message,
91 std::function<void(const std::string&)> on_error)
92 : cli_(cli), connOpts_(connOpts), topic_(std::move(topic)),
93 on_message_(std::move(on_message)), on_error_(std::move(on_error))
94 {}
95 ~Connector() override = default;
96};
97
103 public:
108 Collector();
109
110 ~Collector() = default;
111
115 void run();
116
120 void stop();
121
122 private:
123 std::string api_key;
124 std::string batch_filepath;
125 std::string sensors_filepath;
126 std::string rules_filepath;
127 std::atomic<bool> fatal_error_{false};
129 std::string log_path;
130
135 void retrieve_api_key();
136
141 void process_packet(const std::string& payload);
142
146 void finalize_drain();
147
151 void submitter_loop();
152
158 void retriever_loop();
159
160 std::thread submitter_thread_;
161 std::thread retriever_thread_;
162 std::queue<std::string> job_queue_;
163 std::mutex queue_mutex_;
164 std::condition_variable queue_cv_;
165 std::condition_variable retriever_cv_;
166 bool stopping_ = false;
167
168 std::atomic<bool> draining_{false};
169 std::atomic<bool> drain_done_{false};
170 std::mutex drain_mutex_;
171 std::condition_variable drain_cv_;
172
173 std::unique_ptr<Connector> broker_connector;
174 std::unique_ptr<BatchWriter> batch_writer;
175 std::unique_ptr<JobScheduler> job_scheduler;
176
177 std::unique_ptr<mqtt::async_client> mqtt_client;
178 mqtt::connect_options connOpts;
179
180 std::unique_ptr<Validator> validator;
181};
Aggregates validated packets into per-timestamp rows and flushes them as batches.
SLURM job creation and HPC cluster submission for the AstraLog Collector.
Definition of the Validator class and related structures for telemetry data validation.
Reads sensor packets from MQTT, validates and batches them, and submits completed batches to the remo...
Definition Collector.hpp:102
std::mutex drain_mutex_
Guards the drain_cv_ wait in stop().
Definition Collector.hpp:170
std::unique_ptr< JobScheduler > job_scheduler
Dispatches batches to the remote Engine via SCP/SSH.
Definition Collector.hpp:175
void retrieve_api_key()
Reads API_KEY from the environment and stores it in api_key.
Definition Collector.cpp:97
std::unique_ptr< mqtt::async_client > mqtt_client
The Paho MQTT asynchronous client.
Definition Collector.hpp:177
int batch_size
Packet count per batch; computed as optimal_batch_size() = 10 × num_sensors.
Definition Collector.hpp:128
void finalize_drain()
Flushes any buffered data and enqueues it for submission; no-op after the first call.
Definition Collector.cpp:169
bool stopping_
Signals submitter_loop to exit once job_queue_ is empty; guarded by queue_mutex_.
Definition Collector.hpp:166
std::string api_key
API key for authenticating with the MQTT broker.
Definition Collector.hpp:123
std::string sensors_filepath
Path to the sensors configuration.
Definition Collector.hpp:125
std::condition_variable drain_cv_
Notified by finalize_drain to unblock stop()'s drain wait.
Definition Collector.hpp:171
std::unique_ptr< Connector > broker_connector
Manages the MQTT connection.
Definition Collector.hpp:173
std::unique_ptr< Validator > validator
Validates incoming packets against expected sensors.
Definition Collector.hpp:180
std::atomic< bool > drain_done_
Set once finalize_drain has run; causes process_packet to discard late packets.
Definition Collector.hpp:169
std::string log_path
Path to the collector's log file.
Definition Collector.hpp:129
~Collector()=default
std::thread submitter_thread_
Thread running submitter_loop.
Definition Collector.hpp:160
void stop()
Drains buffered data, disconnects from the broker, and joins the submitter thread.
Definition Collector.cpp:188
void submitter_loop()
Worker thread that dequeues batch files and submits them via SCP/SSH, keeping the MQTT callback non-b...
Definition Collector.cpp:239
void retriever_loop()
Worker thread that periodically downloads results and SLURM logs from the cluster.
Definition Collector.cpp:261
void run()
Connects to the broker and blocks until 'q' is pressed or a fatal error occurs.
Definition Collector.cpp:104
std::atomic< bool > fatal_error_
Set to true by the MQTT or submitter thread to trigger shutdown.
Definition Collector.hpp:127
void process_packet(const std::string &payload)
Validates the payload, writes it to the current batch, and flushes to disk when the batch is full.
Definition Collector.cpp:285
std::string batch_filepath
Path where batches will be temporarily saved.
Definition Collector.hpp:124
std::condition_variable retriever_cv_
Wakes the retriever thread early when stopping_ is set; shares queue_mutex_.
Definition Collector.hpp:165
std::atomic< bool > draining_
Set when shutdown begins; tells process_packet to flush after the current row closes.
Definition Collector.hpp:168
mqtt::connect_options connOpts
Options for the MQTT connection.
Definition Collector.hpp:178
std::thread retriever_thread_
Thread running retriever_loop.
Definition Collector.hpp:161
Collector()
Reads configuration from environment variables and initializes all subcomponents.
Definition Collector.cpp:8
std::queue< std::string > job_queue_
Batch file paths waiting for submission.
Definition Collector.hpp:162
std::unique_ptr< BatchWriter > batch_writer
Buffers incoming packets and writes complete batches to disk.
Definition Collector.hpp:174
std::mutex queue_mutex_
Protects job_queue_ and stopping_.
Definition Collector.hpp:163
std::string rules_filepath
Path to the rules configuration.
Definition Collector.hpp:126
std::condition_variable queue_cv_
Wakes the submitter thread when a batch is enqueued or stopping_ is set.
Definition Collector.hpp:164
MQTT callback handler that forwards incoming messages and manages automatic reconnection on connectio...
Definition Collector.hpp:35
void on_failure(const mqtt::token &tok) override
Increments the retry counter and reconnects, or calls on_error_ after N_RETRY_ATTEMPTS failures.
Definition Collector.cpp:68
~Connector() override=default
void connected(const std::string &cause) override
Subscribes to topic_ once the connection is established.
Definition Collector.cpp:81
void on_success(const mqtt::token &tok) override
Logs the successful connection attempt.
Definition Collector.cpp:77
void message_arrived(mqtt::const_message_ptr msg) override
Forwards the payload string of msg to on_message_.
Definition Collector.cpp:54
std::function< void(const std::string &)> on_error_
Reports fatal connection errors to the Collector.
Definition Collector.hpp:42
void reconnect()
Waits briefly then calls cli_.connect() again.
Definition Collector.cpp:58
std::string topic_
MQTT topic to subscribe to.
Definition Collector.hpp:40
int nretry_
Current retry attempt counter.
Definition Collector.hpp:37
void connection_lost(const std::string &cause) override
Resets the retry counter and initiates reconnection.
Definition Collector.cpp:86
Connector(mqtt::async_client &cli, mqtt::connect_options &connOpts, std::string topic, std::function< void(const std::string &)> on_message, std::function< void(const std::string &)> on_error)
Constructs a Connector.
Definition Collector.hpp:88
mqtt::connect_options & connOpts_
Connection options passed to the client on each connect attempt.
Definition Collector.hpp:39
std::function< void(const std::string &)> on_message_
Forwards received message payloads to the Collector.
Definition Collector.hpp:41
mqtt::async_client & cli_
MQTT client used to connect, subscribe, and reconnect.
Definition Collector.hpp:38
Unified exception hierarchy for AstraLog-HPC.