21#include <condition_variable>
24#include <mqtt/async_client.h>
28static constexpr int N_RETRY_ATTEMPTS = 5;
35class Connector :
public virtual mqtt::callback,
public virtual mqtt::iaction_listener {
59 void on_failure(
const mqtt::token& tok)
override;
65 void on_success(
const mqtt::token& tok)
override;
71 void connected(
const std::string& cause)
override;
88 Connector(mqtt::async_client& cli, mqtt::connect_options& connOpts,
90 std::function<
void(
const std::string&)> on_message,
91 std::function<
void(
const std::string&)> on_error)
Aggregates validated packets into per-timestamp rows and flushes them as batches.
SLURM job creation and HPC cluster submission for the AstraLog Collector.
Definition of the Validator class and related structures for telemetry data validation.
Reads sensor packets from MQTT, validates and batches them, and submits completed batches to the remo...
Definition Collector.hpp:102
std::mutex drain_mutex_
Guards the drain_cv_ wait in stop().
Definition Collector.hpp:170
std::unique_ptr< JobScheduler > job_scheduler
Dispatches batches to the remote Engine via SCP/SSH.
Definition Collector.hpp:175
void retrieve_api_key()
Reads API_KEY from the environment and stores it in api_key.
Definition Collector.cpp:97
std::unique_ptr< mqtt::async_client > mqtt_client
The Paho MQTT asynchronous client.
Definition Collector.hpp:177
int batch_size
Packet count per batch; computed as optimal_batch_size() = 10 × num_sensors.
Definition Collector.hpp:128
void finalize_drain()
Flushes any buffered data and enqueues it for submission; no-op after the first call.
Definition Collector.cpp:169
bool stopping_
Signals submitter_loop to exit once job_queue_ is empty; guarded by queue_mutex_.
Definition Collector.hpp:166
std::string api_key
API key for authenticating with the MQTT broker.
Definition Collector.hpp:123
std::string sensors_filepath
Path to the sensors configuration.
Definition Collector.hpp:125
std::condition_variable drain_cv_
Notified by finalize_drain to unblock stop()'s drain wait.
Definition Collector.hpp:171
std::unique_ptr< Connector > broker_connector
Manages the MQTT connection.
Definition Collector.hpp:173
std::unique_ptr< Validator > validator
Validates incoming packets against expected sensors.
Definition Collector.hpp:180
std::atomic< bool > drain_done_
Set once finalize_drain has run; causes process_packet to discard late packets.
Definition Collector.hpp:169
std::string log_path
Path to the collector's log file.
Definition Collector.hpp:129
std::thread submitter_thread_
Thread running submitter_loop.
Definition Collector.hpp:160
void stop()
Drains buffered data, disconnects from the broker, and joins the submitter thread.
Definition Collector.cpp:188
void submitter_loop()
Worker thread that dequeues batch files and submits them via SCP/SSH, keeping the MQTT callback non-b...
Definition Collector.cpp:239
void retriever_loop()
Worker thread that periodically downloads results and SLURM logs from the cluster.
Definition Collector.cpp:261
void run()
Connects to the broker and blocks until 'q' is pressed or a fatal error occurs.
Definition Collector.cpp:104
std::atomic< bool > fatal_error_
Set to true by the MQTT or submitter thread to trigger shutdown.
Definition Collector.hpp:127
void process_packet(const std::string &payload)
Validates the payload, writes it to the current batch, and flushes to disk when the batch is full.
Definition Collector.cpp:285
std::string batch_filepath
Path where batches will be temporarily saved.
Definition Collector.hpp:124
std::condition_variable retriever_cv_
Wakes the retriever thread early when stopping_ is set; shares queue_mutex_.
Definition Collector.hpp:165
std::atomic< bool > draining_
Set when shutdown begins; tells process_packet to flush after the current row closes.
Definition Collector.hpp:168
mqtt::connect_options connOpts
Options for the MQTT connection.
Definition Collector.hpp:178
std::thread retriever_thread_
Thread running retriever_loop.
Definition Collector.hpp:161
Collector()
Reads configuration from environment variables and initializes all subcomponents.
Definition Collector.cpp:8
std::queue< std::string > job_queue_
Batch file paths waiting for submission.
Definition Collector.hpp:162
std::unique_ptr< BatchWriter > batch_writer
Buffers incoming packets and writes complete batches to disk.
Definition Collector.hpp:174
std::mutex queue_mutex_
Protects job_queue_ and stopping_.
Definition Collector.hpp:163
std::string rules_filepath
Path to the rules configuration.
Definition Collector.hpp:126
std::condition_variable queue_cv_
Wakes the submitter thread when a batch is enqueued or stopping_ is set.
Definition Collector.hpp:164
MQTT callback handler that forwards incoming messages and manages automatic reconnection on connectio...
Definition Collector.hpp:35
void on_failure(const mqtt::token &tok) override
Increments the retry counter and reconnects, or calls on_error_ after N_RETRY_ATTEMPTS failures.
Definition Collector.cpp:68
~Connector() override=default
void connected(const std::string &cause) override
Subscribes to topic_ once the connection is established.
Definition Collector.cpp:81
void on_success(const mqtt::token &tok) override
Logs the successful connection attempt.
Definition Collector.cpp:77
void message_arrived(mqtt::const_message_ptr msg) override
Forwards the payload string of msg to on_message_.
Definition Collector.cpp:54
std::function< void(const std::string &)> on_error_
Reports fatal connection errors to the Collector.
Definition Collector.hpp:42
void reconnect()
Waits briefly then calls cli_.connect() again.
Definition Collector.cpp:58
std::string topic_
MQTT topic to subscribe to.
Definition Collector.hpp:40
int nretry_
Current retry attempt counter.
Definition Collector.hpp:37
void connection_lost(const std::string &cause) override
Resets the retry counter and initiates reconnection.
Definition Collector.cpp:86
Connector(mqtt::async_client &cli, mqtt::connect_options &connOpts, std::string topic, std::function< void(const std::string &)> on_message, std::function< void(const std::string &)> on_error)
Constructs a Connector.
Definition Collector.hpp:88
mqtt::connect_options & connOpts_
Connection options passed to the client on each connect attempt.
Definition Collector.hpp:39
std::function< void(const std::string &)> on_message_
Forwards received message payloads to the Collector.
Definition Collector.hpp:41
mqtt::async_client & cli_
MQTT client used to connect, subscribe, and reconnect.
Definition Collector.hpp:38
Unified exception hierarchy for AstraLog-HPC.