// SPDX-License-Identifier: GPL-3.0-or-later

#include "stream.h"
#include "stream-thread.h"
#include "stream-receiver-internals.h"

#ifdef NETDATA_LOG_STREAM_RECEIVER
#include "stream-trace.h"

void stream_receiver_log_payload(struct receiver_state *rpt, const char *payload, STREAM_TRAFFIC_TYPE type __maybe_unused, bool inbound) {
    if (!rpt || type != STREAM_TRAFFIC_TYPE_REPLICATION) return; // not a streaming parser

    spinlock_lock(&rpt->log.spinlock);

    if (!rpt->log.fp) {
        char filename[FILENAME_MAX + 1];
        snprintfz(
            filename, FILENAME_MAX, "/tmp/stream-receiver-%s.txt", rpt->host ? rrdhost_hostname(rpt->host) : "unknown");

        rpt->log.fp = fopen(filename, "w");

        // Align first_call to wall clock time
        clock_gettime(CLOCK_REALTIME, &rpt->log.first_call);
        rpt->log.first_call.tv_nsec = 0; // Align to the start of the second
    }

    if (rpt->log.fp) {
        struct timespec now;
        clock_gettime(CLOCK_REALTIME, &now);

        char prefix[STREAM_TRACE_PREFIX_SIZE];
        stream_trace_format_elapsed_prefix(prefix, now, rpt->log.first_call);

        const char *line_start = payload;
        const char *line_end;

        while (line_start && *line_start) {
            line_end = strchr(line_start, '\n');
            if (line_end) {
                fprintf(rpt->log.fp, "%s%s%.*s\n", prefix, inbound ? "> " : "< ", (int)(line_end - line_start), line_start);
                line_start = line_end + 1;
            } else {
                fprintf(rpt->log.fp, "%s%s%s\n", prefix, inbound ? "> " : "< ", line_start);
                break;
            }
        }
    }

    fflush(rpt->log.fp);
    spinlock_unlock(&rpt->log.spinlock);
}
#endif

// help the IDE identify use after free
#define stream_receiver_remove(sth, rpt, reason) do {           \
        stream_receiver_remove_internal(sth, rpt, reason);      \
        (rpt) = NULL;                                           \
} while(0)

static void stream_receiver_remove_internal(struct stream_thread *sth, struct receiver_state *rpt, STREAM_HANDSHAKE reason);

// When a child disconnects this is the maximum we will wait
// before we update the cloud that the child is offline
#define MAX_CHILD_DISC_DELAY (30000)
#define MAX_CHILD_DISC_TOLERANCE (125 / 100)

static uint32_t streaming_connected_receivers = 0;

bool plugin_is_enabled(struct plugind *cd);

uint32_t stream_receivers_currently_connected(void) {
    return __atomic_load_n(&streaming_connected_receivers, __ATOMIC_RELAXED);
}

static void streaming_receiver_connected(void) {
    __atomic_add_fetch(&streaming_connected_receivers, 1, __ATOMIC_RELAXED);
}

static void streaming_receiver_disconnected(void) {
    __atomic_sub_fetch(&streaming_connected_receivers, 1, __ATOMIC_RELAXED);
}

// --------------------------------------------------------------------------------------------------------------------

static bool stream_receiver_log_capabilities(BUFFER *wb, void *ptr) {
    struct receiver_state *rpt = ptr;
    if(!rpt)
        return false;

    stream_capabilities_to_string(wb, rpt->capabilities);
    return true;
}

static bool stream_receiver_log_transport(BUFFER *wb, void *ptr) {
    struct receiver_state *rpt = ptr;
    if(!rpt)
        return false;

    buffer_strcat(wb, nd_sock_is_ssl(&rpt->sock) ? "https" : "http");
    return true;
}

// --------------------------------------------------------------------------------------------------------------------

ALWAYS_INLINE
static ssize_t write_stream(struct receiver_state *r, char* buffer, size_t size) {
    if(unlikely(!size)) {
        internal_error(true, "%s() asked to read zero bytes", __FUNCTION__);
        errno_clear();
        return -2;
    }

    ssize_t bytes_written = nd_sock_send_nowait(&r->sock, buffer, size);
    return bytes_written;
}

ALWAYS_INLINE
static ssize_t read_stream(struct receiver_state *r, char* buffer, size_t size) {
    if(unlikely(!size)) {
        internal_error(true, "%s() asked to read zero bytes", __FUNCTION__);
        errno_clear();
        return -2;
    }

    ssize_t bytes_read = nd_sock_revc_nowait(&r->sock, buffer, size);
    return bytes_read;
}

// --------------------------------------------------------------------------------------------------------------------

ALWAYS_INLINE
static ssize_t receiver_read_uncompressed(struct receiver_state *r) {
    ssize_t read_len = r->thread.uncompressed.read_len;
    if(unlikely(read_len < 0)) {
        internal_fatal(true, "The line to read is too big! Already have %zd bytes in read_buffer.", read_len);
        errno_clear();
        return -2;
    }

    size_t read_offset = (size_t)read_len;
    if(unlikely(read_offset >= sizeof(r->thread.uncompressed.read_buffer) - 1)) {
        internal_fatal(true, "The line to read is too big! Already have %zd bytes in read_buffer.", read_len);
        errno_clear();
        return -2;
    }

    internal_fatal(r->thread.uncompressed.read_buffer[read_offset] != '\0',
                   "%s: read_buffer does not start with zero #2", __FUNCTION__ );

    size_t available = sizeof(r->thread.uncompressed.read_buffer) - read_offset - 1;
    char *dst = r->thread.uncompressed.read_buffer + read_offset;
    ssize_t bytes = read_stream(r, dst, available);
    if(bytes > 0) {
        size_t bytes_read = (size_t)bytes;
        if(unlikely(bytes_read > available)) {
            internal_fatal(true, "read_stream() returned %zd bytes with only %zu bytes available", bytes, available);
            errno_clear();
            return -2;
        }

        size_t new_read_len = read_offset + bytes_read;
        worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, (NETDATA_DOUBLE)bytes);
        worker_set_metric(WORKER_RECEIVER_JOB_BYTES_UNCOMPRESSED, (NETDATA_DOUBLE)bytes);

        r->thread.uncompressed.read_len = (ssize_t)new_read_len;
        dst[bytes_read] = '\0';
        r->thread.bytes_received += bytes;
        if(likely(r->host))
            single_writer_atomic_add(&r->host->stream.rcv.status.bytes_in, (uint64_t)bytes);
        pulse_stream_received_bytes(bytes);
    }

    return bytes;
}

typedef enum {
    DECOMPRESS_NEED_MORE_DATA,
    DECOMPRESS_FAILED,
    DECOMPRESS_OK,
} decompressor_status_t;

static inline void receiver_move_compressed(struct receiver_state *r) {
    size_t remaining = r->thread.compressed.used - r->thread.compressed.start;
    if(remaining > 0) {
        memmove(r->thread.compressed.buf, r->thread.compressed.buf + r->thread.compressed.start, remaining);
        r->thread.compressed.start = 0;
        r->thread.compressed.used = remaining;
    }
    else {
        r->thread.compressed.start = 0;
        r->thread.compressed.used = 0;
    }
}

ALWAYS_INLINE_HOT_FLATTEN
static decompressor_status_t receiver_feed_decompressor(struct receiver_state *r) {
    char *buf = r->thread.compressed.buf;
    size_t start = r->thread.compressed.start;
    size_t signature_size = r->thread.compressed.decompressor.signature_size;
    size_t used = r->thread.compressed.used;

    if(start + signature_size > used) {
        // incomplete header, we need to wait for more data
        receiver_move_compressed(r);
        return DECOMPRESS_NEED_MORE_DATA;
    }

    size_t compressed_message_size =
        stream_decompressor_start(&r->thread.compressed.decompressor, buf + start, signature_size);

    if (unlikely(!compressed_message_size)) {
        nd_log(NDLS_DAEMON, NDLP_ERR,
               "STREAM RCV[x] '%s' [from [%s]:%s]: multiplexed uncompressed data in compressed stream!",
               rrdhost_hostname(r->host), r->remote_ip, r->remote_port);
        return DECOMPRESS_FAILED;
    }

    if(unlikely(compressed_message_size > COMPRESSION_MAX_MSG_SIZE)) {
        nd_log(NDLS_DAEMON, NDLP_ERR,
               "STREAM RCV[x] '%s' [from [%s]:%s]: received a compressed message of %zu bytes, "
               "which is bigger than the max compressed message "
               "size supported of %zu. Ignoring message.",
               rrdhost_hostname(r->host), r->remote_ip, r->remote_port,
               compressed_message_size, (size_t)COMPRESSION_MAX_MSG_SIZE);
        return DECOMPRESS_FAILED;
    }

    if(start + signature_size + compressed_message_size > used) {
        // incomplete compressed message, we need to wait for more data
        receiver_move_compressed(r);
        return DECOMPRESS_NEED_MORE_DATA;
    }

    size_t bytes_to_parse =
        stream_decompress(&r->thread.compressed.decompressor, buf + start + signature_size, compressed_message_size);

    if (unlikely(!bytes_to_parse)) {
        nd_log(NDLS_DAEMON, NDLP_ERR,
               "STREAM RCV[x] '%s' [from [%s]:%s]: no bytes to decompress.",
               rrdhost_hostname(r->host), r->remote_ip, r->remote_port);
        return DECOMPRESS_FAILED;
    }

    worker_set_metric(WORKER_RECEIVER_JOB_BYTES_UNCOMPRESSED, (NETDATA_DOUBLE)bytes_to_parse);

    // move the header to the next message
    r->thread.compressed.start += signature_size + compressed_message_size;

    return DECOMPRESS_OK;
}

ALWAYS_INLINE_HOT_FLATTEN
static decompressor_status_t receiver_get_decompressed(struct receiver_state *r) {
    if (unlikely(!stream_decompressed_bytes_in_buffer(&r->thread.compressed.decompressor)))
        return DECOMPRESS_NEED_MORE_DATA;

    ssize_t read_len = r->thread.uncompressed.read_len;
    if(unlikely(read_len < 0)) {
        internal_fatal(true, "The line to read is too big! Already have %zd bytes in read_buffer.", read_len);
        return DECOMPRESS_FAILED;
    }

    size_t read_offset = (size_t)read_len;
    if(unlikely(read_offset >= sizeof(r->thread.uncompressed.read_buffer) - 1)) {
        internal_fatal(true, "The line to read is too big! Already have %zd bytes in read_buffer.", read_len);
        return DECOMPRESS_FAILED;
    }

    size_t available = sizeof(r->thread.uncompressed.read_buffer) - read_offset - 1;
    char *dst = r->thread.uncompressed.read_buffer + read_offset;
    size_t len = stream_decompressor_get(
        &r->thread.compressed.decompressor, dst, available);
    if (unlikely(!len)) {
        internal_error(true, "decompressor returned zero length #1");
        return DECOMPRESS_FAILED;
    }

    if(unlikely(len > available)) {
        internal_fatal(true, "decompressor returned %zu bytes with only %zu bytes available", len, available);
        return DECOMPRESS_FAILED;
    }

    size_t new_read_len = read_offset + len;
    r->thread.uncompressed.read_len = (ssize_t)new_read_len;
    dst[len] = '\0';

    return DECOMPRESS_OK;
}

ALWAYS_INLINE_HOT_FLATTEN
static ssize_t receiver_read_compressed(struct receiver_state *r) {
    ssize_t read_len = r->thread.uncompressed.read_len;
    if(unlikely(read_len < 0)) {
        internal_fatal(true, "The line to read is too big! Already have %zd bytes in read_buffer.", read_len);
        errno_clear();
        return -2;
    }

    size_t read_offset = (size_t)read_len;
    if(unlikely(read_offset >= sizeof(r->thread.uncompressed.read_buffer) - 1)) {
        internal_fatal(true, "The line to read is too big! Already have %zd bytes in read_buffer.", read_len);
        errno_clear();
        return -2;
    }

    internal_fatal(r->thread.uncompressed.read_buffer[read_offset] != '\0',
                   "%s: read_buffer does not start with zero #2", __FUNCTION__ );

    ssize_t bytes = read_stream(r, r->thread.compressed.buf + r->thread.compressed.used,
                                 r->thread.compressed.size - r->thread.compressed.used);

    if(bytes > 0) {
        r->thread.compressed.used += bytes;
        r->thread.bytes_received += bytes;
        if(likely(r->host))
            single_writer_atomic_add(&r->host->stream.rcv.status.bytes_in, (uint64_t)bytes);
        worker_set_metric(WORKER_RECEIVER_JOB_BYTES_READ, (NETDATA_DOUBLE)bytes);
        pulse_stream_received_bytes(bytes);
    }

    return bytes;
}

// --------------------------------------------------------------------------------------------------------------------

static STREAM_HANDSHAKE receiver_set_exit_reason(struct receiver_state *rpt, STREAM_HANDSHAKE reason, bool force) {
    if(force || !rpt->exit.reason)
        rpt->exit.reason = reason;

    return rpt->exit.reason;
}

ALWAYS_INLINE
static bool receiver_should_stop(struct receiver_state *rpt) {
    if(unlikely(__atomic_load_n(&rpt->exit.shutdown, __ATOMIC_ACQUIRE))) {
        receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_SIGNALED_TO_STOP, false);
        return true;
    }

    return false;
}

// --------------------------------------------------------------------------------------------------------------------

ALWAYS_INLINE
void stream_receiver_handle_op(struct stream_thread *sth, struct receiver_state *rpt, struct stream_opcode *msg) {
    ND_LOG_STACK lgs[] = {
        ND_LOG_FIELD_STR(NDF_NIDL_NODE, rpt->host->hostname),
        ND_LOG_FIELD_TXT(NDF_SRC_IP, rpt->remote_ip),
        ND_LOG_FIELD_TXT(NDF_SRC_PORT, rpt->remote_port),
        ND_LOG_FIELD_CB(NDF_SRC_TRANSPORT, stream_receiver_log_transport, rpt),
        ND_LOG_FIELD_CB(NDF_SRC_CAPABILITIES, stream_receiver_log_capabilities, rpt),
        ND_LOG_FIELD_END(),
    };
    ND_LOG_STACK_PUSH(lgs);

    if(msg->opcode & STREAM_OPCODE_RECEIVER_BUFFER_OVERFLOW) {
        worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_OVERFLOW);
        errno_clear();
        spinlock_lock(&rpt->thread.send_to_child.spinlock);
        // copy the statistics
        STREAM_CIRCULAR_BUFFER_STATS stats = *stream_circular_buffer_stats_unsafe(rpt->thread.send_to_child.scb);
        spinlock_unlock(&rpt->thread.send_to_child.spinlock);
        nd_log(NDLS_DAEMON, NDLP_ERR,
               "STREAM RCV[%zu] '%s' [from [%s]:%s]: send buffer is full (buffer size %zu, max %zu, used %zu, available %zu). "
               "Restarting connection.",
               sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip, rpt->remote_port,
               stats.bytes_size, stats.bytes_max_size, stats.bytes_outstanding, stats.bytes_available);

        stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_DISCONNECT_BUFFER_OVERFLOW);
        return;
    }

    nd_log(NDLS_DAEMON, NDLP_ERR,
           "STREAM RCV[%zu]: invalid msg id %u", sth->id, (unsigned)msg->opcode);
}

static ssize_t send_to_child(const char *txt, void *data, STREAM_TRAFFIC_TYPE type) {
    struct receiver_state *rpt = data;
    if(!rpt || rpt->thread.meta.type != POLLFD_TYPE_RECEIVER || !rpt->thread.send_to_child.scb)
        return 0;

    spinlock_lock(&rpt->thread.send_to_child.spinlock);
    STREAM_CIRCULAR_BUFFER *scb = rpt->thread.send_to_child.scb;
    STREAM_CIRCULAR_BUFFER_STATS *stats = stream_circular_buffer_stats_unsafe(scb);
    bool was_empty = stats->bytes_outstanding == 0;
    struct stream_opcode msg = rpt->thread.send_to_child.msg;
    msg.opcode = STREAM_OPCODE_NONE;
    msg.reason = 0;

    size_t size = strlen(txt);
    ssize_t rc = (ssize_t)size;
    if(!stream_circular_buffer_add_unsafe(scb, txt, size, size, type, true)) {
        // should never happen, because of autoscaling
        msg.opcode = STREAM_OPCODE_RECEIVER_BUFFER_OVERFLOW;
        msg.reason = STREAM_HANDSHAKE_DISCONNECT_BUFFER_OVERFLOW;
        rc = -1;
    }
    else {
        stream_receiver_log_payload(rpt, txt, type, false);

        if(was_empty) {
            msg.opcode = STREAM_OPCODE_RECEIVER_POLLOUT;
            msg.reason = 0;
        }
    }

    spinlock_unlock(&rpt->thread.send_to_child.spinlock);

    if(msg.opcode != STREAM_OPCODE_NONE)
        stream_receiver_send_opcode(rpt, msg);

    return rc;
}

// --------------------------------------------------------------------------------------------------------------------

void stream_receiver_move_to_running_unsafe(struct stream_thread *sth, struct receiver_state *rpt) {
    internal_fatal(sth->tid != gettid_cached(), "Function %s() should only be used by the dispatcher thread", __FUNCTION__ );

    worker_is_busy(WORKER_STREAM_JOB_DEQUEUE);

    ND_LOG_STACK lgs[] = {
        ND_LOG_FIELD_STR(NDF_NIDL_NODE, rpt->host->hostname),
        ND_LOG_FIELD_UUID(NDF_MESSAGE_ID, &streaming_from_child_msgid),
        ND_LOG_FIELD_END(),
    };
    ND_LOG_STACK_PUSH(lgs);

    nd_log(NDLS_DAEMON, NDLP_DEBUG,
           "STREAM RCV[%zu] '%s' [from [%s]:%s]: moving host from receiver queue to receiver running...",
           sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip, rpt->remote_port);

    sock_setcloexec(rpt->sock.fd, true);
    sock_enlarge_rcv_buf(rpt->sock.fd);
    sock_enlarge_snd_buf(rpt->sock.fd);
    sock_setcork(rpt->sock.fd, false);
    if(sock_setnonblock(rpt->sock.fd, true) != 1)
        nd_log(NDLS_DAEMON, NDLP_ERR,
               "STREAM RCV '%s' [from [%s]:%s]: failed to set non-blocking mode on socket %d",
               rrdhost_hostname(rpt->host), rpt->remote_ip, rpt->remote_port, rpt->sock.fd);

    __atomic_store_n(&rpt->host->stream.rcv.status.tid, gettid_cached(), __ATOMIC_RELAXED);
    rpt->thread.meta.type = POLLFD_TYPE_RECEIVER;
    rpt->thread.meta.rpt = rpt;

    spinlock_lock(&rpt->thread.send_to_child.spinlock);
    rpt->thread.send_to_child.scb = stream_circular_buffer_create();
    rpt->thread.send_to_child.msg.thread_slot = (int32_t)sth->id;
    rpt->thread.send_to_child.msg.session = os_random32();
    rpt->thread.send_to_child.msg.meta = &rpt->thread.meta;
    spinlock_unlock(&rpt->thread.send_to_child.spinlock);

    internal_fatal(META_GET(&sth->run.meta, (Word_t)&rpt->thread.meta) != NULL, "Receiver to be added is already in the list of receivers");
    META_SET(&sth->run.meta, (Word_t)&rpt->thread.meta, &rpt->thread.meta);

    rpt->thread.wanted = ND_POLL_READ;
    if(!nd_poll_add(sth->run.ndpl, rpt->sock.fd, rpt->thread.wanted, &rpt->thread.meta))
        nd_log(NDLS_DAEMON, NDLP_ERR,
               "STREAM RCV[%zu] '%s' [from [%s]:%s]:"
               "Failed to add receiver socket to nd_poll()",
               sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip, rpt->remote_port);

    rpt->thread.compressed.start = 0;
    rpt->thread.compressed.used = 0;
    rpt->thread.compressed.enabled = stream_decompression_initialize(rpt);
    buffered_reader_init(&rpt->thread.uncompressed);

    // start fresh at admission: the no-traffic timeout must be measured from when we start
    // reading (now), not from when the connection was accepted and queued.
    rpt->thread.bytes_received = 0;
    rpt->thread.last_traffic_ut = now_monotonic_usec();

    rpt->thread.line_buffer = buffer_create(sizeof(rpt->thread.uncompressed.read_buffer), NULL);

    // help preferred_sender_buffer() select the right buffer
    __atomic_store_n(&rpt->host->stream.snd.commit.receiver_tid, gettid_cached(), __ATOMIC_RELAXED);

    rpt->replication.last_progress_ut = now_monotonic_usec();

    PARSER *parser = NULL;
    {
        char buf[CONFIG_MAX_NAME];
        snprintfz(buf, sizeof(buf),  "[%s]:%s", rpt->remote_ip, rpt->remote_port);
        string_freez(rpt->thread.cd.id);
        rpt->thread.cd.id = string_strdupz(buf);

        string_freez(rpt->thread.cd.filename);
        rpt->thread.cd.filename = NULL;

        string_freez(rpt->thread.cd.fullfilename);
        rpt->thread.cd.fullfilename = NULL;

        string_freez(rpt->thread.cd.cmd);
        rpt->thread.cd.cmd = NULL;

        rpt->thread.cd.update_every = (int)nd_profile.update_every;
        spinlock_init(&rpt->thread.cd.unsafe.spinlock);
        rpt->thread.cd.unsafe.running = true;
        rpt->thread.cd.unsafe.enabled = true;
        rpt->thread.cd.started_t = now_realtime_sec();

        PARSER_USER_OBJECT user = {
            .enabled = plugin_is_enabled(&rpt->thread.cd),
            .host = rpt->host,
            .opaque = rpt,
            .cd = &rpt->thread.cd,
            .trust_durations = 1,
            .capabilities = rpt->capabilities,
#ifdef NETDATA_LOG_STREAM_RECEIVER
            .rpt = rpt,
#endif
        };

        parser = parser_init(&user, -1, -1, PARSER_INPUT_SPLIT, &rpt->sock);
        parser->send_to_plugin_data = rpt;
        parser->send_to_plugin_cb = send_to_child;

        pluginsd_keywords_init(parser, PARSER_INIT_STREAMING);

        __atomic_store_n(&rpt->thread.parser, parser, __ATOMIC_RELAXED);
    }

    if(stream_receive.replication.enabled)
        pulse_host_status(rpt->host, PULSE_HOST_STATUS_RCV_REPLICATION_WAIT, 0);
    else
        pulse_host_status(rpt->host, PULSE_HOST_STATUS_RCV_RUNNING, 0);

    // keep this last - it needs everything ready since to sends data to the child
    stream_receiver_send_node_and_claim_id_to_child(rpt->host);
}

void stream_receiver_move_entire_queue_to_running_unsafe(struct stream_thread *sth) {
    internal_fatal(sth->tid != gettid_cached(), "Function %s() should only be used by the dispatcher thread", __FUNCTION__ );

    // process the queue
    Word_t idx = 0;
    for(struct receiver_state *rpt = RECEIVERS_FIRST(&sth->queue.receivers, &idx);
         rpt;
         rpt = RECEIVERS_NEXT(&sth->queue.receivers, &idx)) {
        RECEIVERS_DEL(&sth->queue.receivers, idx);
        stream_receiver_move_to_running_unsafe(sth, rpt);
    }
}

static void stream_receiver_remove_internal(struct stream_thread *sth, struct receiver_state *rpt, STREAM_HANDSHAKE reason) {
    internal_fatal(sth->tid != gettid_cached(), "Function %s() should only be used by the dispatcher thread", __FUNCTION__ );

    receiver_set_exit_reason(rpt, reason, false);

    // rpt->host may be NULL here (e.g. removal on an early handshake failure,
    // before the host is attached) -- this path null-checks it below at the
    // iface/replication reads, so guard the log field too.
    ND_LOG_STACK lgs[] = {
        ND_LOG_FIELD_STR(NDF_NIDL_NODE, rpt->host ? rpt->host->hostname : NULL),
        ND_LOG_FIELD_TXT(NDF_SRC_IP, rpt->remote_ip),
        ND_LOG_FIELD_TXT(NDF_SRC_PORT, rpt->remote_port),
        ND_LOG_FIELD_CB(NDF_SRC_TRANSPORT, stream_receiver_log_transport, rpt),
        ND_LOG_FIELD_CB(NDF_SRC_CAPABILITIES, stream_receiver_log_capabilities, rpt),
        ND_LOG_FIELD_UUID(NDF_MESSAGE_ID, &streaming_from_child_msgid),
        ND_LOG_FIELD_END(),
    };
    ND_LOG_STACK_PUSH(lgs);

    PARSER *parser = __atomic_load_n(&rpt->thread.parser, __ATOMIC_RELAXED);
    size_t count = 0;
    if(parser)
        count = parser->user.data_collections_count;

    // gather diagnostics for a single, uniform disconnect line (same fields for every reason):
    // bytes_in distinguishes "the child sent nothing" (network/silent) from "sent but unparsed";
    // iface exposes the child's connection type (e.g. ppp0 cellular vs eth0) for log-side pivots.
    size_t bytes_out = 0;
    spinlock_lock(&rpt->thread.send_to_child.spinlock);
    if(rpt->thread.send_to_child.scb)
        bytes_out = stream_circular_buffer_stats_unsafe(rpt->thread.send_to_child.scb)->bytes_sent;
    spinlock_unlock(&rpt->thread.send_to_child.spinlock);

    char iface[64] = "";
    if(rpt->host && rpt->host->rrdlabels)
        rrdlabels_get_value_strcpyz(rpt->host->rrdlabels, iface, sizeof(iface), "_net_default_iface");

    time_t connected_s = rpt->connected_since_s ?
        nd_time_t_elapsed_saturating(now_realtime_sec(), rpt->connected_since_s) : 0;
    usec_t idle_ut = rpt->thread.last_traffic_ut ?
        clocks_usec_delta_or_zero(now_monotonic_usec(), rpt->thread.last_traffic_ut) : 0;
    long long idle_s = (long long)(idle_ut / USEC_PER_SEC);
    double repl_pct = rpt->host ? rpt->host->stream.rcv.status.replication.percent : 0.0;

    errno_clear();
    nd_log(NDLS_DAEMON, NDLP_ERR,
           "STREAM RCV[%zu] '%s' [from [%s]:%s]: receiver disconnected: "
           "reason=\"%s\" msgs=%zu bytes_in=%zu bytes_out=%zu connected=%" PRIdMAX "s idle=%llds repl=%.0f%% iface=%s"
           , sth->id
           , rpt->hostname ? rpt->hostname : "-"
           , rpt->remote_ip ? rpt->remote_ip : "-"
           , rpt->remote_port ? rpt->remote_port : "-"
           , stream_handshake_error_to_string(reason)
           , count
           , rpt->thread.bytes_received
           , bytes_out
           , (intmax_t)connected_s
           , idle_s
           , repl_pct
           , iface[0] ? iface : "-");

    internal_fatal(META_GET(&sth->run.meta, (Word_t)&rpt->thread.meta) == NULL,
                   "Receiver to be removed is not found in the list of receivers");

    META_DEL(&sth->run.meta, (Word_t)&rpt->thread.meta);

    rpt->thread.wanted = 0;
    if(!nd_poll_del(sth->run.ndpl, rpt->sock.fd))
        nd_log(NDLS_DAEMON, NDLP_ERR, "Failed to delete receiver socket from nd_poll()");

    __atomic_store_n(&rpt->host->stream.rcv.status.tid, 0, __ATOMIC_RELAXED);

    // make sure send_to_plugin() will not write any data to the socket (or wait for it to finish)
    if(parser) {
        spinlock_lock(&parser->writer.spinlock);
        parser->fd_input = -1;
        parser->fd_output = -1;
        parser->sock = NULL;
        spinlock_unlock(&parser->writer.spinlock);

        parser->user.v2.stream_buffer.wb = NULL;
    }

    stream_thread_node_removed(rpt->host);
    pulse_host_status(rpt->host, PULSE_HOST_STATUS_RCV_OFFLINE, reason);

    // set a default exit reason, if not set
    receiver_set_exit_reason(rpt, reason, false);

    // in case we are connected to netdata cloud,
    // we inform cloud that a child got disconnected
    uint64_t total_reboot = rrdhost_stream_path_total_reboot_time_ms(rpt->host);
    schedule_node_state_update(rpt->host, MIN((total_reboot * MAX_CHILD_DISC_TOLERANCE), MAX_CHILD_DISC_DELAY));

    rrdhost_clear_receiver(rpt, reason);
    rrdhost_set_is_parent_label();

    stream_receiver_free(rpt);
    // DO NOT USE rpt after this point
}

static bool stream_receiver_dequeue_senders(struct stream_thread *sth, struct receiver_state *rpt, usec_t now_ut) {
    internal_fatal(sth->tid != gettid_cached(), "Function %s() should only be used by the dispatcher thread", __FUNCTION__);

    // re-check if we need to send data after reading - if we do, try now
    if(rpt->thread.wanted & ND_POLL_WRITE) {
        worker_is_busy(WORKER_STREAM_JOB_SOCKET_SEND);
        if(!stream_receiver_send_data(sth, rpt, now_ut, false))
            return false;
    }

    if(rpt->host->sender &&                                         // the host has a sender
        rpt->host->stream.snd.status.tid == gettid_cached() &&      // the sender is mine
        (rpt->host->sender->thread.wanted & ND_POLL_WRITE)) {       // the sender needs to send data
        // we return true even if this fais,
        // so that we will not disconnect the receiver because the sender failed
        stream_sender_send_data(sth, rpt->host->sender, now_ut, false);
    }

    return true;
}

static ssize_t
stream_receive_and_process(struct stream_thread *sth, struct receiver_state *rpt, PARSER *parser, usec_t now_ut __maybe_unused, bool *removed) {
    internal_fatal(sth->tid != gettid_cached(), "Function %s() should only be used by the dispatcher thread", __FUNCTION__);
    *removed = false;

    ssize_t rc;
    if(rpt->thread.compressed.enabled) {
        rc = receiver_read_compressed(rpt);
        if(unlikely(rc <= 0))
            return rc;

        while(!nd_thread_signaled_to_cancel() && service_running(SERVICE_STREAMING) && !receiver_should_stop(rpt)) {
            worker_is_busy(WORKER_STREAM_JOB_DECOMPRESS);

            // feed the decompressor with the new data we just read
            decompressor_status_t feed_rc = receiver_feed_decompressor(rpt);

            if(likely(feed_rc == DECOMPRESS_OK)) {
                while (true) {
                    // feed our uncompressed data buffer with new data
                    decompressor_status_t decompress_rc = receiver_get_decompressed(rpt);

                    if (likely(decompress_rc == DECOMPRESS_OK)) {
                        // loop through all the complete lines found in the uncompressed buffer

                        while (buffered_reader_next_line(&rpt->thread.uncompressed, rpt->thread.line_buffer)) {
                            if (unlikely(parser_action(parser, rpt->thread.line_buffer->buffer))) {
                                stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_RCV_DISCONNECT_PARSER_FAILED);
                                *removed = true;
                                return -1;
                            }

                            rpt->thread.line_buffer->len = 0;
                            rpt->thread.line_buffer->buffer[0] = '\0';
                        }
                    }
                    else if (decompress_rc == DECOMPRESS_NEED_MORE_DATA)
                        break;

                    else {
                        stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_RCV_DECOMPRESSION_FAILED);
                        *removed = true;
                        return -1;
                    }
                }
            }
            else if (feed_rc == DECOMPRESS_NEED_MORE_DATA)
                break;
            else {
                stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_RCV_DECOMPRESSION_FAILED);
                *removed = true;
                return -1;
            }
        }

        if(receiver_should_stop(rpt)) {
            stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_DISCONNECT_SIGNALED_TO_STOP);
            *removed = true;
            return -1;
        }
    }
    else {
        rc = receiver_read_uncompressed(rpt);
        if(rc <= 0)
            return rc;

        while(buffered_reader_next_line(&rpt->thread.uncompressed, rpt->thread.line_buffer)) {
            if(unlikely(parser_action(parser, rpt->thread.line_buffer->buffer))) {
                stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_RCV_DISCONNECT_PARSER_FAILED);
                *removed = true;
                return -1;
            }

            rpt->thread.line_buffer->len = 0;
            rpt->thread.line_buffer->buffer[0] = '\0';
        }
    }

    return rc;
}

bool stream_receiver_send_data(struct stream_thread *sth, struct receiver_state *rpt, usec_t now_ut, bool process_opcodes_and_enable_removal) {
    internal_fatal(sth->tid != gettid_cached(), "Function %s() should only be used by the dispatcher thread", __FUNCTION__ );

    EVLOOP_STATUS status = EVLOOP_STATUS_CONTINUE;
    while(status == EVLOOP_STATUS_CONTINUE) {
        if (!spinlock_trylock(&rpt->thread.send_to_child.spinlock)) {
            status = EVLOOP_STATUS_CANT_GET_LOCK;
            break;
        }

        char *chunk;
        STREAM_CIRCULAR_BUFFER *scb = rpt->thread.send_to_child.scb;
        STREAM_CIRCULAR_BUFFER_STATS *stats = stream_circular_buffer_stats_unsafe(scb);
        size_t outstanding = stream_circular_buffer_get_unsafe(scb, &chunk);

        if(!outstanding) {
            status = EVLOOP_STATUS_NO_MORE_DATA;
            spinlock_unlock(&rpt->thread.send_to_child.spinlock);
            continue;
        }

        ssize_t rc = write_stream(rpt, chunk, outstanding);
        if (likely(rc > 0)) {
            pulse_stream_sent_bytes(rc);
            rpt->thread.last_traffic_ut = now_ut;
            stream_circular_buffer_del_unsafe(scb, rc, now_ut);
            if(likely(rpt->host))
                single_writer_atomic_add(&rpt->host->stream.rcv.status.bytes_out, (uint64_t)rc);
            if (!stats->bytes_outstanding) {
                rpt->thread.wanted = ND_POLL_READ;
                if (!nd_poll_upd(sth->run.ndpl, rpt->sock.fd, rpt->thread.wanted))
                    nd_log(NDLS_DAEMON, NDLP_ERR,
                           "STREAM RCV[%zu] '%s' [from [%s]:%s]: cannot update nd_poll()",
                           sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip, rpt->remote_port);

                // recreate the circular buffer if we have to
                stream_circular_buffer_recreate_timed_unsafe(rpt->thread.send_to_child.scb, now_ut, false);
                status = EVLOOP_STATUS_NO_MORE_DATA;
            }
        }
        else if (rc == 0 || errno == ECONNRESET)
            status = EVLOOP_STATUS_SOCKET_CLOSED;

        else if (rc < 0) {
            if (errno == EWOULDBLOCK || errno == EAGAIN || errno == EINTR)
                status = EVLOOP_STATUS_SOCKET_FULL;
            else
                status = EVLOOP_STATUS_SOCKET_ERROR;
        }

        spinlock_unlock(&rpt->thread.send_to_child.spinlock);

        if (status == EVLOOP_STATUS_SOCKET_ERROR || status == EVLOOP_STATUS_SOCKET_CLOSED) {
            STREAM_HANDSHAKE reason;

            if(status == EVLOOP_STATUS_SOCKET_ERROR) {
                worker_is_busy(WORKER_STREAM_JOB_DISCONNECT_SEND_ERROR);
                reason = STREAM_HANDSHAKE_DISCONNECT_SOCKET_WRITE_FAILED;
            }
            else /* if(status == EVLOOP_STATUS_SOCKET_CLOSED) */ {
                worker_is_busy(WORKER_STREAM_JOB_DISCONNECT_REMOTE_CLOSED);
                reason = STREAM_HANDSHAKE_DISCONNECT_SOCKET_CLOSED_BY_REMOTE;
            }

            nd_log(NDLS_DAEMON, NDLP_ERR,
                   "STREAM RCV[%zu] '%s' [from [%s]:%s]: %s (%zd, on fd %d) - closing receiver connection - "
                   "we have sent %zu bytes in %zu operations.",
                   sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip, rpt->remote_port,
                   stream_handshake_error_to_string(reason), rc, rpt->sock.fd, stats->bytes_sent, stats->sends);

            if(process_opcodes_and_enable_removal) {
                // this is not executed from the opcode handling mechanism
                // so we can safely remove the receiver.
                stream_receiver_remove(sth, rpt, reason);
            }
            else {
                receiver_set_exit_reason(rpt, reason, false);

                // protection against this case:
                //
                // 1. receiver gets a replication request
                // 2. parser processes the request
                // 3. parser decides to send back a message to the child (REPLAY_CHART)
                // 4. send_to_child appends the data to the sending circular buffer
                // 5. send_to_child sends opcode to enable sending
                // 6. opcode bypasses the signal and runs this function inline to dispatch immediately
                // 7. sending fails (child disconnected)
                // 8. receiver is removed
                //
                // Point 2 above crashes. The parser is no longer there (freed at point 7)
                // and there is no way for point 2 to know...
            }
        }
        else if(process_opcodes_and_enable_removal &&
                 status == EVLOOP_STATUS_CONTINUE &&
                 stream_thread_process_opcodes(sth, &rpt->thread.meta))
            status = EVLOOP_STATUS_OPCODE_ON_ME;
    }

    return EVLOOP_STATUS_STILL_ALIVE(status);
}

bool stream_receiver_receive_data(struct stream_thread *sth, struct receiver_state *rpt, usec_t now_ut, bool process_opcodes) {
    internal_fatal(sth->tid != gettid_cached(), "Function %s() should only be used by the dispatcher thread", __FUNCTION__ );

    PARSER *parser = __atomic_load_n(&rpt->thread.parser, __ATOMIC_RELAXED);
    if(unlikely(!parser)) {
        stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_RCV_DISCONNECT_PARSER_FAILED);
        return false;
    }

    ND_LOG_STACK lgs[] = {
        ND_LOG_FIELD_CB(NDF_REQUEST, line_splitter_reconstruct_line, &parser->line),
        ND_LOG_FIELD_CB(NDF_NIDL_NODE, parser_reconstruct_node, parser),
        ND_LOG_FIELD_CB(NDF_NIDL_INSTANCE, parser_reconstruct_instance, parser),
        ND_LOG_FIELD_CB(NDF_NIDL_CONTEXT, parser_reconstruct_context, parser),
        ND_LOG_FIELD_END(),
    };
    ND_LOG_STACK_PUSH(lgs);

    size_t count = 1; // how many reads to do per host, before moving to the next host
    EVLOOP_STATUS status = EVLOOP_STATUS_CONTINUE;
    while(status == EVLOOP_STATUS_CONTINUE && count-- > 0) {
        bool removed = false;
        ssize_t rc = stream_receive_and_process(sth, rpt, parser, now_ut, &removed);
        if(unlikely(removed))
            status = EVLOOP_STATUS_PARSER_FAILED;

        else if (likely(rc > 0)) {
            rpt->thread.last_traffic_ut = now_ut;

            if(!stream_receiver_dequeue_senders(sth, rpt, now_ut))
                status = EVLOOP_STATUS_SOCKET_ERROR;
        }
        else if (rc == 0 || errno == ECONNRESET) {
            status = EVLOOP_STATUS_SOCKET_CLOSED;
        }
        else if (rc < 0) {
            if ((errno == EWOULDBLOCK || errno == EAGAIN || errno == EINTR))
                status = EVLOOP_STATUS_SOCKET_FULL;
            else
                status = EVLOOP_STATUS_SOCKET_ERROR;
        }

        if(status == EVLOOP_STATUS_SOCKET_ERROR || status == EVLOOP_STATUS_SOCKET_CLOSED) {
            STREAM_HANDSHAKE reason;

            if(status == EVLOOP_STATUS_SOCKET_ERROR) {
                worker_is_busy(WORKER_STREAM_JOB_DISCONNECT_RECEIVE_ERROR);
                reason = STREAM_HANDSHAKE_DISCONNECT_SOCKET_READ_FAILED;
            }
            else /* if(status == EVLOOP_STATUS_SOCKET_CLOSED) */ {
                worker_is_busy(WORKER_STREAM_JOB_DISCONNECT_REMOTE_CLOSED);
                reason = STREAM_HANDSHAKE_DISCONNECT_SOCKET_CLOSED_BY_REMOTE;
            }

            nd_log(NDLS_DAEMON, NDLP_ERR,
                   "STREAM RCV[%zu] '%s' [from [%s]:%s]: %s (fd %d) - closing receiver connection.",
                   sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip, rpt->remote_port,
                   stream_handshake_error_to_string(reason), rpt->sock.fd);

            stream_receiver_remove(sth, rpt, reason);
            break;
        }
        else if(status == EVLOOP_STATUS_CONTINUE && process_opcodes && stream_thread_process_opcodes(sth, &rpt->thread.meta))
            status = EVLOOP_STATUS_OPCODE_ON_ME;
    }

    return EVLOOP_STATUS_STILL_ALIVE(status);
}

// process poll() events for streaming receivers
// returns true when the receiver is still there, false if it removed it
bool stream_receive_process_poll_events(struct stream_thread *sth, struct receiver_state *rpt, nd_poll_event_t events, usec_t now_ut) {
    internal_fatal(sth->tid != gettid_cached(), "Function %s() should only be used by the dispatcher thread", __FUNCTION__);

    ND_LOG_STACK lgs[] = {
        ND_LOG_FIELD_TXT(NDF_SRC_IP, rpt->remote_ip),
        ND_LOG_FIELD_TXT(NDF_SRC_PORT, rpt->remote_port),
        ND_LOG_FIELD_TXT(NDF_NIDL_NODE, rpt->hostname),
        ND_LOG_FIELD_CB(NDF_SRC_TRANSPORT, stream_receiver_log_transport, rpt),
        ND_LOG_FIELD_CB(NDF_SRC_CAPABILITIES, stream_receiver_log_capabilities, rpt),
        ND_LOG_FIELD_END(),
    };
    ND_LOG_STACK_PUSH(lgs);

    if (receiver_should_stop(rpt)) {
        stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_DISCONNECT_SIGNALED_TO_STOP);
        return false;
    }

    // ND_POLL_HUP together with ND_POLL_READ means the peer disconnected,
    // but data it already delivered is still pending in the socket buffer.
    // Removing the receiver here would discard that data, so keep reading:
    // polling is level-triggered, both flags stay set while data remains,
    // and when the buffer is drained recv() returns 0 and the read path
    // removes the receiver with the same disconnect reason.
    if (unlikely((events & (ND_POLL_ERROR | ND_POLL_INVALID)) ||
                 ((events & ND_POLL_HUP) && !(events & ND_POLL_READ)))) {
        // we have errors on this socket, or EOF with nothing left to read

        worker_is_busy(WORKER_STREAM_JOB_DISCONNECT_SOCKET_ERROR);

        STREAM_HANDSHAKE reason = events & ND_POLL_HUP ? STREAM_HANDSHAKE_DISCONNECT_SOCKET_CLOSED_BY_REMOTE : STREAM_HANDSHAKE_DISCONNECT_SOCKET_ERROR;

        nd_log(NDLS_DAEMON, NDLP_ERR,
               "STREAM RCV[%zu] '%s' [from [%s]:%s]: %s - closing connection",
               sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip, rpt->remote_port,
               stream_handshake_error_to_string(reason));

        stream_receiver_remove(sth, rpt, reason);
        return false;
    }

    if (events & ND_POLL_WRITE) {
        worker_is_busy(WORKER_STREAM_JOB_SOCKET_SEND);
        if(!stream_receiver_send_data(sth, rpt, now_ut, true))
            return false;
    }

    if (events & ND_POLL_READ) {
        worker_is_busy(WORKER_STREAM_JOB_SOCKET_RECEIVE);
        if(!stream_receiver_receive_data(sth, rpt, now_ut, true))
            return false;
    }

    return true;
}

void stream_receiver_check_all_nodes_from_poll(struct stream_thread *sth, usec_t now_ut) {
    internal_fatal(sth->tid != gettid_cached(), "Function %s() should only be used by the dispatcher thread", __FUNCTION__ );

    NETDATA_DOUBLE overall_buffer_ratio = 0.0;

    Word_t idx = 0;
    for(struct pollfd_meta *m = META_FIRST(&sth->run.meta, &idx);
         m;
         m = META_NEXT(&sth->run.meta, &idx)) {
        if (m->type != POLLFD_TYPE_RECEIVER) continue;
        struct receiver_state *rpt = m->rpt;

        // Probe socket to detect dead connections (e.g., from TCP keepalive)
        // Uses nd_sock_peek_nowait() which handles both SSL and plain TCP:
        // - For SSL: uses SSL_peek() to avoid corrupting SSL state
        // - For plain TCP: uses recv(MSG_PEEK | MSG_DONTWAIT)
        char probe_byte;
        ssize_t probe_rc = nd_sock_peek_nowait(&rpt->sock, &probe_byte, 1);
        if (probe_rc == 0 || (probe_rc < 0 && errno == ECONNRESET)) {
            // Connection closed by remote (gracefully or via reset)
            ND_LOG_STACK lgs[] = {
                ND_LOG_FIELD_TXT(NDF_SRC_IP, rpt->remote_ip),
                ND_LOG_FIELD_TXT(NDF_SRC_PORT, rpt->remote_port),
                ND_LOG_FIELD_TXT(NDF_NIDL_NODE, rpt->hostname),
                ND_LOG_FIELD_CB(NDF_SRC_TRANSPORT, stream_receiver_log_transport, rpt),
                ND_LOG_FIELD_CB(NDF_SRC_CAPABILITIES, stream_receiver_log_capabilities, rpt),
                ND_LOG_FIELD_END(),
            };
            ND_LOG_STACK_PUSH(lgs);

            worker_is_busy(WORKER_STREAM_JOB_DISCONNECT_REMOTE_CLOSED);
            nd_log(NDLS_DAEMON, NDLP_ERR,
                   "STREAM RCV[%zu] '%s' [from %s]: socket closed by remote - closing connection",
                   sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip);

            stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_DISCONNECT_SOCKET_CLOSED_BY_REMOTE);
            continue;
        }
        if (probe_rc < 0 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR) {
            // Socket error detected (keepalive timeout, etc.)
            // Save errno immediately as subsequent calls may modify it
            int saved_errno = errno;

            ND_LOG_STACK lgs[] = {
                ND_LOG_FIELD_TXT(NDF_SRC_IP, rpt->remote_ip),
                ND_LOG_FIELD_TXT(NDF_SRC_PORT, rpt->remote_port),
                ND_LOG_FIELD_TXT(NDF_NIDL_NODE, rpt->hostname),
                ND_LOG_FIELD_CB(NDF_SRC_TRANSPORT, stream_receiver_log_transport, rpt),
                ND_LOG_FIELD_CB(NDF_SRC_CAPABILITIES, stream_receiver_log_capabilities, rpt),
                ND_LOG_FIELD_END(),
            };
            ND_LOG_STACK_PUSH(lgs);

            worker_is_busy(WORKER_STREAM_JOB_DISCONNECT_SOCKET_ERROR);
            nd_log(NDLS_DAEMON, NDLP_ERR,
                   "STREAM RCV[%zu] '%s' [from %s]: socket error detected: %s - closing connection",
                   sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip, strerror(saved_errno));

            stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_DISCONNECT_SOCKET_ERROR);
            continue;
        }
        // probe_rc > 0: data available (normal)
        // probe_rc < 0 with EAGAIN/EWOULDBLOCK/EINTR: no data but connection alive

        spinlock_lock(&rpt->thread.send_to_child.spinlock);
        STREAM_CIRCULAR_BUFFER_STATS stats = *stream_circular_buffer_stats_unsafe(rpt->thread.send_to_child.scb);
        spinlock_unlock(&rpt->thread.send_to_child.spinlock);

        if (stats.buffer_ratio > overall_buffer_ratio)
            overall_buffer_ratio = stats.buffer_ratio;

        time_t timeout_s = 600;
        if(unlikely(!rpt->thread.last_traffic_ut) && now_ut)
            rpt->thread.last_traffic_ut = now_ut;
        usec_t idle_ut = clocks_usec_delta_or_zero_with_rebase(now_ut, &rpt->thread.last_traffic_ut);

        if(unlikely(idle_ut > (usec_t)timeout_s * USEC_PER_SEC &&
                     !rrdhost_receiver_replicating_charts(rpt->host))) {

            ND_LOG_STACK lgs[] = {
                ND_LOG_FIELD_TXT(NDF_SRC_IP, rpt->remote_ip),
                ND_LOG_FIELD_TXT(NDF_SRC_PORT, rpt->remote_port),
                ND_LOG_FIELD_TXT(NDF_NIDL_NODE, rpt->hostname),
                ND_LOG_FIELD_CB(NDF_SRC_TRANSPORT, stream_receiver_log_transport, rpt),
                ND_LOG_FIELD_CB(NDF_SRC_CAPABILITIES, stream_receiver_log_capabilities, rpt),
                ND_LOG_FIELD_END(),
            };
            ND_LOG_STACK_PUSH(lgs);

            worker_is_busy(WORKER_STREAM_JOB_DISCONNECT_TIMEOUT);

            char duration[RFC3339_MAX_LENGTH];
            duration_snprintf(duration, sizeof(duration), (int64_t)MIN(idle_ut, (usec_t)INT64_MAX), "us", true);

            char pending[64] = "0";
            if(stats.bytes_outstanding)
                size_snprintf(pending, sizeof(pending), stats.bytes_outstanding, "B", false);

            nd_log(NDLS_DAEMON, NDLP_ERR,
                   "STREAM RCV[%zu] '%s' [from %s]: there was not traffic for %" PRId64 " seconds - closing connection - "
                   "we have sent %zu bytes in %zu operations, it is idle for %s, and we have %s pending to send "
                   "(buffer is used %.2f%%).",
                   sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip, (int64_t)timeout_s,
                   stats.bytes_sent, stats.sends, duration, pending, stats.buffer_ratio);

            stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_DISCONNECT_TIMEOUT);
            continue;
        }

        nd_poll_event_t wanted = ND_POLL_READ | (stats.bytes_outstanding ? ND_POLL_WRITE : 0);
        if(unlikely(rpt->thread.wanted != wanted)) {
//            nd_log(NDLS_DAEMON, NDLP_DEBUG,
//                   "STREAM RCV[%zu] '%s' [from %s]: nd_poll() wanted events mismatch.",
//                   sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip);

            rpt->thread.wanted = wanted;
            if(!nd_poll_upd(sth->run.ndpl, rpt->sock.fd, rpt->thread.wanted))
                nd_log(NDLS_DAEMON, NDLP_ERR,
                       "STREAM RCV[%zu] '%s' [from %s]: failed to update nd_poll().",
                       sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip);
        }
    }
}

static bool stream_receiver_did_replication_progress(struct receiver_state *rpt, usec_t now_ut) {
    RRDHOST *host = rpt->host;

    size_t host_counter_sum =
        __atomic_load_n(&host->stream.rcv.status.replication.counter_in, __ATOMIC_RELAXED) +
        __atomic_load_n(&host->stream.rcv.status.replication.counter_out, __ATOMIC_RELAXED);

    if(rpt->replication.last_counter_sum != host_counter_sum) {
        // there has been some progress
        rpt->replication.last_counter_sum = host_counter_sum;
        rpt->replication.last_progress_ut = now_ut;
        return true;
    }

    if(!host_counter_sum)
        // we have not started yet
        return true;

    if(__atomic_load_n(&host->stream.rcv.status.replication.backfill_pending, __ATOMIC_RELAXED))
        // we still have requests to execute
        return true;

    if(unlikely(!rpt->replication.last_progress_ut)) {
        rpt->replication.last_progress_ut = now_ut;
        return true;
    }

    return clocks_usec_delta_or_zero_with_rebase(now_ut, &rpt->replication.last_progress_ut) <
           10ULL * 60 * USEC_PER_SEC;
}

void stream_receiver_replication_check_from_poll(struct stream_thread *sth, usec_t now_ut) {
    internal_fatal(sth->tid != gettid_cached(), "Function %s() should only be used by the dispatcher thread", __FUNCTION__);

    Word_t idx = 0;
    for(struct pollfd_meta *m = META_FIRST(&sth->run.meta, &idx);
         m;
         m = META_NEXT(&sth->run.meta, &idx)) {
        if (m->type != POLLFD_TYPE_RECEIVER) continue;
        struct receiver_state *rpt = m->rpt;
        RRDHOST *host = rpt->host;


        if(stream_receiver_did_replication_progress(rpt, now_ut)) {
            rpt->replication.last_checked_ut = 0;
            continue;
        }

        if(rpt->replication.last_checked_ut == rpt->replication.last_progress_ut)
            continue;

        size_t stalled = 0, finished = 0;
        RRDSET *st;
        rrdset_foreach_read(st, rpt->host) {
            RRDSET_FLAGS st_flags = rrdset_flag_get(st);
            if(st_flags & RRDSET_FLAG_OBSOLETE)
                continue;

            if(st_flags & RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED) {
                finished++;
                continue;
            }

            nd_log(NDLS_DAEMON, NDLP_DEBUG,
                   "STREAM RCV[%zu] '%s' [from %s]: REPLICATION EXCEPTIONS: instance '%s' %s replication yet.",
                   sth->id, rrdhost_hostname(host), rpt->remote_ip,
                   rrdset_id(st),
                   (st_flags & RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS) ? "has not finished" : "has not started");

            stalled++;
        }
        rrdset_foreach_done(st);

        if(stalled && !stream_receiver_did_replication_progress(rpt, now_ut)) {
            nd_log(NDLS_DAEMON, NDLP_WARNING,
                   "STREAM RCV[%zu] '%s' [from %s]: REPLICATION EXCEPTIONS SUMMARY: node has %zu stalled replication requests (%zu finished). "
                   "We have requested %u and got replies for %u replication commands. "
                   "Disconnecting node to restore streaming.",
                   sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip,
                   stalled, finished,
                   __atomic_load_n(&host->stream.rcv.status.replication.counter_out, __ATOMIC_RELAXED),
                   __atomic_load_n(&host->stream.rcv.status.replication.counter_in, __ATOMIC_RELAXED));

            stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_DISCONNECT_REPLICATION_STALLED);
            continue;
        }

        rpt->replication.last_checked_ut = rpt->replication.last_progress_ut;
    }
}

void stream_receiver_cleanup(struct stream_thread *sth) {
    Word_t idx = 0;
    for(struct pollfd_meta *m = META_FIRST(&sth->run.meta, &idx);
         m;
         m = META_NEXT(&sth->run.meta, &idx)) {
        if (m->type != POLLFD_TYPE_RECEIVER) continue;
        struct receiver_state *rpt = m->rpt;
        receiver_set_exit_reason(rpt, STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN, true);
        stream_receiver_remove(sth, rpt, STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN);
    }
}

static void stream_receiver_replication_reset(RRDHOST *host) {
    RRDSET *st;
    rrdset_foreach_read(st, host) {
        RRDSET_FLAGS old = rrdset_flag_set_and_clear(st, RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED, RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS);
        if(!(old & RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED))
            rrdhost_receiver_replicating_charts_minus_one(host);

#ifdef REPLICATION_TRACKING
        st->stream.rcv.who = REPLAY_WHO_UNKNOWN;
#endif
    }
    rrdset_foreach_done(st);

    if(rrdhost_receiver_replicating_charts(host) != 0) {
        nd_log(NDLS_DAEMON, NDLP_WARNING,
               "STREAM REPLAY ERROR: receiver replication instances counter should be zero, but it is %u"
               " - resetting it to zero",
               rrdhost_receiver_replicating_charts(host));

        rrdhost_receiver_replicating_charts_zero(host);
    }

    __atomic_store_n(&host->stream.rcv.status.replication.counter_in, 0, __ATOMIC_RELAXED);
    __atomic_store_n(&host->stream.rcv.status.replication.counter_out, 0, __ATOMIC_RELAXED);
    __atomic_store_n(&host->stream.rcv.status.replication.backfill_pending, 0, __ATOMIC_RELAXED);
}

RRDHOST_SET_RECEIVER_RESULT rrdhost_set_receiver(RRDHOST *host, struct receiver_state *rpt) {
    bool signal_rrdcontext = false;
    bool set_this = false;

    rrdhost_receiver_lock(host);

    // If the obsolete-all cleanup is running on this host, refuse the attach.
    // The cleanup walks all charts marking them obsolete without holding
    // receiver_lock; attaching mid-pass would let it mark the new receiver's
    // charts obsolete and call ml_host_disconnected() on a connected host.
    // The child reconnects via normal backoff; by then the pass is done.
    if (rrdhost_flag_check(host, RRDHOST_FLAG_OBSOLETE_ALL_IN_PROGRESS)) {
        rrdhost_receiver_unlock(host);
        return RRDHOST_SET_RECEIVER_CLEANUP_BUSY;
    }

    // A vnode has exactly one writer: the local collector that defines it.
    //
    // stream_receiver_accept_connection() already rejects locally collected vnodes, but that check
    // runs before the socket takeover and this attach - the connection can spend seconds in between
    // (stale-receiver wait, host creation, first response). Meanwhile the collector may claim the
    // vnode: it sets RRDHOST_FLAG_VIRTUAL_HOST and then evicts any attached receiver under this
    // same lock (pluginsd_host_claim_as_local_vnode()). Re-checking the flag here is what makes the
    // two paths mutually exclusive: either we attach first and the collector evicts us, or the
    // collector claims first and we are refused. Without it, we would become a second writer.
    if (rrdhost_is_virtual(host)) {
        rrdhost_receiver_unlock(host);
        return RRDHOST_SET_RECEIVER_VNODE_IS_LOCAL;
    }

    if (!host->receiver) {
        object_state_activate_if_not_activated(&host->state_id);

        rrdhost_flag_clear(host, RRDHOST_FLAG_ORPHAN);
        rrdhost_set_health_evloop_iteration(host);

        // atomic: read lock-free by the pulse traversal (this write is under receiver_lock, but the
        // read side has no lock; the connect path is rare so the atomic RMW cost is irrelevant)
        __atomic_add_fetch(&host->stream.rcv.status.connections, 1, __ATOMIC_RELAXED);
        streaming_receiver_connected();

        host->receiver = rpt;
        rpt->host = host;

        host->stream.rcv.status.reason = (STREAM_HANDSHAKE)rpt->capabilities;
        rpt->exit.reason = 0;
        __atomic_store_n(&rpt->exit.shutdown, false, __ATOMIC_RELEASE);
        host->stream.rcv.status.last_connected = now_realtime_sec();
        host->stream.rcv.status.last_disconnected = 0;

        if (rpt->config.health.enabled != CONFIG_BOOLEAN_NO) {
            if (rpt->config.health.delay > 0) {
                host->health.delay_up_to = nd_time_t_add_saturating(now_realtime_sec(), rpt->config.health.delay);
                nd_log(NDLS_DAEMON, NDLP_DEBUG,
                       "STREAM RCV '%s' [from [%s]:%s]: "
                       "Postponing health checks for %" PRId64 " seconds, because it was just connected.",
                       rrdhost_hostname(host), rpt->remote_ip, rpt->remote_port,
                       (int64_t) rpt->config.health.delay);
            }
        }

        host->health_log.health_log_retention_s = rpt->config.health.history;

//         this is a test
//        if(rpt->hops <= host->sender->hops)
//            stream_sender_thread_stop(host, "HOPS MISMATCH", false);

        signal_rrdcontext = true;
        stream_receiver_replication_reset(host);

        rrdhost_flag_set(rpt->host, RRDHOST_FLAG_COLLECTOR_ONLINE);
        aclk_queue_node_info(rpt->host, true);

        stream_parents_host_reset(host, STREAM_HANDSHAKE_SP_PREPARING);

        set_this = true;
    }

    rrdhost_receiver_unlock(host);

    if(signal_rrdcontext)
        rrdcontext_host_child_connected(host);

    if(set_this)
        ml_host_start(host);

    return set_this ? RRDHOST_SET_RECEIVER_OK : RRDHOST_SET_RECEIVER_ALREADY_ATTACHED;
}

void rrdhost_clear_receiver(struct receiver_state *rpt, STREAM_HANDSHAKE reason) {
    RRDHOST *host = rpt->host;
    if(!host) return;

    rrdhost_receiver_lock(host);
    {
        // Make sure that we detach this thread and don't kill a freshly arriving receiver

        if (host->receiver == rpt) {
            rrdhost_flag_clear(host, RRDHOST_FLAG_COLLECTOR_ONLINE);

            rrdhost_receiver_unlock(host);
            {
                // this will wait until all workers finish
                object_state_deactivate(&host->state_id);

                // run all these without having the receiver lock

                rrdhost_set_health_evloop_iteration(host);
                ml_host_stop(host);
                stream_path_child_disconnected(host);
                stream_sender_signal_to_stop_and_wait(host, reason, false);
                rrdcontext_host_child_disconnected(host);

                if (rpt->config.health.enabled)
                    rrdcalc_child_disconnected(host);

                stream_parents_host_reset(host, reason);

                // object_state_deactivate() above made this child's functions unavailable
                // without removing them, so nothing else refreshes the cloud manifest
                aclk_arm_node_manifest(host);
            }
            rrdhost_receiver_lock(host);

            // now we have the lock again

            stream_receiver_replication_reset(host);
            streaming_receiver_disconnected();

            host->stream.rcv.status.reason = rpt->exit.reason;
            rpt->exit.reason = 0;
            __atomic_store_n(&rpt->exit.shutdown, false, __ATOMIC_RELEASE);
            host->stream.rcv.status.last_connected = 0;
            host->stream.rcv.status.last_disconnected = now_realtime_sec();
            host->health.enabled = false;

            rrdhost_flag_set(host, RRDHOST_FLAG_ORPHAN);
            __atomic_store_n(&host->stream.snd.commit.receiver_tid, 0, __ATOMIC_RELAXED);
            host->receiver = NULL;
        }
    }

    // this must be cleared with the receiver lock
    pluginsd_process_cleanup(rpt->thread.parser);
    __atomic_store_n(&rpt->thread.parser, NULL, __ATOMIC_RELAXED);

    rrdhost_receiver_unlock(host);
}

bool stream_receiver_signal_to_stop_and_wait(RRDHOST *host, STREAM_HANDSHAKE reason) {
    bool ret = false;

    rrdhost_receiver_lock(host);

    struct receiver_state *rpt = host->receiver;

    if(rpt) {
        if(!__atomic_load_n(&rpt->exit.shutdown, __ATOMIC_ACQUIRE)) {
            receiver_set_exit_reason(rpt, reason, true);
            __atomic_store_n(&rpt->exit.shutdown, true, __ATOMIC_RELEASE);
            shutdown(rpt->sock.fd, SHUT_RDWR);
        }

        int count = 2000;
        while (host->receiver == rpt && count-- > 0) {
            rrdhost_receiver_unlock(host);

            // let the lock for the receiver thread to exit
            sleep_usec(1 * USEC_PER_MS);

            rrdhost_receiver_lock(host);
        }

        if(host->receiver == rpt)
            netdata_log_error("STREAM RCV[x] '%s' [from [%s]:%s]: "
                              "streaming thread takes too long to stop, giving up..."
                              , rrdhost_hostname(host)
                                  , rpt->remote_ip, rpt->remote_port);
        else
            ret = true;
    }
    else
        ret = true;

    rrdhost_receiver_unlock(host);

    return ret;
}
