blend.cpp File Reference

#include "config.h"
#include "config_unix.h"
#include "config_win32.h"
#include "debug.h"
#include "lib_common.h"
#include "video.h"
#include "video_display.h"
#include <cinttypes>
#include <condition_variable>
#include <chrono>
#include <iostream>
#include <list>
#include <map>
#include <memory>
#include <mutex>
#include <queue>
#include <thread>
#include <unordered_map>

Include dependency graph for blend.cpp:

Go to the source code of this file.

Classes
struct	state_blend
struct	state_blend_common

Functions
static void	check_reconf (struct state_blend_common *s, struct video_desc desc)
static void	display_blend_done (void *state)
static struct display *	display_blend_fork (void *state)
static bool	display_blend_get_property (void *state, int property, void *val, size_t *len)
static struct video_frame *	display_blend_getf (void *state)
static void *	display_blend_init (struct module *parent, const char *fmt, unsigned int flags)
static void	display_blend_probe (struct device_info **available_cards, int *count, void(**deleter)(void *))
static void	display_blend_put_audio_frame (void *state, const struct audio_frame *frame)
static bool	display_blend_putf (void *state, struct video_frame *frame, long long flags)
static bool	display_blend_reconfigure (void *state, struct video_desc desc)
static bool	display_blend_reconfigure_audio (void *state, int quant_samples, int channels, int sample_rate)
static void	display_blend_run (void *state)
	REGISTER_HIDDEN_MODULE (blend, &display_blend_info, LIBRARY_CLASS_VIDEO_DISPLAY, VIDEO_DISPLAY_ABI_VERSION)
static constexpr chrono::milliseconds	SOURCE_TIMEOUT (500)

Variables
static constexpr int	BUFFER_LEN = 5
static const struct video_display_info	display_blend_info
static constexpr unsigned int	IN_QUEUE_MAX_BUFFER_LEN = 5
static constexpr int	SKIP_FIRST_N_FRAMES_IN_STREAM = 5
static constexpr int	TRANSITION_COUNT = 10

Detailed Description

Author

Martin Pulec pulec.nosp@m.@ces.nosp@m.net.c.nosp@m.z

Todo

Rewrite the code to have more clear state machine!

Definition in file blend.cpp.

Function Documentation

check_reconf()

static void check_reconf ( struct state_blend_common * s, struct video_desc desc )

static

Definition at line 164 of file blend.cpp.

{
        if (!video_desc_eq(desc, s->display_desc)) {
                s->display_desc = desc;
                fprintf(stderr, "RECONFIGURED\n");
                display_reconfigure(s->real_display, s->display_desc, VIDEO_NORMAL);
        }
}

References desc, state_blend_common::display_desc, display_reconfigure(), state_blend_common::real_display, video_desc_eq(), and VIDEO_NORMAL.

Referenced by display_blend_run(), and display_conference_worker().

Here is the call graph for this function:

display_blend_done()

static void display_blend_done ( void * state )

static

Definition at line 332 of file blend.cpp.

{
        struct state_blend *s = (struct state_blend *)state;
        delete s;
}

References state.

display_blend_fork()

static struct display * display_blend_fork ( void * state )

static

Definition at line 109 of file blend.cpp.

{
        shared_ptr<struct state_blend_common> s = ((struct state_blend *)state)->common;
        struct display *out;
        char fmt[2 + sizeof(void *) * 2 + 1] = "";
        snprintf(fmt, sizeof fmt, "%p", state);
 
        int rc = initialize_video_display(s->parent,
                        "blend", fmt, 0, NULL, &out);
        if (rc == 0) return out; else return NULL;
 
        return out;
}

References state_blend::common, fmt, initialize_video_display(), and state.

Referenced by display_blend_get_property().

Here is the call graph for this function:

display_blend_get_property()

static bool display_blend_get_property ( void * state, int property, void * val, size_t * len )

static

Definition at line 369 of file blend.cpp.

{
        shared_ptr<struct state_blend_common> s = ((struct state_blend *)state)->common;
        if (property == DISPLAY_PROPERTY_SUPPORTS_MULTI_SOURCES) {
                ((struct multi_sources_supp_info *) val)->val = true;
                ((struct multi_sources_supp_info *) val)->fork_display = display_blend_fork;
                ((struct multi_sources_supp_info *) val)->state = state;
                *len = sizeof(struct multi_sources_supp_info);
                return true;
        }
        return display_ctl_property(s->real_display, property, val, len);
}

References state_blend::common, display_blend_fork(), display_ctl_property(), DISPLAY_PROPERTY_SUPPORTS_MULTI_SOURCES, multi_sources_supp_info::fork_display, len, property, state, and val.

Here is the call graph for this function:

display_blend_getf()

static struct video_frame * display_blend_getf ( void * state )

static

Definition at line 338 of file blend.cpp.

{
        struct state_blend *s = (struct state_blend *)state;
 
        return vf_alloc_desc_data(s->desc);
}

References state_blend::desc, state, and vf_alloc_desc_data().

Here is the call graph for this function:

display_blend_init()

static void * display_blend_init ( struct module * parent, const char * fmt, unsigned int flags )

static

Definition at line 123 of file blend.cpp.

{
        char *fmt_copy = NULL;
        const char *requested_display = "";
        const char *cfg = "";
        int ret;
 
        if (fmt == nullptr || strlen(fmt) == 0 || "help"s == fmt) {
                cout << "blend is a helper display to combine (blend) multiple incoming streams.\n"
                                "Please do not use directly, intended for internal purposes!\n";
                return nullptr;
        }
 
        auto *s = new state_blend();
 
        if (isdigit(fmt[0]) != 0) { // fork
                struct state_blend *orig = nullptr;
                sscanf(fmt, "%p", &orig);
                s->common = orig->common;
                return s;
        }
        fmt_copy = strdup(fmt);
        requested_display = fmt_copy;
        char *delim = strchr(fmt_copy, ':');
        if (delim != nullptr) {
                *delim = '\0';
                cfg = delim + 1;
        }
        s->common = shared_ptr<state_blend_common>(new state_blend_common());
        ret = initialize_video_display(parent, requested_display, cfg, flags, NULL, &s->common->real_display);
        assert(ret == 0 && "Unable to initialize real display for blend");
        free(fmt_copy);
 
        display_run_new_thread(s->common->real_display);
 
        s->common->parent = parent;
        s->common->thread_id = thread(display_blend_run, s);
 
        return s;
}

References state_blend::common, display_blend_run(), display_run_new_thread(), flags, fmt, initialize_video_display(), and parent.

Here is the call graph for this function:

display_blend_probe()

static void display_blend_probe ( struct device_info ** available_cards, int * count, void(** deleter )(void *) )

static

Definition at line 408 of file blend.cpp.

                                                                                                            {
        UNUSED(deleter);
        *available_cards = nullptr;
        *count = 0;
}

References count, deleter, and UNUSED.

display_blend_put_audio_frame()

static void display_blend_put_audio_frame ( void * state, const struct audio_frame * frame )

static

Definition at line 391 of file blend.cpp.

{
        UNUSED(state);
        UNUSED(frame);
}

References frame, state, and UNUSED.

display_blend_putf()

static bool display_blend_putf ( void * state, struct video_frame * frame, long long flags )

static

Definition at line 345 of file blend.cpp.

{
        shared_ptr<struct state_blend_common> s = ((struct state_blend *)state)->common;
 
        if (flags == PUTF_DISCARD) {
                vf_free(frame);
        } else {
                unique_lock<mutex> lg(s->lock);
                if (s->incoming_queue.size() >= IN_QUEUE_MAX_BUFFER_LEN) {
                        fprintf(stderr, "blend: queue full!\n");
                }
                if (flags != PUTF_BLOCKING && s->incoming_queue.size() >= IN_QUEUE_MAX_BUFFER_LEN) {
                        vf_free(frame);
                        return 1;
                }
                s->in_queue_decremented_cv.wait(lg, [s]{return s->incoming_queue.size() < IN_QUEUE_MAX_BUFFER_LEN;});
                s->incoming_queue.push(frame);
                lg.unlock();
                s->cv.notify_one();
        }
 
        return true;
}

References state_blend::common, flags, frame, IN_QUEUE_MAX_BUFFER_LEN, PUTF_BLOCKING, PUTF_DISCARD, state, and vf_free().

Here is the call graph for this function:

display_blend_reconfigure()

static bool display_blend_reconfigure ( void * state, struct video_desc desc )

static

Definition at line 382 of file blend.cpp.

{
        struct state_blend *s = (struct state_blend *) state;
 
        s->desc = desc;
 
        return true;
}

References desc, state_blend::desc, and state.

display_blend_reconfigure_audio()

static bool display_blend_reconfigure_audio ( void * state, int quant_samples, int channels, int sample_rate )

static

Definition at line 397 of file blend.cpp.

{
        UNUSED(state);
        UNUSED(quant_samples);
        UNUSED(channels);
        UNUSED(sample_rate);
 
        return false;
}

References channels, quant_samples, sample_rate, state, and UNUSED.

display_blend_run()

static void display_blend_run ( void * state )

static

Definition at line 173 of file blend.cpp.

{
        shared_ptr<struct state_blend_common> s = ((struct state_blend *)state)->common;
        bool prefill = false;
        int skipped = 0;
 
        while (1) {
                struct video_frame *frame;
                {
                        unique_lock<mutex> lg(s->lock);
                        s->cv.wait(lg, [s]{return s->incoming_queue.size() > 0;});
                        frame = s->incoming_queue.front();
                        s->incoming_queue.pop();
                        s->in_queue_decremented_cv.notify_one();
                }
 
                if (!frame) {
                        display_put_frame(s->real_display, NULL, PUTF_BLOCKING);
                        break;
                }
 
                chrono::system_clock::time_point now = chrono::system_clock::now();
                auto it = s->disabled_ssrc.find(frame->ssrc);
                if (it != s->disabled_ssrc.end()) {
                        it->second = now;
                        vf_free(frame);
                        continue;
                }
 
                it = s->disabled_ssrc.begin();
                while (it != s->disabled_ssrc.end()) {
                        if (chrono::duration_cast<chrono::milliseconds>(now - it->second) > SOURCE_TIMEOUT) {
                                verbose_msg("Source 0x%08" PRIx32 " timeout. Deleting from blend display.\n", it->first);
                                s->disabled_ssrc.erase(it++);
                        } else {
                                ++it;
                        }
                }
 
                if (frame->ssrc != s->current_ssrc && frame->ssrc != s->old_ssrc) {
                        if (skipped >= SKIP_FIRST_N_FRAMES_IN_STREAM) {
                                s->old_ssrc = s->current_ssrc; // if != 0, we will be in transition state
                                s->current_ssrc = frame->ssrc;
                                prefill = true;
                                skipped = 0;
                        } else {
                                skipped++;
                                continue;
                        }
                }
 
                s->frames[frame->ssrc].push_back(frame);
 
                if (s->frames[s->current_ssrc].size() >= BUFFER_LEN) {
                        prefill = false;
                }
 
                // we may receive two streams concurrently, therefore we use the timing according to
                // the later one
                if (frame->ssrc != s->current_ssrc) {
                        continue;
                }
 
                if (s->old_ssrc != 0) {
                        if (prefill) {
                                auto & ssrc_list = s->frames[s->old_ssrc];
                                if (ssrc_list.empty()) {
                                        fprintf(stderr, "SMOLIK1!\n");
                                } else {
                                        frame = ssrc_list.front();
                                        ssrc_list.pop_front();
 
                                        check_reconf(s.get(), video_desc_from_frame(frame));
 
                                        struct video_frame *real_display_frame = display_get_frame(s->real_display);
                                        memcpy(real_display_frame->tiles[0].data, frame->tiles[0].data, frame->tiles[0].data_len);
                                        vf_free(frame);
                                        real_display_frame->ssrc = s->current_ssrc;
                                        display_put_frame(s->real_display, real_display_frame, PUTF_BLOCKING);
                                }
                        } else {
                                auto & old_list = s->frames[s->old_ssrc];
                                auto & new_list = s->frames[s->current_ssrc];
 
                                s->transition += 1;
 
                                if (old_list.empty() || new_list.empty()) {
                                        fprintf(stderr, "SMOLIK2!\n");
                                        // ok here, we do not have nothing to mix, therefore cancel smooth transition
                                        s->transition = TRANSITION_COUNT;
                                } else {
                                        struct video_frame *old_frame, *new_frame;
 
                                        old_frame = old_list.front();
                                        old_list.pop_front();
                                        new_frame = new_list.front();
                                        new_list.pop_front();
 
                                        struct video_desc old_desc = video_desc_from_frame(old_frame),
                                                          new_desc = video_desc_from_frame(new_frame);
 
                                        check_reconf(s.get(), new_desc);
 
                                        struct video_frame *real_display_frame = display_get_frame(s->real_display);
 
                                        if (video_desc_eq(old_desc, new_desc)) {
                                                for (unsigned int i = 0; i < new_frame->tiles[0].data_len; ++i) {
                                                        int old_val = ((unsigned char *) old_frame->tiles[0].data)[i];
                                                        int new_val = ((unsigned char *) new_frame->tiles[0].data)[i];
                                                        ((unsigned char *) real_display_frame->tiles[0].data)[i] =
                                                                ((new_val * s->transition) + (old_val * (TRANSITION_COUNT - s->transition))) / TRANSITION_COUNT;
                                                }
                                        } else {
                                                // new desc is different than old desc!
                                                fprintf(stderr, "SMOLIK4!\n");
                                                memcpy(real_display_frame->tiles[0].data, new_frame->tiles[0].data, new_frame->tiles[0].data_len);
                                        }
                                        vf_free(old_frame);
                                        vf_free(new_frame);
                                        real_display_frame->ssrc = s->current_ssrc;
                                        display_put_frame(s->real_display, real_display_frame, PUTF_BLOCKING);
                                }
                        }
                } else {
                        if (!prefill) {
                                if (s->frames[s->current_ssrc].empty()) {
                                        // this should not happen, heh ?
                                        fprintf(stderr, "SMOLIK3!\n");
                                } else {
                                        frame = s->frames[s->current_ssrc].front();
                                        s->frames[s->current_ssrc].pop_front();
 
                                        check_reconf(s.get(), video_desc_from_frame(frame));
 
                                        struct video_frame *real_display_frame = display_get_frame(s->real_display);
                                        memcpy(real_display_frame->tiles[0].data, frame->tiles[0].data, frame->tiles[0].data_len);
                                        vf_free(frame);
                                        real_display_frame->ssrc = s->current_ssrc;
                                        display_put_frame(s->real_display, real_display_frame, PUTF_BLOCKING);
                                }
                        }
                }
 
                if (s->old_ssrc != 0 && s->transition >= TRANSITION_COUNT) {
                        for (auto && frame : s->frames[s->old_ssrc]) {
                                vf_free(frame);
                        }
 
                        s->frames.erase(s->old_ssrc);
 
                        s->disabled_ssrc[s->old_ssrc] = chrono::system_clock::now();
                        s->old_ssrc = 0u;
                        s->transition = 0;
                }
        }
 
        display_join(s->real_display);
}

References BUFFER_LEN, check_reconf(), state_blend::common, tile::data, tile::data_len, display_get_frame(), display_join(), display_put_frame(), frame, PUTF_BLOCKING, SKIP_FIRST_N_FRAMES_IN_STREAM, SOURCE_TIMEOUT(), video_frame::ssrc, state, video_frame::tiles, TRANSITION_COUNT, verbose_msg, vf_free(), video_desc_eq(), and video_desc_from_frame().

Referenced by display_blend_init().

Here is the call graph for this function:

REGISTER_HIDDEN_MODULE()

REGISTER_HIDDEN_MODULE	(	blend	,
		&	display_blend_info,
		LIBRARY_CLASS_VIDEO_DISPLAY	,
		VIDEO_DISPLAY_ABI_VERSION	)

SOURCE_TIMEOUT()

static constexpr chrono::milliseconds SOURCE_TIMEOUT ( 500 )

staticconstexpr

Referenced by display_blend_run().

Variable Documentation

BUFFER_LEN

int BUFFER_LEN = 5

staticconstexpr

Definition at line 66 of file blend.cpp.

Referenced by display_blend_run().

display_blend_info

const struct video_display_info display_blend_info

static

Initial value:

= {
        display_blend_probe,
        display_blend_init,
        NULL, 
        display_blend_done,
        display_blend_getf,
        display_blend_putf,
        display_blend_reconfigure,
        display_blend_get_property,
        display_blend_put_audio_frame,
        display_blend_reconfigure_audio,
        DISPLAY_NO_GENERIC_FPS_INDICATOR,
}

Definition at line 414 of file blend.cpp.

                                                            {
        display_blend_probe,
        display_blend_init,
        NULL, // _run
        display_blend_done,
        display_blend_getf,
        display_blend_putf,
        display_blend_reconfigure,
        display_blend_get_property,
        display_blend_put_audio_frame,
        display_blend_reconfigure_audio,
        DISPLAY_NO_GENERIC_FPS_INDICATOR,
};

IN_QUEUE_MAX_BUFFER_LEN

unsigned int IN_QUEUE_MAX_BUFFER_LEN = 5

staticconstexpr

Definition at line 68 of file blend.cpp.

Referenced by display_blend_putf(), and display_conference_putf().

SKIP_FIRST_N_FRAMES_IN_STREAM

int SKIP_FIRST_N_FRAMES_IN_STREAM = 5

staticconstexpr

Definition at line 69 of file blend.cpp.

Referenced by display_blend_run().

TRANSITION_COUNT

int TRANSITION_COUNT = 10

staticconstexpr

Definition at line 65 of file blend.cpp.

Referenced by display_blend_run().