server/src/video.h
Rama main · réplica de lectura del repositorio original.
/**
* @file src/video.h
* @brief Declarations for video.
*/
#pragma once
// local includes
#include "input.h"
#include "platform/common.h"
#include "pyrowave_policy.h"
#include "video_policy.h"
#include "thread_safe.h"
#include "video_colorspace.h"
// standard includes
#include <array>
#include <atomic>
#include <optional>
#include <string>
extern "C" {
#include <libavcodec/avcodec.h>
#include <libswscale/swscale.h>
}
struct AVPacket;
namespace video {
enum class capture_source_e : std::uint8_t {
active_output,
exact_output,
synthetic_black,
};
/* Encoding configuration requested by remote client */
struct config_t {
// DO NOT CHANGE ORDER OR ADD FIELDS IN THE MIDDLE!!!!!
// ONLY APPEND NEW FIELD AFTERWARDS!!!!!!!!!
// BIG F WORD to Sunshine!!!!!!!!!
int width; // Video width in pixels
int height; // Video height in pixels
int framerate; // Requested framerate, used in individual frame bitrate budget calculation
int framerateX100; // Optional field for streaming at NTSC or similar rates e.g. 59.94 = 5994
int bitrate; // Video bitrate in kilobits (1000 bits) for requested framerate
int slicesPerFrame; // Number of slices per frame
int numRefFrames; // Max number of reference frames
/* Requested color range and SDR encoding colorspace, HDR encoding colorspace is always BT.2020+ST2084
Color range (encoderCscMode & 0x1) : 0 - limited, 1 - full
SDR encoding colorspace (encoderCscMode >> 1) : 0 - BT.601, 1 - BT.709, 2 - BT.2020 */
int encoderCscMode;
int videoFormat; // 0 - H.264, 1 - HEVC, 2 - AV1, 3 - PyroWave
/* Encoding color depth (bit depth): 0 - 8-bit, 1 - 10-bit
HDR encoding activates when color depth is higher than 8-bit and the display which is being captured is operating in HDR mode */
int dynamicRange;
// Set when we bumped dynamicRange to 10-bit for SDR-only encoding (prefer_10bit_sdr)
bool prefer_sdr_10bit;
// Force SDR colorspace/signaling independently of negotiated encoder bit depth.
bool force_sdr = false;
// Stream-scoped TrueHDR-capable pipeline enable. Runtime foreground-window
// checks can still bypass per-frame conversion while this remains true, which
// keeps capture and encoder format stable across desktop/app focus changes.
bool rtx_hdr_active = false;
int rtx_hdr_peak_nits = 1000;
int chromaSamplingType; // 0 - 4:2:0, 1 - 4:4:4
int enableIntraRefresh; // 0 - disabled, 1 - enabled
int encodingFramerate = 0; // Requested display framerate
bool input_only = false;
// Capture ownership is session-local. Exact remote outputs must not fall
// back through the mutable process-wide output selection, while input-only
// sessions use an in-memory black producer and never initialize capture.
capture_source_e capture_source {capture_source_e::active_output};
std::optional<std::string> capture_output;
// Opt into the smallest supported host-side queues for a launch-qualified VRR
// session. This remains false for clients that do not negotiate the mode.
bool vrr_low_latency = false;
// Original client-requested wire-bandwidth budget in Kbps, before Sunshine
// subtracts FEC/audio/control overhead from `bitrate` for the encoder.
// Same as `bitrate` for clients that don't send maximumBitrateKbps.
int client_requested_bitrate = 0;
// PyroWave only (videoFormat 3): the negotiated RTP packet size, which record
// framing aligns to, and the framing the client asked for in ANNOUNCE.
int packetsize = 0;
pyrowave::policy::framing_e pyrowave_framing = pyrowave::policy::framing_e::records;
};
platf::mem_type_e map_base_dev_type(AVHWDeviceType type);
platf::pix_fmt_e map_pix_fmt(AVPixelFormat fmt);
void free_ctx(AVCodecContext *ctx);
void free_frame(AVFrame *frame);
void free_buffer(AVBufferRef *ref);
using avcodec_ctx_t = util::safe_ptr<AVCodecContext, free_ctx>;
using avcodec_frame_t = util::safe_ptr<AVFrame, free_frame>;
using avcodec_buffer_t = util::safe_ptr<AVBufferRef, free_buffer>;
using sws_t = util::safe_ptr<SwsContext, sws_freeContext>;
using img_event_t = std::shared_ptr<safe::event_t<std::shared_ptr<platf::img_t>>>;
struct encoder_platform_formats_t {
virtual ~encoder_platform_formats_t() = default;
platf::mem_type_e dev_type;
platf::pix_fmt_e pix_fmt_8bit;
platf::pix_fmt_e pix_fmt_10bit;
platf::pix_fmt_e pix_fmt_yuv444_8bit;
platf::pix_fmt_e pix_fmt_yuv444_10bit;
};
struct encoder_platform_formats_avcodec: encoder_platform_formats_t {
using init_buffer_function_t = std::function<util::Either<avcodec_buffer_t, int>(platf::avcodec_encode_device_t *)>;
encoder_platform_formats_avcodec(
const AVHWDeviceType &avcodec_base_dev_type,
const AVHWDeviceType &avcodec_derived_dev_type,
const AVPixelFormat &avcodec_dev_pix_fmt,
const AVPixelFormat &avcodec_pix_fmt_8bit,
const AVPixelFormat &avcodec_pix_fmt_10bit,
const AVPixelFormat &avcodec_pix_fmt_yuv444_8bit,
const AVPixelFormat &avcodec_pix_fmt_yuv444_10bit,
const init_buffer_function_t &init_avcodec_hardware_input_buffer_function
):
avcodec_base_dev_type {avcodec_base_dev_type},
avcodec_derived_dev_type {avcodec_derived_dev_type},
avcodec_dev_pix_fmt {avcodec_dev_pix_fmt},
avcodec_pix_fmt_8bit {avcodec_pix_fmt_8bit},
avcodec_pix_fmt_10bit {avcodec_pix_fmt_10bit},
avcodec_pix_fmt_yuv444_8bit {avcodec_pix_fmt_yuv444_8bit},
avcodec_pix_fmt_yuv444_10bit {avcodec_pix_fmt_yuv444_10bit},
init_avcodec_hardware_input_buffer {init_avcodec_hardware_input_buffer_function} {
dev_type = map_base_dev_type(avcodec_base_dev_type);
pix_fmt_8bit = map_pix_fmt(avcodec_pix_fmt_8bit);
pix_fmt_10bit = map_pix_fmt(avcodec_pix_fmt_10bit);
pix_fmt_yuv444_8bit = map_pix_fmt(avcodec_pix_fmt_yuv444_8bit);
pix_fmt_yuv444_10bit = map_pix_fmt(avcodec_pix_fmt_yuv444_10bit);
}
AVHWDeviceType avcodec_base_dev_type;
AVHWDeviceType avcodec_derived_dev_type;
AVPixelFormat avcodec_dev_pix_fmt;
AVPixelFormat avcodec_pix_fmt_8bit;
AVPixelFormat avcodec_pix_fmt_10bit;
AVPixelFormat avcodec_pix_fmt_yuv444_8bit;
AVPixelFormat avcodec_pix_fmt_yuv444_10bit;
init_buffer_function_t init_avcodec_hardware_input_buffer;
};
struct encoder_platform_formats_nvenc: encoder_platform_formats_t {
encoder_platform_formats_nvenc(
const platf::mem_type_e &dev_type,
const platf::pix_fmt_e &pix_fmt_8bit,
const platf::pix_fmt_e &pix_fmt_10bit,
const platf::pix_fmt_e &pix_fmt_yuv444_8bit,
const platf::pix_fmt_e &pix_fmt_yuv444_10bit
) {
encoder_platform_formats_t::dev_type = dev_type;
encoder_platform_formats_t::pix_fmt_8bit = pix_fmt_8bit;
encoder_platform_formats_t::pix_fmt_10bit = pix_fmt_10bit;
encoder_platform_formats_t::pix_fmt_yuv444_8bit = pix_fmt_yuv444_8bit;
encoder_platform_formats_t::pix_fmt_yuv444_10bit = pix_fmt_yuv444_10bit;
}
};
struct encoder_platform_formats_amf: encoder_platform_formats_t {
encoder_platform_formats_amf(
const platf::mem_type_e &dev_type,
const platf::pix_fmt_e &pix_fmt_8bit,
const platf::pix_fmt_e &pix_fmt_10bit,
const platf::pix_fmt_e &pix_fmt_yuv444_8bit,
const platf::pix_fmt_e &pix_fmt_yuv444_10bit
) {
encoder_platform_formats_t::dev_type = dev_type;
encoder_platform_formats_t::pix_fmt_8bit = pix_fmt_8bit;
encoder_platform_formats_t::pix_fmt_10bit = pix_fmt_10bit;
encoder_platform_formats_t::pix_fmt_yuv444_8bit = pix_fmt_yuv444_8bit;
encoder_platform_formats_t::pix_fmt_yuv444_10bit = pix_fmt_yuv444_10bit;
}
};
struct encoder_t {
std::string_view name;
enum flag_e {
PASSED, ///< Indicates the encoder is supported.
REF_FRAMES_RESTRICT, ///< Set maximum reference frames.
DYNAMIC_RANGE, ///< HDR support.
YUV444, ///< YUV 4:4:4 support.
VUI_PARAMETERS, ///< AMD encoder with VAAPI doesn't add VUI parameters to SPS.
MAX_FLAGS ///< Maximum number of flags.
};
static std::string_view from_flag(flag_e flag) {
#define _CONVERT(x) \
case flag_e::x: \
return std::string_view(#x)
switch (flag) {
_CONVERT(PASSED);
_CONVERT(REF_FRAMES_RESTRICT);
_CONVERT(DYNAMIC_RANGE);
_CONVERT(YUV444);
_CONVERT(VUI_PARAMETERS);
_CONVERT(MAX_FLAGS);
}
#undef _CONVERT
return {"unknown"};
}
struct option_t {
KITTY_DEFAULT_CONSTR_MOVE(option_t)
option_t(const option_t &) = default;
std::string name;
struct optional_int_function_t {
std::function<std::optional<int>()> evaluate;
};
std::variant<int, int *, std::optional<int> *, std::function<int()>, optional_int_function_t, std::string, std::string *, std::function<const std::string(const config_t &)>> value;
option_t(std::string &&name, decltype(value) &&value):
name {std::move(name)},
value {std::move(value)} {
}
};
const std::unique_ptr<const encoder_platform_formats_t> platform_formats;
struct codec_t {
std::vector<option_t> common_options;
std::vector<option_t> sdr_options;
std::vector<option_t> hdr_options;
std::vector<option_t> sdr444_options;
std::vector<option_t> hdr444_options;
std::vector<option_t> fallback_options;
std::string name;
std::bitset<MAX_FLAGS> capabilities;
bool operator[](flag_e flag) const {
return capabilities[(std::size_t) flag];
}
std::bitset<MAX_FLAGS>::reference operator[](flag_e flag) {
return capabilities[(std::size_t) flag];
}
};
codec_t av1;
codec_t hevc;
codec_t h264;
const codec_t &codec_from_config(const config_t &config) const {
switch (config.videoFormat) {
default:
BOOST_LOG(error) << "Unknown video format " << config.videoFormat << ", falling back to H.264";
// fallthrough
case 0:
return h264;
case 1:
return hevc;
case 2:
return av1;
}
}
uint32_t flags;
};
struct encode_session_t {
virtual ~encode_session_t() = default;
virtual int convert(platf::img_t &img) = 0;
virtual void request_idr_frame() = 0;
virtual void request_normal_frame() = 0;
virtual void invalidate_ref_frames(int64_t first_frame, int64_t last_frame) = 0;
/**
* @brief Apply a new encoder bitrate to the live encoder without rebuilding it.
* @param bitrate_kbps New bitrate in kbps.
* @return `true` if the encoder reconfigured itself in place; `false` if the caller must
* rebuild the encode session to apply the change.
*/
virtual bool set_bitrate(int bitrate_kbps) {
return false;
}
/**
* @brief Replace static HDR mastering metadata used for subsequent encoded frames.
* @param metadata New mastering-display and content-light metadata.
*/
virtual void set_hdr_metadata(const SS_HDR_METADATA &) {
}
};
// encoders
extern encoder_t software;
#if !defined(__APPLE__)
extern encoder_t nvenc; // available for windows and linux
#endif
#if defined(__linux__)
extern encoder_t nvenc_legacy;
#endif
#ifdef _WIN32
extern encoder_t amdvce_experimental;
extern encoder_t amdvce_ffmpeg;
extern encoder_t quicksync;
extern encoder_t mediafoundation;
#endif
#if defined(__linux__) || defined(linux) || defined(__linux) || defined(__FreeBSD__)
extern encoder_t vaapi;
#endif
#ifdef __APPLE__
extern encoder_t videotoolbox;
#endif
struct packet_raw_t {
virtual ~packet_raw_t() = default;
virtual bool is_idr() = 0;
virtual int64_t frame_index() = 0;
virtual uint8_t *data() = 0;
virtual size_t data_size() = 0;
struct replace_t {
std::string_view old;
std::string_view _new;
KITTY_DEFAULT_CONSTR_MOVE(replace_t)
replace_t(std::string_view old, std::string_view _new) noexcept:
old {std::move(old)},
_new {std::move(_new)} {
}
};
std::vector<replace_t> *replacements = nullptr;
void *channel_data = nullptr;
bool after_ref_frame_invalidation = false;
// Pacing/scheduled timestamp used for transport timing.
/// PyroWave record framing: frame bytes through the coarsest wavelet level, whose
/// packets stream.cpp protects with parity and announces; 0 when unknown.
std::size_t pyrowave_critical_bytes = 0;
int pyrowave_detail_fec_percentage = 0;
std::size_t pyrowave_frame_wire_budget = 0;
std::optional<std::chrono::steady_clock::time_point> frame_timestamp;
// Raw capture/QPC-derived timestamp before pacing adjustments.
std::optional<std::chrono::steady_clock::time_point> capture_timestamp;
std::optional<std::chrono::steady_clock::time_point> host_processing_timestamp;
std::chrono::steady_clock::time_point packet_enqueue_timestamp = std::chrono::steady_clock::now();
};
struct packet_raw_avcodec: packet_raw_t {
packet_raw_avcodec() {
av_packet = av_packet_alloc();
}
~packet_raw_avcodec() {
av_packet_free(&this->av_packet);
}
bool is_idr() override {
return av_packet->flags & AV_PKT_FLAG_KEY;
}
int64_t frame_index() override {
return av_packet->pts;
}
uint8_t *data() override {
return av_packet->data;
}
size_t data_size() override {
return av_packet->size;
}
AVPacket *av_packet;
};
struct packet_raw_generic: packet_raw_t {
packet_raw_generic(std::vector<uint8_t> &&frame_data, int64_t frame_index, bool idr):
frame_data {std::move(frame_data)},
index {frame_index},
idr {idr} {
}
bool is_idr() override {
return idr;
}
int64_t frame_index() override {
return index;
}
uint8_t *data() override {
return frame_data.data();
}
size_t data_size() override {
return frame_data.size();
}
std::vector<uint8_t> frame_data;
int64_t index;
bool idr;
};
using packet_t = std::unique_ptr<packet_raw_t>;
struct hdr_info_raw_t {
explicit hdr_info_raw_t(bool enabled):
enabled {enabled},
metadata {} {};
explicit hdr_info_raw_t(bool enabled, const SS_HDR_METADATA &metadata):
enabled {enabled},
metadata {metadata} {};
bool enabled;
SS_HDR_METADATA metadata;
};
using hdr_info_t = std::unique_ptr<hdr_info_raw_t>;
extern int active_hevc_mode;
extern int active_av1_mode;
/**
* PyroWave encoding on the probed capture adapter: 0 - not probed, 1 - disabled
* or unsupported, 2 - available (advertised). PyroWave bypasses the encoder_t
* selection, so this is independent of the chosen hardware encoder.
*/
extern std::atomic_int active_pyrowave_mode;
extern bool last_encoder_probe_supported_ref_frames_invalidation;
extern std::array<bool, 3> last_encoder_probe_supported_yuv444_for_codec; // 0 - H.264, 1 - HEVC, 2 - AV1
bool has_attempted_encoder_probe();
bool has_successful_encoder_probe();
bool last_encoder_probe_failed();
struct advertised_encoder_capabilities_t {
int hevc_mode = 0;
int av1_mode = 0;
int pyrowave_mode = 0; ///< Same values as active_pyrowave_mode.
std::array<bool, 3> yuv444_for_codec {};
};
advertised_encoder_capabilities_t advertised_encoder_capabilities(
bool probe_before_negative = false,
bool *probe_complete = nullptr
);
#ifdef _WIN32
// Bridge the interval between selecting a virtual-display render adapter and
// Windows publishing its replacement output. The lease prevents an older
// request from clearing a newer session's identity.
using encoder_probe_adapter_hint_lease_t = std::uint64_t;
encoder_probe_adapter_hint_lease_t set_pending_virtual_display_adapter_hint(const LUID &adapter_luid);
// Once the replacement output is published, capability identity must agree
// with its WGC/DXGI adapter before a successful probe can remain cached.
bool mark_pending_virtual_display_adapter_hint_ready_for_verification(encoder_probe_adapter_hint_lease_t lease);
bool clear_pending_virtual_display_adapter_hint(encoder_probe_adapter_hint_lease_t lease);
#endif
void capture(
safe::mail_t mail,
config_t config,
void *channel_data
);
bool validate_encoder(
encoder_t &encoder,
bool expect_failure,
const std::optional<platf::adapter_id_t> &required_adapter = std::nullopt,
std::optional<platf::adapter_id_t> *actual_adapter = nullptr,
const std::string &probe_display_name = std::string {}
);
/**
* @brief Check if we can allow probing for the encoders.
* @return True if there should be no issues with the probing, false if we should prevent it.
*/
bool allow_encoder_probing();
/**
* @brief Probe encoders and select the preferred encoder.
* This is called once at startup and each time a stream is launched to
* ensure the best encoder is selected. Encoder availability can change
* at runtime due to all sorts of things from driver updates to eGPUs.
*
* @warning This is only safe to call when there is no client actively streaming.
*/
int probe_encoders();
// Several NTSC standard refresh rates are hardcoded here, because their
// true rate requires a denominator of 1001. ffmpeg's av_d2q() would assume it could
// reduce 29.97 to 2997/100 but this would be slightly wrong. We also include
// support for 23.976 film in case someone wants to stream a film at the perfect
// framerate.
inline AVRational framerateX100_to_rational(const int framerateX100) {
const auto rational = policy::framerate_x100_to_rational(framerateX100);
return AVRational {rational.numerator, rational.denominator};
}
} // namespace video