AOMedia AV1 Codec
tpl_model.h
1 /*
2  * Copyright (c) 2019, Alliance for Open Media. All rights reserved
3  *
4  * This source code is subject to the terms of the BSD 2 Clause License and
5  * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6  * was not distributed with this source code in the LICENSE file, you can
7  * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8  * Media Patent License 1.0 was not distributed with this source code in the
9  * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10  */
11 
12 #ifndef AOM_AV1_ENCODER_TPL_MODEL_H_
13 #define AOM_AV1_ENCODER_TPL_MODEL_H_
14 
15 #ifdef __cplusplus
16 extern "C" {
17 #endif
18 
21 struct AV1_PRIMARY;
22 struct AV1_COMP;
23 struct AV1_SEQ_CODING_TOOLS;
24 struct EncodeFrameParams;
25 struct EncodeFrameInput;
26 struct GF_GROUP;
27 struct TPL_INFO;
28 
29 #include "config/aom_config.h"
30 
31 #include "aom_scale/yv12config.h"
32 
33 #include "av1/common/mv.h"
34 #include "av1/common/scale.h"
35 #include "av1/encoder/block.h"
36 #include "av1/encoder/lookahead.h"
37 #include "av1/encoder/ratectrl.h"
38 
39 static INLINE BLOCK_SIZE convert_length_to_bsize(int length) {
40  switch (length) {
41  case 64: return BLOCK_64X64;
42  case 32: return BLOCK_32X32;
43  case 16: return BLOCK_16X16;
44  case 8: return BLOCK_8X8;
45  case 4: return BLOCK_4X4;
46  default:
47  assert(0 && "Invalid block size for tpl model");
48  return BLOCK_16X16;
49  }
50 }
51 
52 typedef struct AV1TplRowMultiThreadSync {
53 #if CONFIG_MULTITHREAD
54  // Synchronization objects for top-right dependency.
55  pthread_mutex_t *mutex_;
56  pthread_cond_t *cond_;
57 #endif
58  // Buffer to store the macroblock whose encoding is complete.
59  // num_finished_cols[i] stores the number of macroblocks which finished
60  // encoding in the ith macroblock row.
61  int *num_finished_cols;
62  // Number of extra macroblocks of the top row to be complete for encoding
63  // of the current macroblock to start. A value of 1 indicates top-right
64  // dependency.
65  int sync_range;
66  // Number of macroblock rows.
67  int rows;
68  // Number of threads processing the current tile.
69  int num_threads_working;
70 } AV1TplRowMultiThreadSync;
71 
72 typedef struct AV1TplRowMultiThreadInfo {
73  // Row synchronization related function pointers.
74  void (*sync_read_ptr)(AV1TplRowMultiThreadSync *tpl_mt_sync, int r, int c);
75  void (*sync_write_ptr)(AV1TplRowMultiThreadSync *tpl_mt_sync, int r, int c,
76  int cols);
77 } AV1TplRowMultiThreadInfo;
78 
79 // TODO(jingning): This needs to be cleaned up next.
80 
81 // TPL stats buffers are prepared for every frame in the GOP,
82 // including (internal) overlays and (internal) arfs.
83 // In addition, frames in the lookahead that are outside of the GOP
84 // are also used.
85 // Thus it should use
86 // (gop_length) + (# overlays) + (MAX_LAG_BUFFERS - gop_len) =
87 // MAX_LAG_BUFFERS + (# overlays)
88 // 2 * MAX_LAG_BUFFERS is therefore a safe estimate.
89 // TODO(bohanli): test setting it to 1.5 * MAX_LAG_BUFFER
90 #define MAX_TPL_FRAME_IDX (2 * MAX_LAG_BUFFERS)
91 // The first REF_FRAMES + 1 buffers are reserved.
92 // tpl_data->tpl_frame starts after REF_FRAMES + 1
93 #define MAX_LENGTH_TPL_FRAME_STATS (MAX_TPL_FRAME_IDX + REF_FRAMES + 1)
94 #define TPL_DEP_COST_SCALE_LOG2 4
95 
96 #define TPL_EPSILON 0.0000001
97 
98 typedef struct TplTxfmStats {
99  int ready; // Whether abs_coeff_mean is ready
100  double abs_coeff_sum[256]; // Assume we are using 16x16 transform block
101  double abs_coeff_mean[256];
102  int txfm_block_count;
103  int coeff_num;
104 } TplTxfmStats;
105 
106 typedef struct TplDepStats {
107  int64_t srcrf_sse;
108  int64_t srcrf_dist;
109  int64_t recrf_sse;
110  int64_t recrf_dist;
111  int64_t intra_sse;
112  int64_t intra_dist;
113  int64_t cmp_recrf_dist[2];
114  int64_t mc_dep_rate;
115  int64_t mc_dep_dist;
116  int64_t pred_error[INTER_REFS_PER_FRAME];
117  int32_t intra_cost;
118  int32_t inter_cost;
119  int32_t srcrf_rate;
120  int32_t recrf_rate;
121  int32_t intra_rate;
122  int32_t cmp_recrf_rate[2];
123  int_mv mv[INTER_REFS_PER_FRAME];
124  int8_t ref_frame_index[2];
125 } TplDepStats;
126 
127 typedef struct TplDepFrame {
128  uint8_t is_valid;
129  TplDepStats *tpl_stats_ptr;
130  const YV12_BUFFER_CONFIG *gf_picture;
131  YV12_BUFFER_CONFIG *rec_picture;
132  int ref_map_index[REF_FRAMES];
133  int stride;
134  int width;
135  int height;
136  int mi_rows;
137  int mi_cols;
138  int base_rdmult;
139  uint32_t frame_display_index;
140 } TplDepFrame;
141 
146 typedef struct TplParams {
150  int ready;
151 
156 
160  uint8_t tpl_bsize_1d;
161 
167  TplDepFrame tpl_stats_buffer[MAX_LENGTH_TPL_FRAME_STATS];
168 
174  TplDepStats *tpl_stats_pool[MAX_LAG_BUFFERS];
175 
182  TplTxfmStats *txfm_stats_list;
183 
189 
193  TplDepFrame *tpl_frame;
194 
198  struct scale_factors sf;
199 
204 
210  const YV12_BUFFER_CONFIG *src_ref_frame[INTER_REFS_PER_FRAME];
211 
217  const YV12_BUFFER_CONFIG *ref_frame[INTER_REFS_PER_FRAME];
218 
223  AV1TplRowMultiThreadSync tpl_mt_sync;
224 
229 
230 } TplParams;
231 
232 #if CONFIG_BITRATE_ACCURACY || CONFIG_RATECTRL_LOG
233 #define VBR_RC_INFO_MAX_FRAMES 500
234 #endif // CONFIG_BITRATE_ACCURACY || CONFIG_RATECTRL_LOG
235 
236 #if CONFIG_BITRATE_ACCURACY
237 
242 typedef struct {
243  int ready;
244  double total_bit_budget; // The total bit budget of the entire video
245  int show_frame_count; // Number of show frames in the entire video
246 
247  int gop_showframe_count; // The number of show frames in the current gop
248  double gop_bit_budget; // The bitbudget for the current gop
249  double scale_factors[FRAME_UPDATE_TYPES]; // Scale factors to improve the
250  // budget estimation
251  double mv_scale_factors[FRAME_UPDATE_TYPES]; // Scale factors to improve
252  // MV entropy estimation
253 
254  // === Below this line are GOP related data that will be updated per GOP ===
255  int base_q_index; // Stores the base q index.
256  int q_index_list_ready;
257  int q_index_list[VBR_RC_INFO_MAX_FRAMES]; // q indices for the current
258  // GOP
259 
260  // Array to store qstep_ratio for each frame in a GOP
261  double qstep_ratio_list[VBR_RC_INFO_MAX_FRAMES];
262 
263 #if CONFIG_THREE_PASS
264  TplTxfmStats txfm_stats_list[VBR_RC_INFO_MAX_FRAMES];
265  FRAME_UPDATE_TYPE update_type_list[VBR_RC_INFO_MAX_FRAMES];
266  int gop_start_idx_list[VBR_RC_INFO_MAX_FRAMES];
267  int gop_length_list[VBR_RC_INFO_MAX_FRAMES];
268  int cur_gop_idx;
269  int total_frame_count;
270  int gop_count;
271 #endif // CONFIG_THREE_PASS
272 } VBR_RATECTRL_INFO;
273 
274 static INLINE void vbr_rc_reset_gop_data(VBR_RATECTRL_INFO *vbr_rc_info) {
275  vbr_rc_info->q_index_list_ready = 0;
276  av1_zero(vbr_rc_info->q_index_list);
277 }
278 
279 void av1_vbr_rc_init(VBR_RATECTRL_INFO *vbr_rc_info, double total_bit_budget,
280  int show_frame_count);
281 
282 int av1_vbr_rc_frame_coding_idx(const VBR_RATECTRL_INFO *vbr_rc_info,
283  int gf_frame_index);
284 
285 void av1_vbr_rc_append_tpl_info(VBR_RATECTRL_INFO *vbr_rc_info,
286  const struct TPL_INFO *tpl_info);
287 
288 void av1_vbr_rc_set_gop_bit_budget(VBR_RATECTRL_INFO *vbr_rc_info,
289  int gop_showframe_count);
290 
291 void av1_vbr_rc_compute_q_indices(int base_q_index, int frame_count,
292  const double *qstep_ratio_list,
293  aom_bit_depth_t bit_depth, int *q_index_list);
294 
303 void av1_vbr_rc_update_q_index_list(VBR_RATECTRL_INFO *vbr_rc_info,
304  const TplParams *tpl_data,
305  const struct GF_GROUP *gf_group,
306  aom_bit_depth_t bit_depth);
307 /*
308  *!\brief Compute the number of bits needed to encode a GOP
309  *
310  * \param[in] base_q_index base layer q_index
311  * \param[in] bit_depth bit depth
312  * \param[in] update_type_scale_factors array of scale factors for each
313  * update_type
314  * \param[in] frame_count size of update_type_list,
315  * qstep_ratio_list stats_list,
316  * q_index_list and
317  * estimated_bitrate_byframe
318  * \param[in] update_type_list array of update_type, one per frame
319  * \param[in] qstep_ratio_list array of qstep_ratio, one per frame
320  * \param[in] stats_list array of transform stats, one per
321  * frame
322  * \param[out] q_index_list array of q_index, one per frame
323  * \param[out] estimated_bitrate_byframe array to keep track of frame
324  * bitrate
325  *
326  * \return The estimated GOP bitrate.
327  *
328  */
329 double av1_vbr_rc_info_estimate_gop_bitrate(
330  int base_q_index, aom_bit_depth_t bit_depth,
331  const double *update_type_scale_factors, int frame_count,
332  const FRAME_UPDATE_TYPE *update_type_list, const double *qstep_ratio_list,
333  const TplTxfmStats *stats_list, int *q_index_list,
334  double *estimated_bitrate_byframe);
335 
357 int av1_vbr_rc_info_estimate_base_q(
358  double bit_budget, aom_bit_depth_t bit_depth,
359  const double *update_type_scale_factors, int frame_count,
360  const FRAME_UPDATE_TYPE *update_type_list, const double *qstep_ratio_list,
361  const TplTxfmStats *stats_list, int *q_index_list,
362  double *estimated_bitrate_byframe);
363 
364 #endif // CONFIG_BITRATE_ACCURACY
365 
366 #if CONFIG_RD_COMMAND
367 typedef enum {
368  RD_OPTION_NONE,
369  RD_OPTION_SET_Q,
370  RD_OPTION_SET_Q_RDMULT
371 } RD_OPTION;
372 
373 typedef struct RD_COMMAND {
374  RD_OPTION option_ls[MAX_LENGTH_TPL_FRAME_STATS];
375  int q_index_ls[MAX_LENGTH_TPL_FRAME_STATS];
376  int rdmult_ls[MAX_LENGTH_TPL_FRAME_STATS];
377  int frame_count;
378  int frame_index;
379 } RD_COMMAND;
380 
381 void av1_read_rd_command(const char *filepath, RD_COMMAND *rd_command);
382 #endif // CONFIG_RD_COMMAND
383 
392 void av1_setup_tpl_buffers(struct AV1_PRIMARY *const ppi,
393  CommonModeInfoParams *const mi_params, int width,
394  int height, int byte_alignment, int lag_in_frames);
395 
407 int av1_tpl_setup_stats(struct AV1_COMP *cpi, int gop_eval,
408  const struct EncodeFrameParams *const frame_params);
409 
412 void av1_tpl_preload_rc_estimate(
413  struct AV1_COMP *cpi, const struct EncodeFrameParams *const frame_params);
414 
415 int av1_tpl_ptr_pos(int mi_row, int mi_col, int stride, uint8_t right_shift);
416 
417 void av1_init_tpl_stats(TplParams *const tpl_data);
418 
419 int av1_tpl_stats_ready(const TplParams *tpl_data, int gf_frame_index);
420 
421 void av1_tpl_rdmult_setup(struct AV1_COMP *cpi);
422 
423 void av1_tpl_rdmult_setup_sb(struct AV1_COMP *cpi, MACROBLOCK *const x,
424  BLOCK_SIZE sb_size, int mi_row, int mi_col);
425 
426 void av1_mc_flow_dispenser_row(struct AV1_COMP *cpi,
427  TplTxfmStats *tpl_txfm_stats, MACROBLOCK *x,
428  int mi_row, BLOCK_SIZE bsize, TX_SIZE tx_size);
429 
442 double av1_exponential_entropy(double q_step, double b);
443 
457 double av1_laplace_entropy(double q_step, double b, double zero_bin_ratio);
458 
476 double av1_laplace_estimate_frame_rate(int q_index, int block_count,
477  const double *abs_coeff_mean,
478  int coeff_num);
479 
480 /*
481  *!\brief Init TplTxfmStats
482  *
483  * \param[in] tpl_txfm_stats a structure for storing transform stats
484  *
485  */
486 void av1_init_tpl_txfm_stats(TplTxfmStats *tpl_txfm_stats);
487 
488 /*
489  *!\brief Accumulate TplTxfmStats
490  *
491  * \param[in] sub_stats a structure for storing sub transform stats
492  * \param[out] accumulated_stats a structure for storing accumulated
493  *transform stats
494  *
495  */
496 void av1_accumulate_tpl_txfm_stats(const TplTxfmStats *sub_stats,
497  TplTxfmStats *accumulated_stats);
498 
499 /*
500  *!\brief Record a transform block into TplTxfmStats
501  *
502  * \param[in] tpl_txfm_stats A structure for storing transform stats
503  * \param[out] coeff An array of transform coefficients. Its size
504  * should equal to tpl_txfm_stats.coeff_num.
505  *
506  */
507 void av1_record_tpl_txfm_block(TplTxfmStats *tpl_txfm_stats,
508  const tran_low_t *coeff);
509 
510 /*
511  *!\brief Update abs_coeff_mean and ready of txfm_stats
512  * If txfm_block_count > 0, this function will use abs_coeff_sum and
513  * txfm_block_count to compute abs_coeff_mean. Moreover, reday flag
514  * will be set to one.
515  *
516  * \param[in] txfm_stats A structure for storing transform stats
517  */
518 void av1_tpl_txfm_stats_update_abs_coeff_mean(TplTxfmStats *txfm_stats);
519 
535 double av1_estimate_coeff_entropy(double q_step, double b,
536  double zero_bin_ratio, int qcoeff);
537 
550 double av1_estimate_txfm_block_entropy(int q_index,
551  const double *abs_coeff_mean,
552  int *qcoeff_arr, int coeff_num);
553 
554 // TODO(angiebird): Add doxygen description here.
555 int64_t av1_delta_rate_cost(int64_t delta_rate, int64_t recrf_dist,
556  int64_t srcrf_dist, int pix_num);
557 
573 int av1_get_overlap_area(int row_a, int col_a, int row_b, int col_b, int width,
574  int height);
575 
585 int av1_tpl_get_q_index(const TplParams *tpl_data, int gf_frame_index,
586  int leaf_qindex, aom_bit_depth_t bit_depth);
587 
595 double av1_tpl_get_frame_importance(const TplParams *tpl_data,
596  int gf_frame_index);
597 
608 double av1_tpl_get_qstep_ratio(const TplParams *tpl_data, int gf_frame_index);
609 
618 int av1_get_q_index_from_qstep_ratio(int leaf_qindex, double qstep_ratio,
619  aom_bit_depth_t bit_depth);
620 
635 int_mv av1_compute_mv_difference(const TplDepFrame *tpl_frame, int row, int col,
636  int step, int tpl_stride, int right_shift);
637 
645 double av1_tpl_compute_frame_mv_entropy(const TplDepFrame *tpl_frame,
646  uint8_t right_shift);
647 
648 #if CONFIG_RATECTRL_LOG
649 typedef struct {
650  int coding_frame_count;
651  int base_q_index;
652 
653  // Encode decision
654  int q_index_list[VBR_RC_INFO_MAX_FRAMES];
655  double qstep_ratio_list[VBR_RC_INFO_MAX_FRAMES];
656  FRAME_UPDATE_TYPE update_type_list[VBR_RC_INFO_MAX_FRAMES];
657 
658  // Frame stats
659  TplTxfmStats txfm_stats_list[VBR_RC_INFO_MAX_FRAMES];
660 
661  // Estimated encode results
662  double est_coeff_rate_list[VBR_RC_INFO_MAX_FRAMES];
663 
664  // Actual encode results
665  double act_rate_list[VBR_RC_INFO_MAX_FRAMES];
666  double act_coeff_rate_list[VBR_RC_INFO_MAX_FRAMES];
667 } RATECTRL_LOG;
668 
669 static INLINE void rc_log_init(RATECTRL_LOG *rc_log) { av1_zero(*rc_log); }
670 
671 static INLINE void rc_log_frame_stats(RATECTRL_LOG *rc_log, int coding_index,
672  const TplTxfmStats *txfm_stats) {
673  rc_log->txfm_stats_list[coding_index] = *txfm_stats;
674 }
675 
676 static INLINE void rc_log_frame_encode_param(RATECTRL_LOG *rc_log,
677  int coding_index,
678  double qstep_ratio, int q_index,
679  FRAME_UPDATE_TYPE update_type) {
680  rc_log->qstep_ratio_list[coding_index] = qstep_ratio;
681  rc_log->q_index_list[coding_index] = q_index;
682  rc_log->update_type_list[coding_index] = update_type;
683  const TplTxfmStats *txfm_stats = &rc_log->txfm_stats_list[coding_index];
684  rc_log->est_coeff_rate_list[coding_index] = 0;
685  if (txfm_stats->ready) {
686  rc_log->est_coeff_rate_list[coding_index] = av1_laplace_estimate_frame_rate(
687  q_index, txfm_stats->txfm_block_count, txfm_stats->abs_coeff_mean,
688  txfm_stats->coeff_num);
689  }
690 }
691 
692 static INLINE void rc_log_frame_entropy(RATECTRL_LOG *rc_log, int coding_index,
693  double act_rate,
694  double act_coeff_rate) {
695  rc_log->act_rate_list[coding_index] = act_rate;
696  rc_log->act_coeff_rate_list[coding_index] = act_coeff_rate;
697 }
698 
699 static INLINE void rc_log_record_chunk_info(RATECTRL_LOG *rc_log,
700  int base_q_index,
701  int coding_frame_count) {
702  rc_log->base_q_index = base_q_index;
703  rc_log->coding_frame_count = coding_frame_count;
704 }
705 
706 static INLINE void rc_log_show(const RATECTRL_LOG *rc_log) {
707  printf("= chunk 1\n");
708  printf("coding_frame_count %d base_q_index %d\n", rc_log->coding_frame_count,
709  rc_log->base_q_index);
710  printf("= frame %d\n", rc_log->coding_frame_count);
711  for (int coding_idx = 0; coding_idx < rc_log->coding_frame_count;
712  coding_idx++) {
713  printf(
714  "coding_idx %d update_type %d q %d qstep_ratio %f est_coeff_rate %f "
715  "act_coeff_rate %f act_rate %f\n",
716  coding_idx, rc_log->update_type_list[coding_idx],
717  rc_log->q_index_list[coding_idx], rc_log->qstep_ratio_list[coding_idx],
718  rc_log->est_coeff_rate_list[coding_idx],
719  rc_log->act_coeff_rate_list[coding_idx],
720  rc_log->act_rate_list[coding_idx]);
721  }
722 }
723 #endif // CONFIG_RATECTRL_LOG
724 
726 #ifdef __cplusplus
727 } // extern "C"
728 #endif
729 
730 #endif // AOM_AV1_ENCODER_TPL_MODEL_H_
enum aom_bit_depth aom_bit_depth_t
Bit depth for codecThis enumeration determines the bit depth of the codec.
int av1_tpl_setup_stats(struct AV1_COMP *cpi, int gop_eval, const struct EncodeFrameParams *const frame_params)
Implements temporal dependency modelling for a GOP (GF/ARF group) and selects between 16 and 32 frame...
Describes look ahead buffer operations.
Top level encoder structure.
Definition: encoder.h:2759
Top level primary encoder structure.
Definition: encoder.h:2469
Params related to MB_MODE_INFO arrays and related info.
Definition: av1_common_int.h:501
Input frames and last input frame.
Definition: encoder.h:3470
contains per-frame encoding parameters decided upon by av1_encode_strategy() and passed down to av1_e...
Definition: encoder.h:3482
Data related to the current GF/ARF group and the individual frames within the group.
Definition: firstpass.h:344
Params related to temporal dependency model.
Definition: tpl_model.h:146
const YV12_BUFFER_CONFIG * src_ref_frame[INTER_REFS_PER_FRAME]
Definition: tpl_model.h:210
struct scale_factors sf
Definition: tpl_model.h:198
int ready
Definition: tpl_model.h:150
TplDepFrame tpl_stats_buffer[MAX_LENGTH_TPL_FRAME_STATS]
Definition: tpl_model.h:167
uint8_t tpl_bsize_1d
Definition: tpl_model.h:160
AV1TplRowMultiThreadSync tpl_mt_sync
Definition: tpl_model.h:223
TplDepFrame * tpl_frame
Definition: tpl_model.h:193
int border_in_pixels
Definition: tpl_model.h:228
TplDepStats * tpl_stats_pool[MAX_LAG_BUFFERS]
Definition: tpl_model.h:174
TplTxfmStats * txfm_stats_list
Definition: tpl_model.h:182
YV12_BUFFER_CONFIG tpl_rec_pool[MAX_LAG_BUFFERS]
Definition: tpl_model.h:188
uint8_t tpl_stats_block_mis_log2
Definition: tpl_model.h:155
int frame_idx
Definition: tpl_model.h:203
const YV12_BUFFER_CONFIG * ref_frame[INTER_REFS_PER_FRAME]
Definition: tpl_model.h:217
Encoder's parameters related to the current coding block.
Definition: block.h:849
YV12 frame buffer data structure.
Definition: yv12config.h:39