AOMedia AV1 Codec
tpl_model.h
1 /*
2  * Copyright (c) 2019, Alliance for Open Media. All rights reserved
3  *
4  * This source code is subject to the terms of the BSD 2 Clause License and
5  * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6  * was not distributed with this source code in the LICENSE file, you can
7  * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8  * Media Patent License 1.0 was not distributed with this source code in the
9  * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10  */
11 
12 #ifndef AOM_AV1_ENCODER_TPL_MODEL_H_
13 #define AOM_AV1_ENCODER_TPL_MODEL_H_
14 
15 #ifdef __cplusplus
16 extern "C" {
17 #endif
18 
21 struct AV1_PRIMARY;
22 struct AV1_COMP;
23 struct AV1_SEQ_CODING_TOOLS;
24 struct EncodeFrameParams;
25 struct EncodeFrameInput;
26 struct GF_GROUP;
27 struct TPL_INFO;
28 
29 #include "config/aom_config.h"
30 
31 #include "aom_scale/yv12config.h"
32 
33 #include "av1/common/mv.h"
34 #include "av1/common/scale.h"
35 #include "av1/encoder/block.h"
36 #include "av1/encoder/lookahead.h"
37 #include "av1/encoder/ratectrl.h"
38 
39 static INLINE BLOCK_SIZE convert_length_to_bsize(int length) {
40  switch (length) {
41  case 64: return BLOCK_64X64;
42  case 32: return BLOCK_32X32;
43  case 16: return BLOCK_16X16;
44  case 8: return BLOCK_8X8;
45  case 4: return BLOCK_4X4;
46  default:
47  assert(0 && "Invalid block size for tpl model");
48  return BLOCK_16X16;
49  }
50 }
51 
52 typedef struct AV1TplRowMultiThreadSync {
53 #if CONFIG_MULTITHREAD
54  // Synchronization objects for top-right dependency.
55  pthread_mutex_t *mutex_;
56  pthread_cond_t *cond_;
57 #endif
58  // Buffer to store the macroblock whose encoding is complete.
59  // num_finished_cols[i] stores the number of macroblocks which finished
60  // encoding in the ith macroblock row.
61  int *num_finished_cols;
62  // Number of extra macroblocks of the top row to be complete for encoding
63  // of the current macroblock to start. A value of 1 indicates top-right
64  // dependency.
65  int sync_range;
66  // Number of macroblock rows.
67  int rows;
68  // Number of threads processing the current tile.
69  int num_threads_working;
70 } AV1TplRowMultiThreadSync;
71 
72 typedef struct AV1TplRowMultiThreadInfo {
73  // Row synchronization related function pointers.
74  void (*sync_read_ptr)(AV1TplRowMultiThreadSync *tpl_mt_sync, int r, int c);
75  void (*sync_write_ptr)(AV1TplRowMultiThreadSync *tpl_mt_sync, int r, int c,
76  int cols);
77 } AV1TplRowMultiThreadInfo;
78 
79 // TODO(jingning): This needs to be cleaned up next.
80 
81 // TPL stats buffers are prepared for every frame in the GOP,
82 // including (internal) overlays and (internal) arfs.
83 // In addition, frames in the lookahead that are outside of the GOP
84 // are also used.
85 // Thus it should use
86 // (gop_length) + (# overlays) + (MAX_LAG_BUFFERS - gop_len) =
87 // MAX_LAG_BUFFERS + (# overlays)
88 // 2 * MAX_LAG_BUFFERS is therefore a safe estimate.
89 // TODO(bohanli): test setting it to 1.5 * MAX_LAG_BUFFER
90 #define MAX_TPL_FRAME_IDX (2 * MAX_LAG_BUFFERS)
91 // The first REF_FRAMES + 1 buffers are reserved.
92 // tpl_data->tpl_frame starts after REF_FRAMES + 1
93 #define MAX_LENGTH_TPL_FRAME_STATS (MAX_TPL_FRAME_IDX + REF_FRAMES + 1)
94 #define TPL_DEP_COST_SCALE_LOG2 4
95 
96 #define TPL_EPSILON 0.0000001
97 
98 typedef struct TplTxfmStats {
99  int ready; // Whether abs_coeff_mean is ready
100  double abs_coeff_sum[256]; // Assume we are using 16x16 transform block
101  double abs_coeff_mean[256];
102  int txfm_block_count;
103  int coeff_num;
104 } TplTxfmStats;
105 
106 typedef struct TplDepStats {
107  int64_t srcrf_sse;
108  int64_t srcrf_dist;
109  int64_t recrf_sse;
110  int64_t recrf_dist;
111  int64_t intra_sse;
112  int64_t intra_dist;
113  int64_t cmp_recrf_dist[2];
114  int64_t mc_dep_rate;
115  int64_t mc_dep_dist;
116  int64_t pred_error[INTER_REFS_PER_FRAME];
117  int32_t intra_cost;
118  int32_t inter_cost;
119  int32_t srcrf_rate;
120  int32_t recrf_rate;
121  int32_t intra_rate;
122  int32_t cmp_recrf_rate[2];
123  int_mv mv[INTER_REFS_PER_FRAME];
124  int8_t ref_frame_index[2];
125 } TplDepStats;
126 
127 typedef struct TplDepFrame {
128  uint8_t is_valid;
129  TplDepStats *tpl_stats_ptr;
130  const YV12_BUFFER_CONFIG *gf_picture;
131  YV12_BUFFER_CONFIG *rec_picture;
132  int ref_map_index[REF_FRAMES];
133  int stride;
134  int width;
135  int height;
136  int mi_rows;
137  int mi_cols;
138  int base_rdmult;
139  uint32_t frame_display_index;
140 } TplDepFrame;
141 
146 typedef struct TplParams {
150  int ready;
151 
156 
160  uint8_t tpl_bsize_1d;
161 
167  TplDepFrame tpl_stats_buffer[MAX_LENGTH_TPL_FRAME_STATS];
168 
174  TplDepStats *tpl_stats_pool[MAX_LAG_BUFFERS];
175 
182  TplTxfmStats *txfm_stats_list;
183 
189 
193  TplDepFrame *tpl_frame;
194 
198  struct scale_factors sf;
199 
204 
210  const YV12_BUFFER_CONFIG *src_ref_frame[INTER_REFS_PER_FRAME];
211 
217  const YV12_BUFFER_CONFIG *ref_frame[INTER_REFS_PER_FRAME];
218 
223  AV1TplRowMultiThreadSync tpl_mt_sync;
224 
229 
230 } TplParams;
231 
232 #if CONFIG_BITRATE_ACCURACY || CONFIG_RATECTRL_LOG
233 #define VBR_RC_INFO_MAX_FRAMES 500
234 #endif // CONFIG_BITRATE_ACCURACY || CONFIG_RATECTRL_LOG
235 
236 #if CONFIG_BITRATE_ACCURACY
237 
242 typedef struct {
243  int ready;
244  double total_bit_budget; // The total bit budget of the entire video
245  int show_frame_count; // Number of show frames in the entire video
246 
247  int gop_showframe_count; // The number of show frames in the current gop
248  double gop_bit_budget; // The bitbudget for the current gop
249  double scale_factors[FRAME_UPDATE_TYPES]; // Scale factors to improve the
250  // budget estimation
251  double mv_scale_factors[FRAME_UPDATE_TYPES]; // Scale factors to improve
252  // MV entropy estimation
253 
254  // === Below this line are GOP related data that will be updated per GOP ===
255  int base_q_index; // Stores the base q index.
256  int q_index_list_ready;
257  int q_index_list[VBR_RC_INFO_MAX_FRAMES]; // q indices for the current
258  // GOP
259 
260  // Array to store qstep_ratio for each frame in a GOP
261  double qstep_ratio_list[VBR_RC_INFO_MAX_FRAMES];
262 
263 #if CONFIG_THREE_PASS
264  TplTxfmStats txfm_stats_list[VBR_RC_INFO_MAX_FRAMES];
265  FRAME_UPDATE_TYPE update_type_list[VBR_RC_INFO_MAX_FRAMES];
266  int gop_start_idx_list[VBR_RC_INFO_MAX_FRAMES];
267  int gop_length_list[VBR_RC_INFO_MAX_FRAMES];
268  int cur_gop_idx;
269  int total_frame_count;
270  int gop_count;
271 #endif // CONFIG_THREE_PASS
272 } VBR_RATECTRL_INFO;
273 
274 static INLINE void vbr_rc_reset_gop_data(VBR_RATECTRL_INFO *vbr_rc_info) {
275  vbr_rc_info->q_index_list_ready = 0;
276  av1_zero(vbr_rc_info->q_index_list);
277 }
278 
279 void av1_vbr_rc_init(VBR_RATECTRL_INFO *vbr_rc_info, double total_bit_budget,
280  int show_frame_count);
281 
282 int av1_vbr_rc_frame_coding_idx(const VBR_RATECTRL_INFO *vbr_rc_info,
283  int gf_frame_index);
284 
285 void av1_vbr_rc_append_tpl_info(VBR_RATECTRL_INFO *vbr_rc_info,
286  const struct TPL_INFO *tpl_info);
287 
288 void av1_vbr_rc_set_gop_bit_budget(VBR_RATECTRL_INFO *vbr_rc_info,
289  int gop_showframe_count);
290 
291 void av1_vbr_rc_compute_q_indices(int base_q_index, int frame_count,
292  const double *qstep_ratio_list,
293  aom_bit_depth_t bit_depth, int *q_index_list);
294 
303 void av1_vbr_rc_update_q_index_list(VBR_RATECTRL_INFO *vbr_rc_info,
304  const TplParams *tpl_data,
305  const struct GF_GROUP *gf_group,
306  aom_bit_depth_t bit_depth);
307 /*
308  *!\brief Compute the number of bits needed to encode a GOP
309  *
310  * \param[in] base_q_index base layer q_index
311  * \param[in] bit_depth bit depth
312  * \param[in] update_type_scale_factors array of scale factors for each
313  * update_type
314  * \param[in] frame_count size of update_type_list,
315  * qstep_ratio_list stats_list,
316  * q_index_list and
317  * estimated_bitrate_byframe
318  * \param[in] update_type_list array of update_type, one per frame
319  * \param[in] qstep_ratio_list array of qstep_ratio, one per frame
320  * \param[in] stats_list array of transform stats, one per
321  * frame
322  * \param[out] q_index_list array of q_index, one per frame
323  * \param[out] estimated_bitrate_byframe array to keep track of frame
324  * bitrate
325  *
326  * \return The estimated GOP bitrate.
327  *
328  */
329 double av1_vbr_rc_info_estimate_gop_bitrate(
330  int base_q_index, aom_bit_depth_t bit_depth,
331  const double *update_type_scale_factors, int frame_count,
332  const FRAME_UPDATE_TYPE *update_type_list, const double *qstep_ratio_list,
333  const TplTxfmStats *stats_list, int *q_index_list,
334  double *estimated_bitrate_byframe);
335 
357 int av1_vbr_rc_info_estimate_base_q(
358  double bit_budget, aom_bit_depth_t bit_depth,
359  const double *update_type_scale_factors, int frame_count,
360  const FRAME_UPDATE_TYPE *update_type_list, const double *qstep_ratio_list,
361  const TplTxfmStats *stats_list, int *q_index_list,
362  double *estimated_bitrate_byframe);
363 
364 #endif // CONFIG_BITRATE_ACCURACY
365 
366 #if CONFIG_RD_COMMAND
367 typedef enum {
368  RD_OPTION_NONE,
369  RD_OPTION_SET_Q,
370  RD_OPTION_SET_Q_RDMULT
371 } RD_OPTION;
372 
373 typedef struct RD_COMMAND {
374  RD_OPTION option_ls[MAX_LENGTH_TPL_FRAME_STATS];
375  int q_index_ls[MAX_LENGTH_TPL_FRAME_STATS];
376  int rdmult_ls[MAX_LENGTH_TPL_FRAME_STATS];
377  int frame_count;
378  int frame_index;
379 } RD_COMMAND;
380 
381 void av1_read_rd_command(const char *filepath, RD_COMMAND *rd_command);
382 #endif // CONFIG_RD_COMMAND
383 
392 void av1_setup_tpl_buffers(struct AV1_PRIMARY *const ppi,
393  CommonModeInfoParams *const mi_params, int width,
394  int height, int byte_alignment, int lag_in_frames);
395 
407 int av1_tpl_setup_stats(struct AV1_COMP *cpi, int gop_eval,
408  const struct EncodeFrameParams *const frame_params);
409 
412 void av1_tpl_preload_rc_estimate(
413  struct AV1_COMP *cpi, const struct EncodeFrameParams *const frame_params);
414 
415 int av1_tpl_ptr_pos(int mi_row, int mi_col, int stride, uint8_t right_shift);
416 
417 void av1_init_tpl_stats(TplParams *const tpl_data);
418 
419 int av1_tpl_stats_ready(const TplParams *tpl_data, int gf_frame_index);
420 
421 void av1_tpl_rdmult_setup(struct AV1_COMP *cpi);
422 
423 void av1_tpl_rdmult_setup_sb(struct AV1_COMP *cpi, MACROBLOCK *const x,
424  BLOCK_SIZE sb_size, int mi_row, int mi_col);
425 
426 void av1_mc_flow_dispenser_row(struct AV1_COMP *cpi,
427  TplTxfmStats *tpl_txfm_stats, MACROBLOCK *x,
428  int mi_row, BLOCK_SIZE bsize, TX_SIZE tx_size);
429 
442 double av1_exponential_entropy(double q_step, double b);
443 
457 double av1_laplace_entropy(double q_step, double b, double zero_bin_ratio);
458 
476 double av1_laplace_estimate_frame_rate(int q_index, int block_count,
477  const double *abs_coeff_mean,
478  int coeff_num);
479 
480 /*
481  *!\brief Init TplTxfmStats
482  *
483  * \param[in] tpl_txfm_stats a structure for storing transform stats
484  *
485  */
486 void av1_init_tpl_txfm_stats(TplTxfmStats *tpl_txfm_stats);
487 
488 #if CONFIG_BITRATE_ACCURACY
489 /*
490  *!\brief Accumulate TplTxfmStats
491  *
492  * \param[in] sub_stats a structure for storing sub transform stats
493  * \param[out] accumulated_stats a structure for storing accumulated
494  *transform stats
495  *
496  */
497 void av1_accumulate_tpl_txfm_stats(const TplTxfmStats *sub_stats,
498  TplTxfmStats *accumulated_stats);
499 
500 /*
501  *!\brief Record a transform block into TplTxfmStats
502  *
503  * \param[in] tpl_txfm_stats A structure for storing transform stats
504  * \param[out] coeff An array of transform coefficients. Its size
505  * should equal to tpl_txfm_stats.coeff_num.
506  *
507  */
508 void av1_record_tpl_txfm_block(TplTxfmStats *tpl_txfm_stats,
509  const tran_low_t *coeff);
510 
511 /*
512  *!\brief Update abs_coeff_mean and ready of txfm_stats
513  * If txfm_block_count > 0, this function will use abs_coeff_sum and
514  * txfm_block_count to compute abs_coeff_mean. Moreover, reday flag
515  * will be set to one.
516  *
517  * \param[in] txfm_stats A structure for storing transform stats
518  */
519 void av1_tpl_txfm_stats_update_abs_coeff_mean(TplTxfmStats *txfm_stats);
520 #endif // CONFIG_BITRATE_ACCURACY
521 
537 double av1_estimate_coeff_entropy(double q_step, double b,
538  double zero_bin_ratio, int qcoeff);
539 
552 double av1_estimate_txfm_block_entropy(int q_index,
553  const double *abs_coeff_mean,
554  int *qcoeff_arr, int coeff_num);
555 
556 // TODO(angiebird): Add doxygen description here.
557 int64_t av1_delta_rate_cost(int64_t delta_rate, int64_t recrf_dist,
558  int64_t srcrf_dist, int pix_num);
559 
575 int av1_get_overlap_area(int row_a, int col_a, int row_b, int col_b, int width,
576  int height);
577 
587 int av1_tpl_get_q_index(const TplParams *tpl_data, int gf_frame_index,
588  int leaf_qindex, aom_bit_depth_t bit_depth);
589 
597 double av1_tpl_get_frame_importance(const TplParams *tpl_data,
598  int gf_frame_index);
599 
610 double av1_tpl_get_qstep_ratio(const TplParams *tpl_data, int gf_frame_index);
611 
620 int av1_get_q_index_from_qstep_ratio(int leaf_qindex, double qstep_ratio,
621  aom_bit_depth_t bit_depth);
622 
637 int_mv av1_compute_mv_difference(const TplDepFrame *tpl_frame, int row, int col,
638  int step, int tpl_stride, int right_shift);
639 
647 double av1_tpl_compute_frame_mv_entropy(const TplDepFrame *tpl_frame,
648  uint8_t right_shift);
649 
650 #if CONFIG_RATECTRL_LOG
651 typedef struct {
652  int coding_frame_count;
653  int base_q_index;
654 
655  // Encode decision
656  int q_index_list[VBR_RC_INFO_MAX_FRAMES];
657  double qstep_ratio_list[VBR_RC_INFO_MAX_FRAMES];
658  FRAME_UPDATE_TYPE update_type_list[VBR_RC_INFO_MAX_FRAMES];
659 
660  // Frame stats
661  TplTxfmStats txfm_stats_list[VBR_RC_INFO_MAX_FRAMES];
662 
663  // Estimated encode results
664  double est_coeff_rate_list[VBR_RC_INFO_MAX_FRAMES];
665 
666  // Actual encode results
667  double act_rate_list[VBR_RC_INFO_MAX_FRAMES];
668  double act_coeff_rate_list[VBR_RC_INFO_MAX_FRAMES];
669 } RATECTRL_LOG;
670 
671 static INLINE void rc_log_init(RATECTRL_LOG *rc_log) { av1_zero(*rc_log); }
672 
673 static INLINE void rc_log_frame_stats(RATECTRL_LOG *rc_log, int coding_index,
674  const TplTxfmStats *txfm_stats) {
675  rc_log->txfm_stats_list[coding_index] = *txfm_stats;
676 }
677 
678 static INLINE void rc_log_frame_encode_param(RATECTRL_LOG *rc_log,
679  int coding_index,
680  double qstep_ratio, int q_index,
681  FRAME_UPDATE_TYPE update_type) {
682  rc_log->qstep_ratio_list[coding_index] = qstep_ratio;
683  rc_log->q_index_list[coding_index] = q_index;
684  rc_log->update_type_list[coding_index] = update_type;
685  const TplTxfmStats *txfm_stats = &rc_log->txfm_stats_list[coding_index];
686  rc_log->est_coeff_rate_list[coding_index] = 0;
687  if (txfm_stats->ready) {
688  rc_log->est_coeff_rate_list[coding_index] = av1_laplace_estimate_frame_rate(
689  q_index, txfm_stats->txfm_block_count, txfm_stats->abs_coeff_mean,
690  txfm_stats->coeff_num);
691  }
692 }
693 
694 static INLINE void rc_log_frame_entropy(RATECTRL_LOG *rc_log, int coding_index,
695  double act_rate,
696  double act_coeff_rate) {
697  rc_log->act_rate_list[coding_index] = act_rate;
698  rc_log->act_coeff_rate_list[coding_index] = act_coeff_rate;
699 }
700 
701 static INLINE void rc_log_record_chunk_info(RATECTRL_LOG *rc_log,
702  int base_q_index,
703  int coding_frame_count) {
704  rc_log->base_q_index = base_q_index;
705  rc_log->coding_frame_count = coding_frame_count;
706 }
707 
708 static INLINE void rc_log_show(const RATECTRL_LOG *rc_log) {
709  printf("= chunk 1\n");
710  printf("coding_frame_count %d base_q_index %d\n", rc_log->coding_frame_count,
711  rc_log->base_q_index);
712  printf("= frame %d\n", rc_log->coding_frame_count);
713  for (int coding_idx = 0; coding_idx < rc_log->coding_frame_count;
714  coding_idx++) {
715  printf(
716  "coding_idx %d update_type %d q %d qstep_ratio %f est_coeff_rate %f "
717  "act_coeff_rate %f act_rate %f\n",
718  coding_idx, rc_log->update_type_list[coding_idx],
719  rc_log->q_index_list[coding_idx], rc_log->qstep_ratio_list[coding_idx],
720  rc_log->est_coeff_rate_list[coding_idx],
721  rc_log->act_coeff_rate_list[coding_idx],
722  rc_log->act_rate_list[coding_idx]);
723  }
724 }
725 #endif // CONFIG_RATECTRL_LOG
726 
728 #ifdef __cplusplus
729 } // extern "C"
730 #endif
731 
732 #endif // AOM_AV1_ENCODER_TPL_MODEL_H_
int border_in_pixels
Definition: tpl_model.h:228
uint8_t tpl_bsize_1d
Definition: tpl_model.h:160
int ready
Definition: tpl_model.h:150
Data related to the current GF/ARF group and the individual frames within the group.
Definition: firstpass.h:354
Params related to temporal dependency model.
Definition: tpl_model.h:146
int av1_tpl_setup_stats(struct AV1_COMP *cpi, int gop_eval, const struct EncodeFrameParams *const frame_params)
Implements temporal dependency modelling for a GOP (GF/ARF group) and selects between 16 and 32 frame...
Top level primary encoder structure.
Definition: encoder.h:2519
const YV12_BUFFER_CONFIG * ref_frame[INTER_REFS_PER_FRAME]
Definition: tpl_model.h:217
contains per-frame encoding parameters decided upon by av1_encode_strategy() and passed down to av1_e...
Definition: encoder.h:3579
Params related to MB_MODE_INFO arrays and related info.
Definition: av1_common_int.h:503
TplDepFrame tpl_stats_buffer[MAX_LENGTH_TPL_FRAME_STATS]
Definition: tpl_model.h:167
AV1TplRowMultiThreadSync tpl_mt_sync
Definition: tpl_model.h:223
struct scale_factors sf
Definition: tpl_model.h:198
TplDepStats * tpl_stats_pool[MAX_LAG_BUFFERS]
Definition: tpl_model.h:174
YV12_BUFFER_CONFIG tpl_rec_pool[MAX_LAG_BUFFERS]
Definition: tpl_model.h:188
Describes look ahead buffer operations.
YV12 frame buffer data structure.
Definition: yv12config.h:44
Input frames and last input frame.
Definition: encoder.h:3567
int frame_idx
Definition: tpl_model.h:203
TplTxfmStats * txfm_stats_list
Definition: tpl_model.h:182
Top level encoder structure.
Definition: encoder.h:2815
enum aom_bit_depth aom_bit_depth_t
Bit depth for codecThis enumeration determines the bit depth of the codec.
Encoder's parameters related to the current coding block.
Definition: block.h:878
TplDepFrame * tpl_frame
Definition: tpl_model.h:193
uint8_t tpl_stats_block_mis_log2
Definition: tpl_model.h:155
const YV12_BUFFER_CONFIG * src_ref_frame[INTER_REFS_PER_FRAME]
Definition: tpl_model.h:210