data_juicer_sandbox.evaluators module#

class data_juicer_sandbox.evaluators.BaseEvaluator(eval_config: dict)[source]#

Bases: object

__init__(eval_config: dict)[source]#
run(eval_type, eval_obj=None, **kwargs) → dict[source]#
conduct the evaluation given specified measurement

on specified target object;

return evaluated results in a dict: {res_name: res_val}

class data_juicer_sandbox.evaluators.Gpt3QualityEvaluator(eval_config: dict)[source]#

Bases: BaseEvaluator

run(eval_type, eval_obj=None, **kwargs)[source]#
conduct the evaluation given specified measurement

on specified target object;

return evaluated results in a dict: {res_name: res_val}

class data_juicer_sandbox.evaluators.InceptionEvaluator(eval_config: dict)[source]#

Bases: BaseEvaluator

run(eval_type, eval_obj=None, **kwargs)[source]#
conduct the evaluation given specified measurement

on specified target object;

return evaluated results in a dict: {res_name: res_val}

class data_juicer_sandbox.evaluators.AccuracyEvaluator(eval_config: dict)[source]#

Bases: BaseEvaluator

A simple evaluator to compare the labels in the predicted ones and ground truth.

The config file for this evaluator should at least include the following items: 1. type: must be “accuracy”. 2. predicted_dataset_path: Required. The dataset path to the data that stores the predicted labels. 3. ground_truth_dataset_path: The dataset path to the data that stores the ground truth labels. If it’s None,

we assume that the ground truth labels are already in the predicted_dataset_path.

  1. predicted_label_key: the key name to store the predicted labels. ‘.’ operator is allowed.

  2. ground_truth_label_key: the key name to store the ground truth labels. ‘.’ operator is allowed.

__init__(eval_config: dict)[source]#
run(eval_type, eval_obj=None, **kwargs)[source]#
conduct the evaluation given specified measurement

on specified target object;

return evaluated results in a dict: {res_name: res_val}

class data_juicer_sandbox.evaluators.MSEEvaluator(eval_config: dict)[source]#

Bases: BaseEvaluator

A simple evaluator to compute the MSE between the predicted values and ground truth.

The config file for this evaluator should at least include the following items: 1. type: must be “mse”. 2. predicted_dataset_path: Required. The dataset path to the data that stores the predicted labels. 3. ground_truth_dataset_path: The dataset path to the data that stores the ground truth labels. If it’s None,

we assume that the ground truth labels are already in the predicted_dataset_path.

  1. predicted_value_key: the key name to store the predicted values. ‘.’ operator is allowed.

  2. ground_truth_value_key: the key name to store the ground truth values. ‘.’ operator is allowed.

__init__(eval_config: dict)[source]#
run(eval_type, eval_obj=None, **kwargs)[source]#
conduct the evaluation given specified measurement

on specified target object;

return evaluated results in a dict: {res_name: res_val}

class data_juicer_sandbox.evaluators.HelmEvaluator(eval_config: dict)[source]#

Bases: BaseEvaluator

run(eval_type, eval_obj=None, **kwargs)[source]#
conduct the evaluation given specified measurement

on specified target object;

return evaluated results in a dict: {res_name: res_val}

class data_juicer_sandbox.evaluators.GptEvaluator(eval_config: dict)[source]#

Bases: BaseEvaluator

run(eval_type, eval_obj=None, **kwargs)[source]#
conduct the evaluation given specified measurement

on specified target object;

return evaluated results in a dict: {res_name: res_val}

class data_juicer_sandbox.evaluators.VideoFvdEvaluator(eval_config: dict)[source]#

Bases: BaseEvaluator

run(eval_type, eval_obj=None, **kwargs)[source]#
conduct the evaluation given specified measurement

on specified target object;

return evaluated results in a dict: {res_name: res_val}

class data_juicer_sandbox.evaluators.Gpt4VEvaluator(eval_config: dict)[source]#

Bases: BaseEvaluator

run(eval_type, eval_obj=None, **kwargs)[source]#
conduct the evaluation given specified measurement

on specified target object;

return evaluated results in a dict: {res_name: res_val}

class data_juicer_sandbox.evaluators.LmHarnessEvaluator(eval_config: dict)[source]#

Bases: BaseEvaluator

run(eval_type, eval_obj=None, **kwargs)[source]#
conduct the evaluation given specified measurement

on specified target object;

return evaluated results in a dict: {res_name: res_val}

class data_juicer_sandbox.evaluators.ModelscopeEvaluator(eval_config: dict)[source]#

Bases: BaseEvaluator

run(eval_type, eval_obj=None, **kwargs)[source]#
conduct the evaluation given specified measurement

on specified target object;

return evaluated results in a dict: {res_name: res_val}