data_juicer.utils.fingerprint_utils module# class data_juicer.utils.fingerprint_utils.Hasher[source]# Bases: object Hasher that accepts python objects as inputs. dispatch: Dict = {}# __init__()[source]# classmethod hash_bytes(value: bytes | List[bytes]) → str[source]# classmethod hash_default(value: Any) → str[source]# Use dill to serialize objects to avoid serialization failures. classmethod hash(value: Any) → str[source]# update(value: Any) → None[source]# hexdigest() → str[source]# data_juicer.utils.fingerprint_utils.update_fingerprint(fingerprint, transform, transform_args)[source]# Combining various objects to update the fingerprint. data_juicer.utils.fingerprint_utils.generate_fingerprint(ds, *args, **kwargs)[source]# Generate new fingerprints by using various kwargs of the dataset.
