Skip to content

plaid.storage.hf_datasets

plaid.storage.hf_datasets

Package for HF_datasets storage.

plaid.storage.hf_datasets.HFBackend

plaid.storage.hf_datasets.HFBackend.name class-attribute instance-attribute 

name = 'hf_datasets'

plaid.storage.hf_datasets.HFBackend.init_from_disk staticmethod 

init_from_disk(path)
Source code in plaid/storage/hf_datasets/__init__.py 
30
31
32
@staticmethod
def init_from_disk(path: Union[str, Path]) -> Mapping[str, Any]:
    return init_datasetdict_from_disk(path=path)

plaid.storage.hf_datasets.HFBackend.download_from_hub staticmethod 

download_from_hub(
    repo_id,
    local_dir,
    split_ids=None,
    features=None,
    overwrite=False,
)
Source code in plaid/storage/hf_datasets/__init__.py 
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
@staticmethod
def download_from_hub(
    repo_id: str,
    local_dir: Union[str, Path],
    split_ids: Optional[dict[str, Iterable[int]]] = None,
    features: Optional[list[str]] = None,  # noqa: ARG001
    overwrite: bool = False,
) -> str:
    return download_datasetdict_from_hub(
        repo_id=repo_id,
        local_dir=local_dir,
        split_ids=split_ids,
        features=features,
        overwrite=overwrite,
    )

plaid.storage.hf_datasets.HFBackend.init_datasetdict_streaming_from_hub staticmethod 

init_datasetdict_streaming_from_hub(
    repo_id, split_ids=None, features=None
)
Source code in plaid/storage/hf_datasets/__init__.py 
50
51
52
53
54
55
56
57
58
@staticmethod
def init_datasetdict_streaming_from_hub(
    repo_id: str,
    split_ids: Optional[dict[str, Iterable[int]]] = None,
    features: Optional[list[str]] = None,
) -> IterableDatasetDict:
    return init_datasetdict_streaming_from_hub(
        repo_id=repo_id, split_ids=split_ids, features=features
    )

plaid.storage.hf_datasets.HFBackend.generate_to_disk staticmethod 

generate_to_disk(
    output_folder,
    generators,
    variable_schema,
    gen_kwargs=None,
    num_proc=1,
    verbose=False,
)
Source code in plaid/storage/hf_datasets/__init__.py 
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
@staticmethod
def generate_to_disk(
    output_folder: Union[str, Path],
    generators: dict,
    variable_schema: dict[str, dict],
    gen_kwargs: Optional[dict[str, dict[str, list]]] = None,
    num_proc: int = 1,
    verbose: bool = False,
) -> None:
    return generate_datasetdict_to_disk(
        output_folder=output_folder,
        generators=generators,
        variable_schema=variable_schema,
        gen_kwargs=gen_kwargs,
        num_proc=num_proc,
        verbose=verbose,
    )

plaid.storage.hf_datasets.HFBackend.push_local_to_hub staticmethod 

push_local_to_hub(repo_id, local_dir, num_workers=1)
Source code in plaid/storage/hf_datasets/__init__.py 
78
79
80
81
82
83
84
@staticmethod
def push_local_to_hub(
    repo_id: str, local_dir: Union[str, Path], num_workers: int = 1
) -> None:
    return push_local_datasetdict_to_hub(
        repo_id=repo_id, local_dir=local_dir, num_workers=num_workers
    )

plaid.storage.hf_datasets.HFBackend.configure_dataset_card staticmethod 

configure_dataset_card(
    repo_id,
    infos,
    local_dir=None,
    viewer=False,
    pretty_name=None,
    dataset_long_description=None,
    illustration_urls=None,
    arxiv_paper_urls=None,
)
Source code in plaid/storage/hf_datasets/__init__.py 
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
@staticmethod
def configure_dataset_card(
    repo_id: str,
    infos: Infos,
    local_dir: Optional[Union[str, Path]] = None,
    viewer: bool = False,
    pretty_name: Optional[str] = None,
    dataset_long_description: Optional[str] = None,
    illustration_urls: Optional[list[str]] = None,
    arxiv_paper_urls: Optional[list[str]] = None,
) -> None:
    return configure_dataset_card(
        repo_id=repo_id,
        infos=infos,
        local_dir=local_dir,
        viewer=viewer,
        pretty_name=pretty_name,
        dataset_long_description=dataset_long_description,
        illustration_urls=illustration_urls,
        arxiv_paper_urls=arxiv_paper_urls,
    )

plaid.storage.hf_datasets.HFBackend.to_var_sample_dict staticmethod 

to_var_sample_dict(
    dataset, idx, features=None, indexers=None
)
Source code in plaid/storage/hf_datasets/__init__.py 
108
109
110
111
112
113
114
115
116
117
@staticmethod
def to_var_sample_dict(
    dataset: Dataset,
    idx: int,
    features: Optional[list[str]] = None,
    indexers: Optional[dict[str, Any]] = None,
) -> dict[str, Optional[np.ndarray]]:
    return to_var_sample_dict(
        ds=dataset, i=idx, features=features, indexers=indexers
    )

plaid.storage.hf_datasets.HFBackend.sample_to_var_sample_dict staticmethod 

sample_to_var_sample_dict(sample)
Source code in plaid/storage/hf_datasets/__init__.py 
119
120
121
@staticmethod
def sample_to_var_sample_dict(sample: dict[str, Any]) -> dict[str, Any]:
    return sample_to_var_sample_dict(hf_sample=sample)