mirror of
https://github.com/pytorch/pytorch.git
synced 2025-10-20 21:14:14 +08:00
docs: fix docstrings for datapipes and other (#112765)
Fixes #112636 Before: 265 ``` torch/utils/data/datapipes/dataframe/structures.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/dataframe/structures.py:8 in public class `DataChunkDF`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/dataframe/structures.py:8 in public class `DataChunkDF`: D208: Docstring is over-indented torch/utils/data/datapipes/dataframe/structures.py:8 in public class `DataChunkDF`: D400: First line should end with a period (not ',') torch/utils/data/datapipes/dataframe/structures.py:13 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/dataframe/structures.py:17 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/datapipe.py:43 in public class `IterDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/datapipe.py:119 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:122 in public method `__getattr__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:135 in public method `register_function`: D102: Missing docstring in public method torch/utils/data/datapipes/datapipe.py:139 in public method `register_datapipe_as_function`: D102: Missing docstring in public method torch/utils/data/datapipes/datapipe.py:161 in public method `__getstate__`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/datapipe.py:161 in public method `__getstate__`: D401: First line should be in imperative mood; try rephrasing (found 'This') torch/utils/data/datapipes/datapipe.py:171 in public method `__reduce_ex__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:180 in public method `set_getstate_hook`: D102: Missing docstring in public method torch/utils/data/datapipes/datapipe.py:186 in public method `set_reduce_ex_hook`: D102: Missing docstring in public method torch/utils/data/datapipes/datapipe.py:191 in public method `__repr__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:197 in public method `__str__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:203 in public method `__dir__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:208 in public method `reset`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/datapipe.py:208 in public method `reset`: D400: First line should end with a period (not ',') torch/utils/data/datapipes/datapipe.py:217 in public class `DFIterDataPipe`: D101: Missing docstring in public class torch/utils/data/datapipes/datapipe.py:223 in public class `MapDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/datapipe.py:261 in public method `__getattr__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:274 in public method `register_function`: D102: Missing docstring in public method torch/utils/data/datapipes/datapipe.py:278 in public method `register_datapipe_as_function`: D102: Missing docstring in public method torch/utils/data/datapipes/datapipe.py:293 in public method `__getstate__`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/datapipe.py:293 in public method `__getstate__`: D401: First line should be in imperative mood; try rephrasing (found 'This') torch/utils/data/datapipes/datapipe.py:303 in public method `__reduce_ex__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:312 in public method `set_getstate_hook`: D102: Missing docstring in public method torch/utils/data/datapipes/datapipe.py:318 in public method `set_reduce_ex_hook`: D102: Missing docstring in public method torch/utils/data/datapipes/datapipe.py:323 in public method `__repr__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:329 in public method `__str__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:335 in public method `__dir__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:392 in public class `DataChunk`: D101: Missing docstring in public class torch/utils/data/datapipes/datapipe.py:393 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/datapipe.py:397 in public method `as_str`: D102: Missing docstring in public method torch/utils/data/datapipes/datapipe.py:401 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:404 in public method `raw_iterator`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/callable.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/callable.py:23 in public class `MapperIterDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/iter/callable.py:23 in public class `MapperIterDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/callable.py:63 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/callable.py:121 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/callable.py:125 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/callable.py:173 in public class `CollatorIterDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/callable.py:213 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/combinatorics.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/combinatorics.py:18 in public class `SamplerIterDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/iter/combinatorics.py:29 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/combinatorics.py:44 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combinatorics.py:47 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combinatorics.py:56 in public class `ShufflerIterDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/iter/combinatorics.py:56 in public class `ShufflerIterDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/combinatorics.py:56 in public class `ShufflerIterDataPipe`: D400: First line should end with a period (not 'r') torch/utils/data/datapipes/iter/combinatorics.py:94 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/combinatorics.py:114 in public method `set_shuffle`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/combinatorics.py:118 in public method `set_seed`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/combinatorics.py:122 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combinatorics.py:137 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combinatorics.py:142 in public method `reset`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/combinatorics.py:150 in public method `__getstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combinatorics.py:165 in public method `__setstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combinatorics.py:179 in public method `__del__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combining.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/combining.py:26 in public class `ConcaterIterDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/iter/combining.py:26 in public class `ConcaterIterDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/combining.py:26 in public class `ConcaterIterDataPipe`: D400: First line should end with a period (not 'l') torch/utils/data/datapipes/iter/combining.py:44 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/combining.py:51 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combining.py:55 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combining.py:64 in public class `ForkerIterDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/iter/combining.py:92 in public method `__new__`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/combining.py:108 in private class `_ContainerTemplate`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/iter/combining.py:108 in private class `_ContainerTemplate`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/combining.py:108 in private class `_ContainerTemplate`: D400: First line should end with a period (not 'd') torch/utils/data/datapipes/iter/combining.py:126 in private method `get_length_by_instance`: D200: One-line docstring should fit on one line with quotes (found 3) torch/utils/data/datapipes/iter/combining.py:126 in private method `get_length_by_instance`: D400: First line should end with a period (not '`') torch/utils/data/datapipes/iter/combining.py:136 in private class `_ForkerIterDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/iter/combining.py:136 in private class `_ForkerIterDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/combining.py:136 in private class `_ForkerIterDataPipe`: D400: First line should end with a period (not 's') torch/utils/data/datapipes/iter/combining.py:275 in private class `_ChildDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/iter/combining.py:275 in private class `_ChildDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/combining.py:275 in private class `_ChildDataPipe`: D400: First line should end with a period (not 's') torch/utils/data/datapipes/iter/combining.py:320 in private method `_set_main_datapipe_valid_iterator_id`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/combining.py:343 in private method `_check_valid_iterator_id`: D200: One-line docstring should fit on one line with quotes (found 3) torch/utils/data/datapipes/iter/combining.py:351 in public class `DemultiplexerIterDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/iter/combining.py:351 in public class `DemultiplexerIterDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/combining.py:351 in public class `DemultiplexerIterDataPipe`: D400: First line should end with a period (not 'n') torch/utils/data/datapipes/iter/combining.py:384 in public method `__new__`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/combining.py:399 in private class `_DemultiplexerIterDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/combining.py:399 in private class `_DemultiplexerIterDataPipe`: D400: First line should end with a period (not 's') torch/utils/data/datapipes/iter/combining.py:534 in public class `MultiplexerIterDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/iter/combining.py:534 in public class `MultiplexerIterDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/combining.py:534 in public class `MultiplexerIterDataPipe`: D400: First line should end with a period (not ',') torch/utils/data/datapipes/iter/combining.py:549 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/combining.py:553 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combining.py:566 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combining.py:572 in public method `reset`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/combining.py:575 in public method `__getstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combining.py:585 in public method `__setstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combining.py:593 in public method `__del__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combining.py:599 in public class `ZipperIterDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/iter/combining.py:599 in public class `ZipperIterDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/combining.py:615 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/combining.py:622 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combining.py:626 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/filelister.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/filelister.py:15 in public class `FileListerIterDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/filelister.py:36 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/filelister.py:58 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/filelister.py:62 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/fileopener.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/fileopener.py:15 in public class `FileOpenerIterDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/fileopener.py:15 in public class `FileOpenerIterDataPipe`: D400: First line should end with a period (not 'm') torch/utils/data/datapipes/iter/fileopener.py:42 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/fileopener.py:66 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/fileopener.py:69 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/grouping.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/grouping.py:31 in public class `BatcherIterDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/iter/grouping.py:31 in public class `BatcherIterDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/grouping.py:31 in public class `BatcherIterDataPipe`: D400: First line should end with a period (not 's') torch/utils/data/datapipes/iter/grouping.py:55 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/grouping.py:68 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/grouping.py:79 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/grouping.py:91 in public class `UnBatcherIterDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/grouping.py:91 in public class `UnBatcherIterDataPipe`: D400: First line should end with a period (not 'l') torch/utils/data/datapipes/iter/grouping.py:112 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/grouping.py:118 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/grouping.py:143 in public class `GrouperIterDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/iter/grouping.py:143 in public class `GrouperIterDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/grouping.py:143 in public class `GrouperIterDataPipe`: D400: First line should end with a period (not ',') torch/utils/data/datapipes/iter/grouping.py:185 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/grouping.py:233 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/grouping.py:257 in public method `reset`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/grouping.py:261 in public method `__getstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/grouping.py:278 in public method `__setstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/grouping.py:294 in public method `__del__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/routeddecoder.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/routeddecoder.py:19 in public class `RoutedDecoderIterDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/routeddecoder.py:19 in public class `RoutedDecoderIterDataPipe`: D400: First line should end with a period (not 'a') torch/utils/data/datapipes/iter/routeddecoder.py:37 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/routeddecoder.py:53 in public method `add_handler`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/routeddecoder.py:56 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/routeddecoder.py:62 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/selecting.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/selecting.py:21 in public class `FilterIterDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/iter/selecting.py:46 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/selecting.py:70 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/sharding.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/sharding.py:17 in public class `SHARDING_PRIORITIES`: D101: Missing docstring in public class torch/utils/data/datapipes/iter/sharding.py:30 in public class `ShardingFilterIterDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/sharding.py:30 in public class `ShardingFilterIterDataPipe`: D400: First line should end with a period (not 's') torch/utils/data/datapipes/iter/sharding.py:39 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/sharding.py:47 in public method `apply_sharding`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/sharding.py:74 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/sharding.py:79 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/streamreader.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/streamreader.py:10 in public class `StreamReaderIterDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/iter/streamreader.py:10 in public class `StreamReaderIterDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/iter/streamreader.py:10 in public class `StreamReaderIterDataPipe`: D400: First line should end with a period (not 'l') torch/utils/data/datapipes/iter/streamreader.py:27 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/streamreader.py:31 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/utils.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/utils.py:9 in public class `IterableWrapperIterDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/iter/utils.py:29 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/utils.py:33 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/utils.py:49 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/callable.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/map/callable.py:14 in public function `default_fn`: D103: Missing docstring in public function torch/utils/data/datapipes/map/callable.py:20 in public class `MapperMapDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/map/callable.py:20 in public class `MapperMapDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/map/callable.py:45 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/map/callable.py:55 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/callable.py:58 in public method `__getitem__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/combinatorics.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/map/combinatorics.py:15 in public class `ShufflerIterDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/map/combinatorics.py:55 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/map/combinatorics.py:68 in public method `set_shuffle`: D102: Missing docstring in public method torch/utils/data/datapipes/map/combinatorics.py:72 in public method `set_seed`: D102: Missing docstring in public method torch/utils/data/datapipes/map/combinatorics.py:76 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/combinatorics.py:85 in public method `reset`: D102: Missing docstring in public method torch/utils/data/datapipes/map/combinatorics.py:92 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/combinatorics.py:95 in public method `__getstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/combinatorics.py:110 in public method `__setstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/combining.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/map/combining.py:12 in public class `ConcaterMapDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/map/combining.py:12 in public class `ConcaterMapDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/map/combining.py:34 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/map/combining.py:43 in public method `__getitem__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/combining.py:52 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/combining.py:58 in public class `ZipperMapDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/map/combining.py:58 in public class `ZipperMapDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/map/combining.py:76 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/map/combining.py:85 in public method `__getitem__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/combining.py:94 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/grouping.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/map/grouping.py:12 in public class `BatcherMapDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/map/grouping.py:12 in public class `BatcherMapDataPipe`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/map/grouping.py:12 in public class `BatcherMapDataPipe`: D400: First line should end with a period (not 's') torch/utils/data/datapipes/map/grouping.py:34 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/map/grouping.py:47 in public method `__getitem__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/grouping.py:60 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/utils.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/map/utils.py:9 in public class `SequenceWrapperMapDataPipe`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/map/utils.py:32 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/map/utils.py:45 in public method `__getitem__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/utils.py:48 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/utils/common.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/utils/common.py:26 in public function `validate_input_col`: D400: First line should end with a period (not 'n') torch/utils/data/datapipes/utils/common.py:26 in public function `validate_input_col`: D401: First line should be in imperative mood (perhaps 'Check', not 'Checks') torch/utils/data/datapipes/utils/common.py:127 in private function `_check_unpickable_fn`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/utils/common.py:127 in private function `_check_unpickable_fn`: D400: First line should end with a period (not 'g') torch/utils/data/datapipes/utils/common.py:127 in private function `_check_unpickable_fn`: D401: First line should be in imperative mood (perhaps 'Check', not 'Checks') torch/utils/data/datapipes/utils/common.py:156 in public function `match_masks`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/common.py:170 in public function `get_file_pathnames_from_root`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/common.py:207 in public function `get_file_binaries_from_pathnames`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/common.py:220 in public function `validate_pathname_binary_tuple`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/common.py:290 in public class `StreamWrapper`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/utils/common.py:290 in public class `StreamWrapper`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/utils/common.py:290 in public class `StreamWrapper`: D400: First line should end with a period (not 'y') torch/utils/data/datapipes/utils/common.py:298 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/utils/common.py:315 in public method `close_streams`: D200: One-line docstring should fit on one line with quotes (found 3) torch/utils/data/datapipes/utils/common.py:331 in public method `__getattr__`: D105: Missing docstring in magic method torch/utils/data/datapipes/utils/common.py:335 in public method `close`: D102: Missing docstring in public method torch/utils/data/datapipes/utils/common.py:351 in public method `autoclose`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/utils/common.py:351 in public method `autoclose`: D400: First line should end with a period (not 's') torch/utils/data/datapipes/utils/common.py:359 in public method `__dir__`: D105: Missing docstring in magic method torch/utils/data/datapipes/utils/common.py:364 in public method `__del__`: D105: Missing docstring in magic method torch/utils/data/datapipes/utils/common.py:368 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/utils/common.py:371 in public method `__next__`: D105: Missing docstring in magic method torch/utils/data/datapipes/utils/common.py:374 in public method `__repr__`: D105: Missing docstring in magic method torch/utils/data/datapipes/utils/common.py:380 in public method `__getstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/utils/common.py:383 in public method `__setstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/utils/decoder.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/utils/decoder.py:31 in public function `basichandlers`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/decoder.py:87 in public function `handle_extension`: D202: No blank lines allowed after function docstring (found 1) torch/utils/data/datapipes/utils/decoder.py:87 in public function `handle_extension`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/utils/decoder.py:87 in public function `handle_extension`: D401: First line should be in imperative mood (perhaps 'Return', not 'Returns') torch/utils/data/datapipes/utils/decoder.py:115 in public class `ImageHandler`: D204: 1 blank line required after class docstring (found 0) torch/utils/data/datapipes/utils/decoder.py:115 in public class `ImageHandler`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/utils/decoder.py:139 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/utils/decoder.py:143 in public method `__call__`: D102: Missing docstring in public method torch/utils/data/datapipes/utils/decoder.py:187 in public function `imagehandler`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/decoder.py:194 in public function `videohandler`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/decoder.py:215 in public function `audiohandler`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/decoder.py:236 in public class `MatHandler`: D101: Missing docstring in public class torch/utils/data/datapipes/utils/decoder.py:237 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/utils/decoder.py:247 in public method `__call__`: D102: Missing docstring in public method torch/utils/data/datapipes/utils/decoder.py:253 in public function `mathandler`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/decoder.py:261 in public function `extension_extract_fn`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/decoder.py:270 in public class `Decoder`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/utils/decoder.py:276 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/utils/decoder.py:282 in public method `add_handler`: D102: Missing docstring in public method torch/utils/data/datapipes/utils/decoder.py:292 in public method `decode1`: D102: Missing docstring in public method torch/utils/data/datapipes/utils/decoder.py:309 in public method `decode`: D102: Missing docstring in public method torch/utils/data/datapipes/utils/decoder.py:326 in public method `__call__`: D102: Missing docstring in public method torch/utils/data/datapipes/utils/snapshot.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/utils/snapshot.py:11 in private function `_simple_graph_snapshot_restoration`: D205: 1 blank line required between summary line and description (found 0) torch/utils/data/datapipes/utils/snapshot.py:11 in private function `_simple_graph_snapshot_restoration`: D400: First line should end with a period (not ',') torch/utils/data/datapipes/utils/snapshot.py:11 in private function `_simple_graph_snapshot_restoration`: D401: First line should be in imperative mood; try rephrasing (found 'This') torch/utils/tensorboard/_convert_np.py:1 at module level: D200: One-line docstring should fit on one line with quotes (found 3) torch/utils/tensorboard/_convert_np.py:9 in public function `make_np`: D205: 1 blank line required between summary line and description (found 0) torch/utils/tensorboard/_convert_np.py:9 in public function `make_np`: D400: First line should end with a period (not ':') 265 ``` After: 166 ``` torch/utils/data/datapipes/dataframe/structures.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/dataframe/structures.py:10 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/dataframe/structures.py:14 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/datapipe.py:120 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:123 in public method `__getattr__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:136 in public method `register_function`: D102: Missing docstring in public method torch/utils/data/datapipes/datapipe.py:140 in public method `register_datapipe_as_function`: D102: Missing docstring in public method torch/utils/data/datapipes/datapipe.py:173 in public method `__reduce_ex__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:182 in public method `set_getstate_hook`: D102: Missing docstring in public method torch/utils/data/datapipes/datapipe.py:188 in public method `set_reduce_ex_hook`: D102: Missing docstring in public method torch/utils/data/datapipes/datapipe.py:193 in public method `__repr__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:199 in public method `__str__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:205 in public method `__dir__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:221 in public class `DFIterDataPipe`: D101: Missing docstring in public class torch/utils/data/datapipes/datapipe.py:266 in public method `__getattr__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:279 in public method `register_function`: D102: Missing docstring in public method torch/utils/data/datapipes/datapipe.py:283 in public method `register_datapipe_as_function`: D102: Missing docstring in public method torch/utils/data/datapipes/datapipe.py:309 in public method `__reduce_ex__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:318 in public method `set_getstate_hook`: D102: Missing docstring in public method torch/utils/data/datapipes/datapipe.py:324 in public method `set_reduce_ex_hook`: D102: Missing docstring in public method torch/utils/data/datapipes/datapipe.py:329 in public method `__repr__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:335 in public method `__str__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:341 in public method `__dir__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:398 in public class `DataChunk`: D101: Missing docstring in public class torch/utils/data/datapipes/datapipe.py:399 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/datapipe.py:403 in public method `as_str`: D102: Missing docstring in public method torch/utils/data/datapipes/datapipe.py:407 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/datapipe.py:410 in public method `raw_iterator`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/callable.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/callable.py:65 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/callable.py:123 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/callable.py:127 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/callable.py:216 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/combinatorics.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/combinatorics.py:30 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/combinatorics.py:45 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combinatorics.py:48 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combinatorics.py:97 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/combinatorics.py:117 in public method `set_shuffle`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/combinatorics.py:121 in public method `set_seed`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/combinatorics.py:125 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combinatorics.py:140 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combinatorics.py:145 in public method `reset`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/combinatorics.py:153 in public method `__getstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combinatorics.py:168 in public method `__setstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combinatorics.py:182 in public method `__del__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combining.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/combining.py:46 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/combining.py:53 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combining.py:57 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combining.py:95 in public method `__new__`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/combining.py:388 in public method `__new__`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/combining.py:556 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/combining.py:560 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combining.py:573 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combining.py:579 in public method `reset`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/combining.py:582 in public method `__getstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combining.py:592 in public method `__setstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combining.py:600 in public method `__del__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combining.py:624 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/combining.py:631 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/combining.py:635 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/filelister.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/filelister.py:37 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/filelister.py:59 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/filelister.py:63 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/fileopener.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/fileopener.py:41 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/fileopener.py:65 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/fileopener.py:68 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/grouping.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/grouping.py:57 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/grouping.py:70 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/grouping.py:81 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/grouping.py:115 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/grouping.py:121 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/grouping.py:190 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/grouping.py:238 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/grouping.py:262 in public method `reset`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/grouping.py:266 in public method `__getstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/grouping.py:283 in public method `__setstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/grouping.py:299 in public method `__del__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/routeddecoder.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/routeddecoder.py:38 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/routeddecoder.py:54 in public method `add_handler`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/routeddecoder.py:57 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/routeddecoder.py:63 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/selecting.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/selecting.py:47 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/selecting.py:71 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/sharding.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/sharding.py:17 in public class `SHARDING_PRIORITIES`: D101: Missing docstring in public class torch/utils/data/datapipes/iter/sharding.py:40 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/sharding.py:48 in public method `apply_sharding`: D102: Missing docstring in public method torch/utils/data/datapipes/iter/sharding.py:75 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/sharding.py:80 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/streamreader.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/streamreader.py:29 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/streamreader.py:33 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/utils.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/iter/utils.py:30 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/iter/utils.py:34 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/iter/utils.py:50 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/callable.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/map/callable.py:14 in public function `default_fn`: D103: Missing docstring in public function torch/utils/data/datapipes/map/callable.py:47 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/map/callable.py:57 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/callable.py:60 in public method `__getitem__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/combinatorics.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/map/combinatorics.py:56 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/map/combinatorics.py:69 in public method `set_shuffle`: D102: Missing docstring in public method torch/utils/data/datapipes/map/combinatorics.py:73 in public method `set_seed`: D102: Missing docstring in public method torch/utils/data/datapipes/map/combinatorics.py:77 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/combinatorics.py:86 in public method `reset`: D102: Missing docstring in public method torch/utils/data/datapipes/map/combinatorics.py:93 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/combinatorics.py:96 in public method `__getstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/combinatorics.py:111 in public method `__setstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/combining.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/map/combining.py:36 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/map/combining.py:45 in public method `__getitem__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/combining.py:54 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/combining.py:80 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/map/combining.py:89 in public method `__getitem__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/combining.py:98 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/grouping.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/map/grouping.py:36 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/map/grouping.py:49 in public method `__getitem__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/grouping.py:62 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/utils.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/map/utils.py:33 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/map/utils.py:46 in public method `__getitem__`: D105: Missing docstring in magic method torch/utils/data/datapipes/map/utils.py:49 in public method `__len__`: D105: Missing docstring in magic method torch/utils/data/datapipes/utils/common.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/utils/common.py:157 in public function `match_masks`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/common.py:171 in public function `get_file_pathnames_from_root`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/common.py:208 in public function `get_file_binaries_from_pathnames`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/common.py:221 in public function `validate_pathname_binary_tuple`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/common.py:300 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/utils/common.py:331 in public method `__getattr__`: D105: Missing docstring in magic method torch/utils/data/datapipes/utils/common.py:335 in public method `close`: D102: Missing docstring in public method torch/utils/data/datapipes/utils/common.py:356 in public method `__dir__`: D105: Missing docstring in magic method torch/utils/data/datapipes/utils/common.py:361 in public method `__del__`: D105: Missing docstring in magic method torch/utils/data/datapipes/utils/common.py:365 in public method `__iter__`: D105: Missing docstring in magic method torch/utils/data/datapipes/utils/common.py:368 in public method `__next__`: D105: Missing docstring in magic method torch/utils/data/datapipes/utils/common.py:371 in public method `__repr__`: D105: Missing docstring in magic method torch/utils/data/datapipes/utils/common.py:377 in public method `__getstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/utils/common.py:380 in public method `__setstate__`: D105: Missing docstring in magic method torch/utils/data/datapipes/utils/decoder.py:1 at module level: D100: Missing docstring in public module torch/utils/data/datapipes/utils/decoder.py:31 in public function `basichandlers`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/decoder.py:141 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/utils/decoder.py:145 in public method `__call__`: D102: Missing docstring in public method torch/utils/data/datapipes/utils/decoder.py:189 in public function `imagehandler`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/decoder.py:196 in public function `videohandler`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/decoder.py:217 in public function `audiohandler`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/decoder.py:238 in public class `MatHandler`: D101: Missing docstring in public class torch/utils/data/datapipes/utils/decoder.py:239 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/utils/decoder.py:249 in public method `__call__`: D102: Missing docstring in public method torch/utils/data/datapipes/utils/decoder.py:255 in public function `mathandler`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/decoder.py:263 in public function `extension_extract_fn`: D103: Missing docstring in public function torch/utils/data/datapipes/utils/decoder.py:279 in public method `__init__`: D107: Missing docstring in __init__ torch/utils/data/datapipes/utils/decoder.py:285 in public method `add_handler`: D102: Missing docstring in public method torch/utils/data/datapipes/utils/decoder.py:295 in public method `decode1`: D102: Missing docstring in public method torch/utils/data/datapipes/utils/decoder.py:312 in public method `decode`: D102: Missing docstring in public method torch/utils/data/datapipes/utils/decoder.py:329 in public method `__call__`: D102: Missing docstring in public method torch/utils/data/datapipes/utils/snapshot.py:1 at module level: D100: Missing docstring in public module 166 ``` Pull Request resolved: https://github.com/pytorch/pytorch/pull/112765 Approved by: https://github.com/ejguan
This commit is contained in:
NVS Abhilash
committed by
PyTorch MergeBot
PyTorch MergeBot
parent
132cb57e47
commit
f58ecd4823
@ -5,10 +5,7 @@ __all__ = ["DataChunkDF", ]
|
||||
|
||||
|
||||
class DataChunkDF(DataChunk):
|
||||
"""
|
||||
DataChunkDF iterating over individual items inside of DataFrame containers,
|
||||
to access DataFrames user `raw_iterator`
|
||||
"""
|
||||
"""DataChunkDF iterating over individual items inside of DataFrame containers, to access DataFrames user `raw_iterator`."""
|
||||
|
||||
def __iter__(self):
|
||||
for df in self.items:
|
||||
|
||||
@ -106,6 +106,7 @@ class IterDataPipe(IterableDataset[T_co], metaclass=_IterDataPipeMeta):
|
||||
0
|
||||
>>> next(it1) # Further usage of `it1` will raise a `RunTimeError`
|
||||
"""
|
||||
|
||||
functions: Dict[str, Callable] = {}
|
||||
reduce_ex_hook: Optional[Callable] = None
|
||||
getstate_hook: Optional[Callable] = None
|
||||
@ -159,7 +160,8 @@ class IterDataPipe(IterableDataset[T_co], metaclass=_IterDataPipeMeta):
|
||||
|
||||
def __getstate__(self):
|
||||
"""
|
||||
This contains special logic to serialize `lambda` functions when `dill` is available.
|
||||
Serialize `lambda` functions when `dill` is available.
|
||||
|
||||
If this doesn't cover your custom DataPipe's use case, consider writing custom methods for
|
||||
`__getstate__` and `__setstate__`, or use `pickle.dumps` for serialization.
|
||||
"""
|
||||
@ -206,8 +208,10 @@ class IterDataPipe(IterableDataset[T_co], metaclass=_IterDataPipeMeta):
|
||||
|
||||
def reset(self) -> None:
|
||||
r"""
|
||||
Reset the `IterDataPipe` to the initial state. By default, no-op. For subclasses of `IterDataPipe`,
|
||||
depending on their functionalities, they may want to override this method with implementations that
|
||||
Reset the `IterDataPipe` to the initial state.
|
||||
|
||||
By default, no-op. For subclasses of `IterDataPipe`, depending on their functionalities,
|
||||
they may want to override this method with implementations that
|
||||
may clear the buffers and reset pointers of the DataPipe.
|
||||
The `reset` method is always called when `__iter__` is called as part of `hook_iterator`.
|
||||
"""
|
||||
@ -252,6 +256,7 @@ class MapDataPipe(Dataset[T_co], metaclass=_DataPipeMeta):
|
||||
>>> list(batch_dp)
|
||||
[[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
|
||||
"""
|
||||
|
||||
functions: Dict[str, Callable] = {}
|
||||
reduce_ex_hook: Optional[Callable] = None
|
||||
getstate_hook: Optional[Callable] = None
|
||||
@ -291,7 +296,8 @@ class MapDataPipe(Dataset[T_co], metaclass=_DataPipeMeta):
|
||||
|
||||
def __getstate__(self):
|
||||
"""
|
||||
This contains special logic to serialize `lambda` functions when `dill` is available.
|
||||
Serialize `lambda` functions when `dill` is available.
|
||||
|
||||
If this doesn't cover your custom DataPipe's use case, consider writing custom methods for
|
||||
`__getstate__` and `__setstate__`, or use `pickle.dumps` for serialization.
|
||||
"""
|
||||
|
||||
@ -22,6 +22,7 @@ T_co = TypeVar("T_co", covariant=True)
|
||||
class MapperIterDataPipe(IterDataPipe[T_co]):
|
||||
r"""
|
||||
Applies a function over each item from the source DataPipe (functional name: ``map``).
|
||||
|
||||
The function can be any regular Python function or partial object. Lambda
|
||||
function is not recommended as it is not supported by pickle.
|
||||
|
||||
@ -57,6 +58,7 @@ class MapperIterDataPipe(IterDataPipe[T_co]):
|
||||
>>> list(map_dp_2)
|
||||
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
||||
"""
|
||||
|
||||
datapipe: IterDataPipe
|
||||
fn: Callable
|
||||
|
||||
@ -172,6 +174,7 @@ def _collate_helper(conversion, item):
|
||||
class CollatorIterDataPipe(MapperIterDataPipe):
|
||||
r"""
|
||||
Collates samples from DataPipe to Tensor(s) by a custom collate function (functional name: ``collate``).
|
||||
|
||||
By default, it uses :func:`torch.utils.data.default_collate`.
|
||||
|
||||
.. note::
|
||||
|
||||
@ -16,13 +16,14 @@ T_co = TypeVar('T_co', covariant=True)
|
||||
|
||||
class SamplerIterDataPipe(IterDataPipe[T_co]):
|
||||
r"""
|
||||
Generates sample elements using the provided ``Sampler`` (defaults to :class:`SequentialSampler`).
|
||||
Generate sample elements using the provided ``Sampler`` (defaults to :class:`SequentialSampler`).
|
||||
|
||||
Args:
|
||||
datapipe: IterDataPipe to sample from
|
||||
sampler: Sampler class to generate sample elements from input DataPipe.
|
||||
Default is :class:`SequentialSampler` for IterDataPipe
|
||||
"""
|
||||
|
||||
datapipe: IterDataPipe
|
||||
sampler: Sampler
|
||||
|
||||
@ -54,8 +55,9 @@ class SamplerIterDataPipe(IterDataPipe[T_co]):
|
||||
@functional_datapipe('shuffle')
|
||||
class ShufflerIterDataPipe(IterDataPipe[T_co]):
|
||||
r"""
|
||||
Shuffles the input DataPipe with a buffer (functional name: ``shuffle``). The buffer
|
||||
with ``buffer_size`` is filled with elements from the datapipe first. Then,
|
||||
Shuffle the input DataPipe with a buffer (functional name: ``shuffle``).
|
||||
|
||||
The buffer with ``buffer_size`` is filled with elements from the datapipe first. Then,
|
||||
each item will be yielded from the buffer by reservoir sampling via iterator.
|
||||
|
||||
``buffer_size`` is required to be larger than ``0``. For ``buffer_size == 1``, the
|
||||
@ -84,6 +86,7 @@ class ShufflerIterDataPipe(IterDataPipe[T_co]):
|
||||
>>> list(shuffle_dp)
|
||||
[0, 4, 1, 6, 3, 2, 9, 5, 7, 8]
|
||||
"""
|
||||
|
||||
datapipe: IterDataPipe[T_co]
|
||||
buffer_size: int
|
||||
_buffer: List[T_co]
|
||||
|
||||
@ -24,8 +24,9 @@ T_co = TypeVar('T_co', covariant=True)
|
||||
@functional_datapipe('concat')
|
||||
class ConcaterIterDataPipe(IterDataPipe):
|
||||
r"""
|
||||
Concatenates multiple Iterable DataPipes (functional name: ``concat``). The resulting DataPipe will
|
||||
yield all the elements from the first input DataPipe, before yielding from the subsequent ones.
|
||||
Concatenates multiple Iterable DataPipes (functional name: ``concat``).
|
||||
|
||||
The resulting DataPipe will yield all the elements from the first input DataPipe, before yielding from the subsequent ones.
|
||||
|
||||
Args:
|
||||
datapipes: Iterable DataPipes being concatenated
|
||||
@ -39,6 +40,7 @@ class ConcaterIterDataPipe(IterDataPipe):
|
||||
>>> list(dp1.concat(dp2))
|
||||
[0, 1, 2, 0, 1, 2, 3, 4]
|
||||
"""
|
||||
|
||||
datapipes: Tuple[IterDataPipe]
|
||||
|
||||
def __init__(self, *datapipes: IterDataPipe):
|
||||
@ -89,6 +91,7 @@ class ForkerIterDataPipe(IterDataPipe):
|
||||
>>> list(dp2)
|
||||
[0, 1, 2, 3, 4]
|
||||
"""
|
||||
|
||||
def __new__(
|
||||
cls,
|
||||
datapipe: IterDataPipe,
|
||||
@ -105,10 +108,8 @@ class ForkerIterDataPipe(IterDataPipe):
|
||||
|
||||
|
||||
class _ContainerTemplate(ABC):
|
||||
r"""
|
||||
Abstract class for container ``DataPipes``. The followings are three required
|
||||
methods.
|
||||
"""
|
||||
r"""Abstract class for container ``DataPipes``. The followings are three required methods."""
|
||||
|
||||
@abstractmethod
|
||||
def get_next_element_by_instance(self, instance_id: int):
|
||||
...
|
||||
@ -123,9 +124,7 @@ class _ContainerTemplate(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def get_length_by_instance(self, instance_id: int):
|
||||
r"""
|
||||
Raise TypeError if it's not supposed to be implemented to support `list(datapipe)`
|
||||
"""
|
||||
r"""Raise TypeError if it's not supposed to be implemented to support `list(datapipe)`."""
|
||||
|
||||
|
||||
def _no_op(x):
|
||||
@ -134,10 +133,12 @@ def _no_op(x):
|
||||
|
||||
class _ForkerIterDataPipe(IterDataPipe, _ContainerTemplate):
|
||||
r"""
|
||||
Container to hold instance-specific information on behalf of ForkerIterDataPipe. It tracks
|
||||
the state of its child DataPipes, maintains the buffer, and yields the next value
|
||||
Container to hold instance-specific information on behalf of ForkerIterDataPipe.
|
||||
|
||||
It tracks the state of its child DataPipes, maintains the buffer, and yields the next value
|
||||
as requested by the child DataPipes.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
datapipe: IterDataPipe,
|
||||
@ -273,8 +274,9 @@ class _ForkerIterDataPipe(IterDataPipe, _ContainerTemplate):
|
||||
|
||||
class _ChildDataPipe(IterDataPipe):
|
||||
r"""
|
||||
Iterable Datapipe that is a child of a main DataPipe. The instance of this class
|
||||
will pass its instance_id to get the next value from its main DataPipe.
|
||||
Iterable Datapipe that is a child of a main DataPipe.
|
||||
|
||||
The instance of this class will pass its instance_id to get the next value from its main DataPipe.
|
||||
|
||||
Note:
|
||||
ChildDataPipe, like all other IterDataPipe, follows the single iterator per IterDataPipe constraint.
|
||||
@ -299,6 +301,7 @@ class _ChildDataPipe(IterDataPipe):
|
||||
main_datapipe: Main DataPipe with a method 'get_next_element_by_instance(instance_id)'
|
||||
instance_id: integer identifier of this instance
|
||||
"""
|
||||
|
||||
_is_child_datapipe: bool = True
|
||||
|
||||
def __init__(self, main_datapipe: IterDataPipe, instance_id: int):
|
||||
@ -319,6 +322,7 @@ class _ChildDataPipe(IterDataPipe):
|
||||
def _set_main_datapipe_valid_iterator_id(self) -> int:
|
||||
r"""
|
||||
Update the valid iterator ID for both this DataPipe object and `main_datapipe`.
|
||||
|
||||
`main_datapipe.reset()` is called when the ID is incremented to a new generation.
|
||||
"""
|
||||
# 1. First time any child iterator is created
|
||||
@ -340,17 +344,16 @@ class _ChildDataPipe(IterDataPipe):
|
||||
|
||||
# This method is called by `hook_iterator` in `_typing.py`.
|
||||
def _check_valid_iterator_id(self, iterator_id) -> bool:
|
||||
r"""
|
||||
Check the valid iterator ID against that of DataPipe object and that of `main_datapipe`.
|
||||
"""
|
||||
r"""Check the valid iterator ID against that of DataPipe object and that of `main_datapipe`."""
|
||||
return iterator_id == self._valid_iterator_id and iterator_id == self.main_datapipe._valid_iterator_id
|
||||
|
||||
|
||||
@functional_datapipe('demux')
|
||||
class DemultiplexerIterDataPipe(IterDataPipe):
|
||||
r"""
|
||||
Splits the input DataPipe into multiple child DataPipes, using the given
|
||||
classification function (functional name: ``demux``). A list of the child DataPipes is returned from this operation.
|
||||
Splits the input DataPipe into multiple child DataPipes, using the given classification function (functional name: ``demux``).
|
||||
|
||||
A list of the child DataPipes is returned from this operation.
|
||||
|
||||
Args:
|
||||
datapipe: Iterable DataPipe being filtered
|
||||
@ -381,6 +384,7 @@ class DemultiplexerIterDataPipe(IterDataPipe):
|
||||
>>> list(dp2)
|
||||
[1, 3]
|
||||
"""
|
||||
|
||||
def __new__(cls, datapipe: IterDataPipe, num_instances: int,
|
||||
classifier_fn: Callable[[T_co], Optional[int]], drop_none: bool = False, buffer_size: int = 1000):
|
||||
if num_instances < 1:
|
||||
@ -397,8 +401,9 @@ class DemultiplexerIterDataPipe(IterDataPipe):
|
||||
|
||||
class _DemultiplexerIterDataPipe(IterDataPipe, _ContainerTemplate):
|
||||
r"""
|
||||
Container to hold instance-specific information on behalf of DemultiplexerIterDataPipe. It tracks
|
||||
the state of its child DataPipes, maintains the buffer, classifies and yields the next correct value
|
||||
Container to hold instance-specific information on behalf of DemultiplexerIterDataPipe.
|
||||
|
||||
It tracks the state of its child DataPipes, maintains the buffer, classifies and yields the next correct value
|
||||
as requested by the child DataPipes.
|
||||
"""
|
||||
|
||||
@ -532,8 +537,9 @@ class _DemultiplexerIterDataPipe(IterDataPipe, _ContainerTemplate):
|
||||
@functional_datapipe('mux')
|
||||
class MultiplexerIterDataPipe(IterDataPipe):
|
||||
r"""
|
||||
Yields one element at a time from each of the input Iterable DataPipes (functional name: ``mux``). As in,
|
||||
one element from the 1st input DataPipe, then one element from the 2nd DataPipe in the next iteration,
|
||||
Yields one element at a time from each of the input Iterable DataPipes (functional name: ``mux``).
|
||||
|
||||
As in, one element from the 1st input DataPipe, then one element from the 2nd DataPipe in the next iteration,
|
||||
and so on. It ends when the shortest input DataPipe is exhausted.
|
||||
|
||||
Args:
|
||||
@ -546,6 +552,7 @@ class MultiplexerIterDataPipe(IterDataPipe):
|
||||
>>> list(dp1.mux(dp2, dp3))
|
||||
[0, 10, 20, 1, 11, 21, 2, 12, 22]
|
||||
"""
|
||||
|
||||
def __init__(self, *datapipes):
|
||||
self.datapipes = datapipes
|
||||
self.buffer: List = [] # Store values to be yielded only when every iterator provides one
|
||||
@ -598,6 +605,7 @@ class MultiplexerIterDataPipe(IterDataPipe):
|
||||
class ZipperIterDataPipe(IterDataPipe[Tuple[T_co]]):
|
||||
r"""
|
||||
Aggregates elements into a tuple from each of the input DataPipes (functional name: ``zip``).
|
||||
|
||||
The output is stopped as soon as the shortest input DataPipe is exhausted.
|
||||
|
||||
Args:
|
||||
@ -610,6 +618,7 @@ class ZipperIterDataPipe(IterDataPipe[Tuple[T_co]]):
|
||||
>>> list(dp1.zip(dp2, dp3))
|
||||
[(0, 10, 20), (1, 11, 21), (2, 12, 22), (3, 13, 23), (4, 14, 24)]
|
||||
"""
|
||||
|
||||
datapipes: Tuple[IterDataPipe]
|
||||
|
||||
def __init__(self, *datapipes: IterDataPipe):
|
||||
|
||||
@ -14,6 +14,7 @@ __all__ = ["FileListerIterDataPipe", ]
|
||||
class FileListerIterDataPipe(IterDataPipe[str]):
|
||||
r"""
|
||||
Given path(s) to the root directory, yields file pathname(s) (path + filename) of files within the root directory.
|
||||
|
||||
Multiple root directories can be provided (functional name: ``list_files``).
|
||||
|
||||
Args:
|
||||
|
||||
@ -13,8 +13,7 @@ __all__ = [
|
||||
@functional_datapipe("open_files")
|
||||
class FileOpenerIterDataPipe(IterDataPipe[Tuple[str, IOBase]]):
|
||||
r"""
|
||||
Given pathnames, opens files and yield pathname and file stream
|
||||
in a tuple (functional name: ``open_files``).
|
||||
Given pathnames, opens files and yield pathname and file stream in a tuple (functional name: ``open_files``).
|
||||
|
||||
Args:
|
||||
datapipe: Iterable datapipe that provides pathnames
|
||||
|
||||
@ -29,8 +29,9 @@ def __getattr__(name: str):
|
||||
@functional_datapipe('batch')
|
||||
class BatcherIterDataPipe(IterDataPipe[DataChunk]):
|
||||
r"""
|
||||
Creates mini-batches of data (functional name: ``batch``). An outer dimension will be added as
|
||||
``batch_size`` if ``drop_last`` is set to ``True``, or ``length % batch_size`` for the
|
||||
Creates mini-batches of data (functional name: ``batch``).
|
||||
|
||||
An outer dimension will be added as ``batch_size`` if ``drop_last`` is set to ``True``, or ``length % batch_size`` for the
|
||||
last batch if ``drop_last`` is set to ``False``.
|
||||
|
||||
Args:
|
||||
@ -48,6 +49,7 @@ class BatcherIterDataPipe(IterDataPipe[DataChunk]):
|
||||
>>> list(dp)
|
||||
[[0, 1, 2], [3, 4, 5], [6, 7, 8]]
|
||||
"""
|
||||
|
||||
datapipe: IterDataPipe
|
||||
batch_size: int
|
||||
drop_last: bool
|
||||
@ -89,8 +91,9 @@ class BatcherIterDataPipe(IterDataPipe[DataChunk]):
|
||||
@functional_datapipe('unbatch')
|
||||
class UnBatcherIterDataPipe(IterDataPipe):
|
||||
r"""
|
||||
Undoes batching of data (functional name: ``unbatch``). In other words, it flattens the data up to the specified level
|
||||
within a batched DataPipe.
|
||||
Undos batching of data (functional name: ``unbatch``).
|
||||
|
||||
In other words, it flattens the data up to the specified level within a batched DataPipe.
|
||||
|
||||
Args:
|
||||
datapipe: Iterable DataPipe being un-batched
|
||||
@ -141,8 +144,9 @@ class UnBatcherIterDataPipe(IterDataPipe):
|
||||
@functional_datapipe('groupby')
|
||||
class GrouperIterDataPipe(IterDataPipe[DataChunk]):
|
||||
r"""
|
||||
Groups data from input IterDataPipe by keys which are generated from ``group_key_fn``,
|
||||
and yields a ``DataChunk`` with batch size up to ``group_size`` if defined (functional name: ``groupby``).
|
||||
Groups data from IterDataPipe by keys from ``group_key_fn``, yielding a ``DataChunk`` with batch size up to ``group_size``.
|
||||
|
||||
(functional name: ``groupby``).
|
||||
|
||||
The samples are read sequentially from the source ``datapipe``, and a batch of samples belonging to the same group
|
||||
will be yielded as soon as the size of the batch reaches ``group_size``. When the buffer is full,
|
||||
@ -182,6 +186,7 @@ class GrouperIterDataPipe(IterDataPipe[DataChunk]):
|
||||
>>> list(dp2)
|
||||
[['a.png', 'a.json'], ['b.png', 'b.json'], ['a.jpg'], ['c.json']]
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
datapipe: IterDataPipe[T_co],
|
||||
group_key_fn: Callable[[T_co], Any],
|
||||
|
||||
@ -17,8 +17,9 @@ __all__ = ["RoutedDecoderIterDataPipe", ]
|
||||
@functional_datapipe('routed_decode')
|
||||
class RoutedDecoderIterDataPipe(IterDataPipe[Tuple[str, Any]]):
|
||||
r"""
|
||||
Decodes binary streams from input DataPipe, yields pathname and decoded data
|
||||
in a tuple (functional name: ``routed_decode``).
|
||||
Decodes binary streams from input DataPipe, yields pathname and decoded data in a tuple.
|
||||
|
||||
(functional name: ``routed_decode``)
|
||||
|
||||
Args:
|
||||
datapipe: Iterable datapipe that provides pathname and binary stream in tuples
|
||||
|
||||
@ -40,6 +40,7 @@ class FilterIterDataPipe(IterDataPipe[T_co]):
|
||||
>>> list(filter_dp)
|
||||
[0, 2, 4]
|
||||
"""
|
||||
|
||||
datapipe: IterDataPipe[T_co]
|
||||
filter_fn: Callable
|
||||
|
||||
|
||||
@ -28,8 +28,9 @@ class _ShardingIterDataPipe(IterDataPipe):
|
||||
@functional_datapipe('sharding_filter')
|
||||
class ShardingFilterIterDataPipe(_ShardingIterDataPipe):
|
||||
r"""
|
||||
Wrapper that allows DataPipe to be sharded (functional name: ``sharding_filter``). After ``apply_sharding`` is
|
||||
called, each instance of the DataPipe (on different workers) will have every `n`-th element of the
|
||||
Wrapper that allows DataPipe to be sharded (functional name: ``sharding_filter``).
|
||||
|
||||
After ``apply_sharding`` is called, each instance of the DataPipe (on different workers) will have every `n`-th element of the
|
||||
original DataPipe, where `n` equals to the number of instances.
|
||||
|
||||
Args:
|
||||
|
||||
@ -8,8 +8,9 @@ __all__ = ["StreamReaderIterDataPipe", ]
|
||||
@functional_datapipe('read_from_stream')
|
||||
class StreamReaderIterDataPipe(IterDataPipe[Tuple[str, bytes]]):
|
||||
r"""
|
||||
Given IO streams and their label names, yields bytes with label
|
||||
name in a tuple (functional name: ``read_from_stream``).
|
||||
Given IO streams and their label names, yield bytes with label name as tuple.
|
||||
|
||||
(functional name: ``read_from_stream``).
|
||||
|
||||
Args:
|
||||
datapipe: Iterable DataPipe provides label/URL and byte stream
|
||||
@ -24,6 +25,7 @@ class StreamReaderIterDataPipe(IterDataPipe[Tuple[str, bytes]]):
|
||||
>>> list(StreamReader(dp, chunk=1))
|
||||
[('alphabet', 'a'), ('alphabet', 'b'), ('alphabet', 'c'), ('alphabet', 'd'), ('alphabet', 'e')]
|
||||
"""
|
||||
|
||||
def __init__(self, datapipe, chunk=None):
|
||||
self.datapipe = datapipe
|
||||
self.chunk = chunk
|
||||
|
||||
@ -26,6 +26,7 @@ class IterableWrapperIterDataPipe(IterDataPipe):
|
||||
>>> list(dp)
|
||||
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
|
||||
"""
|
||||
|
||||
def __init__(self, iterable, deepcopy=True):
|
||||
self.iterable = iterable
|
||||
self.deepcopy = deepcopy
|
||||
|
||||
@ -19,6 +19,7 @@ def default_fn(data):
|
||||
class MapperMapDataPipe(MapDataPipe[T_co]):
|
||||
r"""
|
||||
Apply the input function over each item from the source DataPipe (functional name: ``map``).
|
||||
|
||||
The function can be any regular Python function or partial object. Lambda
|
||||
function is not recommended as it is not supported by pickle.
|
||||
|
||||
@ -39,6 +40,7 @@ class MapperMapDataPipe(MapDataPipe[T_co]):
|
||||
>>> list(map_dp_2)
|
||||
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
|
||||
"""
|
||||
|
||||
datapipe: MapDataPipe
|
||||
fn: Callable
|
||||
|
||||
|
||||
@ -47,6 +47,7 @@ class ShufflerIterDataPipe(IterDataPipe[T_co]):
|
||||
the order of data order for the sake of random reads, but ``IterDataPipe`` depends on the order
|
||||
of data during data-processing.
|
||||
"""
|
||||
|
||||
datapipe: MapDataPipe[T_co]
|
||||
_enabled: bool
|
||||
_seed: Optional[int]
|
||||
|
||||
@ -11,6 +11,7 @@ T_co = TypeVar('T_co', covariant=True)
|
||||
class ConcaterMapDataPipe(MapDataPipe):
|
||||
r"""
|
||||
Concatenate multiple Map DataPipes (functional name: ``concat``).
|
||||
|
||||
The new index of is the cumulative sum of source DataPipes.
|
||||
For example, if there are 2 source DataPipes both with length 5,
|
||||
index 0 to 4 of the resulting `ConcatMapDataPipe` would refer to
|
||||
@ -29,6 +30,7 @@ class ConcaterMapDataPipe(MapDataPipe):
|
||||
>>> list(concat_dp)
|
||||
[0, 1, 2, 0, 1, 2]
|
||||
"""
|
||||
|
||||
datapipes: Tuple[MapDataPipe]
|
||||
|
||||
def __init__(self, *datapipes: MapDataPipe):
|
||||
@ -57,6 +59,7 @@ class ConcaterMapDataPipe(MapDataPipe):
|
||||
class ZipperMapDataPipe(MapDataPipe[Tuple[T_co, ...]]):
|
||||
r"""
|
||||
Aggregates elements into a tuple from each of the input DataPipes (functional name: ``zip``).
|
||||
|
||||
This MataPipe is out of bound as soon as the shortest input DataPipe is exhausted.
|
||||
|
||||
Args:
|
||||
@ -71,6 +74,7 @@ class ZipperMapDataPipe(MapDataPipe[Tuple[T_co, ...]]):
|
||||
>>> list(zip_dp)
|
||||
[(0, 10), (1, 11), (2, 12)]
|
||||
"""
|
||||
|
||||
datapipes: Tuple[MapDataPipe[T_co], ...]
|
||||
|
||||
def __init__(self, *datapipes: MapDataPipe[T_co]) -> None:
|
||||
|
||||
@ -10,9 +10,10 @@ T = TypeVar('T')
|
||||
@functional_datapipe('batch')
|
||||
class BatcherMapDataPipe(MapDataPipe[DataChunk]):
|
||||
r"""
|
||||
Create mini-batches of data (functional name: ``batch``). An outer dimension will be added as
|
||||
``batch_size`` if ``drop_last`` is set to ``True``, or ``length % batch_size`` for the
|
||||
last batch if ``drop_last`` is set to ``False``.
|
||||
Create mini-batches of data (functional name: ``batch``).
|
||||
|
||||
An outer dimension will be added as ``batch_size`` if ``drop_last`` is set to ``True``,
|
||||
or ``length % batch_size`` for the last batch if ``drop_last`` is set to ``False``.
|
||||
|
||||
Args:
|
||||
datapipe: Iterable DataPipe being batched
|
||||
@ -27,6 +28,7 @@ class BatcherMapDataPipe(MapDataPipe[DataChunk]):
|
||||
>>> list(batch_dp)
|
||||
[[0, 1], [2, 3], [4, 5], [6, 7], [8, 9]]
|
||||
"""
|
||||
|
||||
datapipe: MapDataPipe
|
||||
batch_size: int
|
||||
drop_last: bool
|
||||
|
||||
@ -29,6 +29,7 @@ class SequenceWrapperMapDataPipe(MapDataPipe):
|
||||
>>> dp['a']
|
||||
100
|
||||
"""
|
||||
|
||||
def __init__(self, sequence, deepcopy=True):
|
||||
if deepcopy:
|
||||
try:
|
||||
|
||||
@ -24,7 +24,7 @@ __all__ = [
|
||||
|
||||
def validate_input_col(fn: Callable, input_col: Optional[Union[int, tuple, list]]):
|
||||
"""
|
||||
Checks that function used in a callable datapipe works with the input column
|
||||
Check that function used in a callable datapipe works with the input column.
|
||||
|
||||
This simply ensures that the number of positional arguments matches the size
|
||||
of the input column. The function must not contain any non-default
|
||||
@ -125,8 +125,9 @@ def _is_local_fn(fn):
|
||||
|
||||
def _check_unpickable_fn(fn: Callable):
|
||||
"""
|
||||
Checks function is pickable or not. If it is a lambda or local function, a UserWarning
|
||||
will be raised. If it's not a callable function, a TypeError will be raised.
|
||||
Check function is pickable or not.
|
||||
|
||||
If it is a lambda or local function, a UserWarning will be raised. If it's not a callable function, a TypeError will be raised.
|
||||
"""
|
||||
if not callable(fn):
|
||||
raise TypeError(f"A callable function is expected, but {type(fn)} is provided.")
|
||||
@ -288,10 +289,11 @@ def _deprecation_warning(
|
||||
|
||||
class StreamWrapper:
|
||||
"""
|
||||
StreamWrapper is introduced to wrap file handler generated by
|
||||
DataPipe operation like `FileOpener`. StreamWrapper would guarantee
|
||||
the wrapped file handler is closed when it's out of scope.
|
||||
StreamWrapper is introduced to wrap file handler generated by DataPipe operation like `FileOpener`.
|
||||
|
||||
StreamWrapper would guarantee the wrapped file handler is closed when it's out of scope.
|
||||
"""
|
||||
|
||||
session_streams: Dict[Any, int] = {}
|
||||
debug_unclosed_streams: bool = False
|
||||
|
||||
@ -312,9 +314,7 @@ class StreamWrapper:
|
||||
|
||||
@classmethod
|
||||
def close_streams(cls, v, depth=0):
|
||||
"""
|
||||
Traverse structure and attempts to close all found StreamWrappers on best effort basis.
|
||||
"""
|
||||
"""Traverse structure and attempts to close all found StreamWrappers on best effort basis."""
|
||||
if depth > 10:
|
||||
return
|
||||
if isinstance(v, StreamWrapper):
|
||||
@ -348,10 +348,7 @@ class StreamWrapper:
|
||||
self.closed = True
|
||||
|
||||
def autoclose(self):
|
||||
"""
|
||||
Close steam if there is no children, or make it to be automatically closed as soon as
|
||||
all child streams are closed.
|
||||
"""
|
||||
"""Automatically close stream when all child streams are closed or if there are none."""
|
||||
self.close_on_last_child = True
|
||||
if self.child_counter == 0:
|
||||
self.close()
|
||||
|
||||
@ -85,7 +85,8 @@ imagespecs = {
|
||||
|
||||
def handle_extension(extensions, f):
|
||||
"""
|
||||
Returns a decoder handler function for the list of extensions.
|
||||
Return a decoder handler function for the list of extensions.
|
||||
|
||||
Extensions can be a space separated list of extensions.
|
||||
Extensions can contain dots, in which case the corresponding number
|
||||
of extension components must be present in the key given to f.
|
||||
@ -94,7 +95,6 @@ def handle_extension(extensions, f):
|
||||
handle_extension("jpg jpeg", my_decode_jpg) # invoked for any file.jpg
|
||||
handle_extension("seg.jpg", special_case_jpg) # invoked only for file.seg.jpg
|
||||
"""
|
||||
|
||||
extensions = extensions.lower().split()
|
||||
|
||||
def g(key, data):
|
||||
@ -114,6 +114,7 @@ def handle_extension(extensions, f):
|
||||
class ImageHandler:
|
||||
"""
|
||||
Decode image data using the given `imagespec`.
|
||||
|
||||
The `imagespec` specifies whether the image is decoded
|
||||
to numpy/torch/pi, decoded to uint8/float, and decoded
|
||||
to l/rgb/rgba:
|
||||
@ -136,6 +137,7 @@ class ImageHandler:
|
||||
- pilrgb: pil None rgb
|
||||
- pilrgba: pil None rgba
|
||||
"""
|
||||
|
||||
def __init__(self, imagespec):
|
||||
assert imagespec in list(imagespecs.keys()), f"unknown image specification: {imagespec}"
|
||||
self.imagespec = imagespec.lower()
|
||||
@ -269,6 +271,7 @@ def extension_extract_fn(pathname):
|
||||
class Decoder:
|
||||
"""
|
||||
Decode key/data sets using a list of handlers.
|
||||
|
||||
For each key/data item, this iterates through the list of
|
||||
handlers until some handler returns something other than None.
|
||||
"""
|
||||
|
||||
@ -9,8 +9,8 @@ from torch.utils.data.graph_settings import apply_random_seed
|
||||
# lack the option to `set_seed`.
|
||||
def _simple_graph_snapshot_restoration(datapipe: IterDataPipe, n_iterations: int, rng=None) -> None:
|
||||
r"""
|
||||
This function will restore a snapshot by fast-forwarding the given DataPipe by ``n_iterations``,
|
||||
and in the process, fast-forward its parent DataPipes as well at the cost of re-doing every computation.
|
||||
Fast-forward the given DataPipe and its parents by ``n_iterations``, re-doing computations to restore a snapshot.
|
||||
|
||||
For instance, applying this function to the final DataPipe of a graph will restore the snapshot
|
||||
(via fast-forward) every DataPipe within the graph.
|
||||
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
"""
|
||||
This module converts objects into numpy array.
|
||||
"""
|
||||
"""This module converts objects into numpy array."""
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
|
||||
def make_np(x):
|
||||
"""
|
||||
Convert an object into numpy array.
|
||||
|
||||
Args:
|
||||
x: An instance of torch tensor or caffe blob name
|
||||
|
||||
|
||||
Reference in New Issue
Block a user