Upload hf_utils.py with huggingface_hub
Browse files- hf_utils.py +44 -0
hf_utils.py
ADDED
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from datasets import (
|
2 |
+
disable_caching,
|
3 |
+
enable_caching,
|
4 |
+
is_caching_enabled,
|
5 |
+
)
|
6 |
+
from pathlib import Path
|
7 |
+
from .file_utils import get_all_files_in_dir
|
8 |
+
from datasets.utils.py_utils import get_imports
|
9 |
+
|
10 |
+
HF_CACHING_ENABLED = False
|
11 |
+
|
12 |
+
class HFCachingContextManager:
|
13 |
+
def __init__(self, should_cache):
|
14 |
+
self.should_cache = should_cache
|
15 |
+
self.was_caching_enabled = None
|
16 |
+
|
17 |
+
def __enter__(self):
|
18 |
+
self.was_caching_enabled = is_caching_enabled()
|
19 |
+
if self.should_cache:
|
20 |
+
if not self.was_caching_enabled:
|
21 |
+
enable_caching()
|
22 |
+
else:
|
23 |
+
if self.was_caching_enabled:
|
24 |
+
disable_caching()
|
25 |
+
|
26 |
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
27 |
+
if self.was_caching_enabled != is_caching_enabled():
|
28 |
+
if self.was_caching_enabled:
|
29 |
+
enable_caching()
|
30 |
+
else:
|
31 |
+
disable_caching()
|
32 |
+
|
33 |
+
def set_hf_caching(enabled):
|
34 |
+
return HFCachingContextManager(enabled)
|
35 |
+
|
36 |
+
def get_missing_imports(file, exclude=[]):
|
37 |
+
src_dir = Path(__file__).parent
|
38 |
+
python_files = get_all_files_in_dir(src_dir, file_extension='.py')
|
39 |
+
# get only the file without the path and extension
|
40 |
+
required_modules = [Path(p).stem for p in python_files]
|
41 |
+
imports = get_imports(file)
|
42 |
+
imported_modules = [i[1] for i in imports if i[0] == 'internal']
|
43 |
+
missing_imports = [i for i in required_modules if i not in imported_modules and i not in exclude]
|
44 |
+
return missing_imports
|