From aeb3f7b0dc787b971997e7d02e339601a6bb33c8 Mon Sep 17 00:00:00 2001 From: Prinz23 Date: Tue, 31 Oct 2023 17:05:51 +0100 Subject: [PATCH] use context manager for scandir as recommended since python 3.6 --- lib/sg_helpers.py | 47 ++++----- sickgear/__init__.py | 7 +- sickgear/db.py | 14 +-- sickgear/helpers.py | 195 +++++++++++++++++++++----------------- tests/tvinfo_api_tests.py | 19 ++-- 5 files changed, 152 insertions(+), 130 deletions(-) diff --git a/lib/sg_helpers.py b/lib/sg_helpers.py index af315e30..08c2d416 100644 --- a/lib/sg_helpers.py +++ b/lib/sg_helpers.py @@ -1120,29 +1120,30 @@ def scantree(path, # type: AnyStr (r'(?i)%s', include))] has_exclude, has_exclude_dirs, has_include = bool(exclude), bool(exclude_dirs), bool(include) - for entry in scandir(path): - is_dir = entry.is_dir(follow_symlinks=follow_symlinks) - is_file = entry.is_file(follow_symlinks=follow_symlinks) - no_filter = any([None is filter_kind, filter_kind and is_dir, not filter_kind and is_file]) - if ((not has_exclude or rc_exc.search(entry.name)) - and (not has_exclude_dirs or not is_dir or rc_exc_dir.search(entry.name)) - and (not has_include or rc_inc.search(entry.name)) - and (no_filter or (not filter_kind and is_dir and recurse))): - if is_dir and exclude_folders_with_files and any(os.path.isfile(os.path.join(entry.path, e_f)) - for e_f in exclude_folders_with_files): - logger.debug(f'Ignoring Folder: "{entry.path}", because it contains a exclude file' - f' "{", ".join(exclude_folders_with_files)}"') - continue - if recurse and is_dir: - for subentry in scantree( - path=entry.path, exclude=exclude, exclude_dirs=exclude_dirs, include=include, - follow_symlinks=follow_symlinks, filter_kind=filter_kind, recurse=recurse, - exclude_folders_with_files=exclude_folders_with_files, internal_call=True, - rc_exc=rc_exc, rc_exc_dir=rc_exc_dir, rc_inc=rc_inc, has_exclude=has_exclude, - has_exclude_dirs=has_exclude_dirs, has_include=has_include): - yield subentry - if no_filter: - yield entry + with scandir(path) as s_d: + for entry in s_d: + is_dir = entry.is_dir(follow_symlinks=follow_symlinks) + is_file = entry.is_file(follow_symlinks=follow_symlinks) + no_filter = any([None is filter_kind, filter_kind and is_dir, not filter_kind and is_file]) + if ((not has_exclude or rc_exc.search(entry.name)) + and (not has_exclude_dirs or not is_dir or rc_exc_dir.search(entry.name)) + and (not has_include or rc_inc.search(entry.name)) + and (no_filter or (not filter_kind and is_dir and recurse))): + if is_dir and exclude_folders_with_files and any(os.path.isfile(os.path.join(entry.path, e_f)) + for e_f in exclude_folders_with_files): + logger.debug(f'Ignoring Folder: "{entry.path}", because it contains a exclude file' + f' "{", ".join(exclude_folders_with_files)}"') + continue + if recurse and is_dir: + for subentry in scantree( + path=entry.path, exclude=exclude, exclude_dirs=exclude_dirs, include=include, + follow_symlinks=follow_symlinks, filter_kind=filter_kind, recurse=recurse, + exclude_folders_with_files=exclude_folders_with_files, internal_call=True, + rc_exc=rc_exc, rc_exc_dir=rc_exc_dir, rc_inc=rc_inc, has_exclude=has_exclude, + has_exclude_dirs=has_exclude_dirs, has_include=has_include): + yield subentry + if no_filter: + yield entry def copy_file(src_file, dest_file): diff --git a/sickgear/__init__.py b/sickgear/__init__.py index 24901649..9807f01a 100644 --- a/sickgear/__init__.py +++ b/sickgear/__init__.py @@ -1721,9 +1721,10 @@ def init_stage_2(): MEMCACHE['history_tab'] = History.menu_tab(MEMCACHE['history_tab_limit']) try: - for f in scandir(os.path.join(PROG_DIR, 'gui', GUI_NAME, 'images', 'flags')): - if f.is_file(): - MEMCACHE_FLAG_IMAGES[os.path.splitext(f.name)[0].lower()] = True + with scandir(os.path.join(PROG_DIR, 'gui', GUI_NAME, 'images', 'flags')) as s_d: + for f in s_d: + if f.is_file(): + MEMCACHE_FLAG_IMAGES[os.path.splitext(f.name)[0].lower()] = True except (BaseException, Exception): pass diff --git a/sickgear/db.py b/sickgear/db.py index 070d0bbb..cde45047 100644 --- a/sickgear/db.py +++ b/sickgear/db.py @@ -746,13 +746,13 @@ def cleanup_old_db_backups(filename): d, filename = os.path.split(filename) if not d: d = sickgear.DATA_DIR - for f in filter(lambda fn: fn.is_file() and filename in fn.name and - re.search(r'\.db(\.v\d+)?\.r\d+$', fn.name), - scandir(d)): - try: - os.unlink(f.path) - except (BaseException, Exception): - pass + with scandir(d) as s_d: + for f in filter(lambda fn: fn.is_file() and filename in fn.name and + re.search(r'\.db(\.v\d+)?\.r\d+$', fn.name), s_d): + try: + os.unlink(f.path) + except (BaseException, Exception): + pass except (BaseException, Exception): pass diff --git a/sickgear/helpers.py b/sickgear/helpers.py index 0abbe5ae..9c682585 100644 --- a/sickgear/helpers.py +++ b/sickgear/helpers.py @@ -1567,15 +1567,19 @@ def count_files_dirs(base_dir): """ f = d = 0 try: - files = scandir(base_dir) + with scandir(base_dir) as s_d: + try: + files = s_d + except OSError as e: + logger.warning('Unable to count files %s / %s' % (repr(e), ex(e))) + else: + for e in files: + if e.is_file(): + f += 1 + elif e.is_dir(): + d += 1 except OSError as e: logger.warning('Unable to count files %s / %s' % (repr(e), ex(e))) - else: - for e in files: - if e.is_file(): - f += 1 - elif e.is_dir(): - d += 1 return f, d @@ -1620,93 +1624,108 @@ def upgrade_new_naming(): cf = 0 p_text = 'Upgrading %s' % (d, 'banner/poster')[not d] _set_progress(p_text, 0, 0) - for entry in scandir(bd): - if entry.is_file(): - cf += 1 - _set_progress(p_text, cf, step) - b_s = bp_match.search(entry.name) - if b_s: - old_id = int(b_s.group(1)) - tvid = show_list.get(old_id) - if tvid: - nb_dir = os.path.join(sickgear.CACHE_DIR, 'images', 'shows', '%s-%s' % (tvid, old_id), d) - if not os.path.isdir(nb_dir): + with scandir(bd) as s_d: + for entry in scandir(bd): + if entry.is_file(): + cf += 1 + _set_progress(p_text, cf, step) + b_s = bp_match.search(entry.name) + if b_s: + old_id = int(b_s.group(1)) + tvid = show_list.get(old_id) + if tvid: + nb_dir = os.path.join(sickgear.CACHE_DIR, 'images', 'shows', '%s-%s' % (tvid, old_id), d) + if not os.path.isdir(nb_dir): + try: + os.makedirs(nb_dir) + except (BaseException, Exception): + pass + new_name = os.path.join(nb_dir, bp_match.sub(r'\2', entry.name)) try: - os.makedirs(nb_dir) + move_file(entry.path, new_name) + except (BaseException, Exception) as e: + logger.warning('Unable to rename %s to %s: %s / %s' + % (entry.path, new_name, repr(e), ex(e))) + else: + # clean up files without reference in db + try: + os.remove(entry.path) except (BaseException, Exception): pass - new_name = os.path.join(nb_dir, bp_match.sub(r'\2', entry.name)) - try: - move_file(entry.path, new_name) - except (BaseException, Exception) as e: - logger.warning('Unable to rename %s to %s: %s / %s' - % (entry.path, new_name, repr(e), ex(e))) - else: - # clean up files without reference in db - try: - os.remove(entry.path) - except (BaseException, Exception): - pass - elif entry.is_dir(): - if entry.name in ['shows', 'browse']: - continue - elif 'fanart' == entry.name: - _set_progress(p_text, 0, 1) - fc_fan, dc_fan = count_files_dirs(entry.path) - step_fan = dc_fan / float(100) - cf_fan = 0 - p_text = 'Upgrading fanart' - _set_progress(p_text, 0, 0) - try: - entries = scandir(entry.path) - except OSError as e: - logger.warning('Unable to stat dirs %s / %s' % (repr(e), ex(e))) + elif entry.is_dir(): + if entry.name in ['shows', 'browse']: continue - for d_entry in entries: - if d_entry.is_dir(): - cf_fan += 1 - _set_progress(p_text, cf_fan, step_fan) - old_id = try_int(d_entry.name) - if old_id: - new_id = show_list.get(old_id) - if new_id: - new_dir_name = os.path.join(sickgear.CACHE_DIR, 'images', 'shows', - '%s-%s' % (new_id, old_id), 'fanart') - try: - move_file(d_entry.path, new_dir_name) - except (BaseException, Exception) as e: - logger.warning(f'Unable to rename {d_entry.path} to {new_dir_name}:' - f' {repr(e)} / {ex(e)}') - if os.path.isdir(new_dir_name): - try: - f_n = filter(lambda fn: fn.is_file(), scandir(new_dir_name)) - except OSError as e: - logger.warning('Unable to rename %s / %s' % (repr(e), ex(e))) - else: - rename_args = [] - # noinspection PyTypeChecker - for f_entry in f_n: - rename_args += [(f_entry.path, bp_match.sub(r'\2', f_entry.path))] - - for args in rename_args: + elif 'fanart' == entry.name: + _set_progress(p_text, 0, 1) + fc_fan, dc_fan = count_files_dirs(entry.path) + step_fan = dc_fan / float(100) + cf_fan = 0 + p_text = 'Upgrading fanart' + _set_progress(p_text, 0, 0) + try: + with scandir(entry.path) as s_p: + try: + entries = s_p + except OSError as e: + logger.warning('Unable to stat dirs %s / %s' % (repr(e), ex(e))) + continue + for d_entry in entries: + if d_entry.is_dir(): + cf_fan += 1 + _set_progress(p_text, cf_fan, step_fan) + old_id = try_int(d_entry.name) + if old_id: + new_id = show_list.get(old_id) + if new_id: + new_dir_name = os.path.join(sickgear.CACHE_DIR, 'images', 'shows', + '%s-%s' % (new_id, old_id), 'fanart') try: - move_file(*args) + move_file(d_entry.path, new_dir_name) except (BaseException, Exception) as e: - logger.warning(f'Unable to rename {args[0]} to {args[1]}:' - f' {repr(e)} / {ex(e)}') - else: - try: - shutil.rmtree(d_entry.path) - except (BaseException, Exception): - pass - try: - shutil.rmtree(d_entry.path) - except (BaseException, Exception): - pass - try: - os.rmdir(entry.path) - except (BaseException, Exception): - pass + logger.warning(f'Unable to rename {d_entry.path}' + f' to {new_dir_name}: {repr(e)} / {ex(e)}') + if os.path.isdir(new_dir_name): + try: + with scandir(new_dir_name) as s_d_n: + try: + f_n = filter(lambda fn: fn.is_file(), s_d_n) + except OSError as e: + logger.warning( + f'Unable to rename {repr(e)} / {ex(d)}') + else: + rename_args = [] + # noinspection PyTypeChecker + for f_entry in f_n: + rename_args += [ + (f_entry.path, + bp_match.sub(r'\2', f_entry.path))] + + for args in rename_args: + try: + move_file(*args) + except (BaseException, Exception) as e: + logger.warning( + f'Unable to rename {args[0]}' + f' to {args[1]}: {repr(e)} / {ex(e)}') + except OSError as e: + logger.warning( + 'Unable to rename %s / %s' % (repr(e), ex(e))) + else: + try: + shutil.rmtree(d_entry.path) + except (BaseException, Exception): + pass + try: + shutil.rmtree(d_entry.path) + except (BaseException, Exception): + pass + except OSError as e: + logger.warning('Unable to stat dirs %s / %s' % (repr(e), ex(e))) + continue + try: + os.rmdir(entry.path) + except (BaseException, Exception): + pass if 'thumbnails' == d: try: os.rmdir(bd) diff --git a/tests/tvinfo_api_tests.py b/tests/tvinfo_api_tests.py index a008476b..4f0178eb 100644 --- a/tests/tvinfo_api_tests.py +++ b/tests/tvinfo_api_tests.py @@ -592,15 +592,16 @@ class TVInfoTests(test.SickbeardTestDBCase): @classmethod def tearDownClass(cls): super(TVInfoTests, cls).tearDownClass() - files = {_f.name for _f in os.scandir(mock_data_dir) if _f.is_file()} - unused_files = files - used_files - if delete_unused_mock_files: - for _u_f in unused_files: - full_filename = os.path.join(mock_data_dir, _u_f) - try: - os.remove(full_filename) - except (BaseException, Exception) as e: - print('errror deleting: [%s], error: %s' % (full_filename, e)) + with os.scandir(mock_data_dir) as s_d: + files = {_f.name for _f in os.scandir(mock_data_dir) if _f.is_file()} + unused_files = files - used_files + if delete_unused_mock_files: + for _u_f in unused_files: + full_filename = os.path.join(mock_data_dir, _u_f) + try: + os.remove(full_filename) + except (BaseException, Exception) as e: + print('errror deleting: [%s], error: %s' % (full_filename, e)) if unused_files: print('unused files: %s' % unused_files) print('reset mock methods')