monkee.ch/SickGear
1
0
Fork 0
mirror of https://github.com/SickGear/SickGear.git synced 2024-11-24 13:55:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'feature/ChangeScandir' into dev

Browse source
This commit is contained in:
JackDandy 2023-10-31 17:18:57 +00:00
commit 4a62212e57
5 changed files with 152 additions and 130 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
lib/sg_helpers.py
View file

@ -1120,29 +1120,30 @@ def scantree(path, # type: AnyStr
(r'(?i)%s', include))] (r'(?i)%s', include))]
has_exclude, has_exclude_dirs, has_include = bool(exclude), bool(exclude_dirs), bool(include) has_exclude, has_exclude_dirs, has_include = bool(exclude), bool(exclude_dirs), bool(include)
for entry in scandir(path): with scandir(path) as s_d:
is_dir = entry.is_dir(follow_symlinks=follow_symlinks) for entry in s_d:
is_file = entry.is_file(follow_symlinks=follow_symlinks) is_dir = entry.is_dir(follow_symlinks=follow_symlinks)
no_filter = any([None is filter_kind, filter_kind and is_dir, not filter_kind and is_file]) is_file = entry.is_file(follow_symlinks=follow_symlinks)
if ((not has_exclude or rc_exc.search(entry.name)) no_filter = any([None is filter_kind, filter_kind and is_dir, not filter_kind and is_file])
and (not has_exclude_dirs or not is_dir or rc_exc_dir.search(entry.name)) if ((not has_exclude or rc_exc.search(entry.name))
and (not has_include or rc_inc.search(entry.name)) and (not has_exclude_dirs or not is_dir or rc_exc_dir.search(entry.name))
and (no_filter or (not filter_kind and is_dir and recurse))): and (not has_include or rc_inc.search(entry.name))
if is_dir and exclude_folders_with_files and any(os.path.isfile(os.path.join(entry.path, e_f)) and (no_filter or (not filter_kind and is_dir and recurse))):
for e_f in exclude_folders_with_files): if is_dir and exclude_folders_with_files and any(os.path.isfile(os.path.join(entry.path, e_f))
logger.debug(f'Ignoring Folder: "{entry.path}", because it contains a exclude file' for e_f in exclude_folders_with_files):
f' "{", ".join(exclude_folders_with_files)}"') logger.debug(f'Ignoring Folder: "{entry.path}", because it contains a exclude file'
continue f' "{", ".join(exclude_folders_with_files)}"')
if recurse and is_dir: continue
for subentry in scantree( if recurse and is_dir:
path=entry.path, exclude=exclude, exclude_dirs=exclude_dirs, include=include, for subentry in scantree(
follow_symlinks=follow_symlinks, filter_kind=filter_kind, recurse=recurse, path=entry.path, exclude=exclude, exclude_dirs=exclude_dirs, include=include,
exclude_folders_with_files=exclude_folders_with_files, internal_call=True, follow_symlinks=follow_symlinks, filter_kind=filter_kind, recurse=recurse,
rc_exc=rc_exc, rc_exc_dir=rc_exc_dir, rc_inc=rc_inc, has_exclude=has_exclude, exclude_folders_with_files=exclude_folders_with_files, internal_call=True,
has_exclude_dirs=has_exclude_dirs, has_include=has_include): rc_exc=rc_exc, rc_exc_dir=rc_exc_dir, rc_inc=rc_inc, has_exclude=has_exclude,
yield subentry has_exclude_dirs=has_exclude_dirs, has_include=has_include):
if no_filter: yield subentry
yield entry if no_filter:
yield entry
def copy_file(src_file, dest_file): def copy_file(src_file, dest_file):

7
sickgear/__init__.py
View file

@ -1721,9 +1721,10 @@ def init_stage_2():
MEMCACHE['history_tab'] = History.menu_tab(MEMCACHE['history_tab_limit']) MEMCACHE['history_tab'] = History.menu_tab(MEMCACHE['history_tab_limit'])
try: try:
for f in scandir(os.path.join(PROG_DIR, 'gui', GUI_NAME, 'images', 'flags')): with scandir(os.path.join(PROG_DIR, 'gui', GUI_NAME, 'images', 'flags')) as s_d:
if f.is_file(): for f in s_d:
MEMCACHE_FLAG_IMAGES[os.path.splitext(f.name)[0].lower()] = True if f.is_file():
MEMCACHE_FLAG_IMAGES[os.path.splitext(f.name)[0].lower()] = True
except (BaseException, Exception): except (BaseException, Exception):
pass pass

14
sickgear/db.py
View file

@ -746,13 +746,13 @@ def cleanup_old_db_backups(filename):
d, filename = os.path.split(filename) d, filename = os.path.split(filename)
if not d: if not d:
d = sickgear.DATA_DIR d = sickgear.DATA_DIR
for f in filter(lambda fn: fn.is_file() and filename in fn.name and with scandir(d) as s_d:
re.search(r'\.db(\.v\d+)?\.r\d+$', fn.name), for f in filter(lambda fn: fn.is_file() and filename in fn.name and
scandir(d)): re.search(r'\.db(\.v\d+)?\.r\d+$', fn.name), s_d):
try: try:
os.unlink(f.path) os.unlink(f.path)
except (BaseException, Exception): except (BaseException, Exception):
pass pass
except (BaseException, Exception): except (BaseException, Exception):
pass pass

195
sickgear/helpers.py
View file

@ -1567,15 +1567,19 @@ def count_files_dirs(base_dir):
""" """
f = d = 0 f = d = 0
try: try:
files = scandir(base_dir) with scandir(base_dir) as s_d:
try:
files = s_d
except OSError as e:
logger.warning('Unable to count files %s / %s' % (repr(e), ex(e)))
else:
for e in files:
if e.is_file():
f += 1
elif e.is_dir():
d += 1
except OSError as e: except OSError as e:
logger.warning('Unable to count files %s / %s' % (repr(e), ex(e))) logger.warning('Unable to count files %s / %s' % (repr(e), ex(e)))
else:
for e in files:
if e.is_file():
f += 1
elif e.is_dir():
d += 1
return f, d return f, d
@ -1620,93 +1624,108 @@ def upgrade_new_naming():
cf = 0 cf = 0
p_text = 'Upgrading %s' % (d, 'banner/poster')[not d] p_text = 'Upgrading %s' % (d, 'banner/poster')[not d]
_set_progress(p_text, 0, 0) _set_progress(p_text, 0, 0)
for entry in scandir(bd): with scandir(bd) as s_d:
if entry.is_file(): for entry in scandir(bd):
cf += 1 if entry.is_file():
_set_progress(p_text, cf, step) cf += 1
b_s = bp_match.search(entry.name) _set_progress(p_text, cf, step)
if b_s: b_s = bp_match.search(entry.name)
old_id = int(b_s.group(1)) if b_s:
tvid = show_list.get(old_id) old_id = int(b_s.group(1))
if tvid: tvid = show_list.get(old_id)
nb_dir = os.path.join(sickgear.CACHE_DIR, 'images', 'shows', '%s-%s' % (tvid, old_id), d) if tvid:
if not os.path.isdir(nb_dir): nb_dir = os.path.join(sickgear.CACHE_DIR, 'images', 'shows', '%s-%s' % (tvid, old_id), d)
if not os.path.isdir(nb_dir):
try:
os.makedirs(nb_dir)
except (BaseException, Exception):
pass
new_name = os.path.join(nb_dir, bp_match.sub(r'\2', entry.name))
try: try:
os.makedirs(nb_dir) move_file(entry.path, new_name)
except (BaseException, Exception) as e:
logger.warning('Unable to rename %s to %s: %s / %s'
% (entry.path, new_name, repr(e), ex(e)))
else:
# clean up files without reference in db
try:
os.remove(entry.path)
except (BaseException, Exception): except (BaseException, Exception):
pass pass
new_name = os.path.join(nb_dir, bp_match.sub(r'\2', entry.name)) elif entry.is_dir():
try: if entry.name in ['shows', 'browse']:
move_file(entry.path, new_name)
except (BaseException, Exception) as e:
logger.warning('Unable to rename %s to %s: %s / %s'
% (entry.path, new_name, repr(e), ex(e)))
else:
# clean up files without reference in db
try:
os.remove(entry.path)
except (BaseException, Exception):
pass
elif entry.is_dir():
if entry.name in ['shows', 'browse']:
continue
elif 'fanart' == entry.name:
_set_progress(p_text, 0, 1)
fc_fan, dc_fan = count_files_dirs(entry.path)
step_fan = dc_fan / float(100)
cf_fan = 0
p_text = 'Upgrading fanart'
_set_progress(p_text, 0, 0)
try:
entries = scandir(entry.path)
except OSError as e:
logger.warning('Unable to stat dirs %s / %s' % (repr(e), ex(e)))
continue continue
for d_entry in entries: elif 'fanart' == entry.name:
if d_entry.is_dir(): _set_progress(p_text, 0, 1)
cf_fan += 1 fc_fan, dc_fan = count_files_dirs(entry.path)
_set_progress(p_text, cf_fan, step_fan) step_fan = dc_fan / float(100)
old_id = try_int(d_entry.name) cf_fan = 0
if old_id: p_text = 'Upgrading fanart'
new_id = show_list.get(old_id) _set_progress(p_text, 0, 0)
if new_id: try:
new_dir_name = os.path.join(sickgear.CACHE_DIR, 'images', 'shows', with scandir(entry.path) as s_p:
'%s-%s' % (new_id, old_id), 'fanart') try:
try: entries = s_p
move_file(d_entry.path, new_dir_name) except OSError as e:
except (BaseException, Exception) as e: logger.warning('Unable to stat dirs %s / %s' % (repr(e), ex(e)))
logger.warning(f'Unable to rename {d_entry.path} to {new_dir_name}:' continue
f' {repr(e)} / {ex(e)}') for d_entry in entries:
if os.path.isdir(new_dir_name): if d_entry.is_dir():
try: cf_fan += 1
f_n = filter(lambda fn: fn.is_file(), scandir(new_dir_name)) _set_progress(p_text, cf_fan, step_fan)
except OSError as e: old_id = try_int(d_entry.name)
logger.warning('Unable to rename %s / %s' % (repr(e), ex(e))) if old_id:
else: new_id = show_list.get(old_id)
rename_args = [] if new_id:
# noinspection PyTypeChecker new_dir_name = os.path.join(sickgear.CACHE_DIR, 'images', 'shows',
for f_entry in f_n: '%s-%s' % (new_id, old_id), 'fanart')
rename_args += [(f_entry.path, bp_match.sub(r'\2', f_entry.path))]
for args in rename_args:
try: try:
move_file(*args) move_file(d_entry.path, new_dir_name)
except (BaseException, Exception) as e: except (BaseException, Exception) as e:
logger.warning(f'Unable to rename {args[0]} to {args[1]}:' logger.warning(f'Unable to rename {d_entry.path}'
f' {repr(e)} / {ex(e)}') f' to {new_dir_name}: {repr(e)} / {ex(e)}')
else: if os.path.isdir(new_dir_name):
try: try:
shutil.rmtree(d_entry.path) with scandir(new_dir_name) as s_d_n:
except (BaseException, Exception): try:
pass f_n = filter(lambda fn: fn.is_file(), s_d_n)
try: except OSError as e:
shutil.rmtree(d_entry.path) logger.warning(
except (BaseException, Exception): f'Unable to rename {repr(e)} / {ex(d)}')
pass else:
try: rename_args = []
os.rmdir(entry.path) # noinspection PyTypeChecker
except (BaseException, Exception): for f_entry in f_n:
pass rename_args += [
(f_entry.path,
bp_match.sub(r'\2', f_entry.path))]
for args in rename_args:
try:
move_file(*args)
except (BaseException, Exception) as e:
logger.warning(
f'Unable to rename {args[0]}'
f' to {args[1]}: {repr(e)} / {ex(e)}')
except OSError as e:
logger.warning(
'Unable to rename %s / %s' % (repr(e), ex(e)))
else:
try:
shutil.rmtree(d_entry.path)
except (BaseException, Exception):
pass
try:
shutil.rmtree(d_entry.path)
except (BaseException, Exception):
pass
except OSError as e:
logger.warning('Unable to stat dirs %s / %s' % (repr(e), ex(e)))
continue
try:
os.rmdir(entry.path)
except (BaseException, Exception):
pass
if 'thumbnails' == d: if 'thumbnails' == d:
try: try:
os.rmdir(bd) os.rmdir(bd)

19
tests/tvinfo_api_tests.py
View file

@ -592,15 +592,16 @@ class TVInfoTests(test.SickbeardTestDBCase):
@classmethod @classmethod
def tearDownClass(cls): def tearDownClass(cls):
super(TVInfoTests, cls).tearDownClass() super(TVInfoTests, cls).tearDownClass()
files = {_f.name for _f in os.scandir(mock_data_dir) if _f.is_file()} with os.scandir(mock_data_dir) as s_d:
unused_files = files - used_files files = {_f.name for _f in os.scandir(mock_data_dir) if _f.is_file()}
if delete_unused_mock_files: unused_files = files - used_files
for _u_f in unused_files: if delete_unused_mock_files:
full_filename = os.path.join(mock_data_dir, _u_f) for _u_f in unused_files:
try: full_filename = os.path.join(mock_data_dir, _u_f)
os.remove(full_filename) try:
except (BaseException, Exception) as e: os.remove(full_filename)
print('errror deleting: [%s], error: %s' % (full_filename, e)) except (BaseException, Exception) as e:
print('errror deleting: [%s], error: %s' % (full_filename, e))
if unused_files: if unused_files:
print('unused files: %s' % unused_files) print('unused files: %s' % unused_files)
print('reset mock methods') print('reset mock methods')