monkee.ch/SickGear
1
0
Fork 0
mirror of https://github.com/SickGear/SickGear.git synced 2024-12-25 12:13:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'feature/UpdateProfilehooks' into dev

Browse source
This commit is contained in:
JackDandy 2024-06-07 15:47:50 +01:00
commit 3286e4d323
2 changed files with 14 additions and 223 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGES.md
View file

@ -9,6 +9,7 @@
* Update filelock 3.12.4 (c1163ae) to 3.14.0 (8556141)
* Update idna library 3.4 (cab054c) to 3.7 (1d365e1)
* Update imdbpie 5.6.4 (f695e87) to 5.6.5 (f8ed7a0)
* Update profilehooks module 1.12.1 (c3fc078) to 1.13.0.dev0 (99f8a31)
* Update Requests library 2.31.0 (8812812) to 2.32.3 (0e322af)
* Update Tornado Web Server 6.4 (b3f2a4b) to 6.4.1 (2a0e1d1)
* Update urllib3 2.0.7 (56f01e0) to 2.2.1 (54d6edf)

236
lib/profilehooks.py
View file

@ -39,28 +39,12 @@ instead of a detailed (but costly) profile.
Caveats
A thread on python-dev convinced me that hotshot produces bogus numbers.
See https://mail.python.org/pipermail/python-dev/2005-November/058264.html
I don't know what will happen if a decorated function will try to call
another decorated function. All decorators probably need to explicitly
support nested profiling (currently TraceFuncCoverage is the only one
that supports this, while HotShotFuncProfile has support for recursive
functions.)
that supports this.)
Profiling with hotshot creates temporary files (*.prof for profiling,
*.cprof for coverage) in the current directory. These files are not
cleaned up. Exception: when you specify a filename to the profile
decorator (to store the pstats.Stats object for later inspection),
the temporary file will be the filename you specified with '.raw'
appended at the end.
Coverage analysis with hotshot seems to miss some executions resulting
in lower line counts and some lines errorneously marked as never
executed. For this reason coverage analysis now uses trace.py which is
slower, but more accurate.
Copyright (c) 2004--2020 Marius Gedminas <marius@gedmin.as>
Copyright (c) 2004--2023 Marius Gedminas <marius@gedmin.as>
Copyright (c) 2007 Hanno Schlichting
Copyright (c) 2008 Florian Schulze
@ -86,8 +70,6 @@ Released under the MIT licence since December 2006:
(Previously it was distributed under the GNU General Public Licence.)
"""
from __future__ import print_function
import atexit
import dis
import functools
@ -104,19 +86,6 @@ import trace
from profile import Profile
# For hotshot profiling (inaccurate!)
try: # pragma: PY2
import hotshot
import hotshot.stats
except ImportError:
hotshot = None
# For hotshot coverage (inaccurate!; uses undocumented APIs; might break)
if hotshot is not None: # pragma: PY2
import _hotshot
import hotshot.log
# For cProfile profiling (best)
try:
import cProfile
@ -127,38 +96,18 @@ except ImportError:
__author__ = "Marius Gedminas <marius@gedmin.as>"
__copyright__ = "Copyright 2004-2020 Marius Gedminas and contributors"
__license__ = "MIT"
__version__ = '1.12.1.dev0'
__date__ = "2020-08-20"
__version__ = '1.13.0.dev0'
__date__ = "2023-12-18"
# registry of available profilers
AVAILABLE_PROFILERS = {}
__all__ = ['coverage', 'coverage_with_hotshot', 'profile', 'timecall']
# Use tokenize.open() on Python >= 3.2, fall back to open() on Python 2
tokenize_open = getattr(tokenize, 'open', open)
try:
from inspect import unwrap as _unwrap
except ImportError: # pragma: PY2
# inspect.unwrap() doesn't exist on Python 2
def _unwrap(fn):
if not hasattr(fn, '__wrapped__'):
return fn
else: # pragma: nocover
# functools.wraps() doesn't set __wrapped__ on Python 2 either,
# so this branch will only get reached if somebody
# manually sets __wrapped__, hence the pragma: nocover.
# NB: intentionally using recursion here instead of a while loop to
# make cycles fail with a recursion error instead of looping forever.
return _unwrap(fn.__wrapped__)
__all__ = ['coverage', 'profile', 'timecall']
def _identify(fn):
fn = _unwrap(fn)
fn = inspect.unwrap(fn)
funcname = fn.__name__
filename = fn.__code__.co_filename
lineno = fn.__code__.co_firstlineno
@ -171,7 +120,7 @@ def _is_file_like(o):
def profile(fn=None, skip=0, filename=None, immediate=False, dirs=False,
sort=None, entries=40,
profiler=('cProfile', 'profile', 'hotshot'),
profiler=('cProfile', 'profile'),
stdout=True):
"""Mark `fn` for profiling.
@ -208,7 +157,7 @@ def profile(fn=None, skip=0, filename=None, immediate=False, dirs=False,
`profiler` can be used to select the preferred profiler, or specify a
sequence of them, in order of preference. The default is ('cProfile'.
'profile', 'hotshot').
'profile').
If `filename` is specified, the profile stats will be stored in the
named file. You can load them with pstats.Stats(filename) or use a
@ -282,26 +231,7 @@ def coverage(fn):
...
"""
fp = TraceFuncCoverage(fn) # or HotShotFuncCoverage
# We cannot return fp or fp.__call__ directly as that would break method
# definitions, instead we need to return a plain function.
@functools.wraps(fn)
def new_fn(*args, **kw):
return fp(*args, **kw)
return new_fn
def coverage_with_hotshot(fn): # pragma: PY2
"""Mark `fn` for line coverage analysis.
Uses the 'hotshot' module for fast coverage analysis.
BUG: Produces inaccurate results.
See the docstring of `coverage` for usage examples.
"""
fp = HotShotFuncCoverage(fn)
fp = TraceFuncCoverage(fn)
# We cannot return fp or fp.__call__ directly as that would break method
# definitions, instead we need to return a plain function.
@ -427,148 +357,8 @@ if cProfile is not None:
AVAILABLE_PROFILERS['cProfile'] = CProfileFuncProfile
if hotshot is not None: # pragma: PY2
class HotShotFuncProfile(FuncProfile):
"""Profiler for a function (uses hotshot)."""
# This flag is shared between all instances
in_profiler = False
def __init__(self, fn, skip=0, filename=None, immediate=False,
dirs=False, sort=None, entries=40, stdout=True):
"""Creates a profiler for a function.
Every profiler has its own log file (the name of which is derived
from the function name).
HotShotFuncProfile registers an atexit handler that prints
profiling information to sys.stderr when the program terminates.
The log file is not removed and remains there to clutter the
current working directory.
"""
if filename:
self.logfilename = filename + ".raw"
else:
self.logfilename = "%s.%d.prof" % (fn.__name__, os.getpid())
super(HotShotFuncProfile, self).__init__(
fn, skip=skip, filename=filename, immediate=immediate,
dirs=dirs, sort=sort, entries=entries, stdout=stdout)
def __call__(self, *args, **kw):
"""Profile a singe call to the function."""
self.ncalls += 1
if self.skip > 0:
self.skip -= 1
self.skipped += 1
return self.fn(*args, **kw)
if HotShotFuncProfile.in_profiler:
# handle recursive calls
return self.fn(*args, **kw)
if self.profiler is None:
self.profiler = hotshot.Profile(self.logfilename)
try:
HotShotFuncProfile.in_profiler = True
return self.profiler.runcall(self.fn, *args, **kw)
finally:
HotShotFuncProfile.in_profiler = False
if self.immediate:
self.print_stats()
self.reset_stats()
def print_stats(self):
if self.profiler is None:
self.stats = pstats.Stats(Profile())
else:
self.profiler.close()
self.stats = hotshot.stats.load(self.logfilename)
super(HotShotFuncProfile, self).print_stats()
def reset_stats(self):
self.profiler = None
self.ncalls = 0
self.skipped = 0
AVAILABLE_PROFILERS['hotshot'] = HotShotFuncProfile
class HotShotFuncCoverage:
"""Coverage analysis for a function (uses _hotshot).
HotShot coverage is reportedly faster than trace.py, but it appears to
have problems with exceptions; also line counts in coverage reports
are generally lower from line counts produced by TraceFuncCoverage.
Is this my bug, or is it a problem with _hotshot?
"""
def __init__(self, fn):
"""Creates a profiler for a function.
Every profiler has its own log file (the name of which is derived
from the function name).
HotShotFuncCoverage registers an atexit handler that prints
profiling information to sys.stderr when the program terminates.
The log file is not removed and remains there to clutter the
current working directory.
"""
self.fn = fn
self.logfilename = "%s.%d.cprof" % (fn.__name__, os.getpid())
self.profiler = _hotshot.coverage(self.logfilename)
self.ncalls = 0
atexit.register(self.atexit)
def __call__(self, *args, **kw):
"""Profile a singe call to the function."""
self.ncalls += 1
old_trace = sys.gettrace()
try:
return self.profiler.runcall(self.fn, args, kw)
finally: # pragma: nocover
sys.settrace(old_trace)
def atexit(self):
"""Stop profiling and print profile information to sys.stderr.
This function is registered as an atexit hook.
"""
self.profiler.close()
funcname, filename, lineno = _identify(self.fn)
print("")
print("*** COVERAGE RESULTS ***")
print("%s (%s:%s)" % (funcname, filename, lineno))
print("function called %d times" % self.ncalls)
print("")
fs = FuncSource(self.fn)
reader = hotshot.log.LogReader(self.logfilename)
for what, (filename, lineno, funcname), tdelta in reader:
if filename != fs.filename:
continue
if what == hotshot.log.LINE:
fs.mark(lineno)
if what == hotshot.log.ENTER:
# hotshot gives us the line number of the function
# definition and never gives us a LINE event for the first
# statement in a function, so if we didn't perform this
# mapping, the first statement would be marked as never
# executed
if lineno == fs.firstlineno:
lineno = fs.firstcodelineno
fs.mark(lineno)
reader.close()
print(fs)
never_executed = fs.count_never_executed()
if never_executed:
print("%d lines were not executed." % never_executed)
class TraceFuncCoverage:
"""Coverage analysis for a function (uses trace module).
HotShot coverage analysis is reportedly faster, but it appears to have
problems with exceptions.
"""
"""Coverage analysis for a function (uses trace module)."""
# Shared between all instances so that nested calls work
tracer = trace.Trace(count=True, trace=False,
@ -657,11 +447,11 @@ class FuncSource:
strs = self._find_docstrings(self.filename)
lines = {
ln
for off, ln in dis.findlinestarts(_unwrap(self.fn).__code__)
for off, ln in dis.findlinestarts(inspect.unwrap(self.fn).__code__)
# skipping firstlineno because Python 3.11 adds a 'RESUME' opcode
# attributed to the `def` line, but then trace.py never sees it
# getting executed
if ln not in strs and ln != self.firstlineno
if ln is not None and ln not in strs and ln != self.firstlineno
}
for lineno in lines:
self.sourcelines.setdefault(lineno, 0)
@ -676,7 +466,7 @@ class FuncSource:
# Python 3.2 and removed in 3.6.
strs = set()
prev = token.INDENT # so module docstring is detected as docstring
with tokenize_open(filename) as f:
with tokenize.open(filename) as f:
tokens = tokenize.generate_tokens(f.readline)
for ttype, tstr, start, end, line in tokens:
if ttype == token.STRING and prev == token.INDENT: