From 75392efb97d5196beae7080929087e4f1c894a3f Mon Sep 17 00:00:00 2001 From: JackDandy Date: Fri, 7 Jun 2024 15:47:27 +0100 Subject: [PATCH] =?UTF-8?q?Update=20profilehooks=20module=201.12.1=20(c3fc?= =?UTF-8?q?078)=20=E2=86=92=201.13.0.dev0=20(99f8a31).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 1 + lib/profilehooks.py | 236 +++----------------------------------------- 2 files changed, 14 insertions(+), 223 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a21f0bb8..b94c3c4c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,6 +9,7 @@ * Update filelock 3.12.4 (c1163ae) to 3.14.0 (8556141) * Update idna library 3.4 (cab054c) to 3.7 (1d365e1) * Update imdbpie 5.6.4 (f695e87) to 5.6.5 (f8ed7a0) +* Update profilehooks module 1.12.1 (c3fc078) to 1.13.0.dev0 (99f8a31) * Update Requests library 2.31.0 (8812812) to 2.32.3 (0e322af) * Update Tornado Web Server 6.4 (b3f2a4b) to 6.4.1 (2a0e1d1) * Update urllib3 2.0.7 (56f01e0) to 2.2.1 (54d6edf) diff --git a/lib/profilehooks.py b/lib/profilehooks.py index dc2d22c8..2d878f50 100644 --- a/lib/profilehooks.py +++ b/lib/profilehooks.py @@ -39,28 +39,12 @@ instead of a detailed (but costly) profile. Caveats - A thread on python-dev convinced me that hotshot produces bogus numbers. - See https://mail.python.org/pipermail/python-dev/2005-November/058264.html - I don't know what will happen if a decorated function will try to call another decorated function. All decorators probably need to explicitly support nested profiling (currently TraceFuncCoverage is the only one - that supports this, while HotShotFuncProfile has support for recursive - functions.) + that supports this.) - Profiling with hotshot creates temporary files (*.prof for profiling, - *.cprof for coverage) in the current directory. These files are not - cleaned up. Exception: when you specify a filename to the profile - decorator (to store the pstats.Stats object for later inspection), - the temporary file will be the filename you specified with '.raw' - appended at the end. - - Coverage analysis with hotshot seems to miss some executions resulting - in lower line counts and some lines errorneously marked as never - executed. For this reason coverage analysis now uses trace.py which is - slower, but more accurate. - -Copyright (c) 2004--2020 Marius Gedminas +Copyright (c) 2004--2023 Marius Gedminas Copyright (c) 2007 Hanno Schlichting Copyright (c) 2008 Florian Schulze @@ -86,8 +70,6 @@ Released under the MIT licence since December 2006: (Previously it was distributed under the GNU General Public Licence.) """ -from __future__ import print_function - import atexit import dis import functools @@ -104,19 +86,6 @@ import trace from profile import Profile -# For hotshot profiling (inaccurate!) -try: # pragma: PY2 - import hotshot - import hotshot.stats -except ImportError: - hotshot = None - - -# For hotshot coverage (inaccurate!; uses undocumented APIs; might break) -if hotshot is not None: # pragma: PY2 - import _hotshot - import hotshot.log - # For cProfile profiling (best) try: import cProfile @@ -127,38 +96,18 @@ except ImportError: __author__ = "Marius Gedminas " __copyright__ = "Copyright 2004-2020 Marius Gedminas and contributors" __license__ = "MIT" -__version__ = '1.12.1.dev0' -__date__ = "2020-08-20" +__version__ = '1.13.0.dev0' +__date__ = "2023-12-18" # registry of available profilers AVAILABLE_PROFILERS = {} -__all__ = ['coverage', 'coverage_with_hotshot', 'profile', 'timecall'] - - -# Use tokenize.open() on Python >= 3.2, fall back to open() on Python 2 -tokenize_open = getattr(tokenize, 'open', open) - - -try: - from inspect import unwrap as _unwrap -except ImportError: # pragma: PY2 - # inspect.unwrap() doesn't exist on Python 2 - def _unwrap(fn): - if not hasattr(fn, '__wrapped__'): - return fn - else: # pragma: nocover - # functools.wraps() doesn't set __wrapped__ on Python 2 either, - # so this branch will only get reached if somebody - # manually sets __wrapped__, hence the pragma: nocover. - # NB: intentionally using recursion here instead of a while loop to - # make cycles fail with a recursion error instead of looping forever. - return _unwrap(fn.__wrapped__) +__all__ = ['coverage', 'profile', 'timecall'] def _identify(fn): - fn = _unwrap(fn) + fn = inspect.unwrap(fn) funcname = fn.__name__ filename = fn.__code__.co_filename lineno = fn.__code__.co_firstlineno @@ -171,7 +120,7 @@ def _is_file_like(o): def profile(fn=None, skip=0, filename=None, immediate=False, dirs=False, sort=None, entries=40, - profiler=('cProfile', 'profile', 'hotshot'), + profiler=('cProfile', 'profile'), stdout=True): """Mark `fn` for profiling. @@ -208,7 +157,7 @@ def profile(fn=None, skip=0, filename=None, immediate=False, dirs=False, `profiler` can be used to select the preferred profiler, or specify a sequence of them, in order of preference. The default is ('cProfile'. - 'profile', 'hotshot'). + 'profile'). If `filename` is specified, the profile stats will be stored in the named file. You can load them with pstats.Stats(filename) or use a @@ -282,26 +231,7 @@ def coverage(fn): ... """ - fp = TraceFuncCoverage(fn) # or HotShotFuncCoverage - # We cannot return fp or fp.__call__ directly as that would break method - # definitions, instead we need to return a plain function. - - @functools.wraps(fn) - def new_fn(*args, **kw): - return fp(*args, **kw) - return new_fn - - -def coverage_with_hotshot(fn): # pragma: PY2 - """Mark `fn` for line coverage analysis. - - Uses the 'hotshot' module for fast coverage analysis. - - BUG: Produces inaccurate results. - - See the docstring of `coverage` for usage examples. - """ - fp = HotShotFuncCoverage(fn) + fp = TraceFuncCoverage(fn) # We cannot return fp or fp.__call__ directly as that would break method # definitions, instead we need to return a plain function. @@ -427,148 +357,8 @@ if cProfile is not None: AVAILABLE_PROFILERS['cProfile'] = CProfileFuncProfile -if hotshot is not None: # pragma: PY2 - - class HotShotFuncProfile(FuncProfile): - """Profiler for a function (uses hotshot).""" - - # This flag is shared between all instances - in_profiler = False - - def __init__(self, fn, skip=0, filename=None, immediate=False, - dirs=False, sort=None, entries=40, stdout=True): - """Creates a profiler for a function. - - Every profiler has its own log file (the name of which is derived - from the function name). - - HotShotFuncProfile registers an atexit handler that prints - profiling information to sys.stderr when the program terminates. - - The log file is not removed and remains there to clutter the - current working directory. - """ - if filename: - self.logfilename = filename + ".raw" - else: - self.logfilename = "%s.%d.prof" % (fn.__name__, os.getpid()) - super(HotShotFuncProfile, self).__init__( - fn, skip=skip, filename=filename, immediate=immediate, - dirs=dirs, sort=sort, entries=entries, stdout=stdout) - - def __call__(self, *args, **kw): - """Profile a singe call to the function.""" - self.ncalls += 1 - if self.skip > 0: - self.skip -= 1 - self.skipped += 1 - return self.fn(*args, **kw) - if HotShotFuncProfile.in_profiler: - # handle recursive calls - return self.fn(*args, **kw) - if self.profiler is None: - self.profiler = hotshot.Profile(self.logfilename) - try: - HotShotFuncProfile.in_profiler = True - return self.profiler.runcall(self.fn, *args, **kw) - finally: - HotShotFuncProfile.in_profiler = False - if self.immediate: - self.print_stats() - self.reset_stats() - - def print_stats(self): - if self.profiler is None: - self.stats = pstats.Stats(Profile()) - else: - self.profiler.close() - self.stats = hotshot.stats.load(self.logfilename) - super(HotShotFuncProfile, self).print_stats() - - def reset_stats(self): - self.profiler = None - self.ncalls = 0 - self.skipped = 0 - - AVAILABLE_PROFILERS['hotshot'] = HotShotFuncProfile - - class HotShotFuncCoverage: - """Coverage analysis for a function (uses _hotshot). - - HotShot coverage is reportedly faster than trace.py, but it appears to - have problems with exceptions; also line counts in coverage reports - are generally lower from line counts produced by TraceFuncCoverage. - Is this my bug, or is it a problem with _hotshot? - """ - - def __init__(self, fn): - """Creates a profiler for a function. - - Every profiler has its own log file (the name of which is derived - from the function name). - - HotShotFuncCoverage registers an atexit handler that prints - profiling information to sys.stderr when the program terminates. - - The log file is not removed and remains there to clutter the - current working directory. - """ - self.fn = fn - self.logfilename = "%s.%d.cprof" % (fn.__name__, os.getpid()) - self.profiler = _hotshot.coverage(self.logfilename) - self.ncalls = 0 - atexit.register(self.atexit) - - def __call__(self, *args, **kw): - """Profile a singe call to the function.""" - self.ncalls += 1 - old_trace = sys.gettrace() - try: - return self.profiler.runcall(self.fn, args, kw) - finally: # pragma: nocover - sys.settrace(old_trace) - - def atexit(self): - """Stop profiling and print profile information to sys.stderr. - - This function is registered as an atexit hook. - """ - self.profiler.close() - funcname, filename, lineno = _identify(self.fn) - print("") - print("*** COVERAGE RESULTS ***") - print("%s (%s:%s)" % (funcname, filename, lineno)) - print("function called %d times" % self.ncalls) - print("") - fs = FuncSource(self.fn) - reader = hotshot.log.LogReader(self.logfilename) - for what, (filename, lineno, funcname), tdelta in reader: - if filename != fs.filename: - continue - if what == hotshot.log.LINE: - fs.mark(lineno) - if what == hotshot.log.ENTER: - # hotshot gives us the line number of the function - # definition and never gives us a LINE event for the first - # statement in a function, so if we didn't perform this - # mapping, the first statement would be marked as never - # executed - if lineno == fs.firstlineno: - lineno = fs.firstcodelineno - fs.mark(lineno) - reader.close() - print(fs) - never_executed = fs.count_never_executed() - if never_executed: - print("%d lines were not executed." % never_executed) - - class TraceFuncCoverage: - """Coverage analysis for a function (uses trace module). - - HotShot coverage analysis is reportedly faster, but it appears to have - problems with exceptions. - """ + """Coverage analysis for a function (uses trace module).""" # Shared between all instances so that nested calls work tracer = trace.Trace(count=True, trace=False, @@ -657,11 +447,11 @@ class FuncSource: strs = self._find_docstrings(self.filename) lines = { ln - for off, ln in dis.findlinestarts(_unwrap(self.fn).__code__) + for off, ln in dis.findlinestarts(inspect.unwrap(self.fn).__code__) # skipping firstlineno because Python 3.11 adds a 'RESUME' opcode # attributed to the `def` line, but then trace.py never sees it # getting executed - if ln not in strs and ln != self.firstlineno + if ln is not None and ln not in strs and ln != self.firstlineno } for lineno in lines: self.sourcelines.setdefault(lineno, 0) @@ -676,7 +466,7 @@ class FuncSource: # Python 3.2 and removed in 3.6. strs = set() prev = token.INDENT # so module docstring is detected as docstring - with tokenize_open(filename) as f: + with tokenize.open(filename) as f: tokens = tokenize.generate_tokens(f.readline) for ttype, tstr, start, end, line in tokens: if ttype == token.STRING and prev == token.INDENT: