Merge branch 'feature/UpdateProfilehooks' into dev

CHANGES.md

@@ -9,6 +9,7 @@
 * Update filelock 3.12.4 (c1163ae) to 3.14.0 (8556141)
 * Update idna library 3.4 (cab054c) to 3.7 (1d365e1)
 * Update imdbpie 5.6.4 (f695e87) to 5.6.5 (f8ed7a0)
+* Update profilehooks module 1.12.1 (c3fc078) to 1.13.0.dev0 (99f8a31)
 * Update Requests library 2.31.0 (8812812) to 2.32.3 (0e322af)
 * Update Tornado Web Server 6.4 (b3f2a4b) to 6.4.1 (2a0e1d1)
 * Update urllib3 2.0.7 (56f01e0) to 2.2.1 (54d6edf)

lib/profilehooks.py

@@ -39,28 +39,12 @@ instead of a detailed (but costly) profile.
 
 Caveats
 
-  A thread on python-dev convinced me that hotshot produces bogus numbers.
-  See https://mail.python.org/pipermail/python-dev/2005-November/058264.html
-
   I don't know what will happen if a decorated function will try to call
   another decorated function.  All decorators probably need to explicitly
   support nested profiling (currently TraceFuncCoverage is the only one
-  that supports this, while HotShotFuncProfile has support for recursive
+  that supports this.)
-  functions.)
 
-  Profiling with hotshot creates temporary files (*.prof for profiling,
+Copyright (c) 2004--2023 Marius Gedminas <marius@gedmin.as>
-  *.cprof for coverage) in the current directory.  These files are not
-  cleaned up.  Exception: when you specify a filename to the profile
-  decorator (to store the pstats.Stats object for later inspection),
-  the temporary file will be the filename you specified with '.raw'
-  appended at the end.
-
-  Coverage analysis with hotshot seems to miss some executions resulting
-  in lower line counts and some lines errorneously marked as never
-  executed.  For this reason coverage analysis now uses trace.py which is
-  slower, but more accurate.
-
-Copyright (c) 2004--2020 Marius Gedminas <marius@gedmin.as>
 Copyright (c) 2007 Hanno Schlichting
 Copyright (c) 2008 Florian Schulze
 
@@ -86,8 +70,6 @@ Released under the MIT licence since December 2006:
 
 (Previously it was distributed under the GNU General Public Licence.)
 """
-from __future__ import print_function
-
 import atexit
 import dis
 import functools
@@ -104,19 +86,6 @@ import trace
 from profile import Profile
 
 
-# For hotshot profiling (inaccurate!)
-try:  # pragma: PY2
-    import hotshot
-    import hotshot.stats
-except ImportError:
-    hotshot = None
-
-
-# For hotshot coverage (inaccurate!; uses undocumented APIs; might break)
-if hotshot is not None:  # pragma: PY2
-    import _hotshot
-    import hotshot.log
-
 # For cProfile profiling (best)
 try:
     import cProfile
@@ -127,38 +96,18 @@ except ImportError:
 __author__ = "Marius Gedminas <marius@gedmin.as>"
 __copyright__ = "Copyright 2004-2020 Marius Gedminas and contributors"
 __license__ = "MIT"
-__version__ = '1.12.1.dev0'
+__version__ = '1.13.0.dev0'
-__date__ = "2020-08-20"
+__date__ = "2023-12-18"
 
 
 # registry of available profilers
 AVAILABLE_PROFILERS = {}
 
-__all__ = ['coverage', 'coverage_with_hotshot', 'profile', 'timecall']
+__all__ = ['coverage', 'profile', 'timecall']
-
-
-# Use tokenize.open() on Python >= 3.2, fall back to open() on Python 2
-tokenize_open = getattr(tokenize, 'open', open)
-
-
-try:
-    from inspect import unwrap as _unwrap
-except ImportError:  # pragma: PY2
-    # inspect.unwrap() doesn't exist on Python 2
-    def _unwrap(fn):
-        if not hasattr(fn, '__wrapped__'):
-            return fn
-        else:  # pragma: nocover
-            # functools.wraps() doesn't set __wrapped__ on Python 2 either,
-            # so this branch will only get reached if somebody
-            # manually sets __wrapped__, hence the pragma: nocover.
-            # NB: intentionally using recursion here instead of a while loop to
-            # make cycles fail with a recursion error instead of looping forever.
-            return _unwrap(fn.__wrapped__)
 
 
 def _identify(fn):
-    fn = _unwrap(fn)
+    fn = inspect.unwrap(fn)
     funcname = fn.__name__
     filename = fn.__code__.co_filename
     lineno = fn.__code__.co_firstlineno
@@ -171,7 +120,7 @@ def _is_file_like(o):
 
 def profile(fn=None, skip=0, filename=None, immediate=False, dirs=False,
             sort=None, entries=40,
-            profiler=('cProfile', 'profile', 'hotshot'),
+            profiler=('cProfile', 'profile'),
             stdout=True):
     """Mark `fn` for profiling.
 
@@ -208,7 +157,7 @@ def profile(fn=None, skip=0, filename=None, immediate=False, dirs=False,
 
     `profiler` can be used to select the preferred profiler, or specify a
     sequence of them, in order of preference.  The default is ('cProfile'.
-    'profile', 'hotshot').
+    'profile').
 
     If `filename` is specified, the profile stats will be stored in the
     named file.  You can load them with pstats.Stats(filename) or use a
@@ -282,26 +231,7 @@ def coverage(fn):
             ...
 
     """
-    fp = TraceFuncCoverage(fn)  # or HotShotFuncCoverage
+    fp = TraceFuncCoverage(fn)
-    # We cannot return fp or fp.__call__ directly as that would break method
-    # definitions, instead we need to return a plain function.
-
-    @functools.wraps(fn)
-    def new_fn(*args, **kw):
-        return fp(*args, **kw)
-    return new_fn
-
-
-def coverage_with_hotshot(fn):  # pragma: PY2
-    """Mark `fn` for line coverage analysis.
-
-    Uses the 'hotshot' module for fast coverage analysis.
-
-    BUG: Produces inaccurate results.
-
-    See the docstring of `coverage` for usage examples.
-    """
-    fp = HotShotFuncCoverage(fn)
     # We cannot return fp or fp.__call__ directly as that would break method
     # definitions, instead we need to return a plain function.
 
@@ -427,148 +357,8 @@ if cProfile is not None:
     AVAILABLE_PROFILERS['cProfile'] = CProfileFuncProfile
 
 
-if hotshot is not None:  # pragma: PY2
-
-    class HotShotFuncProfile(FuncProfile):
-        """Profiler for a function (uses hotshot)."""
-
-        # This flag is shared between all instances
-        in_profiler = False
-
-        def __init__(self, fn, skip=0, filename=None, immediate=False,
-                     dirs=False, sort=None, entries=40, stdout=True):
-            """Creates a profiler for a function.
-
-            Every profiler has its own log file (the name of which is derived
-            from the function name).
-
-            HotShotFuncProfile registers an atexit handler that prints
-            profiling information to sys.stderr when the program terminates.
-
-            The log file is not removed and remains there to clutter the
-            current working directory.
-            """
-            if filename:
-                self.logfilename = filename + ".raw"
-            else:
-                self.logfilename = "%s.%d.prof" % (fn.__name__, os.getpid())
-            super(HotShotFuncProfile, self).__init__(
-                fn, skip=skip, filename=filename, immediate=immediate,
-                dirs=dirs, sort=sort, entries=entries, stdout=stdout)
-
-        def __call__(self, *args, **kw):
-            """Profile a singe call to the function."""
-            self.ncalls += 1
-            if self.skip > 0:
-                self.skip -= 1
-                self.skipped += 1
-                return self.fn(*args, **kw)
-            if HotShotFuncProfile.in_profiler:
-                # handle recursive calls
-                return self.fn(*args, **kw)
-            if self.profiler is None:
-                self.profiler = hotshot.Profile(self.logfilename)
-            try:
-                HotShotFuncProfile.in_profiler = True
-                return self.profiler.runcall(self.fn, *args, **kw)
-            finally:
-                HotShotFuncProfile.in_profiler = False
-                if self.immediate:
-                    self.print_stats()
-                    self.reset_stats()
-
-        def print_stats(self):
-            if self.profiler is None:
-                self.stats = pstats.Stats(Profile())
-            else:
-                self.profiler.close()
-                self.stats = hotshot.stats.load(self.logfilename)
-            super(HotShotFuncProfile, self).print_stats()
-
-        def reset_stats(self):
-            self.profiler = None
-            self.ncalls = 0
-            self.skipped = 0
-
-    AVAILABLE_PROFILERS['hotshot'] = HotShotFuncProfile
-
-    class HotShotFuncCoverage:
-        """Coverage analysis for a function (uses _hotshot).
-
-        HotShot coverage is reportedly faster than trace.py, but it appears to
-        have problems with exceptions; also line counts in coverage reports
-        are generally lower from line counts produced by TraceFuncCoverage.
-        Is this my bug, or is it a problem with _hotshot?
-        """
-
-        def __init__(self, fn):
-            """Creates a profiler for a function.
-
-            Every profiler has its own log file (the name of which is derived
-            from the function name).
-
-            HotShotFuncCoverage registers an atexit handler that prints
-            profiling information to sys.stderr when the program terminates.
-
-            The log file is not removed and remains there to clutter the
-            current working directory.
-            """
-            self.fn = fn
-            self.logfilename = "%s.%d.cprof" % (fn.__name__, os.getpid())
-            self.profiler = _hotshot.coverage(self.logfilename)
-            self.ncalls = 0
-            atexit.register(self.atexit)
-
-        def __call__(self, *args, **kw):
-            """Profile a singe call to the function."""
-            self.ncalls += 1
-            old_trace = sys.gettrace()
-            try:
-                return self.profiler.runcall(self.fn, args, kw)
-            finally:  # pragma: nocover
-                sys.settrace(old_trace)
-
-        def atexit(self):
-            """Stop profiling and print profile information to sys.stderr.
-
-            This function is registered as an atexit hook.
-            """
-            self.profiler.close()
-            funcname, filename, lineno = _identify(self.fn)
-            print("")
-            print("*** COVERAGE RESULTS ***")
-            print("%s (%s:%s)" % (funcname, filename, lineno))
-            print("function called %d times" % self.ncalls)
-            print("")
-            fs = FuncSource(self.fn)
-            reader = hotshot.log.LogReader(self.logfilename)
-            for what, (filename, lineno, funcname), tdelta in reader:
-                if filename != fs.filename:
-                    continue
-                if what == hotshot.log.LINE:
-                    fs.mark(lineno)
-                if what == hotshot.log.ENTER:
-                    # hotshot gives us the line number of the function
-                    # definition and never gives us a LINE event for the first
-                    # statement in a function, so if we didn't perform this
-                    # mapping, the first statement would be marked as never
-                    # executed
-                    if lineno == fs.firstlineno:
-                        lineno = fs.firstcodelineno
-                    fs.mark(lineno)
-            reader.close()
-            print(fs)
-            never_executed = fs.count_never_executed()
-            if never_executed:
-                print("%d lines were not executed." % never_executed)
-
-
 class TraceFuncCoverage:
-    """Coverage analysis for a function (uses trace module).
+    """Coverage analysis for a function (uses trace module)."""
-
-    HotShot coverage analysis is reportedly faster, but it appears to have
-    problems with exceptions.
-    """
 
     # Shared between all instances so that nested calls work
     tracer = trace.Trace(count=True, trace=False,
@@ -657,11 +447,11 @@ class FuncSource:
         strs = self._find_docstrings(self.filename)
         lines = {
             ln
-            for off, ln in dis.findlinestarts(_unwrap(self.fn).__code__)
+            for off, ln in dis.findlinestarts(inspect.unwrap(self.fn).__code__)
             # skipping firstlineno because Python 3.11 adds a 'RESUME' opcode
             # attributed to the `def` line, but then trace.py never sees it
             # getting executed
-            if ln not in strs and ln != self.firstlineno
+            if ln is not None and ln not in strs and ln != self.firstlineno
         }
         for lineno in lines:
             self.sourcelines.setdefault(lineno, 0)
@@ -676,7 +466,7 @@ class FuncSource:
         # Python 3.2 and removed in 3.6.
         strs = set()
         prev = token.INDENT  # so module docstring is detected as docstring
-        with tokenize_open(filename) as f:
+        with tokenize.open(filename) as f:
             tokens = tokenize.generate_tokens(f.readline)
             for ttype, tstr, start, end, line in tokens:
                 if ttype == token.STRING and prev == token.INDENT: