diff --git a/.hgignore b/.gitignore similarity index 77% rename from .hgignore rename to .gitignore index 4221033..9ec966d 100644 --- a/.hgignore +++ b/.gitignore @@ -1,11 +1,14 @@ -syntax: glob -.gdb_history +# Swap files. +.*.swp +.*.swo +*~ + *.pyc *.pyo -*.pyclbr *.so -.*.swp -.*.swo +*.o +*.a + build/ dist/ _line_profiler.c diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..a8aecb4 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,14 @@ +language: python +python: + - "3.4" + - "3.3" + - "3.2" + - "2.7" +install: + - pip install --install-option='--no-cython-compile' -r dev_requirements.txt + - python setup.py develop +script: + - python -m unittest discover -v tests +notifications: + email: + - robert.kern+travis-ci@gmail.com diff --git a/MANIFEST.in b/MANIFEST.in index 75c2572..a24056f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,8 @@ include LICENSE.txt include LICENSE_Python.txt -include README.txt +include README.rst include python25.pxd include timers.h include _line_profiler.c include unset_trace.h +recursive-include tests *.py diff --git a/README.txt b/README.rst similarity index 82% rename from README.txt rename to README.rst index 40459f5..3f87178 100644 --- a/README.txt +++ b/README.rst @@ -1,13 +1,13 @@ line_profiler and kernprof -------------------------- -line_profiler is a module for doing line-by-line profiling of functions. -kernprof is a convenient script for running either line_profiler or the Python +`line_profiler` is a module for doing line-by-line profiling of functions. +kernprof is a convenient script for running either `line_profiler` or the Python standard library's cProfile or profile modules, depending on what is available. They are available under a `BSD license`_. -.. _BSD license: http://packages.python.org/line_profiler/LICENSE.txt +.. _BSD license: https://raw.githubusercontent.com/rkern/line_profiler/master/LICENSE.txt .. contents:: @@ -15,36 +15,31 @@ They are available under a `BSD license`_. Installation ============ -Source releases and any binaries can be downloaded from the PyPI link. +Releases of `line_profiler` can be installed using pip_:: - http://pypi.python.org/pypi/line_profiler + $ pip install line_profiler -The current release of the kernprof.py script may be downloaded separately here: +Source releases and any binaries can be downloaded from the PyPI link. - http://packages.python.org/line_profiler/kernprof.py + http://pypi.python.org/pypi/line_profiler -To check out the development sources, you can use Mercurial_:: +To check out the development sources, you can use Git_:: - $ hg clone https://bitbucket.org/robertkern/line_profiler + $ git clone https://github.com/rkern/line_profiler.git You may also download source tarballs of any snapshot from that URL. -Source releases will require a C compiler in order to build line_profiler. In -addition, Mercurial checkouts will also require Cython_ >= 0.10. Source releases +Source releases will require a C compiler in order to build `line_profiler`. +In addition, git checkouts will also require Cython_ >= 0.10. Source releases on PyPI should contain the pregenerated C sources, so Cython should not be required in that case. -kernprof.py is a single-file pure Python script and does not require a compiler. -If you wish to use it to run cProfile and not line-by-line profiling, you may -copy it to a directory on your PATH manually and avoid trying to build any -C extensions. +`kernprof` is a single-file pure Python script and does not require +a compiler. If you wish to use it to run cProfile and not line-by-line +profiling, you may copy it to a directory on your `PATH` manually and avoid +trying to build any C extensions. -In order to build and install line_profiler, you will simply use the standard -`build and install`_ for most Python packages:: - - $ python setup.py install - -.. _Mercurial: http://www.selenic.com/mercurial/wiki/ +.. _git: http://git-scm.com/ .. _Cython: http://www.cython.org .. _build and install: http://docs.python.org/install/index.html @@ -52,7 +47,7 @@ In order to build and install line_profiler, you will simply use the standard line_profiler ============= -The current profiling tools supported in Python 2.5 and later only time +The current profiling tools supported in Python 2.7 and later only time function calls. This is a good first step for locating hotspots in one's program and is frequently all one needs to do to optimize the program. However, sometimes the cause of the hotspot is actually a single line in the function, @@ -75,19 +70,21 @@ of each individual line inside those functions. In a typical workflow, one only cares about line timings of a few functions because wading through the results of timing every single line of code would be overwhelming. However, LineProfiler does need to be explicitly told what functions to profile. The easiest way to -get started is to use the kernprof.py script. +get started is to use the `kernprof` script. :: + + $ kernprof -l script_to_profile.py -If you use "kernprof.py [-l/--line-by-line] script_to_profile.py", an instance -of LineProfiler will be created and inserted into the __builtins__ namespace -with the name "profile". It has been written to be used as a decorator, so in -your script, you can decorate any function you want to profile with @profile. :: +`kernprof` will create an instance of LineProfiler and insert it into the +`__builtins__` namespace with the name `profile`. It has been written to be +used as a decorator, so in your script, you can decorate any function you want +to profile with @profile. :: @profile def slow_function(a, b, c): ... -The default behavior of kernprof is to put the results into a binary file -script_to_profile.py.lprof . You can tell kernprof to immediately view the +The default behavior of `kernprof` is to put the results into a binary file +script_to_profile.py.lprof . You can tell `kernprof` to immediately view the formatted results at the terminal with the [-v/--view] option. Otherwise, you can view the results later like so:: @@ -95,7 +92,7 @@ can view the results later like so:: For example, here are the results of profiling a single function from a decorated version of the pystone.py benchmark (the first two lines are output -from pystone.py, not kernprof):: +from `pystone.py`, not `kernprof`):: Pystone(1.1) time for 50000 passes = 2.48 This machine benchmarks at 20161.3 pystones/second @@ -148,21 +145,11 @@ line. There are six columns of information. If you are using IPython, there is an implementation of an %lprun magic command which will let you specify functions to profile and a statement to execute. It will also add its LineProfiler instance into the __builtins__, but typically, -you would not use it like that. For IPython 0.10, you can install it by editing -the IPython configuration file ~/.ipython/ipy_user_conf.py to add the following -lines:: - - # These two lines are standard and probably already there. - import IPython.ipapi - ip = IPython.ipapi.get() - - # These two are the important ones. - import line_profiler - ip.expose_magic('lprun', line_profiler.magic_lprun) +you would not use it like that. For IPython 0.11+, you can install it by editing the IPython configuration file -~/.ipython/profile_default/ipython_config.py to add the `'line_profiler'` item -to the extensions list:: +`~/.ipython/profile_default/ipython_config.py` to add the `'line_profiler'` +item to the extensions list:: c.TerminalIPythonApp.extensions = [ 'line_profiler', @@ -200,7 +187,7 @@ the timer unit. kernprof ======== -kernprof also works with cProfile, its third-party incarnation lsprof, or the +`kernprof` also works with cProfile, its third-party incarnation lsprof, or the pure-Python profile module depending on what is available. It has a few main features: @@ -224,8 +211,7 @@ features: Profiler will be instantiated and inserted into your __builtins__ with the name "profile". Like LineProfiler, it may be used as a decorator, or enabled/disabled with `enable_by_count()` and `disable_by_count()`, or - even as a context manager with the "with profile:" statement in Python 2.5 - and 2.6. + even as a context manager with the "with profile:" statement. * Pre-profiling setup. With the [-s/--setup] option, you can provide a script which will be executed without profiling before executing the @@ -324,32 +310,26 @@ Frequently Asked Questions projects for modules as small as these. However, kernprof.py is a standalone, pure Python script that can be used to do function profiling with just the Python standard library. You may grab it and install it by - itself without line_profiler. + itself without `line_profiler`. -* Do I need a C compiler to build line_profiler? kernprof.py? +* Do I need a C compiler to build `line_profiler`? kernprof.py? You do need a C compiler for line_profiler. kernprof.py is a pure Python script and can be installed separately, though. -* Do I need Cython to build line_profiler? +* Do I need Cython to build `line_profiler`? You should not have to if you are building from a released source tarball. It should contain the generated C sources already. If you are running into problems, that may be a bug; let me know. If you are building from - a Mercurial checkout or snapshot, you will need Cython to generate the + a git checkout or snapshot, you will need Cython to generate the C sources. You will probably need version 0.10 or higher. There is a bug in some earlier versions in how it handles NULL PyObject* pointers. * What version of Python do I need? - Both line_profiler and kernprof have been tested with Python 2.4-2.7. - It might work with Python 2.3, but does not currently work with Python 3.x. - -* I get negative line timings! What's going on? - - There was a bug in 1.0b1 on Windows that resulted in this. It should be - fixed in 1.0b2. If you are still seeing negative numbers, please let me - know. + Both `line_profiler` and `kernprof` have been tested with Python 2.7, and + 3.2-3.4. To Do @@ -367,16 +347,27 @@ now. Maybe later. Contributions accepted! Bugs and Such ============= -If you find a bug, or a missing feature you really want added, please post to -the enthought-dev_ mailing list or email the author at -. +Bugs and pull requested can be submitted on GitHub_. -.. _enthought-dev : https://mail.enthought.com/mailman/listinfo/enthought-dev +.. _GitHub: https://github.com/rkern/line_profiler Changes ======= +1.0 +~~~ +* ENH: `kernprof.py` is now installed as `kernprof`. +* ENH: Python 3 support. Thanks to the long-suffering Mikhail Korobov for being + patient. +* Dropped 2.6 as it was too annoying. +* ENH: The `stripzeros` and `add_module` options. Thanks to Erik Tollerud for + contributing it. +* ENH: Support for IPython cell blocks. Thanks to Michael Forbes for adding + this feature. +* ENH: Better warnings when building without Cython. Thanks to David Cournapeau + for spotting this. + 1.0b3 ~~~~~ @@ -396,4 +387,3 @@ Changes ~~~~~ * Initial release. - diff --git a/_line_profiler.pyx b/_line_profiler.pyx index 8810565..4798d32 100644 --- a/_line_profiler.pyx +++ b/_line_profiler.pyx @@ -58,10 +58,8 @@ cdef class LineTiming: """ cdef public object code cdef public int lineno - # Note: leave at least total_time private. This should help compile under - # Python 2.4. - cdef PY_LONG_LONG total_time - cdef long nhits + cdef public PY_LONG_LONG total_time + cdef public long nhits def __init__(self, object code, int lineno): self.code = code @@ -106,9 +104,9 @@ class LineStats(object): cdef class LineProfiler: """ Time the execution of lines of Python code. """ - cdef public object functions - cdef public object code_map - cdef public object last_time + cdef public list functions + cdef public dict code_map + cdef public dict last_time cdef public double timer_unit cdef public long enable_count @@ -186,21 +184,26 @@ cdef class LastTime: self.time = time -cdef int python_trace_callback(object self, PyFrameObject *py_frame, int what, +cdef int python_trace_callback(object self_, PyFrameObject *py_frame, int what, PyObject *arg): """ The PyEval_SetTrace() callback. """ - cdef object code, line_entries, key + cdef LineProfiler self + cdef object code, key + cdef dict line_entries, last_time cdef LineTiming entry cdef LastTime old cdef PY_LONG_LONG time + self = self_ + last_time = self.last_time + if what == PyTrace_LINE or what == PyTrace_RETURN: code = py_frame.f_code if code in self.code_map: time = hpTimer() - if code in self.last_time: - old = self.last_time[code] + if code in last_time: + old = last_time[code] line_entries = self.code_map[code] key = old.f_lineno if key not in line_entries: @@ -212,11 +215,13 @@ cdef int python_trace_callback(object self, PyFrameObject *py_frame, int what, if what == PyTrace_LINE: # Get the time again. This way, we don't record much time wasted # in this function. - self.last_time[code] = LastTime(py_frame.f_lineno, hpTimer()) + last_time[code] = LastTime(py_frame.f_lineno, hpTimer()) else: # We are returning from a function, not executing a line. Delete - # the last_time record. - del self.last_time[code] + # the last_time record. It may have already been deleted if we + # are profiling a generator that is being pumped past its end. + if code in last_time: + del last_time[code] return 0 diff --git a/dev_requirements.txt b/dev_requirements.txt new file mode 100644 index 0000000..002d1b9 --- /dev/null +++ b/dev_requirements.txt @@ -0,0 +1 @@ +Cython diff --git a/kernprof.py b/kernprof.py index 91e573b..5271a55 100755 --- a/kernprof.py +++ b/kernprof.py @@ -3,6 +3,7 @@ """ Script to conveniently run profilers on code in a variety of circumstances. """ +import functools import optparse import os import sys @@ -75,21 +76,18 @@ def __call__(self, func): # FIXME: refactor this into a utility function so that both it and # line_profiler can use it. if is_generator(func): - f = self.wrap_generator(func) + wrapper = self.wrap_generator(func) else: - f = self.wrap_function(func) - f.__module__ = func.__module__ - f.__name__ = func.__name__ - f.__doc__ = func.__doc__ - f.__dict__.update(getattr(func, '__dict__', {})) - return f + wrapper = self.wrap_function(func) + return wrapper # FIXME: refactor this stuff so that both LineProfiler and # ContextualProfile can use the same implementation. def wrap_generator(self, func): """ Wrap a generator to profile it. """ - def f(*args, **kwds): + @functools.wraps(func) + def wrapper(*args, **kwds): g = func(*args, **kwds) # The first iterate will not be a .send() self.enable_by_count() @@ -106,19 +104,20 @@ def f(*args, **kwds): finally: self.disable_by_count() input = (yield item) - return f + return wrapper def wrap_function(self, func): """ Wrap a function to profile it. """ - def f(*args, **kwds): + @functools.wraps(func) + def wrapper(*args, **kwds): self.enable_by_count() try: result = func(*args, **kwds) finally: self.disable_by_count() return result - return f + return wrapper def __enter__(self): self.enable_by_count() @@ -146,7 +145,9 @@ def find_script(script_name): raise SystemExit(1) -def main(args): +def main(args=None): + if args is None: + args = sys.argv usage = "%prog [-s setupfile] [-o output_file_path] scriptfile [arg] ..." parser = optparse.OptionParser(usage=usage, version="%prog 1.0b2") parser.allow_interspersed_args = False @@ -230,4 +231,3 @@ def main(args): if __name__ == '__main__': sys.exit(main(sys.argv)) - diff --git a/line_profiler.py b/line_profiler.py index 853a9b1..4480c7b 100755 --- a/line_profiler.py +++ b/line_profiler.py @@ -10,6 +10,7 @@ from cStringIO import StringIO except ImportError: from io import StringIO +import functools import inspect import linecache import optparse @@ -60,19 +61,16 @@ def __call__(self, func): """ self.add_function(func) if is_generator(func): - f = self.wrap_generator(func) + wrapper = self.wrap_generator(func) else: - f = self.wrap_function(func) - f.__module__ = func.__module__ - f.__name__ = func.__name__ - f.__doc__ = func.__doc__ - f.__dict__.update(getattr(func, '__dict__', {})) - return f + wrapper = self.wrap_function(func) + return wrapper def wrap_generator(self, func): """ Wrap a generator to profile it. """ - def f(*args, **kwds): + @functools.wraps(func) + def wrapper(*args, **kwds): g = func(*args, **kwds) # The first iterate will not be a .send() self.enable_by_count() @@ -89,19 +87,20 @@ def f(*args, **kwds): finally: self.disable_by_count() input = (yield item) - return f + return wrapper def wrap_function(self, func): """ Wrap a function to profile it. """ - def f(*args, **kwds): + @functools.wraps(func) + def wrapper(*args, **kwds): self.enable_by_count() try: result = func(*args, **kwds) finally: self.disable_by_count() return result - return f + return wrapper def dump_stats(self, filename): """ Dump a representation of the data to a file as a pickled LineStats diff --git a/python25.pxd b/python25.pxd index f478a5d..b421adf 100644 --- a/python25.pxd +++ b/python25.pxd @@ -399,8 +399,12 @@ cdef extern from "Python.h": ctypedef struct PyThreadState: PyFrameObject * frame int recursion_depth - void * curexc_type, * curexc_value, * curexc_traceback - void * exc_type, * exc_value, * exc_traceback + void * curexc_type + void * curexc_value + void * curexc_traceback + void * exc_type + void * exc_value + void * exc_traceback void PyEval_AcquireLock () void PyEval_ReleaseLock () diff --git a/setup.py b/setup.py index 34a1534..a8042c0 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,7 @@ -#!/usr/bin/env python -import os.path +import os + +# Monkeypatch distutils. +import setuptools import distutils.errors from distutils.core import setup @@ -15,10 +17,11 @@ line_profiler_source = '_line_profiler.c' if not os.path.exists(line_profiler_source): raise distutils.errors.DistutilsError("""\ -You need cython to build the line_profiler from a mercurial checkout, or -alternatively use a release tarball from PyPI to build it without cython.""") +You need Cython to build the line_profiler from a git checkout, or +alternatively use a release tarball from PyPI to build it without Cython.""") else: - warn("Could not import Cython. Using pre-generated C file if available.") + warn("Could not import Cython. " + "Using the available pre-generated C file.") long_description = """\ line_profiler will profile the time individual lines of code take to execute. @@ -32,12 +35,12 @@ setup( name = 'line_profiler', - version = '1.0b3', + version = '1.0', author = 'Robert Kern', author_email = 'robert.kern@enthought.com', description = 'Line-by-line profiler.', long_description = long_description, - url = 'http://packages.python.org/line_profiler', + url = 'https://github.com/rkern/line_profiler', ext_modules = [ Extension('_line_profiler', sources=[line_profiler_source, 'timers.c', 'unset_trace.c'], @@ -46,23 +49,26 @@ ], license = "BSD", classifiers = [ - "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Programming Language :: C", "Programming Language :: Python", 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: Implementation :: CPython', "Topic :: Software Development", ], - py_modules = ['line_profiler'], - scripts = ['kernprof.py'], + py_modules = ['line_profiler', 'kernprof'], + entry_points = { + 'console_scripts': [ + 'kernprof=kernprof:main', + ], + }, cmdclass = cmdclass, ) - diff --git a/tests/test_kernprof.py b/tests/test_kernprof.py new file mode 100644 index 0000000..d230428 --- /dev/null +++ b/tests/test_kernprof.py @@ -0,0 +1,74 @@ +import unittest + +from kernprof import ContextualProfile + + +def f(x): + """ A function. """ + y = x + 10 + return y + + +def g(x): + """ A generator. """ + y = yield x + 10 + yield y + 20 + + +class TestKernprof(unittest.TestCase): + + def test_enable_disable(self): + profile = ContextualProfile() + self.assertEqual(profile.enable_count, 0) + profile.enable_by_count() + self.assertEqual(profile.enable_count, 1) + profile.enable_by_count() + self.assertEqual(profile.enable_count, 2) + profile.disable_by_count() + self.assertEqual(profile.enable_count, 1) + profile.disable_by_count() + self.assertEqual(profile.enable_count, 0) + profile.disable_by_count() + self.assertEqual(profile.enable_count, 0) + + with profile: + self.assertEqual(profile.enable_count, 1) + with profile: + self.assertEqual(profile.enable_count, 2) + self.assertEqual(profile.enable_count, 1) + self.assertEqual(profile.enable_count, 0) + + with self.assertRaises(RuntimeError): + self.assertEqual(profile.enable_count, 0) + with profile: + self.assertEqual(profile.enable_count, 1) + raise RuntimeError() + self.assertEqual(profile.enable_count, 0) + + def test_function_decorator(self): + profile = ContextualProfile() + f_wrapped = profile(f) + self.assertEqual(f_wrapped.__name__, f.__name__) + self.assertEqual(f_wrapped.__doc__, f.__doc__) + + self.assertEqual(profile.enable_count, 0) + value = f_wrapped(10) + self.assertEqual(profile.enable_count, 0) + self.assertEqual(value, f(10)) + + def test_gen_decorator(self): + profile = ContextualProfile() + g_wrapped = profile(g) + self.assertEqual(g_wrapped.__name__, g.__name__) + self.assertEqual(g_wrapped.__doc__, g.__doc__) + + self.assertEqual(profile.enable_count, 0) + i = g_wrapped(10) + self.assertEqual(profile.enable_count, 0) + self.assertEqual(next(i), 20) + self.assertEqual(profile.enable_count, 0) + self.assertEqual(i.send(30), 50) + self.assertEqual(profile.enable_count, 0) + with self.assertRaises(StopIteration): + next(i) + self.assertEqual(profile.enable_count, 0) diff --git a/tests/test_line_profiler.py b/tests/test_line_profiler.py new file mode 100644 index 0000000..c734e1b --- /dev/null +++ b/tests/test_line_profiler.py @@ -0,0 +1,99 @@ +import unittest + +from line_profiler import LineProfiler + + +def f(x): + y = x + 10 + return y + + +def g(x): + y = yield x + 10 + yield y + 20 + + +class TestLineProfiler(unittest.TestCase): + + def test_init(self): + lp = LineProfiler() + self.assertEqual(lp.functions, []) + self.assertEqual(lp.code_map, {}) + lp = LineProfiler(f) + self.assertEqual(lp.functions, [f]) + self.assertEqual(lp.code_map, {f.__code__: {}}) + lp = LineProfiler(f, g) + self.assertEqual(lp.functions, [f, g]) + self.assertEqual(lp.code_map, { + f.__code__: {}, + g.__code__: {}, + }) + + def test_enable_disable(self): + lp = LineProfiler() + self.assertEqual(lp.enable_count, 0) + lp.enable_by_count() + self.assertEqual(lp.enable_count, 1) + lp.enable_by_count() + self.assertEqual(lp.enable_count, 2) + lp.disable_by_count() + self.assertEqual(lp.enable_count, 1) + lp.disable_by_count() + self.assertEqual(lp.enable_count, 0) + self.assertEqual(lp.last_time, {}) + lp.disable_by_count() + self.assertEqual(lp.enable_count, 0) + + with lp: + self.assertEqual(lp.enable_count, 1) + with lp: + self.assertEqual(lp.enable_count, 2) + self.assertEqual(lp.enable_count, 1) + self.assertEqual(lp.enable_count, 0) + self.assertEqual(lp.last_time, {}) + + with self.assertRaises(RuntimeError): + self.assertEqual(lp.enable_count, 0) + with lp: + self.assertEqual(lp.enable_count, 1) + raise RuntimeError() + self.assertEqual(lp.enable_count, 0) + self.assertEqual(lp.last_time, {}) + + def test_function_decorator(self): + profile = LineProfiler() + f_wrapped = profile(f) + self.assertEqual(f_wrapped.__name__, 'f') + + self.assertEqual(profile.enable_count, 0) + value = f_wrapped(10) + self.assertEqual(profile.enable_count, 0) + self.assertEqual(value, f(10)) + + timings = profile.code_map[f.__code__] + self.assertEqual(len(timings), 2) + for timing in timings.values(): + self.assertEqual(timing.nhits, 1) + + def test_gen_decorator(self): + profile = LineProfiler() + g_wrapped = profile(g) + self.assertEqual(g_wrapped.__name__, 'g') + timings = profile.code_map[g.__code__] + + self.assertEqual(profile.enable_count, 0) + i = g_wrapped(10) + self.assertEqual(profile.enable_count, 0) + self.assertEqual(next(i), 20) + self.assertEqual(profile.enable_count, 0) + self.assertEqual(len(timings), 1) + self.assertEqual(i.send(30), 50) + self.assertEqual(profile.enable_count, 0) + self.assertEqual(len(timings), 2) + with self.assertRaises(StopIteration): + next(i) + self.assertEqual(profile.enable_count, 0) + + self.assertEqual(len(timings), 2) + for timing in timings.values(): + self.assertEqual(timing.nhits, 1)