From b4f39345ffeba1b28f016a76d9527114422e9d45 Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Sat, 19 Apr 2025 12:55:39 -0500 Subject: [PATCH 01/15] Support profiling modules that import __main___ --- Lib/cProfile.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/cProfile.py b/Lib/cProfile.py index e7c868b8d55543..8503ccbe57f83c 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -6,6 +6,7 @@ import _lsprof import importlib.machinery +import importlib.util import io import profile as _pyprofile @@ -169,17 +170,22 @@ def main(): else: progname = args[0] sys.path.insert(0, os.path.dirname(progname)) - with io.open_code(progname) as fp: - code = compile(fp.read(), progname, 'exec') spec = importlib.machinery.ModuleSpec(name='__main__', loader=None, origin=progname) + loader = importlib.machinery.SourceFileLoader("__main__", progname) + spec.loader = loader + module = importlib.util.module_from_spec(spec) globs = { '__spec__': spec, '__file__': spec.origin, '__name__': spec.name, '__package__': None, '__cached__': None, + 'module': module } + + sys.modules["__main__"] = module + code = "__spec__.loader.exec_module(module)" try: runctx(code, globs, None, options.outfile, options.sort) except BrokenPipeError as exc: From e068beafaffe3e1fb045fb368d80206a96e803f1 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 19 Apr 2025 18:07:35 +0000 Subject: [PATCH 02/15] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst new file mode 100644 index 00000000000000..5bdd54a3fbae17 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst @@ -0,0 +1 @@ +Support profiling modules that import __main___, such as modules that use to pickle. The github issue has an example repro that throws an exception without this change, and succeeds with it. From ca66a0e3e2668711a3cc7e40aae58dc163c708c4 Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Sun, 20 Apr 2025 11:12:23 -0500 Subject: [PATCH 03/15] replace __main__'s namespace instead of creating a new module --- Lib/cProfile.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 8503ccbe57f83c..a0138201573f55 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -98,13 +98,25 @@ def run(self, cmd): return self.runctx(cmd, dict, dict) def runctx(self, cmd, globals, locals): + # cmd has to run in __main__ namespace (or imports from __main__ will + # break). Clear __main__ and replace with the globals provided. + import __main__ + # Save a reference to the current __main__ namespace so that we can + # restore it after cmd completes. + original_main = __main__.__dict__.copy() + __main__.__dict__.clear() + __main__.__dict__.update(globals) + self.enable() try: - exec(cmd, globals, locals) + exec(cmd, __main__.__dict__, locals) finally: self.disable() + __main__.__dict__.clear() + __main__.__dict__.update(original_main) return self + # This method is more useful to profile a single function call. def runcall(self, func, /, *args, **kw): self.enable() @@ -170,22 +182,19 @@ def main(): else: progname = args[0] sys.path.insert(0, os.path.dirname(progname)) + with io.open_code(progname) as fp: + code = compile(fp.read(), progname, 'exec') spec = importlib.machinery.ModuleSpec(name='__main__', loader=None, origin=progname) - loader = importlib.machinery.SourceFileLoader("__main__", progname) - spec.loader = loader - module = importlib.util.module_from_spec(spec) globs = { '__spec__': spec, '__file__': spec.origin, '__name__': spec.name, '__package__': None, '__cached__': None, - 'module': module + '__builtins__': __builtins__, } - sys.modules["__main__"] = module - code = "__spec__.loader.exec_module(module)" try: runctx(code, globs, None, options.outfile, options.sort) except BrokenPipeError as exc: From 1c621a0154480168886b62bfa9d5a611f4f6f319 Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Sun, 20 Apr 2025 11:14:29 -0500 Subject: [PATCH 04/15] Update Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- .../2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst index 5bdd54a3fbae17..892e3ba4b57c81 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst @@ -1 +1 @@ -Support profiling modules that import __main___, such as modules that use to pickle. The github issue has an example repro that throws an exception without this change, and succeeds with it. +Support profiling modules that import __main___, such as modules that use to pickle. From 923cb6e5f22115a7ad6e698c84681784b82d7749 Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Sun, 20 Apr 2025 11:15:37 -0500 Subject: [PATCH 05/15] quote __main__ and fix typo --- .../2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst index 892e3ba4b57c81..bc2731d797b421 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst @@ -1 +1 @@ -Support profiling modules that import __main___, such as modules that use to pickle. +Support profiling modules that import ``__main__``, such as modules that use pickle. From 75a542e93cf5b253ad8dcaf0451907cf76d012dc Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Sun, 20 Apr 2025 13:41:50 -0500 Subject: [PATCH 06/15] Add regression test --- Lib/cProfile.py | 6 ++---- Lib/test/test_cprofile.py | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Lib/cProfile.py b/Lib/cProfile.py index a0138201573f55..13e4dd239a5554 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -175,10 +175,8 @@ def main(): if len(args) > 0: if options.module: code = "run_module(modname, run_name='__main__')" - globs = { - 'run_module': runpy.run_module, - 'modname': args[0] - } + globs = globals().copy() + globs.update({"run_module": runpy.run_module, "modname": args[0]}) else: progname = args[0] sys.path.insert(0, os.path.dirname(progname)) diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index b46edf66bf09f8..b7389adf30c3d0 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -5,8 +5,10 @@ # rip off all interesting stuff from test_profile import cProfile +import tempfile +import textwrap from test.test_profile import ProfileTest, regenerate_expected_output -from test.support.script_helper import assert_python_failure +from test.support.script_helper import assert_python_failure, assert_python_ok from test import support @@ -155,6 +157,22 @@ def test_sort(self): self.assertIn(b"option -s: invalid choice: 'demo'", err) +class TestProfilingScript(unittest.TestCase): + def test_profile_script_importing_main(self): + """Check that scripts that reference __main__ see their own namespace + when being profiled.""" + with tempfile.NamedTemporaryFile("w+") as f: + f.write(textwrap.dedent("""\ + class Foo: + pass + + import __main__ + assert Foo == __main__.Foo + """)) + f.flush() + assert_python_ok('-m', "cProfile", f.name) + + def main(): if '-r' not in sys.argv: unittest.main() From 3665a7f3f676b6f9abe477a680c507d019c9f0bd Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Sun, 20 Apr 2025 13:45:48 -0500 Subject: [PATCH 07/15] Only modify __main__ in CLI invocation --- Lib/cProfile.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 13e4dd239a5554..e07ad711f89e24 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -98,25 +98,13 @@ def run(self, cmd): return self.runctx(cmd, dict, dict) def runctx(self, cmd, globals, locals): - # cmd has to run in __main__ namespace (or imports from __main__ will - # break). Clear __main__ and replace with the globals provided. - import __main__ - # Save a reference to the current __main__ namespace so that we can - # restore it after cmd completes. - original_main = __main__.__dict__.copy() - __main__.__dict__.clear() - __main__.__dict__.update(globals) - self.enable() try: - exec(cmd, __main__.__dict__, locals) + exec(cmd, globals, locals) finally: self.disable() - __main__.__dict__.clear() - __main__.__dict__.update(original_main) return self - # This method is more useful to profile a single function call. def runcall(self, func, /, *args, **kw): self.enable() @@ -192,10 +180,19 @@ def main(): '__cached__': None, '__builtins__': __builtins__, } + # cmd has to run in __main__ namespace (or imports from __main__ will + # break). Clear __main__ and replace with the globals provided. + import __main__ + # Save a reference to the current __main__ namespace so that we can + # restore it after cmd completes. + original_main = __main__.__dict__.copy() + __main__.__dict__.update(globs) try: - runctx(code, globs, None, options.outfile, options.sort) + runctx(code, __main__.__dict__, None, options.outfile, options.sort) except BrokenPipeError as exc: + __main__.__dict__.clear() + __main__.__dict__.update(original_main) # Prevent "Exception ignored" during interpreter shutdown. sys.stdout = None sys.exit(exc.errno) From e6d50bd1e41d18ba867fbc8f26da2389d3654f24 Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Sun, 20 Apr 2025 18:28:10 -0500 Subject: [PATCH 08/15] Avoid leaking cProfile scope into profiled code --- Lib/cProfile.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/cProfile.py b/Lib/cProfile.py index e07ad711f89e24..99736d0d107cf2 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -183,16 +183,12 @@ def main(): # cmd has to run in __main__ namespace (or imports from __main__ will # break). Clear __main__ and replace with the globals provided. import __main__ - # Save a reference to the current __main__ namespace so that we can - # restore it after cmd completes. - original_main = __main__.__dict__.copy() + __main__.__dict__.clear() __main__.__dict__.update(globs) try: runctx(code, __main__.__dict__, None, options.outfile, options.sort) except BrokenPipeError as exc: - __main__.__dict__.clear() - __main__.__dict__.update(original_main) # Prevent "Exception ignored" during interpreter shutdown. sys.stdout = None sys.exit(exc.errno) @@ -202,4 +198,8 @@ def main(): # When invoked as main program, invoke the profiler on a script if __name__ == '__main__': - main() + # Since the code we run might need to modify __main__, we need to ensure + # that cProfile's main function is run under some other namespace, so we + # reimport and execute the main function separately + import cProfile + cProfile.main() From 5607cb8b9fde6470c4b2e57659e5d2018e3f2155 Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Sun, 20 Apr 2025 18:29:58 -0500 Subject: [PATCH 09/15] Remove unused import --- Lib/cProfile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 99736d0d107cf2..d0365a3700da1d 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -6,7 +6,6 @@ import _lsprof import importlib.machinery -import importlib.util import io import profile as _pyprofile From 8d6f37b0d1cd5862dab32959480a6f4d92f7e3c0 Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Sun, 20 Apr 2025 18:31:50 -0500 Subject: [PATCH 10/15] Undo changes to module execution --- Lib/cProfile.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/cProfile.py b/Lib/cProfile.py index d0365a3700da1d..64d589de8393b5 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -162,8 +162,10 @@ def main(): if len(args) > 0: if options.module: code = "run_module(modname, run_name='__main__')" - globs = globals().copy() - globs.update({"run_module": runpy.run_module, "modname": args[0]}) + globs = { + "run_module": runpy.run_module, + "modname": args[0] + } else: progname = args[0] sys.path.insert(0, os.path.dirname(progname)) From 191445cf5bb20f483f48e21d6e1074413ab1b790 Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Sun, 20 Apr 2025 20:25:44 -0500 Subject: [PATCH 11/15] Revert quote change --- Lib/cProfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 64d589de8393b5..6253755a9df660 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -163,8 +163,8 @@ def main(): if options.module: code = "run_module(modname, run_name='__main__')" globs = { - "run_module": runpy.run_module, - "modname": args[0] + 'run_module': runpy.run_module, + 'modname': args[0] } else: progname = args[0] From 2fffb33541c7b15806d1c4dc136780dbc16c85a6 Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Wed, 23 Apr 2025 10:57:19 -0500 Subject: [PATCH 12/15] less hacky --- Lib/cProfile.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 6253755a9df660..654e9fdad13150 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -6,6 +6,7 @@ import _lsprof import importlib.machinery +import importlib.util import io import profile as _pyprofile @@ -173,22 +174,25 @@ def main(): code = compile(fp.read(), progname, 'exec') spec = importlib.machinery.ModuleSpec(name='__main__', loader=None, origin=progname) - globs = { + module = importlib.util.module_from_spec(spec) + # Set __main__ so that importing __main__ in the profiled code will + # return the same namespace that the code is executing under. + sys.modules['__main__'] = module + # Ensure that we're using the same __dict__ instance as the module + # for the global variables so that updates to globals are reflected + # in the module's namespace. + globs = module.__dict__ + globs.update({ '__spec__': spec, '__file__': spec.origin, '__name__': spec.name, '__package__': None, '__cached__': None, '__builtins__': __builtins__, - } - # cmd has to run in __main__ namespace (or imports from __main__ will - # break). Clear __main__ and replace with the globals provided. - import __main__ - __main__.__dict__.clear() - __main__.__dict__.update(globs) + }) try: - runctx(code, __main__.__dict__, None, options.outfile, options.sort) + runctx(code, globs, None, options.outfile, options.sort) except BrokenPipeError as exc: # Prevent "Exception ignored" during interpreter shutdown. sys.stdout = None @@ -199,8 +203,4 @@ def main(): # When invoked as main program, invoke the profiler on a script if __name__ == '__main__': - # Since the code we run might need to modify __main__, we need to ensure - # that cProfile's main function is run under some other namespace, so we - # reimport and execute the main function separately - import cProfile - cProfile.main() + main() From 8f9b8a16f69552ec15fec1f77fb73cb33cbb3e26 Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Wed, 23 Apr 2025 11:19:16 -0500 Subject: [PATCH 13/15] Remove unecessary inclusion of __builtins__ to globals --- Lib/cProfile.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 654e9fdad13150..770d26f79628fe 100644 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -188,7 +188,6 @@ def main(): '__name__': spec.name, '__package__': None, '__cached__': None, - '__builtins__': __builtins__, }) try: From 226d78d330477d60149cf45091c2c9eef0ac43e4 Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Wed, 23 Apr 2025 11:20:48 -0500 Subject: [PATCH 14/15] Requested test changes --- Lib/test/test_cprofile.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index b7389adf30c3d0..192c8eab26ebff 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -156,20 +156,17 @@ def test_sort(self): self.assertGreater(rc, 0) self.assertIn(b"option -s: invalid choice: 'demo'", err) - -class TestProfilingScript(unittest.TestCase): def test_profile_script_importing_main(self): """Check that scripts that reference __main__ see their own namespace when being profiled.""" - with tempfile.NamedTemporaryFile("w+") as f: + with tempfile.NamedTemporaryFile("w+", delete_on_close=False) as f: f.write(textwrap.dedent("""\ class Foo: pass - import __main__ assert Foo == __main__.Foo """)) - f.flush() + f.close() assert_python_ok('-m', "cProfile", f.name) From 15d867437325bb5ed01fb8e17a3a17a53a205916 Mon Sep 17 00:00:00 2001 From: Aneesh Durg Date: Wed, 23 Apr 2025 11:21:22 -0500 Subject: [PATCH 15/15] Update Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst Co-authored-by: Tian Gao --- .../2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst index bc2731d797b421..0f01091782529d 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-18-07-34.gh-issue-132737.9mW1il.rst @@ -1 +1 @@ -Support profiling modules that import ``__main__``, such as modules that use pickle. +Support profiling code that requires ``__main__``, such as :mod:`pickle`.