diff --git a/.cspell.json b/.cspell.json index 2889a518409..57f7baab3f7 100644 --- a/.cspell.json +++ b/.cspell.json @@ -59,6 +59,7 @@ "alnum", "csock", "coro", + "contig", "Crnl", "dedentations", "dedents", diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d832398ebf6..5a8daae06ef 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -554,7 +554,7 @@ jobs: components: rustfmt - name: actionlint - uses: reviewdog/action-actionlint@50842263c20a7c46bd0065b9e624d3c569db061e # v1.73.0 + uses: reviewdog/action-actionlint@d63ba7532e0942965320cd8d73cbae4c7b3c5283 # v1.73.1 - name: zizmor uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2 diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index b93a90e8d94..e3557e3e05f 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -33,7 +33,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 + - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 with: tool: cargo-llvm-cov diff --git a/.github/workflows/oscca-pr.yml b/.github/workflows/oscca-pr.yml new file mode 100644 index 00000000000..38b67625ce0 --- /dev/null +++ b/.github/workflows/oscca-pr.yml @@ -0,0 +1,70 @@ +name: Manage OSCCA pull requests + +on: + pull_request_target: + types: [opened] + +permissions: {} + +jobs: + label-and-assign: + name: Label and assign OSCCA pull request + runs-on: ubuntu-slim + timeout-minutes: 5 + permissions: + pull-requests: write + steps: + - name: Label and assign pull request + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const osccaUsers = new Set( + [ + "2jiyong", + "chestnut1717", + "devyubin", + "fregataa", + "hyoinandout", + "HyoJongPark", + "JaceJung-dev", + "jinmay", + "jiwahn", + "kangdora", + "kim-jaedeok", + "kyokuping", + "leehanjeong", + "lms0806", + "lsahn-gh", + "moreal", + "name-of-okja", + "rlaisqls", + "seungje0612", + "shAn-kor", + "sigmaith", + "teddygood", + "widehyo1", + "YangSiJun528", + "zzarbttoo", + ].map((login) => login.toLowerCase()), + ); + const pullRequest = context.payload.pull_request; + const author = pullRequest.user.login; + + if (!osccaUsers.has(author.toLowerCase())) { + core.info(`${author} is not an OSCCA participant; skipping.`); + return; + } + + const issue = { + ...context.repo, + issue_number: pullRequest.number, + }; + + await github.rest.issues.addLabels({ + ...issue, + labels: ["z-ca-2026"], + }); + await github.rest.issues.addAssignees({ + ...issue, + assignees: [author], + }); diff --git a/.github/zizmor.yml b/.github/zizmor.yml index 33ac61c6489..02ceb805c2c 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -1,4 +1,9 @@ rules: + dangerous-triggers: + ignore: + # pull_request_target is needed to label and assign PRs from forks with issues: write. + # The workflow does not check out or execute pull request code. + - oscca-pr.yml:3 excessive-permissions: ignore: # pull_request_target is needed to post PR comments with pull-requests: write. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9ffc4b8d4fd..750f649403e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: priority: 0 - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.16.1 + rev: v0.16.2 hooks: - id: ruff-format priority: 0 diff --git a/Cargo.lock b/Cargo.lock index 26015d39933..fa5e7b52236 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2307,9 +2307,9 @@ checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] @@ -2766,9 +2766,9 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" dependencies = [ "ar_archive_writer", "cc", @@ -2847,9 +2847,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -4209,18 +4209,18 @@ checksum = "b0f7e269b48f0a7dd0146680fa24b50cc67fc0373f086a5b2f99bd084639b482" [[package]] name = "thiserror" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index bc09329e6de..19582e75716 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -4471,7 +4471,6 @@ def test_flags_overflow(self): class TestPythonBufferProtocol(unittest.TestCase): - @unittest.expectedFailure # TODO: RUSTPYTHON def test_basic(self): class MyBuffer: def __buffer__(self, flags): @@ -4500,7 +4499,6 @@ def __buffer__(self): self.assertRaises(TypeError, memoryview, WrongArity()) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_release_buffer(self): class WhatToRelease: def __init__(self): @@ -4523,7 +4521,6 @@ def __release_buffer__(self, buffer): self.assertEqual(mv.tobytes(), b"hello") self.assertFalse(wr.held) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_same_buffer_returned(self): class WhatToRelease: def __init__(self): @@ -4549,7 +4546,6 @@ def __release_buffer__(self, buffer): self.assertEqual(mv.tobytes(), b"hello") self.assertFalse(wr.held) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_buffer_flags(self): class PossiblyMutable: def __init__(self, data, mutable) -> None: @@ -4589,7 +4585,6 @@ def __buffer__(self, flags): mv[0] = ord(b'x') self.assertEqual(mv.tobytes(), b"hello") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_call_builtins(self): ba = bytearray(b"hello") mv = ba.__buffer__(0) @@ -4651,7 +4646,6 @@ def __buffer__(self, flags): mv = memoryview(a) self.assertEqual(mv.tobytes(), b"hello") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_inheritance_releasebuffer(self): rb_call_count = 0 class B(bytearray): @@ -4668,7 +4662,6 @@ def __release_buffer__(self, view): self.assertEqual(rb_call_count, 0) self.assertEqual(rb_call_count, 1) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_inherit_but_return_something_else(self): class A(bytearray): def __buffer__(self, flags): @@ -4708,7 +4701,6 @@ def __release_buffer__(self, buffer): with memoryview(c) as mv: self.assertEqual(mv.tobytes(), b"hello") - @unittest.expectedFailure # TODO: RUSTPYTHON def test_release_saves_reference(self): smuggled_buffer = None @@ -4736,7 +4728,6 @@ def __release_buffer__(s, buffer: memoryview): with self.assertRaises(ValueError): smuggled_buffer.tobytes() - @unittest.expectedFailure # TODO: RUSTPYTHON def test_release_saves_reference_no_subclassing(self): ba = bytearray(b"hello") @@ -4757,7 +4748,6 @@ def __release_buffer__(self, buffer): c.buffer.release() ba.clear() - @unittest.expectedFailure # TODO: RUSTPYTHON def test_multiple_inheritance_buffer_last(self): class A: def __buffer__(self, flags): @@ -4817,7 +4807,6 @@ def __buffer__(self, flags): c.clear() self.assertIs(c.buffer, None) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_release_buffer_with_exception_set(self): class A: def __buffer__(self, flags): diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 39d85d46274..fb519878cd8 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -128,7 +128,6 @@ def test_indentation_error(self): self.assertIsNone(self.sysmod.last_value.__traceback__) self.assertIs(self.sysmod.last_exc, self.sysmod.last_value) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 'UnicodeDecodeError: invalid utf-8 sequence of 1 bytes from index 1\n\nnow exiti [truncated]... doesn't start with 'UnicodeEncodeError: ' def test_unicode_error(self): self.infunc.side_effect = ["'\ud800'", EOFError('Finished')] self.console.interact() diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 763146c94fc..28ddf0a63b0 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -1067,8 +1067,7 @@ def test_decodehelper_bug36819(self): decoded = input.decode(enc, "test.bug36819") self.assertEqual(decoded, 'abcdx' * 51) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailureIf(sys.platform != "win32", "TODO: RUSTPYTHON") def test_encodehelper_bug36819(self): handler = RepeatedPosReturn() codecs.register_error("test.bug36819", handler.handle) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index b5d3411c71a..c1dadc4e274 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1956,7 +1956,6 @@ class X(ByteString): pass # No metaclass conflict class Z(ByteString, Awaitable): pass - @unittest.expectedFailure # TODO: RUSTPYTHON; Need to implement __buffer__ and __release_buffer__ (https://docs.python.org/3.13/reference/datamodel.html#emulating-buffer-types) def test_Buffer(self): for sample in [bytes, bytearray, memoryview]: self.assertIsInstance(sample(b"x"), Buffer) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 585f6611ade..c1695690b72 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1260,7 +1260,6 @@ def test_takewhile(self): self.assertEqual(list(t), [1, 1, 1]) self.assertRaises(StopIteration, next, t) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_dropwhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] self.assertEqual(list(dropwhile(underten, data)), [20, 2, 4, 6, 8]) @@ -1271,7 +1270,6 @@ def test_dropwhile(self): self.assertRaises(TypeError, next, dropwhile(10, [(4,5)])) self.assertRaises(ValueError, next, dropwhile(errfunc, [(4,5)])) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_tee(self): n = 200 diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index ad4c6095abf..4e5311cd0a2 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -319,7 +319,6 @@ def test_recursion_limit(self): last.append([0]) self.assertRaises(ValueError, marshal.dumps, head) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data def test_reference_loop_list(self): a = [] a.append(a) @@ -331,7 +330,6 @@ def test_reference_loop_list(self): self.assertIsInstance(b, list) self.assertIs(b[0], b) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data def test_reference_loop_dict(self): a = {} a[None] = a @@ -343,7 +341,6 @@ def test_reference_loop_dict(self): self.assertIsInstance(b, dict) self.assertIs(b[None], b) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data def test_reference_loop_tuple(self): a = ([],) a[0].append(a) @@ -356,7 +353,6 @@ def test_reference_loop_tuple(self): self.assertIsInstance(b[0], list) self.assertIs(b[0][0], b) - @unittest.skip("TODO: RUSTPYTHON; unexpected payload for constant python value") def test_reference_loop_code(self): def f(): return 1234.5 @@ -370,7 +366,6 @@ def f(): for v in range(marshal.version + 1): self.assertRaises(ValueError, marshal.dumps, code, v) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised by dumps def test_reference_loop_slice(self): a = slice([], None) a.start.append(a) @@ -387,21 +382,18 @@ def test_reference_loop_slice(self): for v in range(marshal.version + 1): self.assertRaises(ValueError, marshal.dumps, a, v) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data def test_loads_reference_loop_list(self): data = b'\xdb\x01\x00\x00\x00r\x00\x00\x00\x00' # [] a = marshal.loads(data) self.assertIsInstance(a, list) self.assertIs(a[0], a) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data def test_loads_reference_loop_dict(self): data = b'\xfbNr\x00\x00\x00\x000' # {None: } a = marshal.loads(data) self.assertIsInstance(a, dict) self.assertIs(a[None], a) - @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: bad marshal data def test_loads_abnormal_reference_loops(self): # Indirect self-references of tuples. data = b'\xa8\x01\x00\x00\x00[\x01\x00\x00\x00r\x00\x00\x00\x00' # ([],) @@ -547,7 +539,6 @@ def test_deterministic_sets(self): _, dump_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1") self.assertEqual(dump_0, dump_1) - @unittest.skip("TODO: RUSTPYTHON; unexpected payload for constant python value") def test_unmarshallable(self): # Check no crash after encountering unmarshallable objects. # See https://github.com/python/cpython/issues/106287. @@ -748,7 +739,6 @@ class InterningTestCase(unittest.TestCase, HelperMixin): strobj = "this is an interned string" strobj = sys.intern(strobj) - @unittest.expectedFailure # TODO: RUSTPYTHON def testIntern(self): s = marshal.loads(marshal.dumps(self.strobj)) self.assertEqual(s, self.strobj) diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 1683a71fc88..7ad3aa8a527 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -587,7 +587,6 @@ def test_issue5449(self): self.ioclass(initial_bytes=buf) self.assertRaises(TypeError, self.ioclass, buf, foo=None) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B' def test_write_concurrent_close(self): class B: def __buffer__(self, flags): @@ -601,7 +600,6 @@ def __buffer__(self, flags): # concurrently mutates (e.g., closes or exports) 'memio'. # See: https://github.com/python/cpython/issues/143378. - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B' def test_writelines_concurrent_close(self): class B: def __buffer__(self, flags): @@ -611,7 +609,6 @@ def __buffer__(self, flags): memio = self.ioclass() self.assertRaises(ValueError, memio.writelines, [B()]) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B' def test_write_concurrent_export(self): class B: buf = None @@ -622,7 +619,6 @@ def __buffer__(self, flags): memio = self.ioclass() self.assertRaises(BufferError, memio.write, B()) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B' def test_writelines_concurrent_export(self): class B: buf = None @@ -633,7 +629,6 @@ def __buffer__(self, flags): memio = self.ioclass() self.assertRaises(BufferError, memio.writelines, [B()]) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: a bytes-like object is required, not 'B' def test_write_mutating_buffer(self): # Test that buffer is exported only once during write(). # See: https://github.com/python/cpython/issues/143602. @@ -930,9 +925,6 @@ def test_cow_mutable(self): def test_flags(self): return super().test_flags() - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised by write - def test_write(self): - return super().test_write() class CStringIOTest(PyStringIOTest): ioclass = io.StringIO @@ -1004,10 +996,6 @@ def __str__(self): def test_flags(self): return super().test_flags() - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'StringIO' object has no attribute 'newlines'. Did you mean: 'readlines'? - def test_newlines_property(self): - return super().test_newlines_property() - class CStringIOPickleTest(PyStringIOPickleTest): UnsupportedOperation = io.UnsupportedOperation @@ -1017,9 +1005,5 @@ def __new__(cls, *args, **kwargs): def __init__(self, *args, **kwargs): pass - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'StringIO' object has no attribute 'newlines'. Did you mean: 'readlines'? - def test_newlines_property(self): - return super().test_newlines_property() - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 12e3504e42e..707540f299d 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -797,7 +797,6 @@ def __bool__(self): m[0] = MyBool() self.assertEqual(ba[:8], b'\0'*8) - @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'memoryview' object has no attribute '__buffer__' def test_buffer_reference_loop(self): m = memoryview(b'abc').__buffer__(0) o = MyObject() diff --git a/Lib/test/test_named_expressions.py b/Lib/test/test_named_expressions.py index 2e0643484fc..a859e051de2 100644 --- a/Lib/test/test_named_expressions.py +++ b/Lib/test/test_named_expressions.py @@ -365,7 +365,6 @@ def test_named_expression_invalid_dict_comprehension_iterable_expression(self): with self.assertRaisesRegex(SyntaxError, msg): exec(f"lambda: {code}", {}) # Function scope - @unittest.expectedFailure # TODO: RUSTPYTHON; wrong error message def test_named_expression_invalid_mangled_class_variables(self): code = """class Foo: def bar(self): diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index 4d062b42ded..42f11c9eb28 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -330,7 +330,6 @@ def test_cyclical_repr(self): name = repr(s).partition('(')[0] # strip class name self.assertEqual(repr(s), '%s({%s(...)})' % (name, name)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_do_not_rehash_dict_keys(self): n = 10 d = dict.fromkeys(map(HashCountingInt, range(n))) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 68f8969a00b..ef8acd0f338 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -364,7 +364,6 @@ def test_use_after_close(self): with self.cx: pass - @unittest.expectedFailure # TODO: RUSTPYTHON def test_exceptions(self): # Optional DB-API extension. self.assertEqual(self.cx.Warning, sqlite.Warning) @@ -401,7 +400,6 @@ def test_in_transaction_ro(self): with self.assertRaises(AttributeError): self.cx.in_transaction = True - @unittest.expectedFailure # TODO: RUSTPYTHON def test_connection_exceptions(self): exceptions = [ "DataError", @@ -527,7 +525,6 @@ def test_connection_bad_reinit(self): cx.executemany, "insert into t values(?)", ((v,) for v in range(3))) - @unittest.expectedFailure # TODO: RUSTPYTHON; SQLITE_DBCONFIG constants not implemented def test_connection_config(self): op = sqlite.SQLITE_DBCONFIG_ENABLE_FKEY with memory_database() as cx: diff --git a/Lib/test/test_sqlite3/test_transactions.py b/Lib/test/test_sqlite3/test_transactions.py index d777af0ffe6..a3de7a7a82e 100644 --- a/Lib/test/test_sqlite3/test_transactions.py +++ b/Lib/test/test_sqlite3/test_transactions.py @@ -387,7 +387,6 @@ def test_autocommit_setget(self): cx.autocommit = mode self.assertEqual(cx.autocommit, mode) - @unittest.expectedFailure # TODO: RUSTPYTHON; autocommit validation error messages differ def test_autocommit_setget_invalid(self): msg = "autocommit must be True, False, or.*LEGACY" for mode in "a", 12, (), None: diff --git a/Lib/test/test_sqlite3/test_userfunctions.py b/Lib/test/test_sqlite3/test_userfunctions.py index e7cecb85213..d63bccf9696 100644 --- a/Lib/test/test_sqlite3/test_userfunctions.py +++ b/Lib/test/test_sqlite3/test_userfunctions.py @@ -170,7 +170,6 @@ def setUp(self): def tearDown(self): self.con.close() - @unittest.expectedFailure # TODO: RUSTPYTHON; error message differs for invalid num args def test_func_error_on_create(self): with self.assertRaisesRegex(sqlite.ProgrammingError, "not -100"): self.con.create_function("bla", -100, lambda x: 2*x) @@ -514,7 +513,6 @@ def test_win_sum_int(self): self.cur.execute(self.query % "sumint") self.assertEqual(self.cur.fetchall(), self.expected) - @unittest.expectedFailure # TODO: RUSTPYTHON; error message differs for invalid num args def test_win_error_on_create(self): with self.assertRaisesRegex(sqlite.ProgrammingError, "not -100"): self.con.create_window_function("shouldfail", -100, WindowSumInt) @@ -649,7 +647,6 @@ def setUp(self): def tearDown(self): self.con.close() - @unittest.expectedFailure # TODO: RUSTPYTHON; error message differs for invalid num args def test_aggr_error_on_create(self): with self.assertRaisesRegex(sqlite.ProgrammingError, "not -100"): self.con.create_function("bla", -100, AggrSum) @@ -803,13 +800,11 @@ def setUp(self): def tearDown(self): self.con.close() - @unittest.expectedFailure # TODO: RUSTPYTHON; error message differs def test_table_access(self): with self.assertRaises(sqlite.DatabaseError) as cm: self.con.execute("select * from t2") self.assertIn('prohibited', str(cm.exception)) - @unittest.expectedFailure # TODO: RUSTPYTHON; error message differs def test_column_access(self): with self.assertRaises(sqlite.DatabaseError) as cm: self.con.execute("select c2 from t1") diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 31d2e58b108..f828b778659 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -498,12 +498,10 @@ def _test_pack_into(self, pack_into): with self.assertRaises((IndexError, OverflowError)): pack_into(writable_buf, -2**1000, test_string) - @unittest.expectedFailure # TODO: RUSTPYTHON; BufferError: non-contiguous buffer is not a bytes-like object def test_pack_into(self): s = struct.Struct('21s') self._test_pack_into(s.pack_into) - @unittest.expectedFailure # TODO: RUSTPYTHON; BufferError: non-contiguous buffer is not a bytes-like object def test_pack_into_fn(self): pack_into = lambda *args: struct.pack_into('21s', *args) self._test_pack_into(pack_into) @@ -603,7 +601,6 @@ def test_trailing_counter(self): 'spam and eggs') self.assertRaises(struct.error, struct.unpack_from, '14s42', store, 0) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '>h' != '>hh' def test_Struct_reinitialization(self): # Issue 9422: there was a memory leak when reinitializing a # Struct instance. This test can be used to detect the leak @@ -828,7 +825,6 @@ def test_error_propagation(fmt_str): test_error_propagation('N') test_error_propagation('n') - @unittest.expectedFailure # TODO: RUSTPYTHON def test_struct_subclass_instantiation(self): # Regression test for https://github.com/python/cpython/issues/112358 class MyStruct(struct.Struct): diff --git a/Lib/test/test_structseq.py b/Lib/test/test_structseq.py index 8ef6dd2fee8..d4014a784da 100644 --- a/Lib/test/test_structseq.py +++ b/Lib/test/test_structseq.py @@ -87,7 +87,6 @@ def test_fields(self): self.assertEqual(t.n_unnamed_fields, 0) self.assertEqual(t.n_fields, time._STRUCT_TM_ITEMS) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Unexpected keyword argument dict def test_constructor(self): t = time.struct_time @@ -111,7 +110,6 @@ def test_constructor(self): s = "123456789" self.assertEqual("".join(t(s)), s) - @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message def test_constructor_with_duplicate_fields(self): t = time.struct_time @@ -125,7 +123,6 @@ def test_constructor_with_duplicate_fields(self): with self.assertRaisesRegex(TypeError, error_message): t("1234567890", dict={"error": 0, "tm_zone": "some zone", "tm_mon": 1}) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at most 1 arguments, got 2 def test_constructor_with_duplicate_unnamed_fields(self): assert os.stat_result.n_unnamed_fields > 0 n_visible_fields = os.stat_result.n_sequence_fields @@ -142,7 +139,6 @@ def test_constructor_with_duplicate_unnamed_fields(self): re.escape("got duplicate or unexpected field name(s)")): os.stat_result((*range(n_visible_fields), -1.0), {'st_atime': -1.0}) - @unittest.expectedFailure # TODO: RUSTPYTHON; Wrong error message def test_constructor_with_unknown_fields(self): t = time.struct_time @@ -185,7 +181,6 @@ def test_pickling(self): self.assertEqual(t2.tm_year, t.tm_year) self.assertEqual(t2.tm_zone, t.tm_zone) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at most 1 arguments, got 2 def test_pickling_with_unnamed_fields(self): assert os.stat_result.n_unnamed_fields > 0 @@ -220,7 +215,6 @@ def test_copying(self): self.assertIsNot(t3[0], t[0]) self.assertIsNot(t3.tm_year, t.tm_year) - @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at most 1 arguments, got 2 def test_copying_with_unnamed_fields(self): assert os.stat_result.n_unnamed_fields > 0 diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 19ea6fafcf7..42aa7e3d9bb 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -631,7 +631,6 @@ def test_has_strftime_extensions(self): else: self.assertTrue(support.has_strftime_extensions) - @unittest.expectedFailure # TODO: RUSTPYTHON; - _testinternalcapi module not available def test_get_recursion_depth(self): # test support.get_recursion_depth() code = textwrap.dedent(""" diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 8cd1da1e972..16204bc45dd 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -247,7 +247,6 @@ def test_lineno(self): self.assertEqual(self.top.get_lineno(), 0) self.assertEqual(self.spam.get_lineno(), 14) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Lists differ: [] != ['a', 'b', 'internal', 'kw', 'other_internal', 'some_var', 'var', 'x'] def test_function_info(self): func = self.spam self.assertEqual(sorted(func.get_parameters()), ["a", "b", "kw", "var"]) @@ -256,7 +255,6 @@ def test_function_info(self): self.assertEqual(sorted(func.get_globals()), ["bar", "glob", "some_assigned_global_var"]) self.assertEqual(self.internal.get_frees(), ("x",)) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_globals(self): self.assertTrue(self.spam.lookup("glob").is_global()) self.assertFalse(self.spam.lookup("glob").is_declared_global()) @@ -275,7 +273,6 @@ def test_nonlocal(self): expected = ("some_var",) self.assertEqual(self.other_internal.get_nonlocals(), expected) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_local(self): self.assertTrue(self.spam.lookup("x").is_local()) self.assertFalse(self.spam.lookup("bar").is_local()) @@ -283,7 +280,6 @@ def test_local(self): self.assertTrue(self.top.lookup("some_non_assigned_global_var").is_local()) self.assertTrue(self.top.lookup("some_assigned_global_var").is_local()) - @unittest.expectedFailure # TODO: RUSTPYTHON def test_free(self): self.assertTrue(self.internal.lookup("x").is_free()) @@ -328,7 +324,6 @@ def test_assigned(self): self.assertTrue(self.Mine.lookup("a_method").is_assigned()) self.assertFalse(self.internal.lookup("x").is_assigned()) - @unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: list index out of range def test_annotated(self): st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec') st2 = st1.get_children()[1] @@ -493,7 +488,6 @@ def test_symtable_repr(self): self.assertEqual(str(self.top), "") self.assertEqual(str(self.spam), "") - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: AssertionError: "" != "" def test_symbol_repr(self): self.assertEqual(repr(self.spam.lookup("glob")), "") diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 3f056f2b753..c98b99e98e9 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -843,7 +843,6 @@ def test_complex_comprehension_inlining_exec(self): lamb = list(genexp)[0] self.assertEqual(lamb(), 42) - @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '__annotate__' != 'f.__annotate__' def test_annotate_qualname(self): code = """ def f() -> None: diff --git a/crates/capi/src/genericaliasobject.rs b/crates/capi/src/genericaliasobject.rs index bcd31308679..1ab443e13ad 100644 --- a/crates/capi/src/genericaliasobject.rs +++ b/crates/capi/src/genericaliasobject.rs @@ -10,6 +10,6 @@ pub unsafe extern "C" fn Py_GenericAlias( with_vm(|vm| { let origin = unsafe { &*origin }.to_owned(); let args = unsafe { &*args }.to_owned(); - PyGenericAlias::from_args(origin, args, vm).into_pyobject(vm) + PyGenericAlias::from_args(origin, args, vm).map(|alias| alias.into_pyobject(vm)) }) } diff --git a/crates/capi/src/objimpl.rs b/crates/capi/src/objimpl.rs index 99b0be7cc68..aa74bb69379 100644 --- a/crates/capi/src/objimpl.rs +++ b/crates/capi/src/objimpl.rs @@ -9,7 +9,7 @@ pub unsafe extern "C" fn PyObject_GC_Track(op: *mut PyObject) { with_vm(|_vm| { let obj = unsafe { &*op }; if !obj.is_gc_tracked() { - unsafe { gc_state::gc_state().track_object(obj.into()) }; + unsafe { gc_state::gc_state().track_object(obj.into(), gc_state::current_owner()) }; } }) } @@ -36,29 +36,33 @@ pub unsafe extern "C" fn PyObject_GC_IsFinalized(op: *mut PyObject) -> c_int { #[unsafe(no_mangle)] pub extern "C" fn PyGC_Collect() -> isize { - let result = gc_state::gc_state().collect(2); - (result.collected + result.uncollectable) as isize + with_vm(|vm| { + let result = vm.state.gc.collect(2); + (result.collected + result.uncollectable) as isize + }) } #[unsafe(no_mangle)] pub extern "C" fn PyGC_Enable() -> c_int { - let gc = gc_state::gc_state(); - let was_enabled = gc.is_enabled(); - gc.enable(); - was_enabled.into() + with_vm(|vm| { + let was_enabled: c_int = vm.state.gc.is_enabled().into(); + vm.state.gc.enable(); + was_enabled + }) } #[unsafe(no_mangle)] pub extern "C" fn PyGC_Disable() -> c_int { - let gc = gc_state::gc_state(); - let was_enabled = gc.is_enabled(); - gc.disable(); - was_enabled.into() + with_vm(|vm| { + let was_enabled: c_int = vm.state.gc.is_enabled().into(); + vm.state.gc.disable(); + was_enabled + }) } #[unsafe(no_mangle)] pub extern "C" fn PyGC_IsEnabled() -> c_int { - gc_state::gc_state().is_enabled().into() + with_vm(|vm| -> c_int { vm.state.gc.is_enabled().into() }) } #[unsafe(no_mangle)] diff --git a/crates/capi/src/pyerrors.rs b/crates/capi/src/pyerrors.rs index 25b76f33362..a4ead856be9 100644 --- a/crates/capi/src/pyerrors.rs +++ b/crates/capi/src/pyerrors.rs @@ -347,7 +347,7 @@ pub unsafe extern "C" fn PyUnicodeDecodeError_Create( unsafe { slice::from_raw_parts(object.cast::(), length) }.to_vec() }; - let exc = vm.new_unicode_decode_error_real( + let exc = vm.new_unicode_decode_error( vm.ctx.new_str(encoding), vm.ctx.new_bytes(bytes), start, diff --git a/crates/capi/src/pystate.rs b/crates/capi/src/pystate.rs index 865a116443b..cec3bc240b1 100644 --- a/crates/capi/src/pystate.rs +++ b/crates/capi/src/pystate.rs @@ -109,8 +109,8 @@ mod tests { current_vm_is_set(), "This thread did not have a vm attached" ); - vm.state.stop_the_world.stop_the_world(vm); - vm.state.stop_the_world.start_the_world(vm); + vm.state.stop_the_world.stop_the_world(&vm.state); + vm.state.stop_the_world.start_the_world(&vm.state); }); }); }); diff --git a/crates/codegen/src/compile.rs b/crates/codegen/src/compile.rs index ba2d41f12b3..fe0a983187a 100644 --- a/crates/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -2175,7 +2175,7 @@ impl<'warnings> Compiler<'warnings> { /// On success, returns the saved CompileContext to pass to exit_annotation_scope. fn enter_annotation_scope( &mut self, - _func_name: &str, + func_name: &str, loc: TextRange, ) -> CompileResult> { if !self.push_annotation_symbol_table() { @@ -2200,6 +2200,12 @@ impl<'warnings> Compiler<'warnings> { lineno.to_u32(), )?; + // enter_scope() qualified the scope by the enclosing scope only; redo it + // now that the annotated function is known. Only signature annotations + // get this treatment - deferred class and module annotations are + // compiled inside the scope they belong to and are already qualified. + self.set_annotation_qualname(func_name); + // Keep the internal ".format" name; exit_annotation_scope() // renames it to "format" on the final code object. self.current_code_info() @@ -2400,7 +2406,13 @@ impl<'warnings> Compiler<'warnings> { } if let FBlockDatum::FinallyBody(ref body) = info.fb_datum { + // This is an extra copy of the finally body, emitted for the + // path that leaves the try block early. The try statement + // emits its own copies afterwards, so rewind the symbol table + // cursors and leave the nested scopes for those copies. + let symbol_table_cursors = self.current_symbol_table_cursors(); self.compile_statements(body)?; + self.set_symbol_table_cursors(symbol_table_cursors); } if preserve_tos { @@ -2605,11 +2617,24 @@ impl<'warnings> Compiler<'warnings> { /// Set the qualified name for the current code object // = compiler_set_qualname fn set_qualname(&mut self) -> String { - let qualname = self.make_qualname(); + self.set_qualname_for_function(None) + } + + /// Set the qualname of an annotation scope, qualified by the function whose + /// signature it annotates. CPython records that name on the annotation + /// block's symbol table entry (`ste_function_name`) and folds it into the + /// qualname, so `f`'s annotation scope is named `f.__annotate__`. + fn set_annotation_qualname(&mut self, function_name: &str) { + self.set_qualname_for_function(Some(function_name)); + } + + fn set_qualname_for_function(&mut self, function_name: Option<&str>) -> String { + let qualname = self.make_qualname(function_name); self.current_code_info().metadata.qualname = Some(qualname.clone()); qualname } - fn make_qualname(&mut self) -> String { + + fn make_qualname(&mut self, function_name: Option<&str>) -> String { let stack_size = self.code_stack.len(); assert!(stack_size >= 1); @@ -2693,10 +2718,10 @@ impl<'warnings> Compiler<'warnings> { } } - // Build the qualified name - if force_global { + // Build the prefix the current name is qualified by, if any + let base = if force_global { // For global symbols, qualname is just the name - current_obj_name + None } else { // Check parent scope type let parent_obj_name = &parent.metadata.name; @@ -2709,23 +2734,32 @@ impl<'warnings> Compiler<'warnings> { ) ); + // Use parent's qualname if available, otherwise use parent_obj_name + let parent_qualname = parent.metadata.qualname.as_ref().unwrap_or(parent_obj_name); + if is_function_parent { // For functions, append . to parent qualname - // Use parent's qualname if available, otherwise use parent_obj_name - let parent_qualname = parent.metadata.qualname.as_ref().unwrap_or(parent_obj_name); - format!("{parent_qualname}..{current_obj_name}") + Some(format!("{parent_qualname}.")) + } else if parent_qualname == "" { + // Module level, nothing to qualify by + None } else { // For classes and other scopes, use parent's qualname directly - // Use parent's qualname if available, otherwise use parent_obj_name - let parent_qualname = parent.metadata.qualname.as_ref().unwrap_or(parent_obj_name); - if parent_qualname == "" { - // Module level, just use the name - current_obj_name - } else { - // Concatenate parent qualname with current name - format!("{parent_qualname}.{current_obj_name}") - } + Some(parent_qualname.clone()) } + }; + + // An annotation scope is compiled in the scope enclosing the function it + // annotates, so the function itself is missing from the prefix above. + let base = match (base, function_name) { + (Some(base), Some(function_name)) => Some(format!("{base}.{function_name}")), + (None, Some(function_name)) => Some(function_name.to_owned()), + (base, None) => base, + }; + + match base { + Some(base) => format!("{base}.{current_obj_name}"), + None => current_obj_name, } } @@ -5055,6 +5089,14 @@ impl<'warnings> Compiler<'warnings> { func_range: TextRange, ) -> CompileResult { if !self.next_function_annotation_symbol_table_uses_annotations() { + // CPython creates a hidden AnnotationBlock for every function + // signature under `from __future__ import annotations`, including + // an unannotated one. It still belongs to this function: consume + // it so the next function sees its own block rather than remaining + // pinned to this unused entry. + if self.push_annotation_symbol_table() { + self.pop_annotation_symbol_table(); + } return Ok(false); } @@ -10877,9 +10919,7 @@ impl<'warnings> Compiler<'warnings> { if sym.flags.contains(SymbolFlags::DEF_PARAM) { continue; // skip .0 } - let is_local = sym - .flags - .intersects(SymbolFlags::DEF_LOCAL | SymbolFlags::ITER) + let is_local = sym.flags.contains(SymbolFlags::DEF_LOCAL) && !sym.flags.contains(SymbolFlags::DEF_NONLOCAL); if is_local { pushed_locals.push(name.clone()); @@ -31219,6 +31259,25 @@ def f(x: T): pass ); } + #[test] + fn future_unannotated_function_does_not_hide_next_annotation_block() { + let code = compile_exec( + "\ +from __future__ import annotations +def plain(x): pass +def annotated(x: int): pass +", + ); + let annotate = find_direct_child_code(&code, "__annotate__") + .expect("second function must retain its annotation closure"); + assert!( + annotate.constants.iter().any( + |constant| matches!(constant, ConstantData::Str { value } if value.as_str() == Ok("int")) + ), + "annotation closure must belong to the annotated function" + ); + } + #[test] fn deferred_annotation_format_name_does_not_capture_helper_parameter() { let code = compile_exec( diff --git a/crates/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs index a66410f9018..a771e19d36f 100644 --- a/crates/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -312,19 +312,8 @@ bitflags! { Self::DEF_LOCAL.bits() | Self::DEF_PARAM.bits() | Self::DEF_IMPORT.bits() - | Self::ITER.bits() | Self::DEF_TYPE_PARAM.bits() ); - - - // TODO: Remove these, RustPython specific - - // indicates if the symbol gets a value assigned by a named expression in a comprehension - // this is required to correct the scope in the analysis. - const ASSIGNED_IN_COMPREHENSION = 2 << 11; - // indicates that the symbol is used a bound iterator variable. We distinguish this case - // from normal assignment to detect disallowed re-assignment to iterator variables. - const ITER = 2 << 12; } } @@ -1030,7 +1019,6 @@ enum SymbolUsage { AnnotationAssigned, Parameter, AnnotationParameter, - AssignedNamedExprInComprehension, Iter, TypeParam, } @@ -1048,8 +1036,6 @@ struct SymbolTableBuilder { varnames_stack: Vec>, // Track if we're inside an iterable definition expression (for nested comprehensions) in_iter_def_exp: bool, - // Track if we're scanning an inner loop iteration target (not the first generator) - in_comp_inner_loop_target: bool, // yield/yield from inside comprehension scopes is rejected with a // message that names the comprehension kind. comprehension_yield_context: Option<&'static str>, @@ -1084,7 +1070,6 @@ impl SymbolTableBuilder { current_varnames: Vec::new(), varnames_stack: Vec::new(), in_iter_def_exp: false, - in_comp_inner_loop_target: false, comprehension_yield_context: None, in_conditional_block: false, recursion_depth: 0, @@ -2525,24 +2510,8 @@ impl SymbolTableBuilder { self.scan_expression(value, ExpressionContext::Load)?; - // special handling for assigned identifier in named expressions - // that are used in comprehensions. This required to correctly - // propagate the scope of the named assigned named and not to - // propagate inner names. if let Some((id, target_range)) = named_target { - let table = self.tables.last().unwrap(); - if table.typ == CompilerScope::Comprehension { - self.register_name( - id, - SymbolUsage::AssignedNamedExprInComprehension, - target_range, - )?; - } else { - // omit one recursion. When the handling of an store changes for - // Identifiers this needs adapted - more forward safe would be - // calling scan_expression directly. - self.register_name(id, SymbolUsage::Assigned, target_range)?; - } + self.register_name(id, SymbolUsage::Assigned, target_range)?; } else { self.scan_expression(target, ExpressionContext::Store)?; } @@ -2613,9 +2582,7 @@ impl SymbolTableBuilder { } for generator in &generators[1..] { - self.in_comp_inner_loop_target = true; self.scan_expression(&generator.target, ExpressionContext::Iter)?; - self.in_comp_inner_loop_target = false; let was_in_iter_def_exp = self.in_iter_def_exp; self.in_iter_def_exp = true; self.scan_expression(&generator.iter, ExpressionContext::IterDefinitionExp)?; @@ -3037,11 +3004,15 @@ impl SymbolTableBuilder { if self.tables[table_idx] .symbols .get(mangled.as_str()) - .is_some_and(|symbol| symbol.flags.contains(SymbolFlags::ITER)) + .is_some_and(|symbol| { + symbol + .flags + .contains(SymbolFlags::DEF_LOCAL | SymbolFlags::DEF_COMP_ITER) + }) { return Err(SymbolTableError { error: format!( - "assignment expression cannot rebind comprehension iteration variable '{mangled}'" + "assignment expression cannot rebind comprehension iteration variable '{name}'" ), location, }); @@ -3151,7 +3122,6 @@ impl SymbolTableBuilder { | SymbolUsage::AnnotationAssigned | SymbolUsage::Parameter | SymbolUsage::AnnotationParameter - | SymbolUsage::AssignedNamedExprInComprehension | SymbolUsage::Iter | SymbolUsage::TypeParam ) { @@ -3179,16 +3149,16 @@ impl SymbolTableBuilder { let symbol = if let Some(symbol) = table.symbols.get_mut(name.as_ref()) { let flags = &symbol.flags; - // INNER_LOOP_CONFLICT: comprehension inner loop cannot rebind - // a variable that was used as a named expression target + // Mirrors CPython's INNER_LOOP_CONFLICT check. extend_namedexpr_scope() + // marks named-expression targets as global or nonlocal in the comprehension. // Example: [i for i in range(5) if (j := 0) for j in range(5)] // Here 'j' is used in named expr first, then as inner loop iter target - if self.in_comp_inner_loop_target - && flags.contains(SymbolFlags::ASSIGNED_IN_COMPREHENSION) + if matches!(role, SymbolUsage::Iter) + && flags.intersects(SymbolFlags::DEF_GLOBAL | SymbolFlags::DEF_NONLOCAL) { return Err(SymbolTableError { error: format!( - "comprehension inner loop cannot rebind assignment expression target '{name}'" + "comprehension inner loop cannot rebind assignment expression target '{original_name}'" ), location, }); @@ -3345,9 +3315,6 @@ impl SymbolTableBuilder { SymbolUsage::Assigned => { flags.insert(SymbolFlags::DEF_LOCAL); } - SymbolUsage::AssignedNamedExprInComprehension => { - flags.insert(SymbolFlags::DEF_LOCAL | SymbolFlags::ASSIGNED_IN_COMPREHENSION); - } SymbolUsage::Global => { symbol.scope = SymbolScope::GlobalExplicit; flags.insert(SymbolFlags::DEF_GLOBAL); @@ -3356,33 +3323,13 @@ impl SymbolTableBuilder { flags.insert(SymbolFlags::USE); } SymbolUsage::Iter => { - // CPython symtable_add_def_helper() records an inlined - // comprehension target as a local definition as well as a - // comprehension iterator. Keep ITER as the internal - // re-assignment check marker; DEF_LOCAL is part of the public - // ste_symbols flags exposed by _symtable. - flags.insert( - SymbolFlags::DEF_LOCAL | SymbolFlags::ITER | SymbolFlags::DEF_COMP_ITER, - ); + flags.insert(SymbolFlags::DEF_LOCAL | SymbolFlags::DEF_COMP_ITER); } SymbolUsage::TypeParam => { flags.insert(SymbolFlags::DEF_LOCAL | SymbolFlags::DEF_TYPE_PARAM); } } - // and even more checking - // it is not allowed to assign to iterator variables (by named expressions) - if flags.contains(SymbolFlags::ITER) - && flags.contains(SymbolFlags::ASSIGNED_IN_COMPREHENSION) - { - return Err(SymbolTableError { - error: format!( - "assignment expression cannot rebind comprehension iteration variable '{}'", - symbol.name - ), - location, - }); - } Ok(()) } } diff --git a/crates/common/src/borrow.rs b/crates/common/src/borrow.rs index 2be5f8275c8..70d755ff155 100644 --- a/crates/common/src/borrow.rs +++ b/crates/common/src/borrow.rs @@ -34,6 +34,17 @@ impl_from!('a, T, BorrowedValue<'a, T>, ); impl<'a, T: ?Sized> BorrowedValue<'a, T> { + /// Whether reaching the value holds a lock that other threads wait on. + /// + /// An immutable object hands out a plain reference and answers `false`; + /// one whose storage can change hands out a guard. A caller about to wait + /// for something unrelated -- a peer, a file, a signal -- can use this to + /// decide whether it may keep the borrow for the duration. + #[must_use] + pub const fn is_locked(&self) -> bool { + !matches!(self, Self::Ref(_)) + } + pub fn map(s: Self, f: F) -> BorrowedValue<'a, U> where F: FnOnce(&T) -> &U, diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 53a8e0d752b..d1e04b46d57 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -21,6 +21,7 @@ pub mod rc; pub mod refcount; pub mod static_cell; pub mod str; +pub mod wtf8_index; pub use rustpython_wtf8 as wtf8; diff --git a/crates/common/src/str.rs b/crates/common/src/str.rs index c006a5f4db4..39ec7da1de5 100644 --- a/crates/common/src/str.rs +++ b/crates/common/src/str.rs @@ -1,7 +1,8 @@ // spell-checker:ignore uncomputed -use crate::atomic::{PyAtomic, Radium}; +use crate::atomic::{OncePtr, PyAtomic, Radium}; use crate::format::CharLen; use crate::wtf8::{CodePoint, Wtf8, Wtf8Buf}; +use crate::wtf8_index::Wtf8Index; use ascii::{AsciiChar, AsciiStr, AsciiString}; use core::fmt; use core::ops::{Bound, RangeBounds}; @@ -112,11 +113,70 @@ pub enum PyKindStr<'a> { Wtf8(&'a Wtf8), } +/// How far from an end an index is resolved by walking rather than by building +/// the code point index. +/// +/// PyPy spells this `MAX_UNROLL_NEXT_CODEPOINT_POS`, in a guard that also asks +/// the JIT whether the index is a constant, so that the walk unrolls. There is +/// no JIT here to ask, and the walk is short rather than free -- but four steps +/// still beat a pass over the whole buffer, and skipping the build is what +/// keeps `s[0]` and `s[1:-1]` on a long string from paying for a table. +const MAX_WALK_TO_INDEX: usize = 4; + #[derive(Debug, Clone)] pub struct StrData { data: Box, kind: StrKind, len: StrLen, + index: Wtf8IndexSlot, +} + +/// A [`Wtf8Index`] built on first use. +/// +/// The table is a pure function of `data`, so publishing it races benignly: a +/// thread that loses the exchange drops its own copy and reads the winner's. +#[derive(Default)] +struct Wtf8IndexSlot(OncePtr); + +impl Wtf8IndexSlot { + #[inline(always)] + fn new() -> Self { + Self(OncePtr::new()) + } + + #[inline] + fn get_or_build(&self, data: &Wtf8, char_len: usize) -> &Wtf8Index { + let index = self + .0 + .get_or_init(|| Box::new(Wtf8Index::new(data, char_len))); + // The slot owns the table, never replaces it, and outlives the borrow. + unsafe { index.as_ref() } + } +} + +impl fmt::Debug for Wtf8IndexSlot { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self.0.get() { + Some(_) => f.write_str(""), + None => f.write_str(""), + } + } +} + +impl Clone for Wtf8IndexSlot { + /// A fresh slot: the clone copies the buffer, so it has to index that copy, + /// and the table is rebuilt on demand rather than eagerly here. + fn clone(&self) -> Self { + Self::new() + } +} + +impl Drop for Wtf8IndexSlot { + fn drop(&mut self) { + if let Some(index) = self.0.get() { + drop(unsafe { Box::from_raw(index.as_ptr()) }); + } + } } struct StrLen(PyAtomic); @@ -163,6 +223,7 @@ impl Default for StrData { data: >::default(), kind: StrKind::Ascii, len: StrLen::zero(), + index: Wtf8IndexSlot::new(), } } } @@ -193,6 +254,7 @@ impl From> for StrData { len: value.len().into(), data: value.into(), kind: StrKind::Ascii, + index: Wtf8IndexSlot::new(), } } } @@ -212,6 +274,7 @@ impl From for StrData { data: ch.to_string().into(), kind: StrKind::Utf8, len: 1.into(), + index: Wtf8IndexSlot::new(), } } } @@ -226,6 +289,7 @@ impl From for StrData { data: Wtf8Buf::from(ch).into(), kind: StrKind::Wtf8, len: 1.into(), + index: Wtf8IndexSlot::new(), } } } @@ -241,7 +305,12 @@ impl StrData { StrKind::Ascii => data.len().into(), _ => StrLen::uncomputed(), }; - Self { data, kind, len } + Self { + data, + kind, + len, + index: Wtf8IndexSlot::new(), + } } /// # Safety @@ -253,6 +322,7 @@ impl StrData { data, kind, len: char_len.into(), + index: Wtf8IndexSlot::new(), } } @@ -322,11 +392,117 @@ impl StrData { len } + /// The byte offset the `index`-th code point starts at. + /// + /// An `index` at or past the end answers the buffer's byte length, so a + /// caller walking to a bound does not have to special-case it. + /// + /// O(1), but the first call on a non-ASCII string builds an index over the + /// whole buffer, so a caller that resolves a single index and stops is + /// better served by [`Self::nth_char`]. + pub fn char_index_to_byte(&self, index: usize) -> usize { + // For ASCII the two units coincide, and the table would be a Nth entry + // saying N. + if self.kind.is_ascii() { + return index.min(self.data.len()); + } + let char_len = self.char_len(); + if index >= char_len { + return self.data.len(); + } + self.index + .get_or_build(&self.data, char_len) + .byte_offset(&self.data, index) + } + + /// The byte offset of code point `index`, for a caller that resolves one + /// index and stops. + /// + /// Building the table costs a pass over the whole buffer, so it is worth it + /// only for a caller that comes back; an index within + /// [`MAX_WALK_TO_INDEX`] steps of either end is cheaper to walk to, and + /// walking keeps `s[0]` on a long string from paying for a table it will + /// never use again. Anything further in builds, on the reasoning that a + /// string indexed once in the middle tends to be indexed again. + fn char_index_to_byte_once(&self, index: usize) -> usize { + if index <= MAX_WALK_TO_INDEX { + return self + .data + .code_point_indices() + .nth(index) + .map_or(self.data.len(), |(byte, _)| byte); + } + let from_end = self.char_len() - index; + if from_end <= MAX_WALK_TO_INDEX { + return self + .data + .code_point_indices() + .nth_back(from_end - 1) + .map_or(self.data.len(), |(byte, _)| byte); + } + self.char_index_to_byte(index) + } + + /// The byte range spanned by the code points in `range`, whose end must not + /// exceed the string's code point count. + /// + /// A range that reaches within [`MAX_WALK_TO_INDEX`] of *both* ends is + /// walked to for the same reason a single index near one end is -- a slice + /// like `s[1:-1]` should not build a table over the whole string. + #[must_use] + pub fn char_range_to_bytes(&self, range: core::ops::Range) -> core::ops::Range { + if self.kind.is_ascii() { + return range; + } + let from_end = self.char_len() - range.end; + if range.start <= MAX_WALK_TO_INDEX && from_end <= MAX_WALK_TO_INDEX { + // Two walks over disjoint ends, each of at most MAX_WALK_TO_INDEX + // steps -- one iterator driven from both sides would have them meet + // on a short string. + let start = self + .data + .code_point_indices() + .nth(range.start) + .map_or(self.data.len(), |(byte, _)| byte); + let end = match from_end { + 0 => self.data.len(), + n => self + .data + .code_point_indices() + .nth_back(n - 1) + .map_or(self.data.len(), |(byte, _)| byte), + }; + return start..end; + } + self.char_index_to_byte(range.start)..self.char_index_to_byte(range.end) + } + + /// The character index of the character starting at byte offset `bytepos`, + /// the inverse of [`Self::char_index_to_byte`]. + /// + /// `bytepos` must be a character boundary at or before the end. + /// + /// Logarithmic rather than constant, because the index is keyed the other + /// way -- but a search whose bounds came from `char_index_to_byte` has the + /// table already, and this is what turns a byte offset back into the answer + /// a caller asked for in characters. + pub fn byte_to_char_index(&self, bytepos: usize) -> usize { + if self.kind.is_ascii() { + return bytepos; + } + let char_len = self.char_len(); + self.index + .get_or_build(&self.data, char_len) + .char_index_at_byte(&self.data, bytepos, char_len) + } + pub fn nth_char(&self, index: usize) -> CodePoint { match self.as_str_kind() { PyKindStr::Ascii(s) => s[index].into(), - PyKindStr::Utf8(s) => s.chars().nth(index).unwrap().into(), - PyKindStr::Wtf8(w) => w.code_points().nth(index).unwrap(), + _ => self.data[self.char_index_to_byte_once(index)..] + .code_points() + .next() + .unwrap(), } } } @@ -416,20 +592,21 @@ pub fn codepoint_range_end(s: &Wtf8, n_chars: usize) -> Option { } #[must_use] -pub fn zfill(bytes: &[u8], width: usize) -> Vec { +/// Returns `None` for a width whose result cannot be allocated. +pub fn zfill(bytes: &[u8], width: usize) -> Option> { if width <= bytes.len() { - bytes.to_vec() - } else { - let (sign, s) = match bytes.first() { - Some(_sign @ (b'+' | b'-')) => (unsafe { bytes.get_unchecked(..1) }, &bytes[1..]), - _ => (&b""[..], bytes), - }; - let mut filled = Vec::new(); - filled.extend_from_slice(sign); - filled.extend(core::iter::repeat_n(b'0', width - bytes.len())); - filled.extend_from_slice(s); - filled + return Some(bytes.to_vec()); } + let (sign, s) = match bytes.first() { + Some(_sign @ (b'+' | b'-')) => (unsafe { bytes.get_unchecked(..1) }, &bytes[1..]), + _ => (&b""[..], bytes), + }; + let mut filled = Vec::new(); + filled.try_reserve_exact(width).ok()?; + filled.extend_from_slice(sign); + filled.extend(core::iter::repeat_n(b'0', width - bytes.len())); + filled.extend_from_slice(s); + Some(filled) } /// Convert a string to ascii compatible, escaping unicode-s into escape diff --git a/crates/common/src/wtf8_index.rs b/crates/common/src/wtf8_index.rs new file mode 100644 index 00000000000..4b2e5ebee5b --- /dev/null +++ b/crates/common/src/wtf8_index.rs @@ -0,0 +1,299 @@ +// spell-checker:ignore rpython rlib rutf +//! Random access into a WTF-8 buffer. +//! +//! WTF-8 is variable width, so a buffer's n-th code point can only be found by +//! decoding the n-1 before it: [`Wtf8`]'s iterators are sequential, and +//! resolving an index through them is O(n). Code that indexes the same string +//! repeatedly -- a regex scan restarting at successive positions, say -- then +//! walks the whole buffer once per index, which is quadratic in its length. +//! +//! [`Wtf8Index`] is the side table that makes the lookup O(1): one 24-byte +//! group per 64 code points, so 0.375 bytes per code point. It is a cache, and +//! holds no state of its own beyond the buffer's shape -- building it twice for +//! the same buffer yields the same table. +//! +//! The layout is PyPy's `UTF8_INDEX_STORAGE` (`rpython/rlib/rutf8.py`). + +use crate::wtf8::Wtf8; + +/// One group of 64 code points. +#[derive(Clone, Copy)] +struct Group { + /// The byte offset the group's first code point starts at. + base: usize, + /// `ofs[i]` is the byte offset of the group's `4 * i + 1`-th code point, + /// relative to `base`. One entry covers four code points, so the widest + /// offset an entry has to hold is that of the 61st code point of a group, + /// at most `61 * 4 = 244` bytes in -- inside a `u8`, which is what buys the + /// table its density. + ofs: [u8; 16], +} + +/// A code-point-index to byte-offset table for one WTF-8 buffer. +pub struct Wtf8Index { + groups: Box<[Group]>, +} + +impl Wtf8Index { + /// Builds the table for `data`, whose code point count is `char_len`. + /// + /// O(`data.len()`), and touches every byte, so it pays for itself only when + /// the caller goes on to index the buffer more than a couple of times. + #[must_use] + pub fn new(data: &Wtf8, char_len: usize) -> Self { + let mut groups = vec![ + Group { + base: 0, + ofs: [0; 16], + }; + char_len / 64 + 1 + ]; + // Signed: the countdown overshoots the last group -- the loop stops on + // the first negative value rather than at a group boundary. + let mut remaining = char_len as isize; + let mut base = 0; + let mut current = 0; + loop { + groups[current].base = base; + let mut next = base; + let mut group_filled = true; + for i in 0..16 { + // Past the end, step as if one more single-byte code point + // followed, so the entry stays in range and is never read. + next = if remaining == 0 { + next + 1 + } else { + next_pos(data, next) + }; + groups[current].ofs[i] = (next - base) as u8; + remaining -= 4; + if remaining < 0 { + debug_assert_eq!(current + 1, groups.len()); + group_filled = false; + break; + } + next = next_pos(data, next_pos(data, next_pos(data, next))); + } + if !group_filled { + break; + } + current += 1; + base = next; + } + Self { + groups: groups.into_boxed_slice(), + } + } + + /// The byte offset of `data`'s `index`-th code point. + /// + /// `data` must be the buffer the table was built for, and `index` must be + /// below its code point count. + #[inline] + #[must_use] + pub fn byte_offset(&self, data: &Wtf8, index: usize) -> usize { + let group = &self.groups[index >> 6]; + // The entry sits on the 4k+1-th code point of the group, so a lookup is + // one table read plus at most two steps in either direction. + let pos = group.base + group.ofs[(index >> 2) & 0x0F] as usize; + match index & 0x3 { + 0 => prev_pos(data, pos), + 1 => pos, + 2 => next_pos(data, pos), + _ => next_pos(data, next_pos(data, pos)), + } + } + + /// The index of the code point starting at byte offset `bytepos`, the + /// inverse of [`Self::byte_offset`]. + /// + /// `data` must be the buffer the table was built for, `char_len` its code + /// point count, and `bytepos` a code point boundary at or before its end. + /// + /// Logarithmic rather than constant: the table is keyed by code point + /// index, so going the other way is a search through it. The bracketing + /// below is what keeps that search short -- a code point occupies one to + /// four bytes, which pins the answer to a narrow band around `bytepos` + /// before the first comparison. + #[must_use] + pub fn char_index_at_byte(&self, data: &Wtf8, bytepos: usize, char_len: usize) -> usize { + let bytes_remaining = data.len() - bytepos; + // At least one byte per remaining code point, and at most four, so the + // group holding the answer lies between these. + let mut group_min = + usize::max(bytepos / 4, char_len.saturating_sub(bytes_remaining + 1)) >> 6; + let mut group_max = usize::min(bytepos, char_len.saturating_sub(bytes_remaining / 4)) >> 6; + while group_min < group_max { + let middle = group_min.midpoint(group_max) + 1; + if bytepos < self.groups[middle].base { + group_max = middle - 1; + } else { + group_min = middle; + } + } + + let base = self.groups[group_min].base; + if base == bytepos { + return group_min << 6; + } + // Walk the group's entries to the last one at or before `bytepos`, + // then step the remaining code points, of which there are at most + // three -- an entry covers four. + let entries = if group_min == self.groups.len() - 1 { + ((char_len - 1) >> 2) & 0x0F + } else { + 16 + }; + let mut index = group_min << 6; + let mut pos = base; + for entry in 0..entries { + let at = base + self.groups[group_min].ofs[entry] as usize; + if at >= bytepos { + break; + } + pos = at; + index = (group_min << 6) + (entry << 2) + 1; + } + while pos < bytepos { + pos = next_pos(data, pos); + index += 1; + } + index + } + + /// The table's heap footprint, in bytes. + #[must_use] + pub fn byte_size(&self) -> usize { + core::mem::size_of_val(&*self.groups) + } +} + +/// The byte offset of the code point after the one at `pos`. +/// +/// `data` must be well-formed WTF-8 and `pos` a code point boundary before its +/// end -- reading only the lead byte is what makes this branch-light. +#[inline] +fn next_pos(data: &Wtf8, pos: usize) -> usize { + match data.as_bytes()[pos] { + 0x00..=0x7F => pos + 1, + 0x80..=0xDF => pos + 2, + 0xE0..=0xEF => pos + 3, + _ => pos + 4, + } +} + +/// The byte offset of the code point before the one at `pos`, which must not be +/// zero. +/// +/// A `pos` one past the end reads as the extra code point [`Wtf8Index::new`] +/// steps over there. +#[inline] +fn prev_pos(data: &Wtf8, pos: usize) -> usize { + let data = data.as_bytes(); + let mut pos = pos - 1; + if pos >= data.len() || data[pos] <= 0x7F { + return pos; + } + pos -= 1; + if data[pos] >= 0xC0 { + return pos; + } + pos -= 1; + if data[pos] >= 0xC0 { + return pos; + } + pos - 1 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::wtf8::{CodePoint, Wtf8Buf}; + + /// Every index of `s`, both ways, against the offsets its own iterator + /// reports. + fn check(s: &Wtf8) { + let expected: Vec = s + .code_point_indices() + .map(|(byte_offset, _)| byte_offset) + .collect(); + let char_len = expected.len(); + let index = Wtf8Index::new(s, char_len); + for (i, &want) in expected.iter().enumerate() { + assert_eq!( + index.byte_offset(s, i), + want, + "index {i} of {s:?} ({char_len} code points)" + ); + assert_eq!( + index.char_index_at_byte(s, want, char_len), + i, + "byte {want} of {s:?} ({char_len} code points)" + ); + } + // One past the last code point is a boundary too, and the searches that + // use this ask for it as an end bound. + assert_eq!( + index.char_index_at_byte(s, s.len(), char_len), + char_len, + "end of {s:?}" + ); + } + + fn wtf8(s: &str) -> Wtf8Buf { + Wtf8Buf::from(s) + } + + #[test] + fn empty() { + check(wtf8("").as_ref()); + } + + #[test] + fn widths() { + // One case per encoded width, and the boundaries between them. + check(wtf8("abc").as_ref()); + check(wtf8("\u{80}\u{7ff}").as_ref()); + check(wtf8("\u{800}\u{ffff}").as_ref()); + check(wtf8("\u{10000}\u{10ffff}").as_ref()); + check(wtf8("a\u{80}\u{800}\u{10000}").as_ref()); + } + + #[test] + fn group_boundaries() { + // A group covers 64 code points and an entry four, so the interesting + // lengths are the ones on and around both. + for len in [1, 3, 4, 5, 63, 64, 65, 127, 128, 129, 255, 256, 257] { + for unit in ["a", "\u{80}", "\u{800}", "\u{10000}"] { + check(wtf8(&unit.repeat(len)).as_ref()); + } + // Mixed widths, so a group's entries do not share a stride. + check(wtf8(&"a\u{80}\u{800}\u{10000}".repeat(len)).as_ref()); + } + } + + #[test] + fn lone_surrogates() { + let mut s = wtf8("a"); + for cp in [0xD800, 0xDBFF, 0xDC00, 0xDFFF] { + s.push(CodePoint::from_u32(cp).unwrap()); + s.push_str("b"); + } + check(s.as_ref()); + + // Surrogates only, spanning more than one group. + let mut s = wtf8(""); + for i in 0..200 { + s.push(CodePoint::from_u32(0xD800 + (i % 0x400)).unwrap()); + } + check(s.as_ref()); + } + + #[test] + fn byte_size_is_one_group_per_64_code_points() { + let s = wtf8(&"\u{10000}".repeat(200)); + let index = Wtf8Index::new(s.as_ref(), 200); + assert_eq!(index.byte_size(), (200 / 64 + 1) * size_of::()); + assert_eq!(size_of::(), 24); + } +} diff --git a/crates/compiler-core/src/marshal.rs b/crates/compiler-core/src/marshal.rs index a2a23054e4b..754854e7cba 100644 --- a/crates/compiler-core/src/marshal.rs +++ b/crates/compiler-core/src/marshal.rs @@ -19,6 +19,14 @@ pub enum MarshalError { InvalidLocation, /// Bad type marker BadType, + /// A type marker no reader knows + UnknownType, + /// A back reference that names nothing + InvalidRef, + /// A marker that stands for no object at all + NullObject, + /// A container length that is negative or does not fit, named by what it counts + BadSize(&'static str), } impl core::fmt::Display for MarshalError { @@ -29,6 +37,10 @@ impl core::fmt::Display for MarshalError { Self::InvalidUtf8 => f.write_str("invalid utf8"), Self::InvalidLocation => f.write_str("invalid source location"), Self::BadType => f.write_str("bad type marker"), + Self::UnknownType => f.write_str("unknown type code"), + Self::InvalidRef => f.write_str("invalid reference"), + Self::NullObject => f.write_str("NULL object in marshal data for object"), + Self::BadSize(what) => write!(f, "{what} size out of range"), } } } @@ -111,7 +123,7 @@ impl TryFrom for Type { b'A' => Self::AsciiInterned, b'z' => Self::ShortAscii, b'Z' => Self::ShortAsciiInterned, - _ => return Err(MarshalError::BadType), + _ => return Err(MarshalError::UnknownType), }) } } @@ -146,6 +158,13 @@ pub trait Read { fn read_u64(&mut self) -> Result { Ok(u64::from_le_bytes(*self.read_array()?)) } + + /// A length, read the way `r_long` reads one: it is signed, so a value + /// with the top bit set is out of range rather than four billion items. + fn read_len(&mut self, what: &'static str) -> Result { + let len = self.read_u32()? as i32; + usize::try_from(len).map_err(|_| MarshalError::BadSize(what)) + } } pub(crate) trait ReadBorrowed<'a>: Read { @@ -305,7 +324,7 @@ fn reserve_ref_slot(has_flag: bool, refs: &mut Vec>) -> Option(idx: usize, refs: &[Option]) -> Result { refs.get(idx) .and_then(|v| v.clone()) - .ok_or(MarshalError::InvalidBytecode) + .ok_or(MarshalError::InvalidRef) } /// Read a marshal bytes object (TYPE_STRING = b's'), resolving TYPE_REF @@ -408,7 +427,7 @@ fn read_marshal_str_vec( } let n = match type_byte { - b'(' => rdr.read_u32()? as usize, + b'(' => rdr.read_len("tuple")?, b')' => rdr.read_u8()? as usize, _ => return Err(MarshalError::BadType), }; @@ -471,7 +490,7 @@ fn read_marshal_const_tuple( } let n = match type_byte { - b'(' => rdr.read_u32()? as usize, + b'(' => rdr.read_len("tuple")?, b')' => rdr.read_u8()? as usize, _ => return Err(MarshalError::BadType), }; @@ -516,7 +535,7 @@ fn read_const_value( let code = deserialize_code_inner(rdr, bag, depth - 1, refs)?; bag.make_code(code) } else { - deserialize_value_typed(rdr, bag, depth, refs, typ)? + deserialize_value_typed(rdr, bag, depth, refs, typ, slot)? }; if let Some(idx) = slot { refs[idx] = Some(value.clone()); @@ -540,6 +559,10 @@ pub trait MarshalBag: Copy { fn make_str(&self, value: &Wtf8) -> Self::Value; + fn make_interned_str(&self, value: &Wtf8) -> Self::Value { + self.make_str(value) + } + fn make_bytes(&self, value: &[u8]) -> Self::Value; fn make_int(&self, value: BigInt) -> Self::Value; @@ -549,7 +572,19 @@ pub trait MarshalBag: Copy { fn make_code( &self, code: CodeObject<::Constant>, - ) -> Self::Value; + ) -> Result; + + /// Construct a runtime code object while retaining the exact values read + /// from ``co_consts``. Compiler bags ignore this second channel; runtime + /// bags use it for marshalable values (lists, dicts, sets, recursive + /// containers) that their compiler constant representation cannot hold. + fn make_code_with_constants( + &self, + code: CodeObject<::Constant>, + _constants: Vec, + ) -> Result { + self.make_code(code) + } fn make_stop_iter(&self) -> Result; @@ -564,6 +599,55 @@ pub trait MarshalBag: Copy { it: impl Iterator, ) -> Result; + /// Install partially-built containers in the marshal reference table + /// before reading their children, as CPython's `r_object()` does. + /// Runtime bags can opt in; constant bags retain collect-then-construct. + /// + /// `len` comes straight from the input and is only bounded by what a + /// length can hold, so a bag that opts in reports the room it cannot get + /// rather than taking it for granted. + fn make_tuple_placeholder(&self, _len: usize) -> Result> { + Ok(None) + } + + fn set_tuple_item( + &self, + _tuple: &Self::Value, + _index: usize, + _value: Self::Value, + ) -> Result<()> { + Err(MarshalError::BadType) + } + + fn make_list_placeholder(&self, _len: usize) -> Result> { + Ok(None) + } + + fn set_list_item(&self, _list: &Self::Value, _index: usize, _value: Self::Value) -> Result<()> { + Err(MarshalError::BadType) + } + + fn make_set_placeholder(&self) -> Option { + None + } + + fn insert_set_item(&self, _set: &Self::Value, _value: Self::Value) -> Result<()> { + Err(MarshalError::BadType) + } + + fn make_dict_placeholder(&self) -> Option { + None + } + + fn insert_dict_item( + &self, + _dict: &Self::Value, + _key: Self::Value, + _value: Self::Value, + ) -> Result<()> { + Err(MarshalError::BadType) + } + fn make_slice( &self, _start: Self::Value, @@ -581,6 +665,30 @@ pub trait MarshalBag: Copy { ) -> Option<::Constant> { None } + + /// Convert a runtime constant to the compiler-side shape stored in + /// ``CodeObject``. Runtime implementations may return a semantically + /// unused placeholder when the exact value is carried by + /// `make_code_with_constants` instead. + fn code_constant_from_value( + &self, + value: &Self::Value, + ) -> Result<::Constant> { + self.constant_ref_from_value(value) + .ok_or(MarshalError::BadType) + } + + fn bytes_from_value(&self, _value: &Self::Value) -> Option> { + None + } + + fn str_from_value(&self, _value: &Self::Value) -> Option { + None + } + + fn tuple_elements_from_value(&self, _value: &Self::Value) -> Option> { + None + } } impl MarshalBag for Bag { @@ -640,8 +748,8 @@ impl MarshalBag for Bag { fn make_code( &self, code: CodeObject<::Constant>, - ) -> Self::Value { - self.make_code(code) + ) -> Result { + Ok(self.make_code(code)) } fn make_stop_iter(&self) -> Result { @@ -682,6 +790,27 @@ impl MarshalBag for Bag { ) -> Option<::Constant> { Some(value.clone()) } + + fn bytes_from_value(&self, value: &Self::Value) -> Option> { + match value.borrow_constant() { + BorrowedConstant::Bytes { value } => Some(value.to_vec()), + _ => None, + } + } + + fn str_from_value(&self, value: &Self::Value) -> Option { + match value.borrow_constant() { + BorrowedConstant::Str { value } => Some(value.to_string_lossy().into_owned()), + _ => None, + } + } + + fn tuple_elements_from_value(&self, value: &Self::Value) -> Option> { + match value.borrow_constant() { + BorrowedConstant::Tuple { elements } => Some(elements.to_vec()), + _ => None, + } + } } pub const MAX_MARSHAL_STACK_DEPTH: usize = 2000; @@ -724,10 +853,7 @@ fn deserialize_value_after_header( // TYPE_REF: return previously stored object if type_code == Type::Ref as u8 { let idx = rdr.read_u32()? as usize; - return refs - .get(idx) - .and_then(|v| v.clone()) - .ok_or(MarshalError::InvalidBytecode); + return resolve_ref(idx, refs); } // Reserve ref slot before reading (matches write order) @@ -740,22 +866,10 @@ fn deserialize_value_after_header( }; let typ = Type::try_from(type_code)?; - // CPython's r_object() uses one global ref table: TYPE_CODE reserves its - // slot before reading code fields, and those fields may use later TYPE_REF - // indexes. Keep the same indexes even when Bag::Value and Constant differ. let value = if matches!(typ, Type::Code) { - let mut inner_refs: Vec::Constant>> = refs - .iter() - .map(|value| { - value - .as_ref() - .and_then(|value| bag.constant_ref_from_value(value)) - }) - .collect(); - let code = deserialize_code_inner(rdr, bag.constant_bag(), depth - 1, &mut inner_refs)?; - bag.make_code(code) + deserialize_code_value_inner(rdr, bag, depth - 1, refs)? } else { - deserialize_value_typed(rdr, bag, depth, refs, typ)? + deserialize_value_typed(rdr, bag, depth, refs, typ, slot)? }; if let Some(idx) = slot { @@ -764,12 +878,144 @@ fn deserialize_value_after_header( Ok(value) } +/// Decode a code object through the runtime bag. CPython's marshal reader +/// keeps one reference table for the code fields and `co_consts`; using +/// `Bag::Value` here preserves that index space and lets runtime-only +/// constants survive alongside the compiler representation. +fn deserialize_code_value_inner( + rdr: &mut R, + bag: Bag, + depth: usize, + refs: &mut Vec>, +) -> Result { + if depth == 0 { + return Err(MarshalError::InvalidBytecode); + } + let arg_count = rdr.read_u32()?; + let posonlyarg_count = rdr.read_u32()?; + let kwonlyarg_count = rdr.read_u32()?; + let max_stackdepth = rdr.read_u32()?; + let flags = CodeFlags::from_bits_truncate(rdr.read_u32()?); + let child_depth = depth - 1; + + let code_value = deserialize_value_depth(rdr, bag, child_depth, refs)?; + let code_bytes = bag + .bytes_from_value(&code_value) + .ok_or(MarshalError::BadType)?; + + let consts_value = deserialize_value_depth(rdr, bag, child_depth, refs)?; + let constant_values = bag + .tuple_elements_from_value(&consts_value) + .ok_or(MarshalError::BadType)?; + let constants = constant_values + .iter() + .map(|value| bag.code_constant_from_value(value)) + .collect::>>()? + .into_iter() + .collect(); + + let read_strings = + |rdr: &mut R, refs: &mut Vec>| -> Result> { + let tuple = deserialize_value_depth(rdr, bag, child_depth, refs)?; + bag.tuple_elements_from_value(&tuple) + .ok_or(MarshalError::BadType)? + .iter() + .map(|value| bag.str_from_value(value).ok_or(MarshalError::BadType)) + .collect() + }; + let names_raw = read_strings(rdr, refs)?; + let localsplusnames = read_strings(rdr, refs)?; + + let kinds_value = deserialize_value_depth(rdr, bag, child_depth, refs)?; + let localspluskinds = bag + .bytes_from_value(&kinds_value) + .ok_or(MarshalError::BadType)?; + + let read_string = + |rdr: &mut R, refs: &mut Vec>| -> Result { + let value = deserialize_value_depth(rdr, bag, child_depth, refs)?; + bag.str_from_value(&value).ok_or(MarshalError::BadType) + }; + let source_path_raw = read_string(rdr, refs)?; + let obj_name_raw = read_string(rdr, refs)?; + let qualname_raw = read_string(rdr, refs)?; + + let first_line_raw = rdr.read_u32()? as i32; + let first_line_number = if first_line_raw > 0 { + OneIndexed::new(first_line_raw as usize) + } else { + None + }; + let linetable_value = deserialize_value_depth(rdr, bag, child_depth, refs)?; + let linetable = bag + .bytes_from_value(&linetable_value) + .ok_or(MarshalError::BadType)? + .into_boxed_slice(); + let exceptiontable_value = deserialize_value_depth(rdr, bag, child_depth, refs)?; + let exceptiontable = bag + .bytes_from_value(&exceptiontable_value) + .ok_or(MarshalError::BadType)? + .into_boxed_slice(); + + let lp = split_localplus( + &localsplusnames + .iter() + .map(|s| s.as_str()) + .collect::>(), + &localspluskinds, + arg_count, + kwonlyarg_count, + flags, + )?; + let instructions = CodeUnits::try_from(code_bytes.as_slice())?; + let locations = linetable_to_locations(&linetable, first_line_raw, instructions.len()); + let constant_bag = bag.constant_bag(); + let code = CodeObject { + instructions, + locations, + flags, + posonlyarg_count, + arg_count, + kwonlyarg_count, + source_path: constant_bag.make_name(&source_path_raw), + first_line_number, + max_stackdepth, + obj_name: constant_bag.make_name(&obj_name_raw), + qualname: constant_bag.make_name(&qualname_raw), + constants, + names: names_raw + .iter() + .map(|name| constant_bag.make_name(name)) + .collect(), + varnames: lp + .varnames + .iter() + .map(|name| constant_bag.make_name(name)) + .collect(), + cellvars: lp + .cellvars + .iter() + .map(|name| constant_bag.make_name(name)) + .collect(), + freevars: lp + .freevars + .iter() + .map(|name| constant_bag.make_name(name)) + .collect(), + localspluskinds: localspluskinds.into_boxed_slice(), + linetable, + exceptiontable, + }; + bag.make_code_with_constants(code, constant_values) +} + fn deserialize_value_typed( rdr: &mut R, bag: Bag, depth: usize, refs: &mut Vec>, typ: Type, + slot: Option, ) -> Result { if depth == 0 { return Err(MarshalError::InvalidBytecode); @@ -806,71 +1052,141 @@ fn deserialize_value_typed( let value = Complex64 { re, im }; bag.make_complex(value) } - Type::Ascii | Type::AsciiInterned | Type::Unicode | Type::Interned => { - let len = rdr.read_u32()?; - let value = rdr.read_wtf8(len)?; + Type::Ascii | Type::Unicode => { + let len = rdr.read_len("string")?; + let value = rdr.read_wtf8(len as u32)?; bag.make_str(value) } - Type::ShortAscii | Type::ShortAsciiInterned => { + Type::AsciiInterned | Type::Interned => { + let len = rdr.read_len("string")?; + let value = rdr.read_wtf8(len as u32)?; + bag.make_interned_str(value) + } + Type::ShortAscii => { let len = rdr.read_u8()? as u32; let value = rdr.read_wtf8(len)?; bag.make_str(value) } + Type::ShortAsciiInterned => { + let len = rdr.read_u8()? as u32; + let value = rdr.read_wtf8(len)?; + bag.make_interned_str(value) + } Type::SmallTuple => { let len = rdr.read_u8()? as usize; let d = depth - 1; - let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); - itertools::process_results(it, |it| bag.make_tuple(it))? + if let Some(index) = slot + && let Some(tuple) = bag.make_tuple_placeholder(len)? + { + refs[index] = Some(tuple.clone()); + for item_index in 0..len { + let item = deserialize_value_depth(rdr, bag, d, refs)?; + bag.set_tuple_item(&tuple, item_index, item)?; + } + tuple + } else { + let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); + itertools::process_results(it, |it| bag.make_tuple(it))? + } } Type::Null => { - return Err(MarshalError::BadType); + return Err(MarshalError::NullObject); } Type::Ref => { // Handled in deserialize_value_depth before calling this function return Err(MarshalError::BadType); } Type::Tuple => { - let len = rdr.read_u32()?; + let len = rdr.read_len("tuple")?; let d = depth - 1; - let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); - itertools::process_results(it, |it| bag.make_tuple(it))? + if let Some(index) = slot + && let Some(tuple) = bag.make_tuple_placeholder(len)? + { + refs[index] = Some(tuple.clone()); + for item_index in 0..len { + let item = deserialize_value_depth(rdr, bag, d, refs)?; + bag.set_tuple_item(&tuple, item_index, item)?; + } + tuple + } else { + let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); + itertools::process_results(it, |it| bag.make_tuple(it))? + } } Type::List => { - let len = rdr.read_u32()?; + let len = rdr.read_len("list")?; let d = depth - 1; - let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); - itertools::process_results(it, |it| bag.make_list(it))?? + if let Some(index) = slot + && let Some(list) = bag.make_list_placeholder(len)? + { + refs[index] = Some(list.clone()); + for item_index in 0..len { + let item = deserialize_value_depth(rdr, bag, d, refs)?; + bag.set_list_item(&list, item_index, item)?; + } + list + } else { + let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); + itertools::process_results(it, |it| bag.make_list(it))?? + } } Type::Set => { - let len = rdr.read_u32()?; + let len = rdr.read_len("set")?; let d = depth - 1; - let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); - itertools::process_results(it, |it| bag.make_set(it))?? + if let Some(index) = slot + && let Some(set) = bag.make_set_placeholder() + { + refs[index] = Some(set.clone()); + for _ in 0..len { + let item = deserialize_value_depth(rdr, bag, d, refs)?; + bag.insert_set_item(&set, item)?; + } + set + } else { + let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); + itertools::process_results(it, |it| bag.make_set(it))?? + } } Type::FrozenSet => { - let len = rdr.read_u32()?; + let len = rdr.read_len("set")?; let d = depth - 1; let it = (0..len).map(|_| deserialize_value_depth(rdr, bag, d, refs)); itertools::process_results(it, |it| bag.make_frozenset(it))?? } Type::Dict => { let d = depth - 1; - let mut pairs = Vec::new(); - loop { - let raw = rdr.read_u8()?; - if raw & !FLAG_REF == b'0' { - break; + if let Some(index) = slot + && let Some(dict) = bag.make_dict_placeholder() + { + refs[index] = Some(dict.clone()); + loop { + let raw = rdr.read_u8()?; + if raw & !FLAG_REF == b'0' { + break; + } + let key = deserialize_value_after_header(rdr, bag, d, refs, raw)?; + let value = deserialize_value_depth(rdr, bag, d, refs)?; + bag.insert_dict_item(&dict, key, value)?; } - let k = deserialize_value_after_header(rdr, bag, d, refs, raw)?; - let v = deserialize_value_depth(rdr, bag, d, refs)?; - pairs.push((k, v)); + dict + } else { + let mut pairs = Vec::new(); + loop { + let raw = rdr.read_u8()?; + if raw & !FLAG_REF == b'0' { + break; + } + let key = deserialize_value_after_header(rdr, bag, d, refs, raw)?; + let value = deserialize_value_depth(rdr, bag, d, refs)?; + pairs.push((key, value)); + } + bag.make_dict(pairs.into_iter())? } - bag.make_dict(pairs.into_iter())? } Type::Bytes => { // After marshaling, byte arrays are converted into bytes. - let len = rdr.read_u32()?; - let value = rdr.read_slice(len)?; + let len = rdr.read_len("bytes object")?; + let value = rdr.read_slice(len as u32)?; bag.make_bytes(value) } Type::Code => return Err(MarshalError::BadType), @@ -1102,6 +1418,25 @@ pub fn serialize_value( /// Split varnames/cellvars/freevars are reassembled into /// co_localsplusnames/co_localspluskinds. pub fn serialize_code(buf: &mut W, code: &CodeObject) { + serialize_code_with(buf, code, |buf, constant| { + serialize_value(buf, constant.borrow_constant().into()).unwrap_or_else(|x| match x {}); + Ok::<(), core::convert::Infallible>(()) + }) + .unwrap_or_else(|x| match x {}) +} + +/// Serialize a code object, writing each `co_consts` entry through +/// `write_constant`. +/// +/// A runtime caller passes its own object writer so that values its constant +/// representation carries but `BorrowedConstant` cannot describe — lists, +/// dicts, sets — reach the stream, and so a constant shared with the enclosing +/// object keeps its entry in that writer's reference table. +pub fn serialize_code_with( + buf: &mut W, + code: &CodeObject, + mut write_constant: impl FnMut(&mut W, &C) -> core::result::Result<(), E>, +) -> core::result::Result<(), E> { // 1–5: scalar fields buf.write_u32(code.arg_count); buf.write_u32(code.posonlyarg_count); @@ -1118,7 +1453,7 @@ pub fn serialize_code(buf: &mut W, code: &CodeObject) buf.write_u8(Type::Tuple as u8); write_len(buf, code.constants.len()); for constant in &*code.constants { - serialize_value(buf, constant.borrow_constant().into()).unwrap_or_else(|x| match x {}) + write_constant(buf, constant)?; } // 8: co_names (tuple of strings) @@ -1161,6 +1496,7 @@ pub fn serialize_code(buf: &mut W, code: &CodeObject) // 16: co_exceptiontable buf.write_u8(Type::Bytes as u8); write_vec(buf, &code.exceptiontable); + Ok(()) } fn write_marshal_str(buf: &mut W, s: &str) { diff --git a/crates/derive-impl/src/pyclass.rs b/crates/derive-impl/src/pyclass.rs index 809d3164b4a..94bb445fec6 100644 --- a/crates/derive-impl/src/pyclass.rs +++ b/crates/derive-impl/src/pyclass.rs @@ -1162,13 +1162,16 @@ where let slot_ident = Ident::new(&slot_ident.to_string().to_lowercase(), slot_ident.span()); let slot_name = slot_ident.to_string(); let tokens = { - const NON_ATOMIC_SLOTS: &[&str] = &["as_buffer"]; const POINTER_SLOTS: &[&str] = &["as_sequence", "as_mapping"]; const STATIC_GEN_SLOTS: &[&str] = &["as_number"]; - if NON_ATOMIC_SLOTS.contains(&slot_name.as_str()) { + if slot_name == "as_buffer" { + // bf_releasebuffer is not a separate function in RustPython; the + // exporter's BufferMethods already release. Only its presence is + // observable, and AsBuffer declares that. quote_spanned! { span => - slots.#slot_ident = Some(Self::#ident as _); + slots.#slot_ident.store(Some(Self::#ident as _)); + slots.has_release_buffer.store(Self::RELEASE_BUFFER); } } else if POINTER_SLOTS.contains(&slot_name.as_str()) { quote_spanned! { span => diff --git a/crates/host_env/src/io.rs b/crates/host_env/src/io.rs index 6df29bcd6bc..f32ef2f6944 100644 --- a/crates/host_env/src/io.rs +++ b/crates/host_env/src/io.rs @@ -199,6 +199,29 @@ pub fn is_seekable(fd: crt_fd::Borrowed<'_>) -> bool { os::seek_fd(fd, 0, libc::SEEK_CUR).is_ok() } +/// Whether a read from `fd` answers from data the file already holds, rather +/// than waiting for whoever writes the other end. +/// +/// Seeking answers this everywhere but Windows, where a pipe seeks too -- +/// `lseek` on one succeeds and reports a position, so a reader that took +/// seekability for an answer would wait on a peer while holding whatever it +/// holds for the length of the call. +#[cfg(not(windows))] +pub fn reads_without_waiting(fd: crt_fd::Borrowed<'_>) -> bool { + is_seekable(fd) +} + +#[cfg(windows)] +pub fn reads_without_waiting(fd: crt_fd::Borrowed<'_>) -> bool { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{FILE_TYPE_DISK, GetFileType}; + + let Ok(handle) = crt_fd::as_handle(fd) else { + return false; + }; + unsafe { GetFileType(handle.as_raw_handle() as _) == FILE_TYPE_DISK } +} + pub fn validate_whence(whence: i32) -> bool { let standard = (0..=2).contains(&whence); #[cfg(any(target_os = "dragonfly", target_os = "freebsd", target_os = "linux"))] diff --git a/crates/host_env/src/io_unsupported.rs b/crates/host_env/src/io_unsupported.rs index e46f05af900..d9fdc5d0d23 100644 --- a/crates/host_env/src/io_unsupported.rs +++ b/crates/host_env/src/io_unsupported.rs @@ -176,6 +176,10 @@ pub fn is_seekable(_fd: crt_fd::Borrowed<'_>) -> bool { false } +pub fn reads_without_waiting(_fd: crt_fd::Borrowed<'_>) -> bool { + false +} + pub fn validate_whence(whence: i32) -> bool { (0..=2).contains(&whence) } diff --git a/crates/host_env/src/posix.rs b/crates/host_env/src/posix.rs index 50d3f52a674..1e8d4cabe1e 100644 --- a/crates/host_env/src/posix.rs +++ b/crates/host_env/src/posix.rs @@ -1,5 +1,4 @@ use alloc::ffi::CString; -#[cfg(all(unix, not(target_os = "redox")))] use alloc::vec::Vec; use core::ffi::CStr; #[cfg(all(unix, not(target_os = "redox")))] @@ -22,6 +21,12 @@ pub struct UnameInfo { pub machine: String, } +#[derive(Debug)] +pub struct UnameDecodeError { + pub bytes: Vec, + pub error: core::str::Utf8Error, +} + #[cfg(all(unix, not(target_os = "redox")))] #[derive(Clone, Copy, Debug)] pub struct StatVfsInfo { @@ -354,14 +359,23 @@ pub fn fchownat( .map_err(std::io::Error::from) } -pub fn uname_info() -> Result { +pub fn uname_info() -> Result { + fn decode(value: &CStr) -> Result { + core::str::from_utf8(value.to_bytes()) + .map(str::to_owned) + .map_err(|error| UnameDecodeError { + bytes: value.to_bytes().to_vec(), + error, + }) + } + let info = rustix::system::uname(); Ok(UnameInfo { - sysname: info.sysname().to_str()?.into(), - nodename: info.nodename().to_str()?.into(), - release: info.release().to_str()?.into(), - version: info.version().to_str()?.into(), - machine: info.machine().to_str()?.into(), + sysname: decode(info.sysname())?, + nodename: decode(info.nodename())?, + release: decode(info.release())?, + version: decode(info.version())?, + machine: decode(info.machine())?, }) } diff --git a/crates/jit/src/instructions.rs b/crates/jit/src/instructions.rs index 67cf07f6e7f..a3c4ca800c4 100644 --- a/crates/jit/src/instructions.rs +++ b/crates/jit/src/instructions.rs @@ -548,8 +548,22 @@ impl<'a, 'b> FunctionCompiler<'a, 'b> { match self.stack.pop().ok_or(JitCompileError::BadBytecode)? { JitValue::FuncRef(reference) => { let call = self.builder.ins().call(reference, &args); - let returns = self.builder.inst_results(call); - self.stack.push(JitValue::Int(returns[0])); + // The only callable reachable here is this function itself, + // so the result carries the declared return type - it is not + // always an Int. A function whose return type is still + // unknown has no return slot in the signature it was + // declared with, and there is nothing to type the result as. + let ret = match *self.builder.inst_results(call) { + [] => None, + [val] => Some(val), + _ => return Err(JitCompileError::NotSupported), + }; + let val = match (self.sig.ret.clone(), ret) { + (Some(JitType::None), None) => JitValue::None, + (Some(ty), Some(val)) => JitValue::from_type_and_value(ty, val), + _ => return Err(JitCompileError::NotSupported), + }; + self.stack.push(val); Ok(()) } diff --git a/crates/jit/tests/bool_tests.rs b/crates/jit/tests/bool_tests.rs index 8a5f4ea9db3..1874ee4d55d 100644 --- a/crates/jit/tests/bool_tests.rs +++ b/crates/jit/tests/bool_tests.rs @@ -202,4 +202,18 @@ mod tests { assert_eq!(lte(false, 1), Ok(1)); assert_eq!(lte(true, 0), Ok(0)); } + + #[test] + fn recursive_bool() { + let recursive_bool = jit_function! { recursive_bool(n: i64) -> bool => r##" + def recursive_bool(n: int) -> bool: + if n == 0: + return True + return not recursive_bool(n - 1) + "## }; + + assert_eq!(recursive_bool(0), Ok(true)); + assert_eq!(recursive_bool(1), Ok(false)); + assert_eq!(recursive_bool(4), Ok(true)); + } } diff --git a/crates/jit/tests/float_tests.rs b/crates/jit/tests/float_tests.rs index b9bbb3ea63c..f667b1e764a 100644 --- a/crates/jit/tests/float_tests.rs +++ b/crates/jit/tests/float_tests.rs @@ -379,4 +379,18 @@ mod tests { assert_eq!(float_lte(f64::NAN, f64::NAN), Ok(false)); assert_eq!(float_lte(f64::INFINITY, f64::NEG_INFINITY), Ok(false)); } + + #[test] + fn recursive_float() { + let recursive_float = jit_function! { recursive_float(n: i64) -> f64 => r##" + def recursive_float(n: int) -> float: + if n == 0: + return 1.0 + return recursive_float(n - 1) / 2.0 + "## }; + + assert_eq!(recursive_float(0), Ok(1.0)); + assert_eq!(recursive_float(1), Ok(0.5)); + assert_eq!(recursive_float(4), Ok(0.0625)); + } } diff --git a/crates/sre_engine/src/engine.rs b/crates/sre_engine/src/engine.rs index 690801e0d9d..c2a6ac81975 100644 --- a/crates/sre_engine/src/engine.rs +++ b/crates/sre_engine/src/engine.rs @@ -581,7 +581,11 @@ fn _match(req: &Request<'_, S>, state: &mut State, mut ctx: MatchCo ..ctx }; - for _ in group_start..group_end { + // Walk the group itself rather than counting to its + // width: `g_ctx` is already stepping over exactly + // the characters being compared, so its own cursor + // is the loop bound. + while g_ctx.cursor.position < group_end { #[allow(clippy::redundant_closure_call)] if ctx.at_end(req) || $f(ctx.peek_char::()) != $f(g_ctx.peek_char::()) diff --git a/crates/sre_engine/src/string.rs b/crates/sre_engine/src/string.rs index ca7303a2a7f..5cc1b04b9fc 100644 --- a/crates/sre_engine/src/string.rs +++ b/crates/sre_engine/src/string.rs @@ -1,5 +1,12 @@ use rustpython_wtf8::Wtf8; +/// A position in the subject, paired with the byte pointer it resolves to. +/// +/// `position` is a **character index**, never a byte offset. The engine does +/// arithmetic on it directly — it subtracts two positions to get a character +/// count, adds a repeat count to get a bound, and compares one against a +/// lookbehind width — so the unit is part of the [`StrDrive`] contract rather +/// than a detail each implementation may pick. #[derive(Debug, Clone, Copy)] pub struct StringCursor { pub(crate) ptr: *const u8, @@ -15,15 +22,43 @@ impl Default for StringCursor { } } +/// Random access over the subject being matched. +/// +/// An implementation chooses how a character is spelled in memory — one byte +/// for `&[u8]`, one code point for `&str` and `&Wtf8` — but **not** how +/// positions are counted. Every position this trait produces or consumes is a +/// character index: `count` is the subject's length in characters, and +/// `skip(n)` advances a cursor's `position` by exactly `n`. +/// +/// That is load-bearing, not incidental. The engine reads position arithmetic +/// as character arithmetic in several places — `_count` bounds a repeat with +/// `position + max_count` and reports the repeat's length as a difference of +/// positions, `ASSERT` tests `position < back` against a lookbehind width, and +/// `search_info` recovers a match start as `position - (len - 1)`. A drive +/// that stored byte offsets here would leave all of those type-correct and +/// silently wrong, and would index a lookbehind out of bounds. +/// +/// So a drive over a variable-width encoding pays for the mapping: `count` +/// and `create_cursor` have to resolve character indices, and cannot simply +/// hand back byte lengths and byte offsets. pub trait StrDrive: Copy { + /// The subject's length, in characters. fn count(&self) -> usize; + /// A cursor at character index `n`. fn create_cursor(&self, n: usize) -> StringCursor; + /// Move `cursor` to character index `n`, from wherever it is now. fn adjust_cursor(&self, cursor: &mut StringCursor, n: usize); + /// Consume one character, returning it; `position` grows by one. fn advance(cursor: &mut StringCursor) -> u32; + /// The character at `cursor`, without moving it. fn peek(cursor: &StringCursor) -> u32; + /// Skip `n` characters, so `position` grows by exactly `n`. fn skip(cursor: &mut StringCursor, n: usize); + /// Step back over one character, returning it; `position` shrinks by one. fn back_advance(cursor: &mut StringCursor) -> u32; + /// The character before `cursor`, without moving it. fn back_peek(cursor: &StringCursor) -> u32; + /// Step back `n` characters, so `position` shrinks by exactly `n`. fn back_skip(cursor: &mut StringCursor, n: usize); } diff --git a/crates/stdlib/src/_asyncio.rs b/crates/stdlib/src/_asyncio.rs index 3146e39b77d..c3f28590e6a 100644 --- a/crates/stdlib/src/_asyncio.rs +++ b/crates/stdlib/src/_asyncio.rs @@ -12,8 +12,8 @@ pub(crate) mod _asyncio { vm::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{ - PyBaseException, PyBaseExceptionRef, PyDict, PyDictRef, PyGenericAlias, PyList, - PyListRef, PyModule, PySet, PyTuple, PyType, PyTypeRef, + PyBaseException, PyBaseExceptionRef, PyDict, PyGenericAlias, PyList, PyListRef, + PyModule, PySet, PyTuple, PyType, PyTypeRef, }, extend_module, function::{FuncArgs, KwArgs, OptionalArg, OptionalOption, PySetterValue}, @@ -724,47 +724,56 @@ pub(crate) mod _asyncio { /// Add waiter to fut_awaited_by with single-object optimization fn awaited_by_add(&self, waiter: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let mut awaited_by = self.fut_awaited_by.write(); - if awaited_by.is_none() { - // First waiter - store directly - *awaited_by = Some(waiter); - return Ok(()); - } + // Storing a waiter in the set runs its __hash__ and __eq__, which can + // come back to this future, so the field is locked only while it is + // read or written. + let existing = { + let mut awaited_by = self.fut_awaited_by.write(); + match awaited_by.as_ref() { + // First waiter - store directly + None => { + *awaited_by = Some(waiter); + return Ok(()); + } + Some(existing) => existing.clone(), + } + }; if self.fut_awaited_by_is_set.load(Ordering::Relaxed) { // Already a Set - add to it - let set = awaited_by.as_ref().unwrap(); - vm.call_method(set, "add", (waiter,))?; - } else { - // Single object - convert to Set - let existing = awaited_by.take().unwrap(); - let new_set = PySet::default().into_ref(&vm.ctx); - new_set.add(existing, vm)?; - new_set.add(waiter, vm)?; - *awaited_by = Some(new_set.into()); - self.fut_awaited_by_is_set.store(true, Ordering::Relaxed); + return vm.call_method(&existing, "add", (waiter,)).map(drop); } + + // Single object - convert to Set + let new_set = PySet::default().into_ref(&vm.ctx); + new_set.add(existing, vm)?; + new_set.add(waiter, vm)?; + *self.fut_awaited_by.write() = Some(new_set.into()); + self.fut_awaited_by_is_set.store(true, Ordering::Relaxed); Ok(()) } /// Discard waiter from fut_awaited_by with single-object optimization fn awaited_by_discard(&self, waiter: &PyObject, vm: &VirtualMachine) -> PyResult<()> { - let mut awaited_by = self.fut_awaited_by.write(); - if awaited_by.is_none() { - return Ok(()); - } - - let obj = awaited_by.as_ref().unwrap(); - if !self.fut_awaited_by_is_set.load(Ordering::Relaxed) { - // Single object - check if it matches - if obj.is(waiter) { - *awaited_by = None; + // As in awaited_by_add, discarding from the set runs Python. + let set = { + let mut awaited_by = self.fut_awaited_by.write(); + let Some(obj) = awaited_by.as_ref() else { + return Ok(()); + }; + if !self.fut_awaited_by_is_set.load(Ordering::Relaxed) { + // Single object - check if it matches + if obj.is(waiter) { + *awaited_by = None; + } + return Ok(()); } - } else { - // It's a Set - use discard - vm.call_method(obj, "discard", (waiter.to_owned(),))?; - } - Ok(()) + obj.clone() + }; + + // It's a Set - use discard + vm.call_method(&set, "discard", (waiter.to_owned(),)) + .map(drop) } #[pymethod] @@ -779,7 +788,7 @@ pub(crate) mod _asyncio { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -1036,7 +1045,7 @@ pub(crate) mod _asyncio { ))); } - let exc = if exc_type.fast_isinstance(vm.ctx.types.type_type) { + let exc: PyBaseExceptionRef = if exc_type.fast_isinstance(vm.ctx.types.type_type) { // exc_type is a class let exc_class: PyTypeRef = exc_type.clone().downcast().unwrap(); // Must be a subclass of BaseException @@ -1047,12 +1056,23 @@ pub(crate) mod _asyncio { } let val = exc_val.unwrap_or_none(vm); - if vm.is_none(&val) { + let exc = if vm.is_none(&val) { exc_type.call((), vm)? } else if val.fast_isinstance(&exc_class) { val } else { exc_type.call((val,), vm)? + }; + match exc.downcast() { + Ok(exc) => exc, + Err(obj) => { + let exc_class_repr = exc_class.as_object().repr(vm)?; + vm.new_type_error(format!( + "calling {} should have returned an instance of BaseException, not {}", + exc_class_repr.as_wtf8(), + obj.class() + )) + } } } else if exc_type.fast_isinstance(vm.ctx.exceptions.base_exception_type) { // exc_type is an exception instance @@ -1063,7 +1083,7 @@ pub(crate) mod _asyncio { vm.new_type_error("instance exception may not have a separate value") ); } - exc_type + exc_type.downcast().unwrap() } else { // exc_type is neither a class nor an exception instance return Err(vm.new_type_error(format!( @@ -1075,10 +1095,11 @@ pub(crate) mod _asyncio { if let OptionalArg::Present(tb) = exc_tb && !vm.is_none(&tb) { - exc.set_attr(vm.ctx.intern_str("__traceback__"), tb, vm)?; + exc.as_object() + .set_attr(vm.ctx.intern_str("__traceback__"), tb, vm)?; } - Err(exc.downcast().unwrap()) + Err(exc) } #[pymethod] @@ -1840,7 +1861,7 @@ pub(crate) mod _asyncio { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -2405,7 +2426,9 @@ pub(crate) mod _asyncio { // Slow path: look up in the module-level dict for cross-thread queries let current_tasks = get_current_tasks_dict(vm)?; - let dict: PyDictRef = current_tasks.downcast().unwrap(); + let Ok(dict) = current_tasks.downcast::() else { + return Ok(vm.ctx.none()); + }; match dict.get_item(&*loop_obj, vm) { Ok(task) => Ok(task), @@ -2485,15 +2508,17 @@ pub(crate) mod _asyncio { #[pyfunction] fn _enter_task(loop_: PyObjectRef, task: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { // Per-thread check, matching CPython's ts->asyncio_running_task - { - let running_task = vm.asyncio_running_task.borrow(); - if running_task.is_some() { - return Err(vm.new_runtime_error(format!( - "Cannot enter into task {:?} while another task {:?} is being executed.", - task, - running_task.as_ref().unwrap() - ))); - } + let running_task = vm.asyncio_running_task.borrow().clone(); + if let Some(running_task) = running_task { + let task_repr = task.repr(vm)?; + let running_task_repr = running_task.repr(vm)?; + return Err(vm.new_runtime_error(wtf8_concat!( + "Cannot enter into task ", + task_repr.as_wtf8(), + " while another task ", + running_task_repr.as_wtf8(), + " is being executed." + ))); } *vm.asyncio_running_task.borrow_mut() = Some(task.clone()); @@ -2729,16 +2754,20 @@ pub(crate) mod _asyncio { } } + fn get_invalid_state_error_type(vm: &VirtualMachine) -> PyResult { + let module = vm.import("asyncio.exceptions", 0)?; + let exc_type = vm + .get_attribute_opt(module, vm.ctx.intern_str("InvalidStateError"))? + .ok_or_else(|| vm.new_attribute_error("InvalidStateError not found"))?; + exc_type + .downcast() + .map_err(|_| vm.new_type_error("InvalidStateError is not a type")) + } + fn new_invalid_state_error(vm: &VirtualMachine, msg: &str) -> PyBaseExceptionRef { - match vm.import("asyncio.exceptions", 0) { - Ok(module) => { - match vm.get_attribute_opt(module, vm.ctx.intern_str("InvalidStateError")) { - Ok(Some(exc_type)) => match exc_type.call((msg,), vm) { - Ok(exc) => exc.downcast().unwrap(), - Err(_) => vm.new_runtime_error(msg.to_string()), - }, - _ => vm.new_runtime_error(msg.to_string()), - } + match get_invalid_state_error_type(vm) { + Ok(invalid_state_error) => { + vm.new_exception_msg(invalid_state_error, msg.to_string().into()) } Err(_) => vm.new_runtime_error(msg.to_string()), } diff --git a/crates/stdlib/src/_queue.rs b/crates/stdlib/src/_queue.rs index 6b150e4c68b..96e77a34a0b 100644 --- a/crates/stdlib/src/_queue.rs +++ b/crates/stdlib/src/_queue.rs @@ -74,9 +74,20 @@ mod _queue { } } - fn release(&self) { + /// Take `mutex`, detaching first so that blocking on it cannot stall a + /// stop-the-world request. + /// + /// A waiter holds this mutex across its `allow_threads` wait, so it can + /// still hold it when it is stopped. An attached thread blocking on it + /// would then never reach a safepoint, the stop would never complete, + /// and the holder would never be resumed to release it. + fn lock_count(&self, vm: &VirtualMachine) -> parking_lot::MutexGuard<'_, usize> { + vm.allow_threads(|| self.mutex.lock()) + } + + fn release(&self, vm: &VirtualMachine) { { - let mut count = self.mutex.lock(); + let mut count = self.lock_count(vm); *count += 1; } // lock dropped. now we can notify a waiting thread @@ -95,7 +106,7 @@ mod _queue { // Guard must be dropped before check_signals() below, since a // signal handler may call back into this same queue. { - let mut count = self.mutex.lock(); + let mut count = self.lock_count(vm); if *count > 0 { *count -= 1; @@ -151,11 +162,15 @@ mod _queue { } impl PySimpleQueue { - fn push(&self, item: PyObjectRef) { + #[cfg_attr( + not(feature = "threading"), + expect(unused_variables, reason = "only the semaphore needs the vm") + )] + fn push(&self, item: PyObjectRef, vm: &VirtualMachine) { self.buf.lock().push_back(item); #[cfg(feature = "threading")] - self.sem.release(); + self.sem.release(vm); } /// Returns a strong reference from the head of the buffer. @@ -221,14 +236,14 @@ mod _queue { } #[pymethod] - fn put(&self, args: PutArgs) { + fn put(&self, args: PutArgs, vm: &VirtualMachine) { let PutArgs { item, .. } = args; - self.push(item); + self.push(item, vm); } #[pymethod] - fn put_nowait(&self, item: PyObjectRef) { - self.push(item); + fn put_nowait(&self, item: PyObjectRef, vm: &VirtualMachine) { + self.push(item, vm); } #[pymethod] @@ -282,7 +297,7 @@ mod _queue { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/stdlib/src/_sqlite3.rs b/crates/stdlib/src/_sqlite3.rs index 5348cc1f5ec..02d40845058 100644 --- a/crates/stdlib/src/_sqlite3.rs +++ b/crates/stdlib/src/_sqlite3.rs @@ -31,11 +31,11 @@ mod _sqlite3 { sqlite3_column_double, sqlite3_column_int64, sqlite3_column_name, sqlite3_column_text, sqlite3_column_type, sqlite3_complete, sqlite3_context, sqlite3_context_db_handle, sqlite3_create_collation_v2, sqlite3_create_function_v2, sqlite3_create_window_function, - sqlite3_data_count, sqlite3_db_handle, sqlite3_errcode, sqlite3_errmsg, sqlite3_exec, - sqlite3_expanded_sql, sqlite3_extended_errcode, sqlite3_finalize, sqlite3_get_autocommit, - sqlite3_interrupt, sqlite3_last_insert_rowid, sqlite3_libversion, sqlite3_limit, - sqlite3_open_v2, sqlite3_prepare_v2, sqlite3_progress_handler, sqlite3_reset, - sqlite3_result_blob, sqlite3_result_double, sqlite3_result_error, + sqlite3_data_count, sqlite3_db_config, sqlite3_db_handle, sqlite3_errcode, sqlite3_errmsg, + sqlite3_exec, sqlite3_expanded_sql, sqlite3_extended_errcode, sqlite3_finalize, + sqlite3_get_autocommit, sqlite3_interrupt, sqlite3_last_insert_rowid, sqlite3_libversion, + sqlite3_limit, sqlite3_open_v2, sqlite3_prepare_v2, sqlite3_progress_handler, + sqlite3_reset, sqlite3_result_blob, sqlite3_result_double, sqlite3_result_error, sqlite3_result_error_nomem, sqlite3_result_error_toobig, sqlite3_result_int64, sqlite3_result_null, sqlite3_result_text, sqlite3_set_authorizer, sqlite3_sleep, sqlite3_step, sqlite3_stmt, sqlite3_stmt_busy, sqlite3_stmt_readonly, sqlite3_threadsafe, @@ -161,7 +161,17 @@ mod _sqlite3 { SQLITE_ALTER_TABLE, SQLITE_ANALYZE, SQLITE_ATTACH, SQLITE_CREATE_INDEX, SQLITE_CREATE_TABLE, SQLITE_CREATE_TEMP_INDEX, SQLITE_CREATE_TEMP_TABLE, SQLITE_CREATE_TEMP_TRIGGER, SQLITE_CREATE_TEMP_VIEW, SQLITE_CREATE_TRIGGER, - SQLITE_CREATE_VIEW, SQLITE_CREATE_VTABLE, SQLITE_DELETE, SQLITE_DENY, SQLITE_DETACH, + SQLITE_CREATE_VIEW, SQLITE_CREATE_VTABLE, SQLITE_DBCONFIG_DEFENSIVE, + SQLITE_DBCONFIG_DQS_DDL, SQLITE_DBCONFIG_DQS_DML, SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE, + SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE, SQLITE_DBCONFIG_ENABLE_COMMENTS, + SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, + SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_DBCONFIG_ENABLE_QPSG, + SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_DBCONFIG_ENABLE_VIEW, + SQLITE_DBCONFIG_LEGACY_ALTER_TABLE, SQLITE_DBCONFIG_LEGACY_FILE_FORMAT, + SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, SQLITE_DBCONFIG_RESET_DATABASE, + SQLITE_DBCONFIG_REVERSE_SCANORDER, SQLITE_DBCONFIG_STMT_SCANSTATUS, + SQLITE_DBCONFIG_TRIGGER_EQP, SQLITE_DBCONFIG_TRUSTED_SCHEMA, + SQLITE_DBCONFIG_WRITABLE_SCHEMA, SQLITE_DELETE, SQLITE_DENY, SQLITE_DETACH, SQLITE_DROP_INDEX, SQLITE_DROP_TABLE, SQLITE_DROP_TEMP_INDEX, SQLITE_DROP_TEMP_TABLE, SQLITE_DROP_TEMP_TRIGGER, SQLITE_DROP_TEMP_VIEW, SQLITE_DROP_TRIGGER, SQLITE_DROP_VIEW, SQLITE_DROP_VTABLE, SQLITE_FUNCTION, SQLITE_IGNORE, SQLITE_INSERT, SQLITE_LIMIT_ATTACHED, @@ -322,7 +332,7 @@ mod _sqlite3 { ))) } } else { - Err(vm.new_type_error(format!( + Err(vm.new_value_error(format!( "autocommit must be True, False, or sqlite3.LEGACY_TRANSACTION_CONTROL, not {}", obj.class().name() ))) @@ -587,10 +597,10 @@ mod _sqlite3 { ) -> c_int { let (callable, vm) = unsafe { (*data.cast::()).retrieve() }; let f = || -> PyResult { - let arg1 = ptr_to_str(arg1, vm)?; - let arg2 = ptr_to_str(arg2, vm)?; - let db_name = ptr_to_str(db_name, vm)?; - let access = ptr_to_str(access, vm)?; + let arg1 = ptr_to_str_or_none(arg1, vm)?; + let arg2 = ptr_to_str_or_none(arg2, vm)?; + let db_name = ptr_to_str_or_none(db_name, vm)?; + let access = ptr_to_str_or_none(access, vm)?; let val = callable.call((action, arg1, arg2, db_name, access), vm)?; let Some(val) = val.downcast_ref::() else { @@ -1279,6 +1289,7 @@ mod _sqlite3 { SQLITE_UTF8 }; let db = self.db_lock(vm)?; + check_num_params(&db, args.narg, "narg", vm)?; let Some(data) = CallbackData::new(args.func, vm) else { return db.create_function( name.as_ptr(), @@ -1310,6 +1321,7 @@ mod _sqlite3 { fn create_aggregate(&self, args: CreateAggregateArgs, vm: &VirtualMachine) -> PyResult<()> { let name = args.name.to_cstring(vm)?; let db = self.db_lock(vm)?; + check_num_params(&db, args.narg, "n_arg", vm)?; let Some(data) = CallbackData::new(args.aggregate_class, vm) else { return db.create_function( name.as_ptr(), @@ -1392,6 +1404,7 @@ mod _sqlite3 { ) -> PyResult<()> { let name = name.to_cstring(vm)?; let db = self.db_lock(vm)?; + check_num_params(&db, narg, "num_params", vm)?; let Some(data) = CallbackData::new(aggregate_class, vm) else { unsafe { sqlite3_create_window_function( @@ -1516,6 +1529,39 @@ mod _sqlite3 { self.db_lock(vm)?.limit(category, limit, vm) } + #[pymethod] + fn setconfig( + &self, + op: c_int, + enable: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult<()> { + let db = self.db_lock(vm)?; + if !is_int_dbconfig(op) { + return Err(vm.new_value_error(format!("unknown config 'op': {op}"))); + } + let enable = enable.unwrap_or(true) as c_int; + let mut actual: c_int = 0; + let rc = unsafe { sqlite3_db_config(db.db, op, enable, &mut actual) }; + db.check(rc, vm)?; + if enable != actual { + return Err(new_operational_error(vm, "Unable to set config".to_owned())); + } + Ok(()) + } + + #[pymethod] + fn getconfig(&self, op: c_int, vm: &VirtualMachine) -> PyResult { + let db = self.db_lock(vm)?; + if !is_int_dbconfig(op) { + return Err(vm.new_value_error(format!("unknown config 'op': {op}"))); + } + let mut current: c_int = 0; + let rc = unsafe { sqlite3_db_config(db.db, op, -1, &mut current) }; + db.check(rc, vm)?; + Ok(current != 0) + } + #[pymethod] fn __enter__(zelf: PyRef) -> PyRef { zelf @@ -1642,6 +1688,47 @@ mod _sqlite3 { fn total_changes(&self, vm: &VirtualMachine) -> PyResult { self._db_lock(vm).map(|x| x.total_changes()) } + + #[pygetset(name = "Warning")] + fn exc_warning(&self) -> PyTypeRef { + warning_type().to_owned() + } + #[pygetset(name = "Error")] + fn exc_error(&self) -> PyTypeRef { + error_type().to_owned() + } + #[pygetset(name = "InterfaceError")] + fn exc_interface_error(&self) -> PyTypeRef { + interface_error_type().to_owned() + } + #[pygetset(name = "DatabaseError")] + fn exc_database_error(&self) -> PyTypeRef { + database_error_type().to_owned() + } + #[pygetset(name = "DataError")] + fn exc_data_error(&self) -> PyTypeRef { + data_error_type().to_owned() + } + #[pygetset(name = "OperationalError")] + fn exc_operational_error(&self) -> PyTypeRef { + operational_error_type().to_owned() + } + #[pygetset(name = "IntegrityError")] + fn exc_integrity_error(&self) -> PyTypeRef { + integrity_error_type().to_owned() + } + #[pygetset(name = "InternalError")] + fn exc_internal_error(&self) -> PyTypeRef { + internal_error_type().to_owned() + } + #[pygetset(name = "ProgrammingError")] + fn exc_programming_error(&self) -> PyTypeRef { + programming_error_type().to_owned() + } + #[pygetset(name = "NotSupportedError")] + fn exc_not_supported_error(&self) -> PyTypeRef { + not_supported_error_type().to_owned() + } } #[pyattr] @@ -2798,12 +2885,14 @@ mod _sqlite3 { } let sql_cstr = sql.to_cstring(vm)?; - let db = connection.db_lock(vm)?; - - db.sql_limit(sql.byte_len(), vm)?; + let raw = { + let db = connection.db_lock(vm)?; + db.sql_limit(sql.byte_len(), vm)?; + **db + }; let mut tail = null(); - let st = db.prepare(sql_cstr.as_ptr(), &mut tail, vm)?; + let st = raw.prepare(sql_cstr.as_ptr(), &mut tail, vm)?; let Some(st) = st else { return Ok(None); @@ -3475,12 +3564,66 @@ mod _sqlite3 { Ok(obj) } + fn check_num_params( + db: &Sqlite, + n: c_int, + param_name: &str, + vm: &VirtualMachine, + ) -> PyResult<()> { + let limit = unsafe { sqlite3_limit(db.db, SQLITE_LIMIT_FUNCTION_ARG, -1) }; + if n < -1 || n > limit { + return Err(new_programming_error( + vm, + format!("'{param_name}' must be between -1 and {limit}, not {n}"), + )); + } + Ok(()) + } + + fn is_int_dbconfig(op: c_int) -> bool { + use libsqlite3_sys::*; + matches!( + op, + SQLITE_DBCONFIG_ENABLE_FKEY + | SQLITE_DBCONFIG_ENABLE_TRIGGER + | SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER + | SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION + | SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE + | SQLITE_DBCONFIG_ENABLE_QPSG + | SQLITE_DBCONFIG_TRIGGER_EQP + | SQLITE_DBCONFIG_RESET_DATABASE + | SQLITE_DBCONFIG_DEFENSIVE + | SQLITE_DBCONFIG_WRITABLE_SCHEMA + | SQLITE_DBCONFIG_LEGACY_ALTER_TABLE + | SQLITE_DBCONFIG_DQS_DDL + | SQLITE_DBCONFIG_DQS_DML + | SQLITE_DBCONFIG_ENABLE_VIEW + | SQLITE_DBCONFIG_LEGACY_FILE_FORMAT + | SQLITE_DBCONFIG_TRUSTED_SCHEMA + | SQLITE_DBCONFIG_STMT_SCANSTATUS + | SQLITE_DBCONFIG_REVERSE_SCANORDER + | SQLITE_DBCONFIG_ENABLE_ATTACH_CREATE + | SQLITE_DBCONFIG_ENABLE_ATTACH_WRITE + | SQLITE_DBCONFIG_ENABLE_COMMENTS + ) + } + fn ptr_to_str<'a>(p: *const libc::c_char, vm: &VirtualMachine) -> PyResult<&'a str> { if p.is_null() { return Err(vm.new_memory_error("string pointer is null")); } unsafe { CStr::from_ptr(p).to_str() } - .map_err(|_| vm.new_value_error("Invalid UIF-8 codepoint")) + .map_err(|_| vm.new_value_error("Invalid UTF-8 codepoint")) + } + + fn ptr_to_str_or_none(p: *const libc::c_char, vm: &VirtualMachine) -> PyResult { + if p.is_null() { + return Ok(vm.ctx.none()); + } + let s = unsafe { CStr::from_ptr(p) } + .to_str() + .map_err(|_| vm.new_value_error("Invalid UTF-8 codepoint".to_owned()))?; + Ok(vm.ctx.new_str(s).into()) } fn ptr_to_string( diff --git a/crates/stdlib/src/array.rs b/crates/stdlib/src/array.rs index f2a16d72356..b1fa925e16d 100644 --- a/crates/stdlib/src/array.rs +++ b/crates/stdlib/src/array.rs @@ -27,8 +27,8 @@ pub mod array { ArgBytesLike, ArgIntoFloat, ArgIterable, KwArgs, OptionalArg, PyComparisonValue, }, protocol::{ - BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn, - PyMappingMethods, PySequenceMethods, + BufferDescriptor, BufferFlags, BufferMethods, BufferResizeGuard, PyBuffer, + PyIterReturn, PyMappingMethods, PySequenceMethods, }, sequence::{OptionalRangeArgs, SequenceExt, SequenceMutExt}, sliceable::{ @@ -55,6 +55,11 @@ pub mod array { $($n(Vec<$t>),)* } + /// One item, already converted to the array's element type. + enum ArrayItem { + $($n($t),)* + } + impl ArrayContentType { fn from_char(c: char) -> Result { match c { @@ -303,17 +308,31 @@ pub mod array { } } - fn setitem_by_index( + /// Convert an object to the element type of the array with + /// this typecode. This runs the object's conversion methods, + /// which can reach the array, so it takes the typecode by + /// value and holds no lock on it. + fn item_from_object( + typecode: char, + value: PyObjectRef, + vm: &VirtualMachine + ) -> PyResult { + match typecode { + $($c => Ok(ArrayItem::$n(<$t>::try_into_from_object(vm, value)?)),)* + _ => unreachable!("array has a typecode"), + } + } + + fn setitem_by_item( &mut self, i: isize, - value: PyObjectRef, + item: ArrayItem, vm: &VirtualMachine ) -> PyResult<()> { - match self { - $(ArrayContentType::$n(v) => { - let value = <$t>::try_into_from_object(vm, value)?; - v.setitem_by_index(vm, i, value) - })* + match (self, item) { + $((ArrayContentType::$n(v), ArrayItem::$n(value)) => + v.setitem_by_index(vm, i, value),)* + _ => unreachable!("item was converted for this array"), } } @@ -638,7 +657,7 @@ pub mod array { impl ToPyResult for WideChar { fn to_pyresult(self, vm: &VirtualMachine) -> PyResult { Ok(CodePoint::try_from(self) - .map_err(|e| vm.new_unicode_encode_error(e))? + .map_err(|e| vm.new_value_error(e))? .to_pyobject(vm)) } } @@ -732,12 +751,12 @@ pub mod array { } } else if init.downcastable::() || init.downcastable::() { init.try_bytes_like(vm, |x| array.frombytes(x))?; - } else if let Ok(iter) = ArgIterable::try_from_object(vm, init.clone()) { + } else { + // Everything else is taken item by item, buffer or not. + let iter = ArgIterable::try_from_object(vm, init)?; for obj in iter.iter(vm)? { array.push(obj?, vm)?; } - } else { - init.try_bytes_like(vm, |x| array.frombytes(x))?; } } @@ -906,6 +925,11 @@ pub mod array { #[pymethod] fn frombytes(&self, b: ArgBytesLike, vm: &VirtualMachine) -> PyResult<()> { + // The source is read as bytes, so items of any other width would + // be reinterpreted rather than appended. + if b.itemsize() != 1 { + return Err(vm.new_type_error("a bytes-like object is required")); + } let b = b.borrow_buf(); let itemsize = self.read().itemsize(); self._from_bytes(&b, itemsize, vm) @@ -1047,7 +1071,11 @@ pub mod array { vm: &VirtualMachine, ) -> PyResult<()> { match SequenceIndex::try_from_borrowed_object(vm, needle, "array")? { - SequenceIndex::Int(i) => zelf.write().setitem_by_index(i, value, vm), + SequenceIndex::Int(i) => { + let typecode = zelf.read().typecode(); + let item = ArrayContentType::item_from_object(typecode, value, vm)?; + zelf.write().setitem_by_item(i, item, vm) + } SequenceIndex::Slice(slice) => { let cloned; let guard; @@ -1234,7 +1262,7 @@ pub mod array { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -1291,20 +1319,42 @@ pub mod array { } } + impl PyArray { + fn buffer_desc(&self) -> BufferDescriptor { + let array = self.read(); + BufferDescriptor::format( + array.len() * array.itemsize(), + false, + array.itemsize(), + array.typecode_str().into(), + ) + } + } + impl AsBuffer for PyArray { + const RELEASE_BUFFER: bool = true; + + // array_buffer_getbuf, which reports the type code only when the request + // asked for a format. + fn slot_as_buffer( + zelf: &PyObject, + flags: BufferFlags, + vm: &VirtualMachine, + ) -> PyResult { + let zelf = zelf + .downcast_ref::() + .ok_or_else(|| vm.new_type_error("unexpected payload for as_buffer"))?; + let desc = zelf.buffer_desc().projected(flags); + flags.check_writable(desc.readonly, "Object is not writable.", vm)?; + Ok(PyBuffer::new(zelf.to_owned().into(), desc, &BUFFER_METHODS)) + } + fn as_buffer(zelf: &Py, _vm: &VirtualMachine) -> PyResult { - let array = zelf.read(); - let buf = PyBuffer::new( + Ok(PyBuffer::new( zelf.to_owned().into(), - BufferDescriptor::format( - array.len() * array.itemsize(), - false, - array.itemsize(), - array.typecode_str().into(), - ), + zelf.buffer_desc(), &BUFFER_METHODS, - ); - Ok(buf) + )) } } @@ -1386,7 +1436,9 @@ pub mod array { ass_item: atomic_func!(|seq, i, value, vm| { let zelf = PyArray::sequence_downcast(seq); if let Some(value) = value { - zelf.write().setitem_by_index(i, value, vm) + let typecode = zelf.read().typecode(); + let item = ArrayContentType::item_from_object(typecode, value, vm)?; + zelf.write().setitem_by_item(i, item, vm) } else { zelf.write().delitem_by_index(i, vm) } @@ -1421,8 +1473,15 @@ pub mod array { type Resizable<'a> = PyRwLockWriteGuard<'a, ArrayContentType>; fn try_resizable_opt(&self) -> Option> { - let w = self.write(); - (self.exports.load(atomic::Ordering::SeqCst) == 0).then_some(w) + // An export is a borrow someone else still holds, so it is + // answered before the lock rather than by waiting on it. + (self.exports.load(atomic::Ordering::SeqCst) == 0).then(|| self.write()) + } + + fn try_resizable(&self, vm: &VirtualMachine) -> PyResult> { + self.try_resizable_opt().ok_or_else(|| { + vm.new_buffer_error("cannot resize an array that is exporting buffers") + }) } } @@ -1696,8 +1755,17 @@ pub mod array { })?, MachineFormatCode::Utf16 { big_endian } => { let utf16: Vec<_> = chunks.map(|b| chunk_to_obj!(b, u16, big_endian)).collect(); - let s = String::from_utf16(&utf16) - .map_err(|_| vm.new_unicode_encode_error("items cannot decode as utf16"))?; + let s = String::from_utf16(&utf16).map_err(|_| { + let (index, reason) = invalid_utf16(&utf16).unwrap(); + vm.new_unicode_decode_error( + vm.ctx + .new_str(if big_endian { "utf-16-be" } else { "utf-16-le" }), + args.items.clone(), + index * 2, + index * 2 + 2, + vm.ctx.new_str(reason), + ) + })?; let bytes = PyArray::_unicode_to_wchar_bytes((*s).as_ref(), array.itemsize()); array.frombytes_move(bytes); } @@ -1713,6 +1781,25 @@ pub mod array { PyArray::from(array).into_ref_with_type(vm, cls) } + fn invalid_utf16(units: &[u16]) -> Option<(usize, &'static str)> { + let mut index = 0; + while index < units.len() { + let unit = units[index]; + if (0xd800..=0xdbff).contains(&unit) { + match units.get(index + 1) { + Some(next) if (0xdc00..=0xdfff).contains(next) => index += 2, + Some(_) => return Some((index, "illegal UTF-16 surrogate")), + None => return Some((index, "unexpected end of data")), + } + } else if (0xdc00..=0xdfff).contains(&unit) { + return Some((index, "illegal encoding")); + } else { + index += 1; + } + } + None + } + // Register array.array as collections.abc.MutableSequence pub(crate) fn module_exec( vm: &VirtualMachine, diff --git a/crates/stdlib/src/blake2.rs b/crates/stdlib/src/blake2.rs index 382aec826b1..83504435674 100644 --- a/crates/stdlib/src/blake2.rs +++ b/crates/stdlib/src/blake2.rs @@ -5,7 +5,7 @@ pub(crate) use _blake2::module_def; #[pymodule] mod _blake2 { use crate::hashlib::_hashlib::{BlakeHashArgs, local_blake2b, local_blake2s}; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; + use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; #[pyattr(name = "_GIL_MINSIZE")] const GIL_MINSIZE: u16 = 2048; @@ -43,4 +43,11 @@ mod _blake2 { fn blake2s(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult { Ok(local_blake2s(args, vm)?.into_pyobject(vm)) } + + #[expect(clippy::unnecessary_wraps, reason = "Needs to comply with a signature")] + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + let _ = vm.import("_hashlib", 0); + __module_exec(vm, module); + Ok(()) + } } diff --git a/crates/stdlib/src/contextvars.rs b/crates/stdlib/src/contextvars.rs index 0a6e0f12314..e3823f6ac59 100644 --- a/crates/stdlib/src/contextvars.rs +++ b/crates/stdlib/src/contextvars.rs @@ -15,16 +15,16 @@ mod _contextvars { AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, builtins::{PyGenericAlias, PyList, PyStrRef, PyType, PyTypeRef}, class::StaticType, - common::{hash::PyHash, lock::LazyLock, wtf8::Wtf8Buf}, + common::{ + hash::PyHash, + lock::{LazyLock, PyMutex}, + wtf8::Wtf8Buf, + }, function::{ArgCallable, FuncArgs, OptionalArg}, protocol::{PyMappingMethods, PySequenceMethods}, types::{AsMapping, AsSequence, Constructor, Hashable, Iterable, Representable}, }; - use core::{ - cell::{Cell, RefCell, UnsafeCell}, - sync::atomic::Ordering, - }; - use crossbeam_utils::atomic::AtomicCell; + use core::sync::atomic::{AtomicBool, AtomicI64, AtomicUsize, Ordering}; use indexmap::IndexMap; // TODO: Real hamt implementation @@ -33,7 +33,7 @@ mod _contextvars { #[pyclass(no_attr, name = "Hamt", module = "contextvars")] #[derive(Debug, PyPayload)] pub(crate) struct HamtObject { - hamt: RefCell, + hamt: PyMutex, } #[pyclass] @@ -42,23 +42,19 @@ mod _contextvars { impl Default for HamtObject { fn default() -> Self { Self { - hamt: RefCell::new(Hamt::default()), + hamt: PyMutex::new(Hamt::default()), } } } - unsafe impl Sync for HamtObject {} - #[derive(Debug)] struct ContextInner { - idx: Cell, + idx: AtomicUsize, vars: PyRef, // PyObject *ctx_weakreflist; - entered: Cell, + entered: AtomicBool, } - unsafe impl Sync for ContextInner {} - #[pyattr] #[pyclass(name = "Context")] #[derive(Debug, PyPayload)] @@ -71,23 +67,30 @@ mod _contextvars { fn empty(vm: &VirtualMachine) -> Self { Self { inner: ContextInner { - idx: Cell::new(usize::MAX), + idx: AtomicUsize::new(usize::MAX), vars: HamtObject::default().into_ref(&vm.ctx), - entered: Cell::new(false), + entered: AtomicBool::new(false), }, } } - fn borrow_vars(&self) -> impl core::ops::Deref + '_ { - self.inner.vars.hamt.borrow() + fn borrow_vars(&self) -> impl core::ops::DerefMut + '_ { + self.inner.vars.hamt.lock() } fn borrow_vars_mut(&self) -> impl core::ops::DerefMut + '_ { - self.inner.vars.hamt.borrow_mut() + self.inner.vars.hamt.lock() } fn enter(zelf: &Py, vm: &VirtualMachine) -> PyResult<()> { - if zelf.inner.entered.get() { + // A context is entered by one thread at a time, so the check and the + // claim have to be a single step. + if zelf + .inner + .entered + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { return Err(vm.new_runtime_error(format!( "cannot enter context: {} is already entered", zelf.as_object().repr(vm)? @@ -95,16 +98,15 @@ mod _contextvars { } super::CONTEXTS.with_borrow_mut(|ctxs| { - zelf.inner.idx.set(ctxs.len()); + zelf.inner.idx.store(ctxs.len(), Ordering::Relaxed); ctxs.push(zelf.to_owned()); }); - zelf.inner.entered.set(true); Ok(()) } fn exit(zelf: &Py, vm: &VirtualMachine) -> PyResult<()> { - if !zelf.inner.entered.get() { + if !zelf.inner.entered.load(Ordering::Acquire) { return Err(vm.new_runtime_error(format!( "cannot exit context: {} is not entered", zelf.as_object().repr(vm)? @@ -120,7 +122,7 @@ mod _contextvars { ) }) })?; - zelf.inner.entered.set(false); + zelf.inner.entered.store(false, Ordering::Release); Ok(()) } @@ -131,8 +133,8 @@ mod _contextvars { ctx.clone() } else { let ctx = Self::empty(vm); - ctx.inner.idx.set(0); - ctx.inner.entered.set(true); + ctx.inner.idx.store(0, Ordering::Relaxed); + ctx.inner.entered.store(true, Ordering::Release); let ctx = ctx.into_ref(&vm.ctx); ctxs.push(ctx); ctxs[0].clone() @@ -170,13 +172,13 @@ mod _contextvars { fn copy(&self, vm: &VirtualMachine) -> Self { // Deep copy the vars - clone the underlying Hamt data, not just the PyRef let vars_copy = HamtObject { - hamt: RefCell::new(self.inner.vars.hamt.borrow().clone()), + hamt: PyMutex::new(self.inner.vars.hamt.lock().clone()), }; Self { inner: ContextInner { - idx: Cell::new(usize::MAX), + idx: AtomicUsize::new(usize::MAX), vars: vars_copy.into_ref(&vm.ctx), - entered: Cell::new(false), + entered: AtomicBool::new(false), }, } } @@ -186,11 +188,8 @@ mod _contextvars { var: PyRef, vm: &VirtualMachine, ) -> PyResult { - let vars = self.borrow_vars(); - let item = vars - .get(&*var) - .ok_or_else(|| vm.new_key_error(var.into()))?; - Ok(item.to_owned()) + let item = self.borrow_vars().get(&*var).map(|item| item.to_owned()); + item.ok_or_else(|| vm.new_key_error(var.into())) } fn __len__(&self) -> usize { @@ -290,11 +289,11 @@ mod _contextvars { name: String, default: Option, #[pytraverse(skip)] - cached: AtomicCell>, + cached: PyMutex>, #[pytraverse(skip)] - cached_id: core::sync::atomic::AtomicUsize, // cached_tsid in CPython + cached_id: AtomicUsize, // cached_tsid in CPython #[pytraverse(skip)] - hash: UnsafeCell, + hash: AtomicI64, } impl core::fmt::Debug for ContextVar { @@ -303,8 +302,6 @@ mod _contextvars { } } - unsafe impl Sync for ContextVar {} - impl PartialEq for ContextVar { fn eq(&self, other: &Self) -> bool { core::ptr::eq(self, other) @@ -320,12 +317,15 @@ mod _contextvars { impl ContextVar { fn delete(zelf: &Py, vm: &VirtualMachine) -> PyResult<()> { - zelf.cached.store(None); + let cached = zelf.cached.lock().take(); + drop(cached); let ctx = PyContext::current(vm); - let mut vars = ctx.borrow_vars_mut(); - if vars.swap_remove(zelf).is_none() { + let removed = ctx.borrow_vars_mut().swap_remove(zelf); + let existed = removed.is_some(); + drop(removed); + if !existed { // TODO: // PyErr_SetObject(PyExc_LookupError, (PyObject *)var); return Err(vm.new_lookup_error(zelf.as_object().repr(vm)?.as_wtf8().to_owned())); @@ -338,16 +338,17 @@ mod _contextvars { fn set_inner(zelf: &Py, value: PyObjectRef, vm: &VirtualMachine) { let ctx = PyContext::current(vm); - let mut vars = ctx.borrow_vars_mut(); - vars.insert(zelf.to_owned(), value.clone()); + let replaced = ctx.borrow_vars_mut().insert(zelf.to_owned(), value.clone()); + drop(replaced); zelf.cached_id.store(ctx.get_id(), Ordering::SeqCst); let cache = ContextVarCache { object: value, - idx: ctx.inner.idx.get(), + idx: ctx.inner.idx.load(Ordering::Relaxed), }; - zelf.cached.store(Some(cache)); + let replaced = zelf.cached.lock().replace(cache); + drop(replaced); } fn generate_hash(zelf: &Py, vm: &VirtualMachine) -> PyHash { @@ -370,28 +371,32 @@ mod _contextvars { default: OptionalArg, vm: &VirtualMachine, ) -> PyResult> { - let found = super::CONTEXTS.with_borrow(|ctxs| { - let ctx = ctxs.last()?; - let cached_ptr = zelf.cached.as_ptr(); - debug_assert!(!cached_ptr.is_null()); - if let Some(cached) = unsafe { &*cached_ptr } + // The replaced cache entry comes back out so that dropping it, which + // can run a __del__ that calls back in, happens with no lock held. + let (found, replaced) = super::CONTEXTS.with_borrow(|ctxs| { + let Some(ctx) = ctxs.last() else { + return (None, None); + }; + let mut cached = zelf.cached.lock(); + if let Some(cached) = &*cached && zelf.cached_id.load(Ordering::SeqCst) == ctx.get_id() && cached.idx + 1 == ctxs.len() { - return Some(cached.object.clone()); + return (Some(cached.object.clone()), None); } - let vars = ctx.borrow_vars(); - let obj = vars.get(zelf)?; + let Some(obj) = ctx.borrow_vars().get(zelf).map(|obj| obj.to_owned()) else { + return (None, None); + }; zelf.cached_id.store(ctx.get_id(), Ordering::SeqCst); - // TODO: ensure cached is not changed - let _removed = zelf.cached.swap(Some(ContextVarCache { + let replaced = cached.replace(ContextVarCache { object: obj.clone(), idx: ctxs.len() - 1, - })); + }); - Some(obj.clone()) + (Some(obj), replaced) }); + drop(replaced); let value = if let Some(value) = found { value @@ -425,7 +430,7 @@ mod _contextvars { #[pymethod] fn reset(zelf: &Py, token: PyRef, vm: &VirtualMachine) -> PyResult<()> { - if token.used.get() { + if token.used.load(Ordering::Acquire) { return Err(vm.new_runtime_error(format!( "{} has already been used once", token.as_object().repr(vm)? @@ -447,7 +452,7 @@ mod _contextvars { ))); } - token.used.set(true); + token.used.store(true, Ordering::Release); if let Some(old_value) = &token.old_value { Self::set_inner(zelf, old_value.clone(), vm); @@ -462,7 +467,7 @@ mod _contextvars { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -484,15 +489,13 @@ mod _contextvars { name: args.name.to_string(), default: args.default.into_option(), cached_id: 0.into(), - cached: AtomicCell::new(None), - hash: UnsafeCell::new(0), + cached: PyMutex::new(None), + hash: AtomicI64::new(0), }; let py_var = var.into_ref_with_type(vm, cls)?; - unsafe { - // SAFETY: py_var is not exposed to python memory model yet - *py_var.hash.get() = Self::generate_hash(&py_var, vm) - }; + let hash = Self::generate_hash(&py_var, vm); + py_var.hash.store(hash, Ordering::Relaxed); Ok(py_var.into()) } @@ -504,14 +507,14 @@ mod _contextvars { impl core::hash::Hash for ContextVar { #[inline] fn hash(&self, state: &mut H) { - unsafe { *self.hash.get() }.hash(state) + self.hash.load(Ordering::Relaxed).hash(state) } } impl Hashable for ContextVar { #[inline] fn hash(zelf: &Py, _vm: &VirtualMachine) -> PyResult { - Ok(unsafe { *zelf.hash.get() }) + Ok(zelf.hash.load(Ordering::Relaxed)) } } @@ -537,11 +540,9 @@ mod _contextvars { ctx: PyRef, // tok_ctx in CPython var: PyRef, // tok_var in CPython old_value: Option, // tok_oldval in CPython - used: Cell, + used: AtomicBool, } - unsafe impl Sync for ContextToken {} - #[pyclass(with(Constructor, Representable))] impl ContextToken { #[pygetset] @@ -562,7 +563,7 @@ mod _contextvars { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } @@ -598,7 +599,11 @@ mod _contextvars { impl Representable for ContextToken { #[inline] fn repr_wtf8(zelf: &Py, vm: &VirtualMachine) -> PyResult { - let used = if zelf.used.get() { " used" } else { "" }; + let used = if zelf.used.load(Ordering::Acquire) { + " used" + } else { + "" + }; let var = Representable::repr_wtf8(&zelf.var, vm)?; let ptr = zelf.as_object().get_id() as *const u8; let mut result = Wtf8Buf::from(format!(" Ok(self.update_py_dialect(o.clone())), - DialectItem::None => { - let g = GLOBAL_HASHMAP.lock(); - let res = g.get("excel").unwrap().clone(); - Ok(self.update_py_dialect(res)) - } + DialectItem::None => Ok(self.update_py_dialect(PyDialect { + delimiter: b',', + quotechar: Some(b'"'), + escapechar: None, + doublequote: true, + skipinitialspace: false, + lineterminator: "\r\n".to_owned(), + quoting: QuoteStyle::Minimal, + strict: false, + })), } } diff --git a/crates/stdlib/src/faulthandler.rs b/crates/stdlib/src/faulthandler.rs index 6edd023f1eb..3fbb8391bec 100644 --- a/crates/stdlib/src/faulthandler.rs +++ b/crates/stdlib/src/faulthandler.rs @@ -260,8 +260,8 @@ mod decl { use core::sync::atomic::Ordering; let current_tid = rustpython_vm::stdlib::_thread::get_ident(); { - vm.state.stop_the_world.stop_the_world(vm); - scopeguard::defer! { vm.state.stop_the_world.start_the_world(vm); } + vm.state.stop_the_world.stop_the_world(&vm.state); + scopeguard::defer! { vm.state.stop_the_world.start_the_world(&vm.state); } let registry = vm.state.thread_frames.lock(); #[expect( clippy::iter_over_hash_type, diff --git a/crates/stdlib/src/hashlib.rs b/crates/stdlib/src/hashlib.rs index c2153b08a59..d7f94cc2796 100644 --- a/crates/stdlib/src/hashlib.rs +++ b/crates/stdlib/src/hashlib.rs @@ -847,15 +847,15 @@ pub(crate) mod _hashlib { if len < 1 { return Err(vm.new_value_error("key length must be greater than 0.")); } - usize::try_from(len) - .map_err(|_| vm.new_overflow_error("key length is too great."))? + i32::try_from(len).map_err(|_| vm.new_overflow_error("key length is too great."))? + as usize } None => hash_digest_size(&name).ok_or_else(|| unsupported_hash(&name, vm))?, }; let password_buf = args.password.borrow_buf(); let salt_buf = args.salt.borrow_buf(); - let mut dk = vec![0u8; dklen]; + let mut dk = vm.new_zeroed_bytes(dklen)?; macro_rules! do_pbkdf2 { ($hash_ty:ty) => {{ diff --git a/crates/stdlib/src/lzma.rs b/crates/stdlib/src/lzma.rs index 0b699baddbb..6e8a913abaa 100644 --- a/crates/stdlib/src/lzma.rs +++ b/crates/stdlib/src/lzma.rs @@ -337,40 +337,43 @@ mod _lzma { } fn parse_filter_chain_spec( - filter_specs: Vec, + filter_specs: PyObjectRef, vm: &VirtualMachine, ) -> PyResult { const LZMA_FILTERS_MAX: usize = 4; - if filter_specs.len() > LZMA_FILTERS_MAX { + let filter_specs_len = filter_specs.length(vm)?; + if filter_specs_len > LZMA_FILTERS_MAX { return Err(new_lzma_error( format!("Too many filters - liblzma supports a maximum of {LZMA_FILTERS_MAX}"), vm, )); } + let filter_specs = filter_specs.try_sequence(vm)?; let mut filters = Filters::new(); - for spec in &filter_specs { - let filter_id = get_dict_opt_u64(spec, "id", vm)? + for i in 0..filter_specs_len { + let spec = filter_specs.get_item(i as isize, vm)?; + let filter_id = get_dict_opt_u64(&spec, "id", vm)? .ok_or_else(|| vm.new_value_error("Filter specifier must have an \"id\" entry"))?; match filter_id { FILTER_LZMA1 => { - let opts = parse_filter_spec_lzma(spec, vm)?; + let opts = parse_filter_spec_lzma(&spec, vm)?; filters.lzma1(&opts); } FILTER_LZMA2 => { - let opts = parse_filter_spec_lzma(spec, vm)?; + let opts = parse_filter_spec_lzma(&spec, vm)?; filters.lzma2(&opts); } FILTER_DELTA => { - let dist = parse_filter_spec_delta(spec, vm)?; + let dist = parse_filter_spec_delta(&spec, vm)?; filters .delta_properties(&[(dist - 1) as u8]) .map_err(|e| catch_lzma_error(e, vm))?; } FILTER_X86 | FILTER_POWERPC | FILTER_IA64 | FILTER_ARM | FILTER_ARMTHUMB | FILTER_SPARC => { - let start_offset = parse_filter_spec_bcj(spec, vm)?; + let start_offset = parse_filter_spec_bcj(&spec, vm)?; add_bcj_filter(&mut filters, filter_id, start_offset) .map_err(|e| catch_lzma_error(e, vm))?; } @@ -570,7 +573,7 @@ mod _lzma { #[pyarg(any, optional)] memlimit: Option, #[pyarg(any, optional)] - filters: Option>, + filters: Option, } impl Constructor for LZMADecompressor { @@ -735,7 +738,7 @@ mod _lzma { fn init_xz( check: i32, preset: u32, - filters: Option>, + filters: Option, vm: &VirtualMachine, ) -> PyResult { let real_check = @@ -751,10 +754,11 @@ mod _lzma { fn init_alone( preset: u32, - filter_specs: Option>, + filter_specs: Option, vm: &VirtualMachine, ) -> PyResult { - if let Some(_filter_specs) = filter_specs { + if let Some(filter_specs) = filter_specs { + filter_specs.length(vm)?; // TODO: validate single LZMA1 filter and use its options let options = LzmaOptions::new_preset(preset).map_err(|_| { new_lzma_error(format!("Invalid compression preset: {preset}"), vm) @@ -768,10 +772,7 @@ mod _lzma { } } - fn init_raw( - filter_specs: Option>, - vm: &VirtualMachine, - ) -> PyResult { + fn init_raw(filter_specs: Option, vm: &VirtualMachine) -> PyResult { let filter_specs = filter_specs .ok_or_else(|| vm.new_value_error("Must specify filters for FORMAT_RAW"))?; let filters = parse_filter_chain_spec(filter_specs, vm)?; @@ -788,7 +789,7 @@ mod _lzma { #[pyarg(any, optional)] preset: Option, #[pyarg(any, optional)] - filters: Option>, + filters: Option, } impl Constructor for LZMACompressor { diff --git a/crates/stdlib/src/math.rs b/crates/stdlib/src/math.rs index 92c2a66e93e..3fe1ffd3e63 100644 --- a/crates/stdlib/src/math.rs +++ b/crates/stdlib/src/math.rs @@ -727,25 +727,20 @@ mod math { } // Generic Python path - let (p_i, q_i) = (p_i.unwrap(), q_i.unwrap()); - - // Collect current + remaining elements - let p_remaining: Result, _> = - core::iter::once(Ok(p_i)).chain(p_iter).collect(); - let q_remaining: Result, _> = - core::iter::once(Ok(q_i)).chain(q_iter).collect(); - let (p_vec, q_vec) = (p_remaining?, q_remaining?); - - if p_vec.len() != q_vec.len() { - return Err(vm.new_value_error("Inputs are not the same length")); - } - + let (mut p_i, mut q_i) = (p_i.unwrap(), q_i.unwrap()); let mut total = obj_total.unwrap_or_else(|| vm.ctx.new_int(0).into()); - for (p_item, q_item) in p_vec.into_iter().zip(q_vec) { - let prod = vm._mul(&p_item, &q_item)?; + loop { + let prod = vm._mul(&p_i, &q_i)?; total = vm._add(&total, &prod)?; + + let next_p = p_iter.next().transpose()?; + let next_q = q_iter.next().transpose()?; + match (next_p, next_q) { + (Some(next_p), Some(next_q)) => (p_i, q_i) = (next_p, next_q), + (None, None) => return Ok(total), + _ => return Err(vm.new_value_error("Inputs are not the same length")), + } } - return Ok(total); } Ok(obj_total.unwrap_or_else(|| vm.ctx.new_int(0).into())) diff --git a/crates/stdlib/src/md5.rs b/crates/stdlib/src/md5.rs index 2ff6cd24ff7..0339bf8ace7 100644 --- a/crates/stdlib/src/md5.rs +++ b/crates/stdlib/src/md5.rs @@ -3,10 +3,17 @@ pub(crate) use _md5::module_def; #[pymodule] mod _md5 { use crate::hashlib::_hashlib::{HashArgs, local_md5}; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; + use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; #[pyfunction] fn md5(args: HashArgs, vm: &VirtualMachine) -> PyResult { Ok(local_md5(args, vm)?.into_pyobject(vm)) } + + #[expect(clippy::unnecessary_wraps, reason = "Needs to comply with a signature")] + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + let _ = vm.import("_hashlib", 0); + __module_exec(vm, module); + Ok(()) + } } diff --git a/crates/stdlib/src/mmap.rs b/crates/stdlib/src/mmap.rs index 312d35a4ed4..14957ad904e 100644 --- a/crates/stdlib/src/mmap.rs +++ b/crates/stdlib/src/mmap.rs @@ -611,6 +611,8 @@ mod mmap { }; impl AsBuffer for PyMmap { + const RELEASE_BUFFER: bool = true; + fn as_buffer(zelf: &Py, _vm: &VirtualMachine) -> PyResult { let readonly = matches!(zelf.access, AccessMode::Read); let buf = PyBuffer::new( @@ -777,7 +779,10 @@ mod mmap { let start = options .start .map_or_else(|| self.pos(), |start| start.saturated_at(size)); - let end = options.end.map_or(size, |end| end.saturated_at(size)); + let end = options + .end + .map_or(size, |end| end.saturated_at(size)) + .max(start); (start, end) } @@ -886,7 +891,7 @@ mod mmap { let dest = dest.try_to_primitive(vm).ok()?; let src = src.try_to_primitive(vm).ok()?; let cnt = cnt.try_to_primitive(vm).ok()?; - if size - dest < cnt || size - src < cnt { + if dest > size || src > size || size - dest < cnt || size - src < cnt { return None; } Some((dest, src, cnt)) @@ -1147,24 +1152,35 @@ mod mmap { } #[pymethod] - fn write(&self, bytes: ArgBytesLike, vm: &VirtualMachine) -> PyResult { - let pos = self.pos(); - let size = self.__len__(); - - let data = bytes.borrow_buf(); + fn write(zelf: &Py, bytes: ArgBytesLike, vm: &VirtualMachine) -> PyResult { + let self_ = &**zelf; + let pos = self_.pos(); + let size = self_.__len__(); + + // Writing locks the map, and reading a source that views this same + // map locks it too, so such a source is copied out first. + let copied; + let borrowed; + let data: &[u8] = if bytes.source_object().is(zelf.as_object()) { + copied = bytes.borrow_buf().to_vec(); + &copied + } else { + borrowed = bytes.borrow_buf(); + &borrowed + }; if pos > size || size - pos < data.len() { return Err(vm.new_value_error("data out of range")); } - let len = self.try_writable(vm, |mmap| { + let len = self_.try_writable(vm, |mmap| { (&mut mmap[pos..(pos + data.len())]) - .write(&data) + .write(data) .map_err(|err| err.to_pyexception(vm))?; Ok(data.len()) })??; - self.advance_pos(len); + self_.advance_pos(len); Ok(PyInt::from(len).into_ref(&vm.ctx)) } diff --git a/crates/stdlib/src/overlapped.rs b/crates/stdlib/src/overlapped.rs index 86ac24e3a0f..6cdd0014604 100644 --- a/crates/stdlib/src/overlapped.rs +++ b/crates/stdlib/src/overlapped.rs @@ -12,7 +12,7 @@ mod _overlapped { builtins::{PyBaseExceptionRef, PyBytesRef, PyModule, PyStrRef, PyTupleRef, PyType}, common::lock::PyMutex, convert::{ToPyException, ToPyObject}, - function::OptionalArg, + function::{ArgBytesLike, ArgMemoryBuffer, OptionalArg}, object::{Traverse, TraverseFn}, protocol::PyBuffer, types::{Constructor, Destructor}, @@ -428,12 +428,14 @@ mod _overlapped { fn ReadFileInto( zelf: &Py, handle: isize, - buf: PyBuffer, + // w*, as _overlapped.Overlapped.ReadFileInto takes + buf: ArgMemoryBuffer, vm: &VirtualMachine, ) -> PyResult { use host_winapi::{ ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS, }; + let buf: PyBuffer = buf.into(); let mut inner = zelf.inner.lock(); if !matches!(inner.data, OverlappedData::None) { @@ -530,13 +532,15 @@ mod _overlapped { fn WSARecvInto( zelf: &Py, handle: isize, - buf: PyBuffer, + // w*, as _overlapped.Overlapped.WSARecvInto takes + buf: ArgMemoryBuffer, flags: u32, vm: &VirtualMachine, ) -> PyResult { use host_winapi::{ ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS, }; + let buf: PyBuffer = buf.into(); let mut inner = zelf.inner.lock(); if !matches!(inner.data, OverlappedData::None) { @@ -583,10 +587,12 @@ mod _overlapped { fn WriteFile( zelf: &Py, handle: isize, - buf: PyBuffer, + // y*, as _overlapped.Overlapped.WriteFile takes + buf: ArgBytesLike, vm: &VirtualMachine, ) -> PyResult { use host_winapi::{ERROR_IO_PENDING, ERROR_SUCCESS}; + let buf: PyBuffer = buf.into(); let mut inner = zelf.inner.lock(); if !matches!(inner.data, OverlappedData::None) { @@ -629,11 +635,13 @@ mod _overlapped { fn WSASend( zelf: &Py, handle: isize, - buf: PyBuffer, + // y*, as _overlapped.Overlapped.WSASend takes + buf: ArgBytesLike, flags: u32, vm: &VirtualMachine, ) -> PyResult { use host_winapi::{ERROR_IO_PENDING, ERROR_SUCCESS}; + let buf: PyBuffer = buf.into(); let mut inner = zelf.inner.lock(); if !matches!(inner.data, OverlappedData::None) { @@ -870,12 +878,14 @@ mod _overlapped { fn WSASendTo( zelf: &Py, handle: isize, - buf: PyBuffer, + // y*, as _overlapped.Overlapped.WSASendTo takes + buf: ArgBytesLike, flags: u32, address: PyTupleRef, vm: &VirtualMachine, ) -> PyResult { use host_winapi::{ERROR_IO_PENDING, ERROR_SUCCESS}; + let buf: PyBuffer = buf.into(); let mut inner = zelf.inner.lock(); if !matches!(inner.data, OverlappedData::None) { @@ -1001,7 +1011,8 @@ mod _overlapped { fn WSARecvFromInto( zelf: &Py, handle: isize, - buf: PyBuffer, + // w*, as _overlapped.Overlapped.WSARecvFromInto takes + buf: ArgMemoryBuffer, size: u32, flags: OptionalArg, vm: &VirtualMachine, @@ -1009,6 +1020,7 @@ mod _overlapped { use host_winapi::{ ERROR_BROKEN_PIPE, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_SUCCESS, }; + let buf: PyBuffer = buf.into(); let mut inner = zelf.inner.lock(); if !matches!(inner.data, OverlappedData::None) { diff --git a/crates/stdlib/src/pystruct.rs b/crates/stdlib/src/pystruct.rs index c525942e35e..496b448e5e8 100644 --- a/crates/stdlib/src/pystruct.rs +++ b/crates/stdlib/src/pystruct.rs @@ -10,13 +10,14 @@ pub(crate) use _struct::module_def; #[pymodule] pub(crate) mod _struct { use crate::vm::{ - AsObject, Py, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, + AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, buffer::{FormatSpec, new_struct_error, struct_error_type}, builtins::{PyBytes, PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef}, - function::{ArgBytesLike, ArgMemoryBuffer, PosArgs}, + common::lock::{PyMappedRwLockReadGuard, PyRwLock, PyRwLockReadGuard}, + function::{ArgBytesLike, ArgMemoryBuffer, FuncArgs, PosArgs}, match_class, protocol::PyIterReturn, - types::{Constructor, IterNext, Iterable, Representable, SelfIter}, + types::{Constructor, Initializer, IterNext, Iterable, Representable, SelfIter}, }; use crossbeam_utils::atomic::AtomicCell; use rustpython_common::wtf8::{Wtf8Buf, wtf8_concat}; @@ -251,41 +252,76 @@ pub(crate) mod _struct { Ok(fmt.format_spec(vm)?.size) } + /// What a `Struct` is once a format has been read into it. Held apart + /// from the object because `__new__` hands out a `Struct` that `__init__` + /// has not filled in yet, and `__init__` may be called again on one that + /// already holds a format. + #[derive(Debug)] + struct StructSpec { + spec: FormatSpec, + format: PyStrRef, + } + #[pyattr] #[pyclass(name = "Struct", traverse)] #[derive(Debug, PyPayload)] struct PyStruct { #[pytraverse(skip)] - spec: FormatSpec, - format: PyStrRef, + inner: PyRwLock>, } impl Constructor for PyStruct { + type Args = FuncArgs; + + fn py_new(_cls: &Py, _args: Self::Args, _vm: &VirtualMachine) -> PyResult { + Ok(Self { + inner: PyRwLock::new(None), + }) + } + } + + impl Initializer for PyStruct { type Args = IntoStructFormatBytes; - fn py_new(_cls: &Py, fmt: Self::Args, vm: &VirtualMachine) -> PyResult { + fn init(zelf: PyRef, fmt: Self::Args, vm: &VirtualMachine) -> PyResult<()> { + // The format is read before anything is replaced, so a format that + // cannot be read leaves the object as it was. let spec = fmt.format_spec(vm)?; - let format = fmt.0; - Ok(Self { spec, format }) + *zelf.inner.write() = Some(StructSpec { + spec, + format: fmt.0, + }); + Ok(()) } } - #[pyclass(with(Constructor, Representable))] + #[pyclass(with(Constructor, Initializer, Representable), flags(BASETYPE))] impl PyStruct { + /// The format this was initialized with, or an error if `__init__` + /// never ran. + fn ready(&self, vm: &VirtualMachine) -> PyResult> { + PyRwLockReadGuard::try_map(self.inner.read(), Option::as_ref) + .map_err(|_| vm.new_runtime_error("Struct object is not initialized")) + } + #[pygetset] - fn format(&self) -> PyStrRef { - self.format.clone() + fn format(&self, vm: &VirtualMachine) -> PyResult { + Ok(self.ready(vm)?.format.clone()) } + /// The size an uninitialized `Struct` reports, which no format has + /// yet given a value. #[pygetset] - #[inline] - const fn size(&self) -> usize { - self.spec.size + fn size(&self) -> isize { + self.inner + .read() + .as_ref() + .map_or(-1, |inner| inner.spec.size as isize) } #[pymethod] fn pack(&self, args: PosArgs, vm: &VirtualMachine) -> PyResult> { - self.spec.pack(args.into_vec(), vm) + self.ready(vm)?.spec.pack(args.into_vec(), vm) } #[pymethod] @@ -296,23 +332,28 @@ pub(crate) mod _struct { args: PosArgs, vm: &VirtualMachine, ) -> PyResult<()> { - let offset = get_buffer_offset(buffer.len(), offset, self.size(), true, vm)?; + let inner = self.ready(vm)?; + let offset = get_buffer_offset(buffer.len(), offset, inner.spec.size, true, vm)?; buffer.with_ref(|data| { - self.spec + inner + .spec .pack_into(&mut data[offset..], args.into_vec(), vm) }) } #[pymethod] fn unpack(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult { - data.with_ref(|buf| self.spec.unpack(buf, vm)) + let inner = self.ready(vm)?; + data.with_ref(|buf| inner.spec.unpack(buf, vm)) } #[pymethod] fn unpack_from(&self, args: UpdateFromArgs, vm: &VirtualMachine) -> PyResult { - let offset = get_buffer_offset(args.buffer.len(), args.offset, self.size(), false, vm)?; + let inner = self.ready(vm)?; + let size = inner.spec.size; + let offset = get_buffer_offset(args.buffer.len(), args.offset, size, false, vm)?; args.buffer - .with_ref(|buf| self.spec.unpack(&buf[offset..][..self.size()], vm)) + .with_ref(|buf| inner.spec.unpack(&buf[offset..][..size], vm)) } #[pymethod] @@ -321,14 +362,19 @@ pub(crate) mod _struct { buffer: ArgBytesLike, vm: &VirtualMachine, ) -> PyResult { - UnpackIterator::with_buffer(vm, self.spec.clone(), buffer) + let spec = self.ready(vm)?.spec.clone(); + UnpackIterator::with_buffer(vm, spec, buffer) } } impl Representable for PyStruct { #[inline] - fn repr_wtf8(zelf: &Py, _vm: &VirtualMachine) -> PyResult { - Ok(wtf8_concat!("Struct('", zelf.format.as_wtf8(), "')")) + fn repr_wtf8(zelf: &Py, vm: &VirtualMachine) -> PyResult { + Ok(wtf8_concat!( + "Struct('", + zelf.ready(vm)?.format.as_wtf8(), + "')" + )) } } diff --git a/crates/stdlib/src/select.rs b/crates/stdlib/src/select.rs index c1f10f3ecc2..84ec92927e8 100644 --- a/crates/stdlib/src/select.rs +++ b/crates/stdlib/src/select.rs @@ -79,16 +79,26 @@ mod decl { } let deadline = timeout.map(|s| time::time(vm).unwrap() + s); + let max_fds: usize = cfg_select! { + windows => FD_SETSIZE as usize, + _ => FD_SETSIZE, + }; + let seq2set = |list: &PyObject| -> PyResult<(Vec, FdSet)> { - let v: Vec = list.try_to_value(vm)?; - - let too_many_fds = cfg_select! { - windows => v.len() > FD_SETSIZE as usize, - _ => v.len() > FD_SETSIZE, - }; - if too_many_fds { - return Err(vm.new_value_error("too many file descriptors in select()")); - } + // The limit is answered while the sequence is walked rather than + // from the length of the result. fileno() runs Python and can + // append to the very list being walked, and a walk that re-reads + // the list each step -- which is what `seq2set` does -- then never + // reaches a length to check. + let seen = core::cell::Cell::new(0usize); + let v: Vec = vm.extract_elements_with(list, |obj| { + let selectable = Selectable::try_from_object(vm, obj)?; + seen.set(seen.get() + 1); + if seen.get() > max_fds { + return Err(vm.new_value_error("too many file descriptors in select()")); + } + Ok(selectable) + })?; let mut fds = FdSet::new(); for fd in &v { @@ -304,7 +314,10 @@ mod decl { timeout: OptionalArg>, vm: &VirtualMachine, ) -> PyResult> { - let mut fds = self.fds.lock(); + // Poll a copy: the wait releases the GIL-equivalent and runs + // signal handlers, which can register or unregister on the same + // object, and a held lock would deadlock them. + let mut fds = self.fds.lock().clone(); let TimeoutArg(timeout) = timeout.unwrap_or_default(); let timeout_ms = match timeout { Some(d) => i32::try_from(d.as_millis()) diff --git a/crates/stdlib/src/sha1.rs b/crates/stdlib/src/sha1.rs index 3e3d4928c79..71495435e56 100644 --- a/crates/stdlib/src/sha1.rs +++ b/crates/stdlib/src/sha1.rs @@ -3,10 +3,17 @@ pub(crate) use _sha1::module_def; #[pymodule] mod _sha1 { use crate::hashlib::_hashlib::{HashArgs, local_sha1}; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; + use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; #[pyfunction] fn sha1(args: HashArgs, vm: &VirtualMachine) -> PyResult { Ok(local_sha1(args, vm)?.into_pyobject(vm)) } + + #[expect(clippy::unnecessary_wraps, reason = "Needs to comply with a signature")] + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + let _ = vm.import("_hashlib", 0); + __module_exec(vm, module); + Ok(()) + } } diff --git a/crates/stdlib/src/sha3.rs b/crates/stdlib/src/sha3.rs index 0eb2dfa84d5..642ed838a4d 100644 --- a/crates/stdlib/src/sha3.rs +++ b/crates/stdlib/src/sha3.rs @@ -6,7 +6,7 @@ mod _sha3 { HashArgs, local_sha3_224, local_sha3_256, local_sha3_384, local_sha3_512, local_shake_128, local_shake_256, }; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; + use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; #[pyfunction] fn sha3_224(args: HashArgs, vm: &VirtualMachine) -> PyResult { @@ -37,4 +37,11 @@ mod _sha3 { fn shake_256(args: HashArgs, vm: &VirtualMachine) -> PyResult { Ok(local_shake_256(args, vm)?.into_pyobject(vm)) } + + #[expect(clippy::unnecessary_wraps, reason = "Needs to comply with a signature")] + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + let _ = vm.import("_hashlib", 0); + __module_exec(vm, module); + Ok(()) + } } diff --git a/crates/stdlib/src/socket.rs b/crates/stdlib/src/socket.rs index f78bec69dc5..a1998ba7c3b 100644 --- a/crates/stdlib/src/socket.rs +++ b/crates/stdlib/src/socket.rs @@ -40,7 +40,6 @@ mod _socket { } use core::{ - mem::MaybeUninit, net::{Ipv4Addr, Ipv6Addr, SocketAddr}, time::Duration, }; @@ -1589,7 +1588,10 @@ mod _socket { vm: &VirtualMachine, ) -> Result, IoOrPyException> { let flags = flags.unwrap_or(0); - let mut buffer = Vec::with_capacity(bufsize); + let mut buffer = Vec::new(); + buffer + .try_reserve_exact(bufsize) + .map_err(|_| vm.new_memory_error(""))?; let sock = self.sock()?; let n = self.sock_op(vm, SockWaitKind::Read, || { sock.recv_with_flags(buffer.spare_capacity_mut(), flags) @@ -1608,8 +1610,6 @@ mod _socket { ) -> Result { let flags = flags.unwrap_or(0); let sock = self.sock()?; - let mut buf = buf.borrow_buf_mut(); - let buf = &mut *buf; // Handle nbytes parameter let read_len = if let OptionalArg::Present(nbytes) = nbytes { @@ -1621,10 +1621,13 @@ mod _socket { buf.len() }; - let buf = &mut buf[..read_len]; - self.sock_op(vm, SockWaitKind::Read, || { - sock.recv_with_flags(unsafe { slice_as_uninit(buf) }, flags) - }) + let mut scratch = alloc_recv_scratch(read_len, vm)?; + let n = self.sock_op(vm, SockWaitKind::Read, || { + sock.recv_with_flags(&mut scratch.spare_capacity_mut()[..read_len], flags) + })?; + unsafe { scratch.set_len(n) }; + buf.borrow_buf_mut()[..n].copy_from_slice(&scratch); + Ok(n) } #[pymethod] @@ -1638,7 +1641,10 @@ mod _socket { let bufsize = bufsize .to_usize() .ok_or_else(|| vm.new_value_error("negative buffersize in recvfrom"))?; - let mut buffer = Vec::with_capacity(bufsize); + let mut buffer = Vec::new(); + buffer + .try_reserve_exact(bufsize) + .map_err(|_| vm.new_memory_error(""))?; let (n, addr) = self.sock_op(vm, SockWaitKind::Read, || { self.sock()? .recv_from_with_flags(buffer.spare_capacity_mut(), flags) @@ -1655,24 +1661,28 @@ mod _socket { flags: OptionalArg, vm: &VirtualMachine, ) -> Result<(usize, PyObjectRef), IoOrPyException> { - let mut buf = buf.borrow_buf_mut(); - let buf = &mut *buf; - let buf = match nbytes { + let read_len = match nbytes { OptionalArg::Present(i) => { let i = i.to_usize().ok_or_else(|| { vm.new_value_error("negative buffersize in recvfrom_into") })?; - buf.get_mut(..i).ok_or_else(|| { - vm.new_value_error("nbytes is greater than the length of the buffer") - })? + if i > buf.len() { + return Err(vm + .new_value_error("nbytes is greater than the length of the buffer") + .into()); + } + i } - OptionalArg::Missing => buf, + OptionalArg::Missing => buf.len(), }; let flags = flags.unwrap_or(0); let sock = self.sock()?; + let mut scratch = alloc_recv_scratch(read_len, vm)?; let (n, addr) = self.sock_op(vm, SockWaitKind::Read, || { - sock.recv_from_with_flags(unsafe { slice_as_uninit(buf) }, flags) + sock.recv_from_with_flags(&mut scratch.spare_capacity_mut()[..read_len], flags) })?; + unsafe { scratch.set_len(n) }; + buf.borrow_buf_mut()[..n].copy_from_slice(&scratch); Ok((n, get_addr_tuple(&addr, vm))) } @@ -1684,7 +1694,7 @@ mod _socket { vm: &VirtualMachine, ) -> Result { let flags = flags.unwrap_or(0); - let buf = bytes.borrow_buf(); + let buf = bytes.borrow_buf_unlocked(vm)?; let buf = &*buf; self.sock_op(vm, SockWaitKind::Write, || { self.sock()?.send_with_flags(buf, flags) @@ -1704,7 +1714,7 @@ mod _socket { let deadline = timeout.map(Deadline::new); - let buf = bytes.borrow_buf(); + let buf = bytes.borrow_buf_unlocked(vm)?; let buf = &*buf; let mut buf_offset = 0; // now we have like 3 layers of interrupt loop :) @@ -1741,7 +1751,7 @@ mod _socket { OptionalArg::Missing => (0, arg2), }; let addr = self.extract_address(address, "sendto", vm)?; - let buf = bytes.borrow_buf(); + let buf = bytes.borrow_buf_unlocked(vm)?; let buf = &*buf; self.sock_op(vm, SockWaitKind::Write, || { self.sock()?.send_to_with_flags(buf, &addr, flags) @@ -1771,8 +1781,8 @@ mod _socket { let buffers = buffers .iter() - .map(|buf| buf.borrow_buf()) - .collect::>(); + .map(|buf| buf.borrow_buf_unlocked(vm)) + .collect::>>()?; let buffers = buffers .iter() .map(|buf| io::IoSlice::new(buf)) @@ -2380,8 +2390,21 @@ mod _socket { Ok(s.to_string_lossy().into_owned()) } - unsafe fn slice_as_uninit(v: &mut [T]) -> &mut [MaybeUninit] { - unsafe { &mut *(v as *mut [T] as *mut [MaybeUninit]) } + /// Room to receive into that belongs to no Python object. + /// + /// A peer may never send, so the wait for it is unbounded. The export of + /// the caller's buffer is held for the whole call, which is what keeps it + /// from being resized, but the borrow that reaches its bytes is a lock + /// every other thread touching that object waits on, and a thread waiting + /// on a lock never reaches a safepoint — holding it across the wait stops + /// the world from being stopped at all. The bytes are copied over once + /// they have arrived. + fn alloc_recv_scratch(len: usize, vm: &VirtualMachine) -> PyResult> { + let mut scratch = Vec::new(); + scratch + .try_reserve_exact(len) + .map_err(|_| vm.new_memory_error(""))?; + Ok(scratch) } enum IoOrPyException { @@ -2601,7 +2624,7 @@ mod _socket { Some(ArgStrOrBytesLike::Buf(b)) => { let bytes = b.borrow_buf(); let host_str = core::str::from_utf8(&bytes).map_err(|e| { - vm.new_unicode_decode_error_real( + vm.new_unicode_decode_error( vm.ctx.new_str("utf-8"), vm.ctx.new_bytes(bytes.to_vec()), e.valid_up_to(), @@ -2643,7 +2666,7 @@ mod _socket { let bytes = b.borrow_buf(); core::str::from_utf8(&bytes) .map_err(|e| { - vm.new_unicode_decode_error_real( + vm.new_unicode_decode_error( vm.ctx.new_str("utf-8"), vm.ctx.new_bytes(bytes.to_vec()), e.valid_up_to(), diff --git a/crates/stdlib/src/ssl.rs b/crates/stdlib/src/ssl.rs index 04b905d544e..b942e27fc69 100644 --- a/crates/stdlib/src/ssl.rs +++ b/crates/stdlib/src/ssl.rs @@ -323,15 +323,17 @@ mod _ssl { #[pyattr] const ALERT_DESCRIPTION_NO_APPLICATION_PROTOCOL: i32 = 120; - // Version info - reporting as OpenSSL 3.3.0 for compatibility + // `ssl.py` still requires OpenSSL-shaped numeric compatibility fields even + // for non-OpenSSL TLS providers. Keep them in the supported 3.x ABI range, + // but report the actual rustls/AWS-LC backend in the human-readable string. #[pyattr] - const OPENSSL_VERSION_NUMBER: i32 = 0x30300000; // OpenSSL 3.3.0 (808452096) + const OPENSSL_VERSION_NUMBER: i32 = 0x30300000; #[pyattr] - const OPENSSL_VERSION: &str = "OpenSSL 3.3.0 (rustls/0.23)"; + const OPENSSL_VERSION: &str = "OpenSSL 3.3.0-compatible (AWS-LC/rustls 0.23)"; #[pyattr] - const OPENSSL_VERSION_INFO: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15); // 3.3.0 release + const OPENSSL_VERSION_INFO: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15); #[pyattr] - const _OPENSSL_API_VERSION: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15); // 3.3.0 release + const _OPENSSL_API_VERSION: (i32, i32, i32, i32, i32) = (3, 3, 0, 0, 15); // Default cipher list for rustls - using modern secure ciphers #[pyattr] @@ -1156,19 +1158,19 @@ mod _ssl { let pwd_result = callable.call((), vm)?; // Convert callable result to string - let password_from_callable = if let Ok(pwd_str) = - PyUtf8StrRef::try_from_object(vm, pwd_result.clone()) - { - pwd_str.as_str().to_owned() - } else if let Ok(pwd_bytes_like) = ArgBytesLike::try_from_object(vm, pwd_result) { - String::from_utf8(pwd_bytes_like.borrow_buf().to_vec()).map_err(|_| { - vm.new_type_error("password callback returned invalid UTF-8 bytes") - })? - } else { - return Err( - vm.new_type_error("password callback must return a string or bytes") - ); - }; + let password_from_callable = + if let Ok(pwd_str) = PyUtf8StrRef::try_from_object(vm, pwd_result.clone()) { + pwd_str.as_str().to_owned() + } else if pwd_result.check_buffer() { + let pwd_bytes_like = ArgBytesLike::try_from_object(vm, pwd_result)?; + String::from_utf8(pwd_bytes_like.borrow_buf().to_vec()).map_err(|_| { + vm.new_type_error("password callback returned invalid UTF-8 bytes") + })? + } else { + return Err( + vm.new_type_error("password callback must return a string or bytes") + ); + }; // Validate callable password length if password_from_callable.len() > PEM_BUFSIZE { @@ -1806,7 +1808,8 @@ mod _ssl { // Validate filepath is str or bytes let path_str = if let Ok(s) = PyUtf8StrRef::try_from_object(vm, filepath.clone()) { s.as_str().to_owned() - } else if let Ok(b) = ArgBytesLike::try_from_object(vm, filepath) { + } else if filepath.check_buffer() { + let b = ArgBytesLike::try_from_object(vm, filepath)?; String::from_utf8(b.borrow_buf().to_vec()) .map_err(|_| vm.new_value_error("Invalid path encoding"))? } else { @@ -1861,7 +1864,8 @@ mod _ssl { // Validate name is str or bytes let curve_name = if let Ok(s) = PyUtf8StrRef::try_from_object(vm, name.clone()) { s.as_str().to_owned() - } else if let Ok(b) = ArgBytesLike::try_from_object(vm, name) { + } else if name.check_buffer() { + let b = ArgBytesLike::try_from_object(vm, name)?; String::from_utf8(b.borrow_buf().to_vec()) .map_err(|_| vm.new_value_error("Invalid curve name encoding"))? } else { @@ -2104,8 +2108,8 @@ mod _ssl { Ok((Some(pwd_str.as_str().to_owned()), None)) } // Try bytes-like - else if let Ok(pwd_bytes_like) = ArgBytesLike::try_from_object(vm, p.clone()) - { + else if p.check_buffer() { + let pwd_bytes_like = ArgBytesLike::try_from_object(vm, p.clone())?; let pwd = String::from_utf8(pwd_bytes_like.borrow_buf().to_vec()) .map_err(|_| vm.new_type_error("password bytes must be valid UTF-8"))?; Ok((Some(pwd), None)) @@ -2816,8 +2820,8 @@ mod _ssl { super::compat::SslError::create_ssl_error_with_reason( vm, Some("SSL"), - "CALLBACK_FAILED", - "[SSL: CALLBACK_FAILED] callback failed", + "PARSE_TLSEXT", + "[SSL: PARSE_TLSEXT] SNI callback owner is no longer available", ) })?; let server_name_py: PyObjectRef = match sni_name { diff --git a/crates/stdlib/src/ssl/cert.rs b/crates/stdlib/src/ssl/cert.rs index f12f4307239..47d11f730b2 100644 --- a/crates/stdlib/src/ssl/cert.rs +++ b/crates/stdlib/src/ssl/cert.rs @@ -287,9 +287,11 @@ pub(super) fn is_ca_certificate(cert_der: &[u8]) -> bool { return ext.value.ca; } - // No Basic Constraints extension -> NOT a CA certificate - // (matches OpenSSL X509_check_ca() behavior) - false + // X509_check_ca() also retains OpenSSL's legacy trust-anchor rule: a + // self-issued X.509v1 certificate has no extensions at all, but is still + // classified as a CA. CPython's test CA at capath/4e1295a3.0 exercises + // precisely this case. + cert.version().0 == 0 && cert.subject() == cert.issuer() } /// Convert an X509Name to Python nested tuple format for SSL certificate dicts @@ -867,26 +869,36 @@ impl ServerCertVerifier for NoVerifier { fn verify_tls12_signature( &self, - _message: &[u8], - _cert: &CertificateDer<'_>, - _dss: &DigitallySignedStruct, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, ) -> Result { - // Accept all signatures without verification - Ok(HandshakeSignatureValid::assertion()) + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &CryptoExt::get_provider().signature_verification_algorithms, + ) } fn verify_tls13_signature( &self, - _message: &[u8], - _cert: &CertificateDer<'_>, - _dss: &DigitallySignedStruct, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, ) -> Result { - // Accept all signatures without verification - Ok(HandshakeSignatureValid::assertion()) + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &CryptoExt::get_provider().signature_verification_algorithms, + ) } fn supported_verify_schemes(&self) -> Vec { - ALL_SIGNATURE_SCHEMES.to_vec() + CryptoExt::get_provider() + .signature_verification_algorithms + .supported_schemes() } } diff --git a/crates/stdlib/src/suggestions.rs b/crates/stdlib/src/suggestions.rs index e0667dfb553..bfde00d2bb9 100644 --- a/crates/stdlib/src/suggestions.rs +++ b/crates/stdlib/src/suggestions.rs @@ -2,19 +2,25 @@ pub(crate) use _suggestions::module_def; #[pymodule] mod _suggestions { - use rustpython_vm::VirtualMachine; + use rustpython_vm::{PyResult, VirtualMachine, builtins::PyList}; use crate::vm::PyObjectRef; #[pyfunction] fn _generate_suggestions( - candidates: Vec, + candidates: PyObjectRef, name: PyObjectRef, vm: &VirtualMachine, - ) -> PyObjectRef { - match crate::vm::suggestion::calculate_suggestions(candidates.iter(), &name) { - Some(suggestion) => suggestion.into(), - None => vm.ctx.none(), - } + ) -> PyResult { + let candidates = candidates + .downcast::() + .map_err(|_| vm.new_type_error("candidates must be a list"))?; + let candidates = candidates.borrow_vec(); + Ok( + match crate::vm::suggestion::calculate_suggestions(candidates.iter(), &name) { + Some(suggestion) => suggestion.into(), + None => vm.ctx.none(), + }, + ) } } diff --git a/crates/stdlib/src/tkinter.rs b/crates/stdlib/src/tkinter.rs index 653d6edb71d..ca70561b3ab 100644 --- a/crates/stdlib/src/tkinter.rs +++ b/crates/stdlib/src/tkinter.rs @@ -162,9 +162,18 @@ mod _tkinter { if let Some(tcl_obj) = obj.downcast_ref::() { let c_str = unsafe { tk_sys::Tcl_GetString(tcl_obj.value) }; - let varname = unsafe { ffi::CStr::from_ptr(c_str as _) } - .to_str() - .map_err(|e| vm.new_unicode_decode_error(e.to_string()))? + let bytes = unsafe { ffi::CStr::from_ptr(c_str as _) }.to_bytes(); + let varname = core::str::from_utf8(bytes) + .map_err(|e| { + vm.new_unicode_decode_error( + vm.ctx.new_str("utf-8"), + vm.ctx.new_bytes(bytes.to_vec()), + e.valid_up_to(), + e.error_len() + .map_or(bytes.len(), |len| e.valid_up_to() + len), + vm.ctx.new_str(e.to_string()), + ) + })? .to_owned(); return Ok(varname); } diff --git a/crates/vm/src/anystr.rs b/crates/vm/src/anystr.rs index 69ba525267a..4896f2789bd 100644 --- a/crates/vm/src/anystr.rs +++ b/crates/vm/src/anystr.rs @@ -4,7 +4,7 @@ use num_traits::{cast::ToPrimitive, sign::Signed}; use rustpython_unicode::case; use crate::{ - Py, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, + AsObject, PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::{PyIntRef, PyTuple}, convert::TryFromBorrowedObject, function::OptionalOption, @@ -27,7 +27,7 @@ pub struct SplitLinesArgs { #[derive(FromArgs)] pub struct ExpandTabsArgs { #[pyarg(any, default = 8)] - tabsize: isize, + tabsize: i32, } impl ExpandTabsArgs { @@ -132,6 +132,11 @@ where { fn new() -> Self; fn with_capacity(capacity: usize) -> Self; + /// `with_capacity`, reporting a capacity that cannot be allocated instead + /// of aborting the process on it. + fn try_with_capacity(capacity: usize) -> Option + where + Self: Sized; fn push_str(&mut self, s: &S); } @@ -147,7 +152,11 @@ pub(crate) trait AnyStr { fn as_bytes(&self) -> &[u8]; fn elements(&self) -> impl Iterator; fn get_bytes(&self, range: Range) -> &Self; - // FIXME: get_chars is expensive for str + /// The characters in `range`, which for a `str` payload means walking to + /// both bounds -- the payload does not carry the string's character index. + /// `PyStr` therefore converts its own ranges and does not reach the search + /// helpers below through this; what remains are the byte strings, where a + /// character range is already a byte range. fn get_chars(&self, range: Range) -> &Self; fn bytes_len(&self) -> usize; // NOTE: str::chars().count() consumes the O(n) time. But pystr::char_len does cache. @@ -281,27 +290,29 @@ pub(crate) trait AnyStr { } } - fn py_pad(&self, left: usize, right: usize, fillchar: Self::Char) -> Self::Container { - let mut u = Self::Container::with_capacity( - (left + right) * fillchar.bytes_len() + self.bytes_len(), - ); + fn py_pad(&self, left: usize, right: usize, fillchar: Self::Char) -> Option { + let capacity = left + .checked_add(right)? + .checked_mul(fillchar.bytes_len())? + .checked_add(self.bytes_len())?; + let mut u = Self::Container::try_with_capacity(capacity)?; u.extend(core::iter::repeat_n(fillchar, left)); u.push_str(self); u.extend(core::iter::repeat_n(fillchar, right)); - u + Some(u) } - fn py_center(&self, width: usize, fillchar: Self::Char, len: usize) -> Self::Container { + fn py_center(&self, width: usize, fillchar: Self::Char, len: usize) -> Option { let marg = width - len; let left = marg / 2 + (marg & width & 1); self.py_pad(left, marg - left, fillchar) } - fn py_ljust(&self, width: usize, fillchar: Self::Char, len: usize) -> Self::Container { + fn py_ljust(&self, width: usize, fillchar: Self::Char, len: usize) -> Option { self.py_pad(0, width - len, fillchar) } - fn py_rjust(&self, width: usize, fillchar: Self::Char, len: usize) -> Self::Container { + fn py_rjust(&self, width: usize, fillchar: Self::Char, len: usize) -> Option { self.py_pad(width - len, 0, fillchar) } @@ -398,7 +409,7 @@ pub(crate) trait AnyStr { elements } - fn py_zfill(&self, width: isize) -> Vec { + fn py_zfill(&self, width: isize) -> Option> { let width = width.to_usize().unwrap_or(0); let char_len = self.elements().count(); let width = self @@ -481,19 +492,25 @@ where F: Fn(T) -> PyResult, M: Fn(&PyObject) -> String, { - if let Ok(single) = obj.try_to_value::(vm) { - (predicate)(single) - } else { - let tuple: &Py = obj - .try_to_value(vm) - .map_err(|_| vm.new_type_error((message)(obj)))?; - - for obj in tuple { - if single_or_tuple_any(obj, predicate, message, vm)? { + // _Py_bytes_tailmatch: a tuple is taken apart before anything is converted, and + // each item is converted on its own terms, so a tuple of tuples is not an affix. + if let Some(tuple) = obj.downcast_ref::() { + for item in tuple { + if (predicate)(item.try_to_value::(vm)?)? { return Ok(true); } } - - Ok(false) + return Ok(false); } + + // Only the argument simply being the wrong kind of object is reported as such; + // whatever the conversion itself raised belongs to the caller. + let single = obj.try_to_value::(vm).map_err(|exc| { + if exc.fast_isinstance(vm.ctx.exceptions.type_error) { + vm.new_type_error((message)(obj)) + } else { + exc + } + })?; + (predicate)(single) } diff --git a/crates/vm/src/buffer.rs b/crates/vm/src/buffer.rs index dc3691b5421..038e7cae9f3 100644 --- a/crates/vm/src/buffer.rs +++ b/crates/vm/src/buffer.rs @@ -16,7 +16,7 @@ use malachite_bigint::BigInt; use num_traits::{PrimInt, ToPrimitive}; use std::os::raw; -type PackFunc = fn(&VirtualMachine, PyObjectRef, &mut [u8]) -> PyResult<()>; +type PackFunc = fn(&VirtualMachine, FormatType, PyObjectRef, &mut [u8]) -> PyResult<()>; type UnpackFunc = fn(&VirtualMachine, &[u8]) -> PyObjectRef; static OVERFLOW_MSG: &str = "total struct size too long"; // not a const to reduce code size @@ -490,7 +490,7 @@ impl FormatSpec { let pack = code.info.pack.unwrap(); for arg in args.by_ref().take(code.repeat) { let (item_buf, rest) = buffer.split_at_mut(code.info.size); - pack(vm, arg, item_buf)?; + pack(vm, code.code, arg, item_buf)?; buffer = rest; } } @@ -549,7 +549,12 @@ impl FormatSpec { } trait Packable { - fn pack(vm: &VirtualMachine, arg: PyObjectRef, data: &mut [u8]) -> PyResult<()>; + fn pack( + vm: &VirtualMachine, + code: FormatType, + arg: PyObjectRef, + data: &mut [u8], + ) -> PyResult<()>; fn unpack(vm: &VirtualMachine, data: &[u8]) -> PyObjectRef; } @@ -576,10 +581,11 @@ macro_rules! make_pack_prim_int { impl Packable for $T { fn pack( vm: &VirtualMachine, + code: FormatType, arg: PyObjectRef, data: &mut [u8], ) -> PyResult<()> { - let i: $T = get_int_or_index(vm, arg)?; + let i: $T = get_int_or_index(vm, code, arg)?; i.pack_int::(data); Ok(()) } @@ -592,16 +598,28 @@ macro_rules! make_pack_prim_int { }; } -fn get_int_or_index(vm: &VirtualMachine, arg: PyObjectRef) -> PyResult +fn get_int_or_index(vm: &VirtualMachine, code: FormatType, arg: PyObjectRef) -> PyResult where - T: PrimInt + for<'a> TryFrom<&'a BigInt>, + T: PrimInt + fmt::Display + for<'a> TryFrom<&'a BigInt>, { let index = arg .try_index_opt(vm) .unwrap_or_else(|| Err(new_struct_error(vm, "required argument is not an integer")))?; - index - .try_to_primitive(vm) - .map_err(|_| new_struct_error(vm, "argument out of range")) + index.try_to_primitive(vm).map_err(|_| { + // A pointer is converted rather than checked against the range of a + // named format, so what it reports is the conversion failing. + let msg = if code == FormatType::VoidP { + "int too large to convert".to_owned() + } else { + format!( + "'{}' format requires {} <= number <= {}", + code as u8 as char, + T::min_value(), + T::max_value() + ) + }; + new_struct_error(vm, msg) + }) } make_pack_prim_int!(i8); @@ -620,6 +638,7 @@ macro_rules! make_pack_float { impl Packable for $T { fn pack( vm: &VirtualMachine, + _code: FormatType, arg: PyObjectRef, data: &mut [u8], ) -> PyResult<()> { @@ -648,7 +667,12 @@ make_pack_float!(f32, "f"); make_pack_float!(f64, "d"); impl Packable for f16 { - fn pack(vm: &VirtualMachine, arg: PyObjectRef, data: &mut [u8]) -> PyResult<()> { + fn pack( + vm: &VirtualMachine, + _code: FormatType, + arg: PyObjectRef, + data: &mut [u8], + ) -> PyResult<()> { let f_64 = ArgIntoFloat::try_from_object(vm, arg)?.into_float(); // "from_f64 should be preferred in any non-`const` context" except it gives the wrong result :/ let f_16 = Self::from_f64_const(f_64); @@ -666,8 +690,13 @@ impl Packable for f16 { } impl Packable for *mut raw::c_void { - fn pack(vm: &VirtualMachine, arg: PyObjectRef, data: &mut [u8]) -> PyResult<()> { - usize::pack::(vm, arg, data) + fn pack( + vm: &VirtualMachine, + code: FormatType, + arg: PyObjectRef, + data: &mut [u8], + ) -> PyResult<()> { + usize::pack::(vm, code, arg, data) } fn unpack(vm: &VirtualMachine, rdr: &[u8]) -> PyObjectRef { @@ -676,7 +705,12 @@ impl Packable for *mut raw::c_void { } impl Packable for bool { - fn pack(vm: &VirtualMachine, arg: PyObjectRef, data: &mut [u8]) -> PyResult<()> { + fn pack( + vm: &VirtualMachine, + _code: FormatType, + arg: PyObjectRef, + data: &mut [u8], + ) -> PyResult<()> { let v = ArgIntoBool::try_from_object(vm, arg)?.into_bool() as u8; v.pack_int::(data); Ok(()) @@ -688,7 +722,12 @@ impl Packable for bool { } } -fn pack_char(vm: &VirtualMachine, arg: PyObjectRef, data: &mut [u8]) -> PyResult<()> { +fn pack_char( + vm: &VirtualMachine, + _code: FormatType, + arg: PyObjectRef, + data: &mut [u8], +) -> PyResult<()> { let v = PyBytesRef::try_from_object(vm, arg)?; let ch = *v .as_bytes() diff --git a/crates/vm/src/builtins/asyncgenerator.rs b/crates/vm/src/builtins/asyncgenerator.rs index b53e59d58c1..7ea43f389c6 100644 --- a/crates/vm/src/builtins/asyncgenerator.rs +++ b/crates/vm/src/builtins/asyncgenerator.rs @@ -144,7 +144,11 @@ impl PyAsyncGen { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/bool.rs b/crates/vm/src/builtins/bool.rs index 4bb980d71a2..1cfa8cc27ee 100644 --- a/crates/vm/src/builtins/bool.rs +++ b/crates/vm/src/builtins/bool.rs @@ -34,6 +34,7 @@ impl<'a> TryFromBorrowedObject<'a> for bool { impl PyObjectRef { /// Convert Python bool into Rust bool. + #[inline(always)] pub fn try_to_bool(self, vm: &VirtualMachine) -> PyResult { if self.is(&vm.ctx.true_value) { return Ok(true); @@ -41,6 +42,12 @@ impl PyObjectRef { return Ok(false); } + self.try_to_bool_slow(vm) + } + + #[cold] + #[inline(never)] + fn try_to_bool_slow(self, vm: &VirtualMachine) -> PyResult { let slots = &self.class().slots; // 1. Try nb_bool slot first diff --git a/crates/vm/src/builtins/builtin_func.rs b/crates/vm/src/builtins/builtin_func.rs index eabe8d4ea27..b34447b79bf 100644 --- a/crates/vm/src/builtins/builtin_func.rs +++ b/crates/vm/src/builtins/builtin_func.rs @@ -247,9 +247,9 @@ fn vectorcall_native_function( let mut all_args = Vec::with_capacity(args.len() + 1); all_args.push(self_obj); all_args.extend(args); - FuncArgs::from_vectorcall(&all_args, nargs + 1, kwnames) + FuncArgs::from_vectorcall_owned(all_args, nargs + 1, kwnames) } else { - FuncArgs::from_vectorcall(&args, nargs, kwnames) + FuncArgs::from_vectorcall_owned(args, nargs, kwnames) }; (zelf.value.func)(vm, func_args) diff --git a/crates/vm/src/builtins/bytearray.rs b/crates/vm/src/builtins/bytearray.rs index a649fe9d8d5..93046b4932e 100644 --- a/crates/vm/src/builtins/bytearray.rs +++ b/crates/vm/src/builtins/bytearray.rs @@ -1,7 +1,7 @@ //! Implementation of the python bytearray object. use super::{ - PositionIterInternal, PyBytes, PyDictRef, PyGenericAlias, PyIntRef, PyStrRef, PyTuple, - PyTupleRef, PyType, PyTypeRef, iter::builtins_iter, + PositionIterInternal, PyBytes, PyDictRef, PyGenericAlias, PyStrRef, PyTuple, PyTupleRef, + PyType, PyTypeRef, iter::builtins_iter, }; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, @@ -11,7 +11,8 @@ use crate::{ byte::{bytes_from_object, value_from_object}, bytes_inner::{ ByteInnerFindOptions, ByteInnerHexOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, - ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner, bytes_decode, + ByteInnerSplitOptions, ByteInnerSub, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner, + bytes_decode, }, class::PyClassImpl, common::{ @@ -23,10 +24,10 @@ use crate::{ }, convert::{ToPyObject, ToPyResult}, function::{ - ArgBytesLike, ArgIterable, ArgSize, Either, OptionalArg, OptionalOption, PyComparisonValue, + ArgBytesLike, ArgIterable, ArgSize, OptionalArg, OptionalOption, PyComparisonValue, }, protocol::{ - BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn, + BufferDescriptor, BufferFlags, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods, }, sliceable::{SequenceIndex, SliceableSequenceMutOp, SliceableSequenceOp}, @@ -228,11 +229,8 @@ impl PyByteArray { self.inner().add(&other.borrow_buf()).into() } - fn __contains__( - &self, - needle: Either, - vm: &VirtualMachine, - ) -> PyResult { + fn __contains__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let needle = ByteInnerSub::from_contains_arg(needle, vm)?; self.inner().contains(needle, vm) } @@ -356,7 +354,10 @@ impl PyByteArray { #[pymethod] fn join(&self, iter: ArgIterable, vm: &VirtualMachine) -> PyResult { - Ok(self.inner().join(iter, vm)?.into()) + // Driving the iterable runs Python, which can reach this bytearray, + // so the separator is taken by value rather than left borrowed. + let separator = self.inner().clone(); + Ok(separator.join(iter, vm)?.into()) } #[pymethod] @@ -499,8 +500,8 @@ impl PyByteArray { } #[pymethod] - fn zfill(&self, width: isize) -> Self { - self.inner().zfill(width).into() + fn zfill(&self, width: isize, vm: &VirtualMachine) -> PyResult { + Ok(self.inner().zfill(width, vm)?.into()) } #[pymethod] @@ -534,7 +535,10 @@ impl PyByteArray { } fn __mod__(&self, values: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let formatted = self.inner().cformat(values, vm)?; + // Formatting calls the values' conversion methods, which can reach + // this bytearray, so the format is taken by value. + let format = self.inner().clone(); + let formatted = format.cformat(values, vm)?; Ok(formatted.into()) } @@ -554,7 +558,11 @@ impl PyByteArray { // TODO: Uncomment when Python adds __class_getitem__ to bytearray // #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -609,12 +617,34 @@ impl Py { #[pymethod] fn extend(&self, object: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { if self.is(&object) { - PyByteArray::irepeat(self, 2, vm) - } else { - let items = bytes_from_object(vm, &object)?; - self.try_resizable(vm)?.elements.extend(items); - Ok(()) + return PyByteArray::irepeat(self, 2, vm); } + // bytearray_setslice keeps the export alive across the resize, so a value + // looking at this bytearray is what stops it from growing. + let buffer = object + .check_buffer() + .then(|| { + PyBuffer::from_object(vm, &object, BufferFlags::SIMPLE).map_err(|_| { + // What an exporter refuses to hand out leaves the value simply + // not usable here, whatever the exporter's own complaint was. + vm.new_type_error(format!( + "can't set bytearray slice from {}", + object.class().name() + )) + }) + }) + .transpose()?; + let items = match &buffer { + Some(buffer) => buffer + .as_contiguous() + .ok_or_else(|| { + vm.new_buffer_error("non-contiguous buffer is not a bytes-like object") + })? + .to_vec(), + None => bytes_from_object(vm, &object)?, + }; + self.try_resizable(vm)?.elements.extend(items); + Ok(()) } #[pymethod] @@ -727,6 +757,20 @@ static BUFFER_METHODS: BufferMethods = BufferMethods { }; impl AsBuffer for PyByteArray { + const RELEASE_BUFFER: bool = true; + + fn slot_as_buffer( + zelf: &PyObject, + flags: BufferFlags, + vm: &VirtualMachine, + ) -> PyResult { + let zelf = zelf + .downcast_ref::() + .ok_or_else(|| vm.new_type_error("unexpected payload for as_buffer"))?; + flags.fill_info_check(false, vm)?; + Self::as_buffer(zelf, vm) + } + fn as_buffer(zelf: &Py, _vm: &VirtualMachine) -> PyResult { Ok(PyBuffer::new( zelf.to_owned().into(), @@ -740,8 +784,9 @@ impl BufferResizeGuard for PyByteArray { type Resizable<'a> = PyRwLockWriteGuard<'a, PyBytesInner>; fn try_resizable_opt(&self) -> Option> { - let w = self.inner.write(); - (self.exports.load(Ordering::SeqCst) == 0).then_some(w) + // An export is a borrow someone else still holds, so it is answered + // before the lock rather than by waiting on it. + (self.exports.load(Ordering::SeqCst) == 0).then(|| self.inner.write()) } } @@ -797,9 +842,7 @@ impl AsSequence for PyByteArray { } }), contains: atomic_func!(|seq, other, vm| { - let other = - >::try_from_object(vm, other.to_owned())?; - PyByteArray::sequence_downcast(seq).__contains__(other, vm) + PyByteArray::sequence_downcast(seq).__contains__(other.to_owned(), vm) }), inplace_concat: atomic_func!(|seq, other, vm| { let other = ArgBytesLike::try_from_object(vm, other.to_owned())?; diff --git a/crates/vm/src/builtins/bytes.rs b/crates/vm/src/builtins/bytes.rs index d4c30a7e94d..48c0e431229 100644 --- a/crates/vm/src/builtins/bytes.rs +++ b/crates/vm/src/builtins/bytes.rs @@ -1,27 +1,28 @@ use super::{ - PositionIterInternal, PyDictRef, PyGenericAlias, PyIntRef, PyStrRef, PyTuple, PyTupleRef, - PyType, PyTypeRef, iter::builtins_iter, + PositionIterInternal, PyDictRef, PyGenericAlias, PyStrRef, PyTuple, PyTupleRef, PyType, + PyTypeRef, iter::builtins_iter, }; use crate::common::lock::LazyLock; use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, - TryFromBorrowedObject, TryFromObject, VirtualMachine, + TryFromBorrowedObject, VirtualMachine, anystr::{self, AnyStr}, atomic_func, bytes_inner::{ ByteInnerFindOptions, ByteInnerHexOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, - ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner, bytes_decode, + ByteInnerSplitOptions, ByteInnerSub, ByteInnerTranslateOptions, DecodeArgs, PyBytesInner, + bytes_decode, }, class::PyClassImpl, common::{hash::PyHash, lock::PyMutex}, convert::{ToPyObject, ToPyResult}, function::{ - ArgBytesLike, ArgIndex, ArgIterable, Either, FuncArgs, OptionalArg, OptionalOption, + ArgBytesLike, ArgIndex, ArgIterable, FuncArgs, OptionalArg, OptionalOption, PyComparisonValue, }, protocol::{ - BufferDescriptor, BufferMethods, PyBuffer, PyIterReturn, PyMappingMethods, PyNumberMethods, - PySequenceMethods, + BufferDescriptor, BufferFlags, BufferMethods, PyBuffer, PyIterReturn, PyMappingMethods, + PyNumberMethods, PySequenceMethods, }, sliceable::{SequenceIndex, SliceableSequenceOp}, types::{ @@ -246,11 +247,8 @@ impl PyBytes { self.inner.add(&other.borrow_buf()) } - fn __contains__( - &self, - needle: Either, - vm: &VirtualMachine, - ) -> PyResult { + fn __contains__(&self, needle: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let needle = ByteInnerSub::from_contains_arg(needle, vm)?; self.inner.contains(needle, vm) } @@ -507,8 +505,8 @@ impl PyBytes { } #[pymethod] - fn zfill(&self, width: isize) -> Self { - self.inner.zfill(width).into() + fn zfill(&self, width: isize, vm: &VirtualMachine) -> PyResult { + Ok(self.inner.zfill(width, vm)?.into()) } #[pymethod] @@ -544,7 +542,11 @@ impl PyBytes { // TODO: Uncomment when Python adds __class_getitem__ to bytes // #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -623,6 +625,18 @@ static BUFFER_METHODS: BufferMethods = BufferMethods { }; impl AsBuffer for PyBytes { + fn slot_as_buffer( + zelf: &PyObject, + flags: BufferFlags, + vm: &VirtualMachine, + ) -> PyResult { + let zelf = zelf + .downcast_ref::() + .ok_or_else(|| vm.new_type_error("unexpected payload for as_buffer"))?; + flags.fill_info_check(true, vm)?; + Self::as_buffer(zelf, vm) + } + fn as_buffer(zelf: &Py, _vm: &VirtualMachine) -> PyResult { let buf = PyBuffer::new( zelf.to_owned().into(), @@ -669,9 +683,7 @@ impl AsSequence for PyBytes { .map(|x| vm.ctx.new_bytes(vec![x]).into()) }), contains: atomic_func!(|seq, other, vm| { - let other = - >::try_from_object(vm, other.to_owned())?; - PyBytes::sequence_downcast(seq).__contains__(other, vm) + PyBytes::sequence_downcast(seq).__contains__(other.to_owned(), vm) }), ..PySequenceMethods::NOT_IMPLEMENTED }); diff --git a/crates/vm/src/builtins/classmethod.rs b/crates/vm/src/builtins/classmethod.rs index eb0e15ece01..26dcd251251 100644 --- a/crates/vm/src/builtins/classmethod.rs +++ b/crates/vm/src/builtins/classmethod.rs @@ -27,7 +27,7 @@ use crate::{ /// /// Class methods are different than C++ or Java static methods. /// If you want those, see the staticmethod builtin. -#[pyclass(module = false, name = "classmethod")] +#[pyclass(module = false, name = "classmethod", traverse)] #[derive(Debug)] pub struct PyClassMethod { callable: PyMutex, @@ -187,7 +187,11 @@ impl PyClassMethod { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/coroutine.rs b/crates/vm/src/builtins/coroutine.rs index d472f1a0bfa..0fc50fb1356 100644 --- a/crates/vm/src/builtins/coroutine.rs +++ b/crates/vm/src/builtins/coroutine.rs @@ -103,7 +103,11 @@ impl PyCoroutine { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/descriptor.rs b/crates/vm/src/builtins/descriptor.rs index 5c0662e9fef..50bdc841abf 100644 --- a/crates/vm/src/builtins/descriptor.rs +++ b/crates/vm/src/builtins/descriptor.rs @@ -542,6 +542,10 @@ pub enum SlotFunc { NumBinaryRight(PyNumberBinaryFunc), // __radd__, __rsub__, etc. (swapped args) NumTernary(PyNumberTernaryFunc), // __pow__ NumTernaryRight(PyNumberTernaryFunc), // __rpow__ (swapped first two args) + + // Buffer protocol + GetBuffer(crate::types::AsBufferFunc), // __buffer__ + ReleaseBuffer, // __release_buffer__ } impl core::fmt::Debug for SlotFunc { @@ -582,6 +586,8 @@ impl core::fmt::Debug for SlotFunc { Self::NumBinaryRight(_) => write!(f, "SlotFunc::NumBinaryRight(...)"), Self::NumTernary(_) => write!(f, "SlotFunc::NumTernary(...)"), Self::NumTernaryRight(_) => write!(f, "SlotFunc::NumTernaryRight(...)"), + Self::GetBuffer(_) => write!(f, "SlotFunc::GetBuffer(...)"), + Self::ReleaseBuffer => write!(f, "SlotFunc::ReleaseBuffer"), } } } @@ -758,10 +764,41 @@ impl SlotFunc { let z = z.unwrap_or_else(|| vm.ctx.none()); func(&y, &obj, &z, vm) // Swapped: y ** obj % z } + // Buffer protocol + Self::GetBuffer(func) => { + let (flags_obj,): (PyObjectRef,) = args.bind(vm)?; + let buffer = func(&obj, parse_buffer_flags(flags_obj, vm)?, vm)?; + crate::builtins::PyMemoryView::from_buffer(buffer, vm) + .map(|mv| mv.into_pyobject(vm)) + } + Self::ReleaseBuffer => { + let (mv_obj,): (PyObjectRef,) = args.bind(vm)?; + let mv = mv_obj + .downcast::() + .map_err(|_| vm.new_type_error("expected a memoryview object"))?; + crate::builtins::memory::release_buffer_from_python(&obj, mv, vm)?; + Ok(vm.ctx.none()) + } } } } +/// Parse the `flags` argument of `__buffer__`. wrap_buffer +fn parse_buffer_flags( + arg: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult { + use num_traits::ToPrimitive; + let idx = arg.try_index(vm)?; + let flags = idx + .as_bigint() + .to_isize() + .ok_or_else(|| vm.new_overflow_error("cannot fit 'int' into an index-sized integer"))?; + let flags = + i32::try_from(flags).map_err(|_| vm.new_overflow_error("buffer flags out of range"))?; + Ok(crate::protocol::BufferFlags::from_bits_retain(flags as u32)) +} + /// wrapper_descriptor: wraps a slot function as a Python method // = PyWrapperDescrObject #[pyclass(name = "wrapper_descriptor", module = false)] diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index 5db071e1d8f..d2b9dea31fa 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -1,6 +1,6 @@ use super::{ IterStatus, PositionIterInternal, PyBaseExceptionRef, PyGenericAlias, PyMappingProxy, PySet, - PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, set::PySetInner, + PyStr, PyStrRef, PyTupleRef, PyType, PyTypeRef, set, set::PySetInner, }; use crate::common::lock::LazyLock; use crate::object::{Traverse, TraverseFn}; @@ -9,7 +9,7 @@ use crate::{ TryFromObject, atomic_func, builtins::{PyList, PyTuple, iter::builtins_iter, type_::PyAttributes}, class::{PyClassDef, PyClassImpl}, - common::ascii, + common::{ascii, hash::PyHash}, dict_inner::{self, DictKey}, function::{ArgIterable, FuncArgs, KwArgs, OptionalArg, PyArithmeticValue, PyComparisonValue}, iter::PyExactSizeIterator, @@ -114,11 +114,6 @@ impl PyDict { &self.entries } - /// Monotonically increasing version for mutation tracking. - pub(crate) fn version(&self) -> u64 { - self.entries.version() - } - /// Returns all keys as a Vec, atomically under a single read lock. /// Thread-safe: prevents "dictionary changed size during iteration" errors. pub fn keys_vec(&self) -> Vec { @@ -354,6 +349,20 @@ impl PyDict { ) -> PyResult> { self.entries.get(vm, key) } + + /// Keys of `obj` with their stored hashes, or `None` if it must be iterated + /// generically. Only exact dicts and sets qualify, as in CPython's + /// `_PyDict_FromKeys`: a subclass may override `__iter__`. + fn fromkeys_known_hashes( + obj: &PyObject, + vm: &VirtualMachine, + ) -> Option> { + if let Some(dict) = obj.downcast_ref_if_exact::(vm) { + Some(dict.entries.keys_with_hashes()) + } else { + set::exact_set_keys_with_hashes(obj, vm) + } + } } // Python dict methods: @@ -384,8 +393,16 @@ impl PyDict { let d = PyType::call(&class, ().into(), vm)?; match d.downcast_exact::(vm) { Ok(pydict) => { - for key in iterable.iter(vm)? { - pydict.__setitem__(key?, value.clone(), vm)?; + if let Some(keys) = Self::fromkeys_known_hashes(iterable.as_object(), vm) { + for (key, hash) in keys { + pydict + .entries + .insert_known_hash(vm, &*key, hash, value.clone())?; + } + } else { + for key in iterable.iter(vm)? { + pydict.__setitem__(key?, value.clone(), vm)?; + } } Ok(pydict.into_pyref().into()) } @@ -513,7 +530,11 @@ impl PyDict { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -795,18 +816,15 @@ impl Py { } } - /// Fast lookup using a cached entry index hint. - pub(crate) fn get_item_opt_hint( + /// Read a cached exact-dict entry after validating its key-layout stamp. + #[inline] + pub(crate) fn get_item_by_index_and_keys_version( &self, - key: &K, - hint: u16, - vm: &VirtualMachine, - ) -> PyResult> { - if self.exact_dict(vm) { - self.entries.get_hint(vm, key, usize::from(hint)) - } else { - self.get_item_opt(key, vm) - } + version: u16, + index: u16, + ) -> Option { + self.entries + .get_index_if_keys_version(u32::from(version), usize::from(index)) } /// Lookup trying a cached entry index hint first. @@ -1072,6 +1090,7 @@ macro_rules! dict_view { $class_name: literal, $iter_class_name: literal, $reverse_iter_class_name: literal, + $project_fn: expr, $result_fn: expr ) => { #[pyclass(module = false, name = $class_name)] @@ -1094,7 +1113,7 @@ macro_rules! dict_view { } fn item(vm: &VirtualMachine, key: PyObjectRef, value: PyObjectRef) -> PyObjectRef { - $result_fn(vm, key, value) + $result_fn(vm, $project_fn(&key, &value)) } fn __reversed__(&self) -> Self::ReverseIter { @@ -1180,7 +1199,7 @@ macro_rules! dict_view { while let Some((next_position, key, value)) = dict.entries.next_entry(position) { - entries.push(($result_fn)(vm, key, value)); + entries.push(($result_fn)(vm, ($project_fn)(&key, &value))); position = next_position; } entries @@ -1197,18 +1216,22 @@ macro_rules! dict_view { fn next(zelf: &Py, vm: &VirtualMachine) -> PyResult { let mut internal = zelf.internal.lock(); let next = if let IterStatus::Active(dict) = &internal.status { - if dict.entries.has_changed_size(&zelf.size) { - internal.status = IterStatus::Exhausted; - return Err( - vm.new_runtime_error("dictionary changed size during iteration") - ); - } - match dict.entries.next_entry(internal.position) { - Some((position, key, value)) => { + match dict.entries.next_entry_checked( + internal.position, + &zelf.size, + $project_fn, + ) { + Err(dict_inner::DictChanged) => { + internal.status = IterStatus::Exhausted; + return Err( + vm.new_runtime_error("dictionary changed size during iteration") + ); + } + Ok(Some((position, item))) => { internal.position = position; - PyIterReturn::Return(($result_fn)(vm, key, value)) + PyIterReturn::Return(($result_fn)(vm, item)) } - None => { + Ok(None) => { internal.status = IterStatus::Exhausted; PyIterReturn::StopIteration(None) } @@ -1256,7 +1279,7 @@ macro_rules! dict_view { while let Some((found_index, key, value)) = dict.entries.prev_entry(position) { - entries.push(($result_fn)(vm, key, value)); + entries.push(($result_fn)(vm, ($project_fn)(&key, &value))); if found_index == 0 { break; } @@ -1283,22 +1306,26 @@ macro_rules! dict_view { fn next(zelf: &Py, vm: &VirtualMachine) -> PyResult { let mut internal = zelf.internal.lock(); let next = if let IterStatus::Active(dict) = &internal.status { - if dict.entries.has_changed_size(&zelf.size) { - internal.status = IterStatus::Exhausted; - return Err( - vm.new_runtime_error("dictionary changed size during iteration") - ); - } - match dict.entries.prev_entry(internal.position) { - Some((found_index, key, value)) => { + match dict.entries.prev_entry_checked( + internal.position, + &zelf.size, + $project_fn, + ) { + Err(dict_inner::DictChanged) => { + internal.status = IterStatus::Exhausted; + return Err( + vm.new_runtime_error("dictionary changed size during iteration") + ); + } + Ok(Some((found_index, item))) => { if found_index == 0 { internal.status = IterStatus::Exhausted; } else { internal.position = found_index - 1; } - PyIterReturn::Return(($result_fn)(vm, key, value)) + PyIterReturn::Return(($result_fn)(vm, item)) } - None => { + Ok(None) => { internal.status = IterStatus::Exhausted; PyIterReturn::StopIteration(None) } @@ -1322,7 +1349,8 @@ dict_view! { "dict_keys", "dict_keyiterator", "dict_reversekeyiterator", - |_vm: &VirtualMachine, key: PyObjectRef, _value: PyObjectRef| key + |key: &PyObjectRef, _value: &PyObjectRef| key.clone(), + |_vm: &VirtualMachine, key: PyObjectRef| key } dict_view! { @@ -1335,7 +1363,8 @@ dict_view! { "dict_values", "dict_valueiterator", "dict_reversevalueiterator", - |_vm: &VirtualMachine, _key: PyObjectRef, value: PyObjectRef| value + |_key: &PyObjectRef, value: &PyObjectRef| value.clone(), + |_vm: &VirtualMachine, value: PyObjectRef| value } dict_view! { @@ -1348,7 +1377,9 @@ dict_view! { "dict_items", "dict_itemiterator", "dict_reverseitemiterator", - |vm: &VirtualMachine, key: PyObjectRef, value: PyObjectRef| + |key: &PyObjectRef, value: &PyObjectRef| (key.clone(), value.clone()), + // Builds a tuple, so it runs after the dict's read guard is released. + |vm: &VirtualMachine, (key, value): (PyObjectRef, PyObjectRef)| vm.new_tuple((key, value)).into() } diff --git a/crates/vm/src/builtins/enumerate.rs b/crates/vm/src/builtins/enumerate.rs index 96073ba7667..95e144dad21 100644 --- a/crates/vm/src/builtins/enumerate.rs +++ b/crates/vm/src/builtins/enumerate.rs @@ -57,7 +57,11 @@ impl Constructor for PyEnumerate { #[pyclass(with(Py, IterNext, Iterable, Constructor), flags(BASETYPE))] impl PyEnumerate { #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/float.rs b/crates/vm/src/builtins/float.rs index 1c861b14fc6..0b739694623 100644 --- a/crates/vm/src/builtins/float.rs +++ b/crates/vm/src/builtins/float.rs @@ -176,16 +176,18 @@ impl Constructor for PyFloat { type Args = OptionalArg; fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + // Bind before the fast path so FromArgs::arity decides how many arguments + // are acceptable, rather than a count repeated here. + let arg: Self::Args = args.bind(vm)?; + // Optimization: return exact float as-is if cls.is(vm.ctx.types.float_type) - && args.kwargs.is_empty() - && let Some(first) = args.args.first() + && let OptionalArg::Present(first) = &arg && first.class().is(vm.ctx.types.float_type) { return Ok(first.clone()); } - let arg: Self::Args = args.bind(vm)?; let payload = Self::py_new(&cls, arg, vm)?; payload.into_ref_with_type(vm, cls).map(Into::into) } diff --git a/crates/vm/src/builtins/frame.rs b/crates/vm/src/builtins/frame.rs index 94ec827b7a6..e4b3aa2b184 100644 --- a/crates/vm/src/builtins/frame.rs +++ b/crates/vm/src/builtins/frame.rs @@ -512,7 +512,7 @@ impl FrameObject { let live = self.find_live_source_iframe(); if !live.is_null() { // Read live prev_line. Use read_volatile to bypass LLVM noalias - // on the &mut InterpreterFrame borrow in with_iframe. + // on the &mut InterpreterFrame borrow held by the running frame. let prev = unsafe { let field_ptr = core::ptr::addr_of!((*live).prev_line); core::ptr::read_volatile(field_ptr as *const u32) @@ -897,32 +897,19 @@ impl Py { { // Enter STW before dereferencing `prev` — the owning thread may // return and free the stack-allocated iframe at any time. - vm.state.stop_the_world.stop_the_world(vm); - scopeguard::defer! { vm.state.stop_the_world.start_the_world(vm); } + vm.state.stop_the_world.stop_the_world(&vm.state); + scopeguard::defer! { vm.state.stop_the_world.start_the_world(&vm.state); } let prev_ref = unsafe { &*prev }; // Fast path: already materialized. if let Some(fo) = prev_ref.frame_obj() { fo.mark_escaped(); return Some(fo.to_owned()); } - // Slow path: materialize the entire chain and link retained_back. - let mut cur = prev; - let mut child_fo: Option> = None; - while !cur.is_null() { - let iframe = unsafe { &*cur }; - let fo = iframe.materialize(vm).to_owned(); - if let Some(child) = child_fo.take() { - let mut guard = child.iframe().cold().retained_back.lock(); - if guard.is_none() { - *guard = Some(fo.clone()); - } - } - child_fo = Some(fo); - cur = iframe.previous(); - } - let fo = prev_ref.materialize(vm); + // Slow path: copy the whole chain, linked through retained_back. + // SAFETY: the world is stopped, so the owning thread is parked. + let fo = unsafe { prev_ref.materialize_detached_chain(vm) }; fo.mark_escaped(); - return Some(fo.to_owned()); + return Some(fo); } #[allow(unreachable_code)] diff --git a/crates/vm/src/builtins/function.rs b/crates/vm/src/builtins/function.rs index 90315bcb194..a5342d1df3a 100644 --- a/crates/vm/src/builtins/function.rs +++ b/crates/vm/src/builtins/function.rs @@ -550,6 +550,20 @@ impl Py { self.code.flags.contains(bytecode::CodeFlags::OPTIMIZED) } + /// Whether this function currently has native JIT code. Adaptive Python + /// call specializations must yield to that entry point. + #[inline] + pub(crate) fn is_jitted(&self) -> bool { + #[cfg(feature = "jit")] + { + self.jitted_code.lock().is_some() + } + #[cfg(not(feature = "jit"))] + { + false + } + } + pub fn invoke_with_locals( &self, func_args: FuncArgs, @@ -643,8 +657,8 @@ impl Py { .and_then(|()| vm.run_frame_fast(iframe)); // Release data stack memory — must happen on both success and error. unsafe { - if let Some(base) = iframe.release_datastack_frame() { - vm.datastack_pop(base); + if let Some((base, size)) = iframe.release_datastack_frame() { + vm.datastack_pop_frame(base, size); } } result @@ -669,7 +683,10 @@ impl Py { ); // SAFETY: the frame is alive (held by `frame`) and untracked. unsafe { - crate::gc_state::gc_state().track_object(core::ptr::NonNull::from(frame.as_object())); + crate::gc_state::gc_state().track_object( + core::ptr::NonNull::from(frame.as_object()), + crate::gc_state::current_owner(), + ); } frame.set_generator(&obj); obj @@ -820,8 +837,8 @@ impl Py { let result = vm.run_frame_fast(iframe); unsafe { - if let Some(base) = iframe.release_datastack_frame() { - vm.datastack_pop(base); + if let Some((base, size)) = iframe.release_datastack_frame() { + vm.datastack_pop_frame(base, size); } } result @@ -1616,6 +1633,16 @@ pub(crate) fn vectorcall_function( let code: &Py = &zelf.code; let has_kwargs = kwnames.is_some_and(|kw| !kw.is_empty()); + if zelf.is_jitted() { + let func_args = if has_kwargs { + FuncArgs::from_vectorcall_owned(args, nargs, kwnames) + } else { + args.truncate(nargs); + FuncArgs::from(args) + }; + return zelf.invoke(func_args, vm); + } + let is_simple = !has_kwargs && code.flags.contains(bytecode::CodeFlags::OPTIMIZED) && !code.flags.contains(bytecode::CodeFlags::VARARGS) @@ -1640,7 +1667,7 @@ pub(crate) fn vectorcall_function( // SLOW PATH: construct FuncArgs from owned Vec and delegate to invoke() let func_args = if has_kwargs { - FuncArgs::from_vectorcall(&args, nargs, kwnames) + FuncArgs::from_vectorcall_owned(args, nargs, kwnames) } else { args.truncate(nargs); FuncArgs::from(args) diff --git a/crates/vm/src/builtins/generator.rs b/crates/vm/src/builtins/generator.rs index 52db3c9522a..b06a3a45ea7 100644 --- a/crates/vm/src/builtins/generator.rs +++ b/crates/vm/src/builtins/generator.rs @@ -99,7 +99,11 @@ impl PyGenerator { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/genericalias.rs b/crates/vm/src/builtins/genericalias.rs index b6f6012fd43..8004bd535be 100644 --- a/crates/vm/src/builtins/genericalias.rs +++ b/crates/vm/src/builtins/genericalias.rs @@ -68,7 +68,7 @@ impl Constructor for PyGenericAlias { } else { PyTuple::new_ref(vec![arguments], &vm.ctx) }; - Ok(Self::new(origin, args, false, vm)) + Self::new(origin, args, false, vm) } } @@ -92,14 +92,14 @@ impl PyGenericAlias { args: PyTupleRef, starred: bool, vm: &VirtualMachine, - ) -> Self { - let parameters = make_parameters(&args, vm); - Self { + ) -> PyResult { + let parameters = make_parameters(&args, vm)?; + Ok(Self { origin: origin.into(), args, parameters, starred, - } + }) } /// Create a GenericAlias from an origin and PyObjectRef arguments (helper for compatibility) @@ -107,7 +107,7 @@ impl PyGenericAlias { origin: impl Into, args: PyObjectRef, vm: &VirtualMachine, - ) -> Self { + ) -> PyResult { let args = if let Ok(tuple) = args.try_to_ref::(vm) { tuple.to_owned() } else { @@ -228,7 +228,7 @@ impl PyGenericAlias { vm, )?; - Ok(Self::new(zelf.origin.clone(), new_args, false, vm).into_pyobject(vm)) + Ok(Self::new(zelf.origin.clone(), new_args, false, vm)?.into_pyobject(vm)) } #[pymethod] @@ -247,7 +247,7 @@ impl PyGenericAlias { if zelf.starred { // (next, (iter(GenericAlias(origin, args)),)) let next_fn = vm.builtins.get_attr("next", vm)?; - let non_starred = Self::new(zelf.origin.clone(), zelf.args.clone(), false, vm); + let non_starred = Self::new(zelf.origin.clone(), zelf.args.clone(), false, vm)?; let iter_obj = PyGenericAliasIterator { obj: crate::common::lock::PyMutex::new(Some(non_starred.into_pyobject(vm))), } @@ -292,11 +292,11 @@ impl PyGenericAlias { } } -pub(crate) fn make_parameters(args: &Py, vm: &VirtualMachine) -> PyTupleRef { +pub(crate) fn make_parameters(args: &Py, vm: &VirtualMachine) -> PyResult { make_parameters_from_slice(args.as_slice(), vm) } -fn make_parameters_from_slice(args: &[PyObjectRef], vm: &VirtualMachine) -> PyTupleRef { +fn make_parameters_from_slice(args: &[PyObjectRef], vm: &VirtualMachine) -> PyResult { let mut parameters: Vec = Vec::with_capacity(args.len()); for arg in args { @@ -326,7 +326,9 @@ fn make_parameters_from_slice(args: &[PyObjectRef], vm: &VirtualMachine) -> PyTu let list = arg.downcast_ref::().unwrap(); list.borrow_vec().to_vec() }; - let sub = make_parameters_from_slice(&items, vm); + let sub = vm.with_recursion("while computing __parameters__", || { + make_parameters_from_slice(&items, vm) + })?; for sub_param in sub.iter() { if tuple_index(¶meters, sub_param).is_none() { parameters.push(sub_param.clone()); @@ -335,7 +337,7 @@ fn make_parameters_from_slice(args: &[PyObjectRef], vm: &VirtualMachine) -> PyTu } } - PyTuple::new_ref(parameters, &vm.ctx) + Ok(PyTuple::new_ref(parameters, &vm.ctx)) } #[inline] @@ -716,7 +718,7 @@ impl crate::types::IterNext for PyGenericAliasIterator { let alias = obj .downcast_ref::() .ok_or_else(|| vm.new_type_error("generic_alias_iterator expected GenericAlias"))?; - let starred = PyGenericAlias::new(alias.origin.clone(), alias.args.clone(), true, vm); + let starred = PyGenericAlias::new(alias.origin.clone(), alias.args.clone(), true, vm)?; Ok(PyIterReturn::Return(starred.into_pyobject(vm))) } } diff --git a/crates/vm/src/builtins/int.rs b/crates/vm/src/builtins/int.rs index 278a9cecbb1..60463ed0d58 100644 --- a/crates/vm/src/builtins/int.rs +++ b/crates/vm/src/builtins/int.rs @@ -3,7 +3,7 @@ use crate::{ AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, TryFromBorrowedObject, VirtualMachine, builtins::PyUtf8StrRef, - bytes_inner::PyBytesInner, + byte::bytes_from_object, class::PyClassImpl, common::{ format::FormatSpec, @@ -305,6 +305,22 @@ impl PyInt { &self.value } + /// Extract the inline magnitude without the generic primitive-conversion path. + #[inline(always)] + pub(crate) fn try_to_i64_fast(&self) -> Option { + let bits = self.value.bits(); + if bits > i64::BITS as u64 { + return None; + } + let magnitude = self.value.iter_u64_digits().next().unwrap_or(0); + let signed_magnitude = i64::try_from(magnitude).ok(); + match self.value.sign() { + Sign::Minus if magnitude == 1u64 << 63 => Some(i64::MIN), + Sign::Minus => signed_magnitude.map(|value| -value), + Sign::NoSign | Sign::Plus => signed_magnitude, + } + } + /// Fast decimal string conversion, using i64 path when possible. #[inline] #[must_use] @@ -487,7 +503,9 @@ impl PyInt { return vm.ctx.new_int(rounded); } } - zelf + // No rounding to do, but an int subclass must still be normalized to an + // exact int, the way CPython's long_long() does. + zelf.__int__(vm).into_pyref() } #[pymethod] @@ -554,13 +572,13 @@ impl PyInt { vm: &VirtualMachine, ) -> PyResult> { let signed = args.signed.map_or(false, Into::into); + // PyObject_Bytes, so an iterable of ints is as good as a buffer + let bytes = bytes_from_object(vm, &args.bytes)?; let value = match (args.byteorder, signed) { - (ArgByteOrder::Big, true) => BigInt::from_signed_bytes_be(args.bytes.as_bytes()), - (ArgByteOrder::Big, false) => BigInt::from_bytes_be(Sign::Plus, args.bytes.as_bytes()), - (ArgByteOrder::Little, true) => BigInt::from_signed_bytes_le(args.bytes.as_bytes()), - (ArgByteOrder::Little, false) => { - BigInt::from_bytes_le(Sign::Plus, args.bytes.as_bytes()) - } + (ArgByteOrder::Big, true) => BigInt::from_signed_bytes_be(&bytes), + (ArgByteOrder::Big, false) => BigInt::from_bytes_be(Sign::Plus, &bytes), + (ArgByteOrder::Little, true) => BigInt::from_signed_bytes_le(&bytes), + (ArgByteOrder::Little, false) => BigInt::from_bytes_le(Sign::Plus, &bytes), }; Self::with_value(cls, value, vm) } @@ -784,7 +802,7 @@ pub(crate) struct IntOptions { #[derive(FromArgs)] struct IntFromByteArgs { - bytes: PyBytesInner, + bytes: PyObjectRef, #[pyarg(any, default = ArgByteOrder::Big)] byteorder: ArgByteOrder, #[pyarg(named, optional)] diff --git a/crates/vm/src/builtins/interpolation.rs b/crates/vm/src/builtins/interpolation.rs index 0ae1b33120b..5d5f3774640 100644 --- a/crates/vm/src/builtins/interpolation.rs +++ b/crates/vm/src/builtins/interpolation.rs @@ -144,7 +144,11 @@ impl PyInterpolation { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } diff --git a/crates/vm/src/builtins/list.rs b/crates/vm/src/builtins/list.rs index c2059e28806..fe674a45821 100644 --- a/crates/vm/src/builtins/list.rs +++ b/crates/vm/src/builtins/list.rs @@ -421,7 +421,11 @@ impl PyList { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/mappingproxy.rs b/crates/vm/src/builtins/mappingproxy.rs index c8b891f7972..dd8c689facb 100644 --- a/crates/vm/src/builtins/mappingproxy.rs +++ b/crates/vm/src/builtins/mappingproxy.rs @@ -177,7 +177,11 @@ impl PyMappingProxy { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } diff --git a/crates/vm/src/builtins/memory.rs b/crates/vm/src/builtins/memory.rs index ee5a071287b..a4c29fe443c 100644 --- a/crates/vm/src/builtins/memory.rs +++ b/crates/vm/src/builtins/memory.rs @@ -1,6 +1,6 @@ use super::{ PositionIterInternal, PyBytes, PyBytesRef, PyGenericAlias, PyInt, PyListRef, PySlice, PyStr, - PyTuple, PyTupleRef, PyType, PyTypeRef, PyUtf8StrRef, iter::builtins_iter, + PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, PyUtf8StrRef, iter::builtins_iter, }; use crate::common::lock::LazyLock; use crate::{ @@ -8,7 +8,7 @@ use crate::{ TryFromBorrowedObject, TryFromObject, VirtualMachine, atomic_func, buffer::FormatSpec, bytes_inner::{ByteInnerHexOptions, bytes_to_hex}, - class::PyClassImpl, + class::{PyClassImpl, StaticType}, common::{ borrow::{BorrowedValue, BorrowedValueMut}, hash::PyHash, @@ -16,9 +16,9 @@ use crate::{ }, convert::ToPyObject, function::Either, - function::{FuncArgs, OptionalArg, PyComparisonValue}, + function::{ArgIndex, FuncArgs, OptionalArg, PyComparisonValue}, protocol::{ - BufferDescriptor, BufferMethods, PyBuffer, PyIterReturn, PyMappingMethods, + BufferDescriptor, BufferFlags, BufferMethods, PyBuffer, PyIterReturn, PyMappingMethods, PySequenceMethods, VecBuffer, }, sliceable::SequenceIndexOp, @@ -27,7 +27,7 @@ use crate::{ PyComparisonOp, Representable, SelfIter, }, }; -use core::{cmp::Ordering, fmt::Debug, mem::ManuallyDrop, ops::Range}; +use core::{cmp::Ordering, fmt::Debug, ops::Range}; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; use rustpython_common::lock::PyMutex; @@ -37,18 +37,24 @@ pub struct PyMemoryViewNewArgs { object: PyObjectRef, } +#[derive(FromArgs)] +struct PyMemoryViewFromFlagsArgs { + object: PyObjectRef, + flags: ArgIndex, +} + #[pyclass(module = false, name = "memoryview")] #[derive(Debug)] pub struct PyMemoryView { - // avoid double release when memoryview had released the buffer before drop - buffer: ManuallyDrop, + /// One share of the acquisition this view is looking at, given up when the + /// view is released or dropped. + buffer: PyBuffer, // the released memoryview does not mean the buffer is destroyed // because the possible another memoryview is viewing from it released: AtomicCell, - // start does NOT mean the bytes before start will not be visited, - // it means the point we starting to get the absolute position via - // the needle - start: usize, + /// Forbids handing out anything that outlives this view, for the window + /// passed to `__release_buffer__`. + restricted: AtomicCell, format_spec: FormatSpec, // memoryview's options could be different from buffer's options desc: BufferDescriptor, @@ -71,13 +77,54 @@ impl PyMemoryView { FormatSpec::parse(format.as_bytes(), vm) } + /// The single native format character a cast is allowed to name, with an + /// optional `@` in front of it. get_native_fmtchar + fn native_fmtchar(format: &str) -> Option { + let format = format.strip_prefix('@').unwrap_or(format); + let [c] = *format.as_bytes() else { + return None; + }; + matches!( + c, + b'c' | b'b' + | b'B' + | b'h' + | b'H' + | b'i' + | b'I' + | b'l' + | b'L' + | b'q' + | b'Q' + | b'n' + | b'N' + | b'f' + | b'd' + | b'e' + | b'?' + | b'P' + ) + .then_some(c) + } + /// this should be the main entrance to create the memoryview /// to avoid the chained memoryview pub fn from_object(obj: &PyObject, vm: &VirtualMachine) -> PyResult { + Self::from_object_with_flags(obj, BufferFlags::FULL_RO, vm) + } + + // PyMemoryView_FromObjectAndFlags + pub fn from_object_with_flags( + obj: &PyObject, + flags: BufferFlags, + vm: &VirtualMachine, + ) -> PyResult { if let Some(other) = obj.downcast_ref::() { + other.try_not_released(vm)?; + other.try_not_restricted(vm)?; Ok(other.new_view()) } else { - let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?; + let buffer = PyBuffer::from_object(vm, obj, flags)?; Self::from_buffer(buffer, vm) } } @@ -93,9 +140,9 @@ impl PyMemoryView { let desc = buffer.desc.clone(); Ok(Self { - buffer: ManuallyDrop::new(buffer), + buffer, released: AtomicCell::new(false), - start: 0, + restricted: AtomicCell::new(false), format_spec, desc, hash: OnceCell::new(), @@ -120,16 +167,34 @@ impl PyMemoryView { /// this should be the only way to create a memoryview from another memoryview. #[must_use] pub fn new_view(&self) -> Self { - let zelf = Self { + Self { buffer: self.buffer.clone(), released: AtomicCell::new(false), - start: self.start, + restricted: AtomicCell::new(false), format_spec: self.format_spec.clone(), desc: self.desc.clone(), hash: OnceCell::new(), - }; - zelf.buffer.retain(); - zelf + } + } + + /// A view for a temporary that never reaches Python. It counts as no export, + /// so the exporter stays exactly as resizable as it already was, the way a + /// `Py_buffer dest = *view` copy does. + #[must_use] + fn borrowed_view(&self) -> Self { + Self { + buffer: self.buffer.detached(), + released: AtomicCell::new(false), + restricted: AtomicCell::new(false), + format_spec: self.format_spec.clone(), + desc: self.desc.clone(), + hash: OnceCell::new(), + } + } + + /// The object this view looks at, whose storage it borrows. + pub fn viewed_object(&self) -> &PyObject { + &self.buffer.obj } fn try_not_released(&self, vm: &VirtualMachine) -> PyResult<()> { @@ -140,22 +205,83 @@ impl PyMemoryView { } } + fn try_not_restricted(&self, vm: &VirtualMachine) -> PyResult<()> { + if self.restricted.load() { + Err(vm.new_value_error("cannot create new view on restricted memoryview")) + } else { + Ok(()) + } + } + + fn try_usable(&self, vm: &VirtualMachine) -> PyResult<()> { + self.try_not_released(vm)?; + self.try_not_restricted(vm) + } + + /// Reject a request this view cannot serve. memory_getbuf + fn check_buffer_request(&self, flags: BufferFlags, vm: &VirtualMachine) -> PyResult<()> { + let c_contiguous = self.desc.is_contiguous(); + flags.check_writable( + self.desc.readonly, + "memoryview: underlying buffer is not writable", + vm, + )?; + if flags.contains(BufferFlags::C_CONTIGUOUS) && !c_contiguous { + return Err(vm.new_buffer_error("memoryview: underlying buffer is not C-contiguous")); + } + if flags.contains(BufferFlags::F_CONTIGUOUS) && !self.desc.is_fortran_contiguous() { + return Err( + vm.new_buffer_error("memoryview: underlying buffer is not Fortran contiguous") + ); + } + if flags.contains(BufferFlags::ANY_CONTIGUOUS) + && !c_contiguous + && !self.desc.is_fortran_contiguous() + { + return Err(vm.new_buffer_error("memoryview: underlying buffer is not contiguous")); + } + // No exporter here produces a suboffset, so this is a guard rather than a + // reachable rejection. + if !flags.contains(BufferFlags::INDIRECT) && self.desc.has_suboffsets() { + return Err(vm.new_buffer_error("memoryview: underlying buffer requires suboffsets")); + } + if !flags.contains(BufferFlags::STRIDES) && !c_contiguous { + return Err(vm.new_buffer_error("memoryview: underlying buffer is not C-contiguous")); + } + if !flags.contains(BufferFlags::ND) && flags.intersects(BufferFlags::FORMAT) { + return Err(vm.new_buffer_error( + "memoryview: cannot cast to unsigned bytes if the format flag is present", + )); + } + Ok(()) + } + + /// The descriptor this view exports for `flags`, or an error if it cannot + /// serve the request. memory_getbuf + fn requested_desc( + &self, + flags: BufferFlags, + vm: &VirtualMachine, + ) -> PyResult { + self.check_buffer_request(flags, vm)?; + Ok(self.desc.projected(flags)) + } + fn getitem_by_idx(&self, i: isize, vm: &VirtualMachine) -> PyResult { if self.desc.ndim() != 1 { return Err( vm.new_not_implemented_error("multi-dimensional sub-views are not implemented") ); } - let (shape, stride, suboffset) = self.desc.dim_desc[0]; + let (shape, _, _) = self.desc.dim_desc[0]; let index = i .wrapped_at(shape) .ok_or_else(|| vm.new_index_error("index out of range"))?; - let index = index as isize * stride + suboffset; - let pos = (index + self.start as isize) as usize; - self.unpack_single(pos, vm) + self.unpack_single(self.desc.fast_position(&[index]) as usize, vm) } fn getitem_by_slice(&self, slice: &PySlice, vm: &VirtualMachine) -> PyResult { + self.try_not_restricted(vm)?; let mut other = self.new_view(); other.init_slice(slice, 0, vm)?; other.init_len(); @@ -166,20 +292,22 @@ impl PyMemoryView { fn getitem_by_multi_idx(&self, indexes: &[isize], vm: &VirtualMachine) -> PyResult { let pos = self.pos_from_multi_index(indexes, vm)?; let bytes = self.buffer.obj_bytes(); - format_unpack(&self.format_spec, &bytes[pos..pos + self.desc.itemsize], vm) + format_unpack( + &self.format_spec, + &bytes[pos..pos + self.format_spec.size()], + vm, + ) } fn setitem_by_idx(&self, i: isize, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { if self.desc.ndim() != 1 { return Err(vm.new_not_implemented_error("sub-views are not implemented")); } - let (shape, stride, suboffset) = self.desc.dim_desc[0]; + let (shape, _, _) = self.desc.dim_desc[0]; let index = i .wrapped_at(shape) .ok_or_else(|| vm.new_index_error("index out of range"))?; - let index = index as isize * stride + suboffset; - let pos = (index + self.start as isize) as usize; - self.pack_single(pos, value, vm) + self.pack_single(self.desc.fast_position(&[index]) as usize, value, vm) } fn setitem_by_multi_idx( @@ -193,7 +321,9 @@ impl PyMemoryView { } fn pack_single(&self, pos: usize, value: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let mut bytes = self.buffer.obj_bytes_mut(); + // The value is converted before the destination is borrowed, because the + // conversion runs `__index__` or `__float__`, which can read or write the + // same buffer. // TODO: Optimize let data = self.format_spec.pack(vec![value], vm).map_err(|_| { vm.new_type_error(format!( @@ -201,15 +331,23 @@ impl PyMemoryView { self.desc.format )) })?; - bytes[pos..pos + self.desc.itemsize].copy_from_slice(&data); + // The conversion, and the index that produced `pos`, could have released + // the view; `pos` addresses a buffer that is no longer there. + // CHECK_RELEASED_INT_AGAIN + self.try_not_released(vm)?; + let mut bytes = self.buffer.obj_bytes_mut(); + bytes[pos..pos + self.format_spec.size()].copy_from_slice(&data); Ok(()) } fn unpack_single(&self, pos: usize, vm: &VirtualMachine) -> PyResult { + // The index that produced `pos` could have released the view. + // CHECK_RELEASED_AGAIN + self.try_not_released(vm)?; let bytes = self.buffer.obj_bytes(); // TODO: Optimize self.format_spec - .unpack(&bytes[pos..pos + self.desc.itemsize], vm) + .unpack(&bytes[pos..pos + self.format_spec.size()], vm) .map(|x| { if x.len() == 1 { x[0].to_owned() @@ -234,9 +372,7 @@ impl PyMemoryView { Ordering::Equal => (), } - let pos = self.desc.position(indexes, vm)?; - let pos = (pos + self.start as isize) as usize; - Ok(pos) + Ok(self.desc.position(indexes, vm)? as usize) } fn init_len(&mut self) { @@ -244,50 +380,38 @@ impl PyMemoryView { self.desc.len = product * self.desc.itemsize; } + /// Move this view by `delta` bytes. The offset moves, unless a dimension + /// outside `dim` is reached through a pointer, in which case its suboffset + /// does. + fn adjust_position(&mut self, dim: usize, delta: isize) { + match self.desc.dim_desc[..dim] + .iter() + .rposition(|&(_, _, suboffset)| suboffset != 0) + { + Some(n) => self.desc.dim_desc[n].2 += delta, + None => self.desc.offset += delta, + } + } + fn init_range(&mut self, range: Range, dim: usize) { let (shape, stride, _) = self.desc.dim_desc[dim]; debug_assert!(shape >= range.len()); - let mut is_adjusted = false; - for (_, _, suboffset) in self.desc.dim_desc.iter_mut().rev() { - if *suboffset != 0 { - *suboffset += stride * range.start as isize; - is_adjusted = true; - break; - } - } - if !is_adjusted { - // no suboffset set, stride must be positive - self.start += stride as usize * range.start; - } - let new_len = range.len(); - self.desc.dim_desc[dim].0 = new_len; + self.adjust_position(dim, stride * range.start as isize); + self.desc.dim_desc[dim].0 = range.len(); } + // init_slice fn init_slice(&mut self, slice: &PySlice, dim: usize, vm: &VirtualMachine) -> PyResult<()> { let (shape, stride, _) = self.desc.dim_desc[dim]; let slice = slice.to_saturated(vm)?; - let (range, step, slice_len) = slice.adjust_indices(shape); - - let mut is_adjusted_suboffset = false; - for (_, _, suboffset) in self.desc.dim_desc.iter_mut().rev() { - if *suboffset != 0 { - *suboffset += stride * range.start as isize; - is_adjusted_suboffset = true; - break; - } - } - if !is_adjusted_suboffset { - // no suboffset set, stride must be positive - self.start += stride as usize - * if step.is_negative() { - range.end - 1 - } else { - range.start - }; - } + let (start, slice_len) = slice.adjust_indices_start(shape); + + // Repeated slicing multiplies the stride by the step every time, which + // overflows after about twenty rounds; C wraps there and so does this. + self.adjust_position(dim, stride.wrapping_mul(start)); self.desc.dim_desc[dim].0 = slice_len; - self.desc.dim_desc[dim].1 *= step; + self.desc.dim_desc[dim].1 = stride.wrapping_mul(slice.step()); Ok(()) } @@ -303,10 +427,12 @@ impl PyMemoryView { if dim + 1 == self.desc.ndim() { let mut v = Vec::with_capacity(shape); for _ in 0..shape { - let pos = index + suboffset; - let pos = (pos + self.start as isize) as usize; - let obj = - format_unpack(&self.format_spec, &bytes[pos..pos + self.desc.itemsize], vm)?; + let pos = (index + suboffset) as usize; + let obj = format_unpack( + &self.format_spec, + &bytes[pos..pos + self.format_spec.size()], + vm, + )?; v.push(obj); index += stride; } @@ -330,29 +456,42 @@ impl PyMemoryView { return Ok(false); } - if let Some(other) = other.downcast_ref::() - && other.released.load() - { - return Ok(false); - } - - let other = match PyBuffer::try_from_borrowed_object(vm, other) { - Ok(buf) => buf, - Err(_) => return Ok(false), + let other = if let Some(mv) = other.downcast_ref::() { + if mv.released.load() { + return Ok(false); + } + // Another view's buffer is read where it lies rather than acquired, + // so that a restricted view still compares. memory_richcompare + let mut view = mv.buffer.detached(); + view.desc = mv.desc.clone(); + view + } else { + match PyBuffer::try_from_borrowed_object(vm, other) { + Ok(buf) => buf, + Err(_) => return Ok(false), + } }; if !is_equiv_shape(&zelf.desc, &other.desc) { return Ok(false); } - let a_itemsize = zelf.desc.itemsize; - let b_itemsize = other.desc.itemsize; let a_format_spec = &zelf.format_spec; let b_format_spec = &Self::parse_format(&other.desc.format, vm)?; + // An element is as wide as its format, which a projected descriptor can + // make narrower than the item size it steps by. + let a_itemsize = a_format_spec.size(); + let b_itemsize = b_format_spec.size(); if zelf.desc.ndim() == 0 { - let a_val = format_unpack(a_format_spec, &zelf.buffer.obj_bytes()[..a_itemsize], vm)?; - let b_val = format_unpack(b_format_spec, &other.obj_bytes()[..b_itemsize], vm)?; + let a_pos = zelf.desc.offset as usize; + let b_pos = other.desc.offset as usize; + let a_bytes = zelf.buffer.obj_bytes(); + let a_val = format_unpack(a_format_spec, &a_bytes[a_pos..a_pos + a_itemsize], vm)?; + drop(a_bytes); + let b_bytes = other.obj_bytes(); + let b_val = format_unpack(b_format_spec, &b_bytes[b_pos..b_pos + b_itemsize], vm)?; + drop(b_bytes); return vm.bool_eq(&a_val, &b_val); } @@ -361,9 +500,8 @@ impl PyMemoryView { let a_bytes = zelf.buffer.obj_bytes(); let b_bytes = other.obj_bytes(); zelf.desc.zip_eq(&other.desc, false, |a_range, b_range| { - let a_range = (a_range.start + zelf.start as isize) as usize - ..(a_range.end + zelf.start as isize) as usize; - let b_range = b_range.start as usize..b_range.end as usize; + let a_range = a_range.start as usize..a_range.start as usize + a_itemsize; + let b_range = b_range.start as usize..b_range.start as usize + b_itemsize; let a_val = match format_unpack(a_format_spec, &a_bytes[a_range], vm) { Ok(val) => val, Err(e) => { @@ -384,39 +522,17 @@ impl PyMemoryView { ret } - fn obj_bytes(&self) -> BorrowedValue<'_, [u8]> { - if self.desc.is_contiguous() { - BorrowedValue::map(self.buffer.obj_bytes(), |x| { - &x[self.start..self.start + self.desc.len] - }) - } else { - BorrowedValue::map(self.buffer.obj_bytes(), |x| &x[self.start..]) - } - } - - fn obj_bytes_mut(&self) -> BorrowedValueMut<'_, [u8]> { - if self.desc.is_contiguous() { - BorrowedValueMut::map(self.buffer.obj_bytes_mut(), |x| { - &mut x[self.start..self.start + self.desc.len] - }) - } else { - BorrowedValueMut::map(self.buffer.obj_bytes_mut(), |x| &mut x[self.start..]) - } - } - fn as_contiguous(&self) -> Option> { self.desc.is_contiguous().then(|| { - BorrowedValue::map(self.buffer.obj_bytes(), |x| { - &x[self.start..self.start + self.desc.len] - }) + let range = self.desc.contiguous_range(); + BorrowedValue::map(self.buffer.obj_bytes(), |x| &x[range]) }) } fn _as_contiguous_mut(&self) -> Option> { self.desc.is_contiguous().then(|| { - BorrowedValueMut::map(self.buffer.obj_bytes_mut(), |x| { - &mut x[self.start..self.start + self.desc.len] - }) + let range = self.desc.contiguous_range(); + BorrowedValueMut::map(self.buffer.obj_bytes_mut(), |x| &mut x[range]) }) } @@ -427,9 +543,7 @@ impl PyMemoryView { buf.reserve(self.desc.len); let bytes = &*self.buffer.obj_bytes(); self.desc.for_each_segment(true, |range| { - let start = (range.start + self.start as isize) as usize; - let end = (range.end + self.start as isize) as usize; - buf.extend_from_slice(&bytes[start..end]); + buf.extend_from_slice(&bytes[range.start as usize..range.end as usize]); }) } } @@ -454,27 +568,7 @@ impl PyMemoryView { let mut data = vec![]; self.append_to(&mut data); - if self.desc.ndim() == 0 { - return VecBuffer::from(data) - .into_ref(&vm.ctx) - .into_pybuffer_with_descriptor(self.desc.clone()); - } - - let mut dim_desc = self.desc.dim_desc.clone(); - dim_desc.last_mut().unwrap().1 = self.desc.itemsize as isize; - dim_desc.last_mut().unwrap().2 = 0; - for i in (0..dim_desc.len() - 1).rev() { - dim_desc[i].1 = dim_desc[i + 1].1 * dim_desc[i + 1].0 as isize; - dim_desc[i].2 = 0; - } - - let desc = BufferDescriptor { - len: self.desc.len, - readonly: self.desc.readonly, - itemsize: self.desc.itemsize, - format: self.desc.format.clone(), - dim_desc, - }; + let desc = self.desc.contiguous(); VecBuffer::from(data) .into_ref(&vm.ctx) @@ -493,7 +587,7 @@ impl Py { return Err(vm.new_not_implemented_error("sub-view are not implemented")); } - let mut dest = self.new_view(); + let mut dest = self.borrowed_view(); dest.init_slice(slice, 0, vm)?; dest.init_len(); @@ -508,15 +602,11 @@ impl Py { }; }; - let src = if let Some(src) = src.downcast_ref::() { - if self.buffer.obj.is(&src.buffer.obj) { - src.to_contiguous(vm) - } else { - AsBuffer::as_buffer(src, vm)? - } - } else { - PyBuffer::try_from_object(vm, src)? - }; + // PyObject_GetBuffer(value, &src, PyBUF_FULL_RO) + let src = PyBuffer::try_from_object(vm, src)?; + // Acquiring the source ran `__buffer__`, which can release this view. + // copy_single: CHECK_RELEASED_INT_AGAIN + self.try_not_released(vm)?; if !is_equiv_structure(&src.desc, &dest.desc) { return Err(vm.new_value_error( @@ -524,11 +614,21 @@ impl Py { )); } + // copy_buffer reads the source as it stood before the copy began, which an + // overlapping assignment depends on and which also keeps the two borrows + // below off the same storage. + let src = if root_exporter(&src).is(&root_exporter(&dest.buffer)) { + let owned = src.to_contiguous(vm); + drop(src); + owned + } else { + src + }; + let mut bytes_mut = dest.buffer.obj_bytes_mut(); let src_bytes = src.obj_bytes(); dest.desc.zip_eq(&src.desc, true, |a_range, b_range| { - let a_range = (a_range.start + dest.start as isize) as usize - ..(a_range.end + dest.start as isize) as usize; + let a_range = a_range.start as usize..a_range.end as usize; let b_range = b_range.start as usize..b_range.end as usize; bytes_mut[a_range].copy_from_slice(&src_bytes[b_range]); false @@ -554,10 +654,25 @@ impl Py { )] impl PyMemoryView { #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } + #[pyclassmethod] + fn _from_flags( + _cls: PyTypeRef, + args: PyMemoryViewFromFlagsArgs, + vm: &VirtualMachine, + ) -> PyResult> { + let flags = + BufferFlags::from_bits_retain(args.flags.as_ref().try_to_primitive::(vm)? as u32); + Self::from_object_with_flags(&args.object, flags, vm).map(|mv| mv.into_ref(&vm.ctx)) + } + #[pymethod] pub fn release(&self) { if self.released.compare_exchange(false, true).is_ok() { @@ -567,7 +682,14 @@ impl PyMemoryView { #[pygetset] fn obj(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm).map(|_| self.buffer.obj.clone()) + self.try_not_released(vm)?; + // A window over a buffer being released exposes no exporter, like a + // Py_buffer whose obj is NULL. + Ok(if self.buffer.obj.downcastable::() { + vm.ctx.none() + } else { + self.buffer.obj.clone() + }) } #[pygetset] @@ -643,7 +765,8 @@ impl PyMemoryView { #[pygetset] fn contiguous(&self, vm: &VirtualMachine) -> PyResult { - self.try_not_released(vm).map(|_| self.desc.is_contiguous()) + self.try_not_released(vm) + .map(|_| self.desc.is_contiguous() || self.desc.is_fortran_contiguous()) } #[pygetset] @@ -653,9 +776,8 @@ impl PyMemoryView { #[pygetset] fn f_contiguous(&self, vm: &VirtualMachine) -> PyResult { - // TODO: column-major order self.try_not_released(vm) - .map(|_| self.desc.ndim() <= 1 && self.desc.is_contiguous()) + .map(|_| self.desc.is_fortran_contiguous()) } #[pymethod] @@ -678,7 +800,7 @@ impl PyMemoryView { if let Some(tuple) = needle.downcast_ref::() && tuple.is_empty() { - return zelf.unpack_single(0, vm); + return zelf.unpack_single(zelf.desc.offset as usize, vm); } return Err(vm.new_type_error("invalid indexing of 0-dim memory")); } @@ -709,30 +831,56 @@ impl PyMemoryView { } #[pymethod] - fn tobytes(&self, vm: &VirtualMachine) -> PyResult { + fn tobytes(&self, args: ToBytesArgs, vm: &VirtualMachine) -> PyResult { self.try_not_released(vm)?; + let order = match &args.order { + None => Order::C, + Some(order) => match order.to_str() { + Some("C") => Order::C, + Some("F") => Order::Fortran, + Some("A") => Order::Any, + _ => return Err(vm.new_value_error("order must be 'C', 'F' or 'A'")), + }, + }; + let mut v = vec![]; - self.append_to(&mut v); + // 'A' asks for the memory as it is laid out, which is what appending a + // contiguous view does. Only a Fortran walk of a view that is not + // already Fortran-contiguous reorders anything, and a view of fewer + // than two dimensions has one layout under either name. + if order == Order::Fortran && self.desc.ndim() > 1 { + v.reserve(self.desc.len); + let bytes = &*self.buffer.obj_bytes(); + self.desc.for_each_segment_fortran(|range| { + v.extend_from_slice(&bytes[range.start as usize..range.end as usize]); + }); + } else { + self.append_to(&mut v); + } Ok(PyBytes::from(v).into_ref(&vm.ctx)) } #[pymethod] - fn tolist(&self, vm: &VirtualMachine) -> PyResult { + // memory_tolist + fn tolist(&self, vm: &VirtualMachine) -> PyResult { self.try_not_released(vm)?; let bytes = self.buffer.obj_bytes(); if self.desc.ndim() == 0 { - return Ok(vm.ctx.new_list(vec![format_unpack( + // A 0-dim view holds one element, which is what it unpacks to. + let pos = self.desc.offset as usize; + return format_unpack( &self.format_spec, - &bytes[..self.desc.itemsize], + &bytes[pos..pos + self.format_spec.size()], vm, - )?])); + ); } - self._to_list(&bytes, 0, 0, vm) + self._to_list(&bytes, self.desc.offset, 0, vm) + .map(Into::into) } #[pymethod] fn toreadonly(&self, vm: &VirtualMachine) -> PyResult> { - self.try_not_released(vm)?; + self.try_usable(vm)?; let mut other = self.new_view(); other.desc.readonly = true; Ok(other.into_ref(&vm.ctx)) @@ -804,31 +952,45 @@ impl PyMemoryView { fn cast_to_1d(&self, format: PyUtf8StrRef, vm: &VirtualMachine) -> PyResult { let format_str = format.as_str(); + let Some(dest_char) = Self::native_fmtchar(format_str) else { + return Err(vm.new_value_error( + "memoryview: destination format must be a native single character format prefixed with an optional '@'", + )); + }; + // One side has to be bytes. Casting between two item types would + // reinterpret the items rather than re-divide the memory, and the + // source items were written by something that chose their type. + let source_is_bytes = Self::native_fmtchar(&self.desc.format).is_some_and(is_byte_fmtchar); + if !source_is_bytes && !is_byte_fmtchar(dest_char) { + return Err(vm.new_type_error("memoryview: cannot cast between two non-byte formats")); + } let format_spec = Self::parse_format(format_str, vm)?; let itemsize = format_spec.size(); if !self.desc.len.is_multiple_of(itemsize) { return Err(vm.new_type_error("memoryview: length is not a multiple of itemsize")); } - Ok(Self { + let zelf = Self { buffer: self.buffer.clone(), released: AtomicCell::new(false), - start: self.start, + restricted: AtomicCell::new(false), format_spec, desc: BufferDescriptor { len: self.desc.len, + offset: self.desc.offset, readonly: self.desc.readonly, itemsize, format: format_str.to_owned().into(), dim_desc: vec![(self.desc.len / itemsize, itemsize as isize, 0)], }, hash: OnceCell::new(), - }) + }; + Ok(zelf) } #[pymethod] fn cast(&self, args: CastArgs, vm: &VirtualMachine) -> PyResult> { - self.try_not_released(vm)?; + self.try_usable(vm)?; if !self.desc.is_contiguous() { return Err(vm.new_type_error("memoryview: casts are restricted to C-contiguous views")); } @@ -866,10 +1028,14 @@ impl PyMemoryView { let mut other = self.cast_to_1d(format, vm)?; let itemsize = other.desc.itemsize; - // 0 ndim is single item + // 0 ndim is single item, so the buffer has to be that one item if shape_ndim == 0 { + if itemsize != other.desc.len { + return Err( + vm.new_type_error("memoryview: product(shape) * itemsize != buffer size") + ); + } other.desc.dim_desc = vec![]; - other.desc.len = itemsize; return Ok(other.into_ref(&vm.ctx)); } @@ -877,7 +1043,19 @@ impl PyMemoryView { let mut dim_descriptor = Vec::with_capacity(shape_ndim); for x in shape { - let x = usize::try_from_borrowed_object(vm, x)?; + let x = x + .downcast_ref::() + .ok_or_else(|| { + vm.new_type_error("memoryview.cast(): elements of shape must be integers") + })? + .try_to_primitive::(vm) + .ok() + .filter(|x| *x > 0) + .ok_or_else(|| { + vm.new_value_error( + "memoryview.cast(): elements of shape must be integers > 0", + ) + })?; if x > isize::MAX as usize / product_shape { return Err(vm.new_value_error("memoryview.cast(): product(shape) > SSIZE_MAX")); @@ -925,11 +1103,11 @@ impl Py { if self.desc.ndim() == 0 { // TODO: merge branches when we got conditional if let if needle.is(&vm.ctx.ellipsis) { - return self.pack_single(0, value, vm); + return self.pack_single(self.desc.offset as usize, value, vm); } else if let Some(tuple) = needle.downcast_ref::() && tuple.is_empty() { - return self.pack_single(0, value, vm); + return self.pack_single(self.desc.offset as usize, value, vm); } return Err(vm.new_type_error("invalid indexing of 0-dim memory")); } @@ -951,6 +1129,20 @@ impl Py { } } +#[derive(FromArgs)] +struct ToBytesArgs { + #[pyarg(any, default)] + order: Option, +} + +/// The layout a copy of a view is written in. +#[derive(PartialEq, Eq)] +enum Order { + C, + Fortran, + Any, +} + #[derive(FromArgs)] struct CastArgs { #[pyarg(any)] @@ -998,33 +1190,43 @@ impl TryFromObject for SubscriptNeedle { } static BUFFER_METHODS: BufferMethods = BufferMethods { - obj_bytes: |buffer| buffer.obj_as::().obj_bytes(), - obj_bytes_mut: |buffer| buffer.obj_as::().obj_bytes_mut(), - release: |buffer| buffer.obj_as::().buffer.release(), - retain: |buffer| buffer.obj_as::().buffer.retain(), + obj_bytes: |buffer| buffer.obj_as::().buffer.obj_bytes(), + obj_bytes_mut: |buffer| buffer.obj_as::().buffer.obj_bytes_mut(), + // memory_releasebuf / memory_getbuf: a consumer's export of this view is a + // share of the acquisition the view is looking at. + release: |buffer| buffer.obj_as::().buffer.release_share(), + retain: |buffer| buffer.obj_as::().buffer.retain_share(), }; impl AsBuffer for PyMemoryView { - fn as_buffer(zelf: &Py, vm: &VirtualMachine) -> PyResult { - if zelf.released.load() { - Err(vm.new_value_error("operation forbidden on released memoryview object")) - } else { - Ok(PyBuffer::new( - zelf.to_owned().into(), - zelf.desc.clone(), - &BUFFER_METHODS, - )) - } + const RELEASE_BUFFER: bool = true; + + // memory_getbuf + fn slot_as_buffer( + zelf: &PyObject, + flags: BufferFlags, + vm: &VirtualMachine, + ) -> PyResult { + let zelf = zelf + .downcast_ref::() + .ok_or_else(|| vm.new_type_error("unexpected payload for as_buffer"))?; + zelf.try_usable(vm)?; + Ok(PyBuffer::new( + zelf.to_owned().into(), + zelf.requested_desc(flags, vm)?, + &BUFFER_METHODS, + )) } -} -impl Drop for PyMemoryView { - fn drop(&mut self) { - if self.released.load() { - unsafe { self.buffer.drop_without_release() }; - } else { - unsafe { ManuallyDrop::drop(&mut self.buffer) }; - } + fn as_buffer(zelf: &Py, vm: &VirtualMachine) -> PyResult { + zelf.try_usable(vm)?; + // memory_getbuf: *view = *base — the descriptor already says where the + // view starts. + Ok(PyBuffer::new( + zelf.to_owned().into(), + zelf.desc.clone(), + &BUFFER_METHODS, + )) } } @@ -1099,6 +1301,13 @@ impl Hashable for PyMemoryView { if !zelf.desc.readonly { return Err(vm.new_value_error("cannot hash writable memoryview object")); } + // The hash is over the bytes, so it agrees with the hash of the same + // bytes only where an item is a byte. + if !Self::native_fmtchar(&zelf.desc.format).is_some_and(is_byte_fmtchar) { + return Err( + vm.new_value_error("memoryview: hashing is restricted to formats 'B', 'b' or 'c'") + ); + } let val = zelf.contiguous_or_collect(|bytes| vm.state.hash_secret.hash_bytes(bytes)); let _ = zelf.hash.set(val); Ok(*zelf.hash.get().unwrap()) @@ -1127,6 +1336,211 @@ impl Representable for PyMemoryView { pub(crate) fn init(ctx: &'static Context) { PyMemoryView::extend_class(ctx, ctx.types.memoryview_type); PyMemoryViewIterator::extend_class(ctx, ctx.types.memoryviewiterator_type); + let wrapper_type = PyBufferWrapper::init_builtin_type(); + // bufferwrapper_as_buffer: bf_releasebuffer and no bf_getbuffer, so the type + // has `__release_buffer__` but no `__buffer__`. + wrapper_type.slots.has_release_buffer.store(true); + PyBufferWrapper::extend_class(ctx, wrapper_type); + PyBufferWindow::extend_class(ctx, PyBufferWindow::init_builtin_type()); +} + +#[pyclass(module = false, name = "_buffer_wrapper")] +#[derive(Debug)] +struct PyBufferWrapper { + // bw->obj: the object whose `__buffer__` produced the view + exporter: PyObjectRef, + // bw->mv: the memoryview `__buffer__` returned, dropped with the last export + returned_mv: PyMutex>>, + /// Memory of `returned_mv`, held on behalf of every live export. The wrapper + /// forwards shares of it rather than owning one. + view: PyBuffer, + /// Exports handed out for this wrapper; the wrapper is spent at zero. + exports: AtomicCell, +} + +impl PyPayload for PyBufferWrapper { + fn class(_ctx: &Context) -> &'static Py { + Self::static_type() + } +} + +#[pyclass(flags(DISALLOW_INSTANTIATION))] +impl PyBufferWrapper {} + +static BUFFER_WRAPPER_METHODS: BufferMethods = BufferMethods { + obj_bytes: |buffer| buffer.obj_as::().view.obj_bytes(), + obj_bytes_mut: |buffer| buffer.obj_as::().view.obj_bytes_mut(), + retain: |buffer| { + let wrapper = buffer.obj_as::(); + wrapper.exports.fetch_add(1); + wrapper.view.retain_share(); + }, + // bufferwrapper_releasebuf + release: |buffer| { + let wrapper = buffer.obj_as::(); + wrapper.view.release_share(); + if wrapper.exports.fetch_sub(1) != 1 { + return; + } + let Some(mv) = wrapper.returned_mv.lock().take() else { + return; + }; + // A native release runs when the memoryview itself is torn down; only a + // Python-level hook on a foreign exporter has to be called here. + if !mv.buffer.obj.is(&wrapper.exporter) + && wrapper.exporter.class().slots.python_release_buffer.load() + { + call_python_release_buffer(&wrapper.exporter, mv.clone()); + } + // Py_CLEAR(bw->mv): the view outlives this only if user code kept it. + drop(mv); + }, +}; + +// Read-only window over an exporter, handed to `__release_buffer__`. It owns no +// export, like a `Py_buffer` whose `obj` is NULL, so releasing it is inert and +// cannot recurse back into the hook. +#[pyclass(module = false, name = "_buffer_window")] +#[derive(Debug)] +struct PyBufferWindow { + source: PyBuffer, +} + +impl PyPayload for PyBufferWindow { + fn class(_ctx: &Context) -> &'static Py { + Self::static_type() + } +} + +#[pyclass(flags(DISALLOW_INSTANTIATION))] +impl PyBufferWindow {} + +static BUFFER_WINDOW_METHODS: BufferMethods = BufferMethods { + obj_bytes: |buffer| buffer.obj_as::().source.obj_bytes(), + obj_bytes_mut: |buffer| buffer.obj_as::().source.obj_bytes_mut(), + retain: |_buffer| {}, + release: |_buffer| {}, +}; + +/// The object that ultimately owns the bytes a buffer reads, seen through the +/// payloads that only forward to another export: a view, the wrapper holding what +/// a `__buffer__` returned, and the window handed to `__release_buffer__`. +/// +/// Two buffers that resolve to the same object address the same storage, so +/// borrowing one for writing while the other is borrowed for reading would +/// deadlock on it. +fn root_exporter(buffer: &PyBuffer) -> PyObjectRef { + let mut obj = buffer.obj.clone(); + loop { + let next = if let Some(view) = obj.downcast_ref::() { + view.buffer.obj.clone() + } else if let Some(wrapper) = obj.downcast_ref::() { + wrapper.view.obj.clone() + } else if let Some(window) = obj.downcast_ref::() { + window.source.obj.clone() + } else { + return obj; + }; + obj = next; + } +} + +// slot_bf_getbuffer +pub(crate) fn buffer_from_python_getbuffer( + obj: &PyObject, + flags: BufferFlags, + vm: &VirtualMachine, +) -> PyResult { + let flags_obj = vm.ctx.new_int(flags.bits() as i32); + let ret = vm.call_special_method(obj, identifier!(vm, __buffer__), (flags_obj,))?; + let mv = ret + .downcast::() + .map_err(|_| vm.new_type_error("__buffer__ returned non-memoryview object"))?; + + // PyObject_GetBuffer(ret, buffer, flags): the returned view has to satisfy + // the request in its own right. + mv.try_usable(vm)?; + let desc = mv.requested_desc(flags, vm)?; + let wrapper = PyBufferWrapper { + exporter: obj.to_owned(), + view: mv.buffer.detached(), + returned_mv: PyMutex::new(Some(mv)), + exports: AtomicCell::new(0), + } + .into_pyobject(vm); + + // PyBuffer::new retains once through BUFFER_WRAPPER_METHODS. + Ok(PyBuffer::new(wrapper, desc, &BUFFER_WRAPPER_METHODS)) +} + +// wrap_releasebuffer +pub(crate) fn release_buffer_from_python( + obj: &PyObject, + mv: PyRef, + vm: &VirtualMachine, +) -> PyResult<()> { + let view_obj = &mv.buffer.obj; + if view_obj.downcastable::() { + // A window exports nothing, so there is nothing left to release, as for + // a `Py_buffer` whose `obj` is NULL. + return Ok(()); + } + let exports_obj = view_obj.is(obj) + || view_obj + .downcast_ref::() + .is_some_and(|wrapper| wrapper.exporter.is(obj)); + if !exports_obj { + return Err(vm.new_value_error("memoryview's buffer is not this object")); + } + if mv.released.load() { + return Err(vm.new_value_error("memoryview's buffer has already been released")); + } + mv.release(); + Ok(()) +} + +// releasebuffer_call_python, for a buffer acquired from a native exporter +pub(crate) fn release_buffer_call_python(buffer: &PyBuffer) { + crate::vm::thread::try_with_current_vm(|vm| { + let exporter = buffer.obj.clone(); + let window = PyBufferWindow { + source: buffer.detached(), + } + .into_pyobject(vm); + let window = PyBuffer::new(window, buffer.desc.clone(), &BUFFER_WINDOW_METHODS); + let mv = match PyMemoryView::from_buffer(window, vm) { + Ok(mv) => mv, + Err(exc) => { + let msg = format!( + "Exception ignored in bf_releasebuffer of {}", + exporter.class().name() + ); + return vm.run_unraisable(exc, Some(msg), vm.ctx.none()); + } + }; + // Restricted, so user code cannot keep anything addressing the memory + // that is about to go away. + mv.restricted.store(true); + let mv = mv.into_ref(&vm.ctx); + call_python_release_buffer(&exporter, mv.clone()); + // The window does not outlive the release it was made for. + mv.release(); + }); +} + +fn call_python_release_buffer(exporter: &PyObject, mv: PyRef) { + crate::vm::thread::try_with_current_vm(|vm| { + let method = vm.get_special_method(exporter, identifier!(vm, __release_buffer__)); + if let Ok(Some(method)) = method + && let Err(exc) = method.invoke((mv,), vm) + { + let msg = format!( + "Exception ignored in __release_buffer__ of {}", + exporter.class().name() + ); + vm.run_unraisable(exc, Some(msg), vm.ctx.none()); + } + }); } fn format_unpack( @@ -1143,6 +1557,10 @@ fn format_unpack( }) } +/// Whether `ch` names a format whose items are single bytes. +const fn is_byte_fmtchar(ch: u8) -> bool { + matches!(ch, b'c' | b'b' | b'B') +} fn is_equiv_shape(a: &BufferDescriptor, b: &BufferDescriptor) -> bool { if a.ndim() != b.ndim() { return false; diff --git a/crates/vm/src/builtins/property.rs b/crates/vm/src/builtins/property.rs index cff5a8a60d0..65ae48222fa 100644 --- a/crates/vm/src/builtins/property.rs +++ b/crates/vm/src/builtins/property.rs @@ -1,7 +1,7 @@ /*! Python `property` descriptor class. */ -use super::{PyStrRef, PyType}; +use super::PyType; use crate::common::lock::PyRwLock; use crate::function::{IntoFuncArgs, PosArgs}; use crate::{ @@ -41,8 +41,6 @@ pub struct PropertyArgs { fdel: Option, #[pyarg(any, default)] doc: Option, - #[pyarg(any, default)] - name: Option, } impl GetDescriptor for PyProperty { @@ -221,7 +219,6 @@ impl PyProperty { fset: new_setter.or_else(|| zelf.fset()), fdel: new_deleter.or_else(|| zelf.fdel()), doc, - name: None, }; // Create new property using py_new and init @@ -401,7 +398,6 @@ impl Initializer for PyProperty { *zelf.getter.write() = args.fget; *zelf.setter.write() = args.fset; *zelf.deleter.write() = args.fdel; - *zelf.name.write() = args.name.map(|a| a.as_object().to_owned()); zelf.getter_doc.store(getter_doc, Ordering::Relaxed); Ok(()) diff --git a/crates/vm/src/builtins/range.rs b/crates/vm/src/builtins/range.rs index 415d34fdb05..5962f90e521 100644 --- a/crates/vm/src/builtins/range.rs +++ b/crates/vm/src/builtins/range.rs @@ -364,7 +364,11 @@ impl PyRange { // TODO: Uncomment when Python adds __class_getitem__ to range // #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/set.rs b/crates/vm/src/builtins/set.rs index cd724abc5c1..d737612b158 100644 --- a/crates/vm/src/builtins/set.rs +++ b/crates/vm/src/builtins/set.rs @@ -195,6 +195,26 @@ impl PySetInner { Ok(set) } + /// Build a set from an arbitrary object, reusing stored hashes when the + /// source is a set/frozenset/dict. + fn from_object(iterable: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let set = Self::default(); + set.update_internal(iterable, vm)?; + Ok(set) + } + + /// Elements of `obj` with their stored hashes, or `None` if `obj` keeps + /// none and must be iterated generically. Mirrors the `PyAnySet_Check` / + /// `PyDict_CheckExact` fast paths in CPython's `set_update_internal`. + fn cached_hashes(obj: &PyObject, vm: &VirtualMachine) -> Option> { + if let Some(set) = extract_set(obj) { + Some(set.content.keys_with_hashes()) + } else { + obj.downcast_ref_if_exact::(vm) + .map(|dict| dict._as_dict_inner().keys_with_hashes()) + } + } + fn fold_op( &self, others: impl core::iter::Iterator, @@ -228,6 +248,17 @@ impl PySetInner { Self::wrap_unhashable_error(result, needle, vm) } + /// [`Self::contains`] with a known hash. Such a needle came out of a + /// set/dict, so it is hashable and needs no frozenset retry. + fn contains_known_hash( + &self, + needle: &PyObject, + hash: PyHash, + vm: &VirtualMachine, + ) -> PyResult { + self.content.contains_known_hash(vm, needle, hash) + } + fn compare(&self, other: &Self, op: PyComparisonOp, vm: &VirtualMachine) -> PyResult { if op == PyComparisonOp::Ne { return self.compare(other, PyComparisonOp::Eq, vm).map(|eq| !eq); @@ -251,6 +282,12 @@ impl PySetInner { pub(super) fn union(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult { let set = self.clone(); + if let Some(elements) = Self::cached_hashes(other.as_object(), vm) { + for (item, hash) in elements { + set.add_known_hash(item, hash, vm)?; + } + return Ok(set); + } for item in other.iter(vm)? { set.add(item?, vm)?; } @@ -260,6 +297,14 @@ impl PySetInner { pub(super) fn intersection(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult { let set = Self::default(); + if let Some(elements) = Self::cached_hashes(other.as_object(), vm) { + for (obj, hash) in elements { + if self.contains_known_hash(&obj, hash, vm)? { + set.add_known_hash(obj, hash, vm)?; + } + } + return Ok(set); + } for item in other.iter(vm)? { let obj = item?; if self.contains(&obj, vm)? { @@ -271,6 +316,12 @@ impl PySetInner { pub(super) fn difference(&self, other: ArgIterable, vm: &VirtualMachine) -> PyResult { let set = self.copy(); + if let Some(elements) = Self::cached_hashes(other.as_object(), vm) { + for (item, hash) in elements { + set.content.delete_if_exists_known_hash(vm, &*item, hash)?; + } + return Ok(set); + } for item in other.iter(vm)? { set.content.delete_if_exists(vm, &*item?)?; } @@ -284,6 +335,16 @@ impl PySetInner { ) -> PyResult { let new_inner = self.clone(); + if let Some(elements) = Self::cached_hashes(other.as_object(), vm) { + // the source is already duplicate-free + for (item, hash) in elements { + new_inner + .content + .delete_or_insert_known_hash(vm, &item, hash, ())?; + } + return Ok(new_inner); + } + // We want to remove duplicates in other let other_set = Self::from_iter(other.iter(vm)?, vm)?; @@ -325,7 +386,8 @@ impl PySetInner { } fn repr(&self, class_name: Option<&str>, vm: &VirtualMachine) -> PyResult { - collection_repr(class_name, "{", "}", self.elements().iter(), vm) + let empty = format!("{}()", class_name.unwrap_or("set")); + collection_repr(class_name, "{", "}", &empty, self.elements().iter(), vm) } fn add(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { @@ -333,6 +395,12 @@ impl PySetInner { Self::wrap_unhashable_error(result, &item, vm) } + /// [`Self::add`] with a known hash. + fn add_known_hash(&self, item: PyObjectRef, hash: PyHash, vm: &VirtualMachine) -> PyResult<()> { + let result = self.content.insert_known_hash(vm, &*item, hash, ()); + Self::wrap_unhashable_error(result, &item, vm) + } + fn remove(&self, item: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { let result = self.retry_op_with_frozenset(&item, vm, |item, vm| self.content.delete(vm, item)); @@ -393,15 +461,15 @@ impl PySetInner { } fn merge_set(&self, any_set: AnySet, vm: &VirtualMachine) -> PyResult<()> { - for item in any_set.as_inner().elements() { - self.add(item, vm)?; + for (item, hash) in any_set.as_inner().content.keys_with_hashes() { + self.add_known_hash(item, hash, vm)?; } Ok(()) } fn merge_dict(&self, dict: PyDictRef, vm: &VirtualMachine) -> PyResult<()> { - for (key, _value) in dict { - self.add(key, vm)?; + for (key, hash) in dict._as_dict_inner().keys_with_hashes() { + self.add_known_hash(key, hash, vm)?; } Ok(()) } @@ -413,8 +481,8 @@ impl PySetInner { ) -> PyResult<()> { let temp_inner = self.fold_op(others, Self::intersection, vm)?; self.clear(); - for obj in temp_inner.elements() { - self.add(obj, vm)?; + for (obj, hash) in temp_inner.content.keys_with_hashes() { + self.add_known_hash(obj, hash, vm)?; } Ok(()) } @@ -425,6 +493,12 @@ impl PySetInner { vm: &VirtualMachine, ) -> PyResult<()> { for iterable in others { + if let Some(elements) = Self::cached_hashes(iterable.as_object(), vm) { + for (item, hash) in elements { + self.content.delete_if_exists_known_hash(vm, &*item, hash)?; + } + continue; + } let items = iterable.iter(vm)?.collect::, _>>()?; for item in items { self.content.delete_if_exists(vm, &*item)?; @@ -439,6 +513,14 @@ impl PySetInner { vm: &VirtualMachine, ) -> PyResult<()> { for iterable in others { + if let Some(elements) = Self::cached_hashes(iterable.as_object(), vm) { + // the source is already duplicate-free + for (item, hash) in elements { + self.content + .delete_or_insert_known_hash(vm, &item, hash, ())?; + } + continue; + } // We want to remove duplicates in iterable let iterable_set = Self::from_iter(iterable.iter(vm)?, vm)?; for item in iterable_set.elements() { @@ -524,6 +606,23 @@ fn extract_set(obj: &PyObject) -> Option<&PySetInner> { }) } +/// Elements of `obj` with their stored hashes, or `None` unless `obj` is exactly +/// a `set` or `frozenset` — `PyAnySet_CheckExact`, where [`extract_set`] is the +/// subclass-inclusive `PyAnySet_Check`. +pub(super) fn exact_set_keys_with_hashes( + obj: &PyObject, + vm: &VirtualMachine, +) -> Option> { + let inner = obj + .downcast_ref_if_exact::(vm) + .map(|set| &set.inner) + .or_else(|| { + obj.downcast_ref_if_exact::(vm) + .map(|frozen| &frozen.inner) + })?; + Some(inner.content.keys_with_hashes()) +} + fn reduce_set(zelf: &PyObject, vm: &VirtualMachine) -> (PyTypeRef, PyTupleRef, Option) { ( zelf.class().to_owned(), @@ -783,7 +882,11 @@ impl PySet { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -955,7 +1058,7 @@ impl Representable for PySet { } impl Constructor for PyFrozenSet { - type Args = Vec; + type Args = OptionalArg; fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { let is_exact_frozenset = cls.is(vm.ctx.types.frozenset_type); @@ -988,11 +1091,11 @@ impl Constructor for PyFrozenSet { return Ok(input.clone()); } - iterable.into_option() + iterable } else { match &args.args[..] { - [] => None, - [iterable] => Some(iterable.clone()), + [] => OptionalArg::Missing, + [iterable] => OptionalArg::Present(iterable.clone()), slice => { return Err(vm.new_type_error(format!( "frozenset expected at most 1 argument, got {}", @@ -1002,23 +1105,25 @@ impl Constructor for PyFrozenSet { } }; - let elements = if let Some(iterable) = iterable_opt { - iterable.try_to_value(vm)? - } else { - vec![] - }; + let payload = Self::py_new(&cls, iterable_opt, vm)?; // Return empty frozenset singleton - if is_exact_frozenset && elements.is_empty() { + if is_exact_frozenset && payload.inner.len() == 0 { return Ok(vm.ctx.empty_frozenset.clone().into()); } - let payload = Self::py_new(&cls, elements, vm)?; payload.into_ref_with_type(vm, cls).map(Into::into) } - fn py_new(_cls: &Py, elements: Self::Args, vm: &VirtualMachine) -> PyResult { - Self::from_iter(vm, elements) + fn py_new(_cls: &Py, iterable: Self::Args, vm: &VirtualMachine) -> PyResult { + let inner = match iterable { + OptionalArg::Present(iterable) => PySetInner::from_object(iterable, vm)?, + OptionalArg::Missing => PySetInner::default(), + }; + Ok(Self { + inner, + ..Default::default() + }) } } @@ -1186,7 +1291,11 @@ impl PyFrozenSet { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -1424,16 +1533,16 @@ impl IterNext for PySetIterator { fn next(zelf: &crate::Py, vm: &VirtualMachine) -> PyResult { let mut internal = zelf.internal.lock(); let next = if let IterStatus::Active(dict) = &internal.status { - if dict.has_changed_size(&zelf.size) { - internal.status = IterStatus::Exhausted; - return Err(vm.new_runtime_error("set changed size during iteration")); - } - match dict.next_entry(internal.position) { - Some((position, key, _)) => { + match dict.next_entry_checked(internal.position, &zelf.size, |key, ()| key.clone()) { + Err(crate::dict_inner::DictChanged) => { + internal.status = IterStatus::Exhausted; + return Err(vm.new_runtime_error("set changed size during iteration")); + } + Ok(Some((position, key))) => { internal.position = position; PyIterReturn::Return(key) } - None => { + Ok(None) => { internal.status = IterStatus::Exhausted; PyIterReturn::StopIteration(None) } diff --git a/crates/vm/src/builtins/slice.rs b/crates/vm/src/builtins/slice.rs index 3c5f13b382d..026b976b65e 100644 --- a/crates/vm/src/builtins/slice.rs +++ b/crates/vm/src/builtins/slice.rs @@ -260,7 +260,11 @@ impl PySlice { // TODO: Uncomment when Python adds __class_getitem__ to slice // #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/staticmethod.rs b/crates/vm/src/builtins/staticmethod.rs index addfe8a4e2b..8ae31b67b5c 100644 --- a/crates/vm/src/builtins/staticmethod.rs +++ b/crates/vm/src/builtins/staticmethod.rs @@ -163,7 +163,11 @@ impl PyStaticMethod { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/builtins/str.rs b/crates/vm/src/builtins/str.rs index 07325159a39..6e774f7e652 100644 --- a/crates/vm/src/builtins/str.rs +++ b/crates/vm/src/builtins/str.rs @@ -9,7 +9,7 @@ use super::{ use crate::{ AsObject, Context, Py, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, TryFromBorrowedObject, VirtualMachine, - anystr::{self, AnyStr, AnyStrContainer, AnyStrWrapper, adjust_indices}, + anystr::{self, AnyStr, AnyStrContainer, AnyStrWrapper, StringRange, adjust_indices}, atomic_func, bytes_inner::{swapcase_ascii, title_ascii}, cformat::cformat_string, @@ -23,7 +23,9 @@ use crate::{ function::{ArgIterable, ArgSize, FuncArgs, OptionalArg, OptionalOption, PyComparisonValue}, intern::PyInterned, object::{MaybeTraverse, Traverse, TraverseFn}, - protocol::{PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods}, + protocol::{ + BufferFlags, PyBuffer, PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods, + }, sequence::SequenceExt, sliceable::{SequenceIndex, SliceableSequenceOp}, types::{ @@ -441,15 +443,24 @@ impl Constructor for PyStr { if input.fast_isinstance(vm.ctx.types.str_type) { return Err(vm.new_type_error("decoding str is not supported")); } - if !input.fast_isinstance(vm.ctx.types.bytes_type) - && !input.fast_isinstance(vm.ctx.types.bytearray_type) - && crate::protocol::PyBuffer::try_from_borrowed_object(vm, &input).is_err() + let input = if input.fast_isinstance(vm.ctx.types.bytes_type) + || input.fast_isinstance(vm.ctx.types.bytearray_type) { - return Err(vm.new_type_error(format!( - "decoding to str: need a bytes-like object, {} found", - input.class().name() - ))); - } + input + } else { + // PyUnicode_FromEncodedObject: whatever an exporter + // complains about, the argument is simply not bytes-like. + let buffer = PyBuffer::from_object(vm, &input, BufferFlags::SIMPLE) + .map_err(|_| { + vm.new_type_error(format!( + "decoding to str: need a bytes-like object, {} found", + input.class().name() + )) + })?; + vm.ctx + .new_bytes(buffer.contiguous_or_collect(<[u8]>::to_vec)) + .into() + }; let enc_str = encoding.as_ref().map_or("utf-8", |e| e.as_str()); let s = vm .state @@ -712,6 +723,20 @@ impl PyStr { self.data.char_len() } + /// The byte offset the `index`-th character starts at, or the string's byte + /// length if `index` is at or past its end. + #[inline] + pub fn char_index_to_byte(&self, index: usize) -> usize { + self.data.char_index_to_byte(index) + } + + /// The character index of the character starting at byte offset `bytepos`, + /// which must be a character boundary at or before the end. + #[inline] + pub fn byte_to_char_index(&self, bytepos: usize) -> usize { + self.data.byte_to_char_index(bytepos) + } + #[pymethod] #[inline(always)] pub const fn isascii(&self) -> bool { @@ -917,11 +942,12 @@ impl PyStr { #[pymethod] fn endswith(&self, options: anystr::StartsEndsWithArgs, vm: &VirtualMachine) -> PyResult { - let (affix, substr) = - match options.prepare(self.as_wtf8(), self.len(), |s, r| s.get_chars(r)) { - Some(x) => x, - None => return Ok(false), - }; + let (affix, substr) = match options.prepare(self.as_wtf8(), self.len(), |s, r| { + &s[self.data.char_range_to_bytes(r)] + }) { + Some(x) => x, + None => return Ok(false), + }; substr.py_starts_ends_with( &affix, "endswith", @@ -937,11 +963,12 @@ impl PyStr { options: anystr::StartsEndsWithArgs, vm: &VirtualMachine, ) -> PyResult { - let (affix, substr) = - match options.prepare(self.as_wtf8(), self.len(), |s, r| s.get_chars(r)) { - Some(x) => x, - None => return Ok(false), - }; + let (affix, substr) = match options.prepare(self.as_wtf8(), self.len(), |s, r| { + &s[self.data.char_range_to_bytes(r)] + }) { + Some(x) => x, + None => return Ok(false), + }; substr.py_starts_ends_with( &affix, "startswith", @@ -1160,42 +1187,52 @@ impl PyStr { Ok(vm.ctx.new_str(joined)) } - // FIXME: two traversals of str is expensive + /// The bytes the character range `range` spans and the byte offset it + /// starts at, or `None` if the range is inverted. + /// + /// The bounds go through the string's character index, so reaching a range + /// deep in the subject costs a lookup rather than a walk to it. #[inline] - fn _to_char_idx(r: &Wtf8, byte_idx: usize) -> usize { - r[..byte_idx].code_points().count() + fn char_range_bytes(&self, range: Range) -> Option<(usize, &Wtf8)> { + if !range.is_normal() { + return None; + } + let bytes = self.data.char_range_to_bytes(range); + Some((bytes.start, &self.as_wtf8()[bytes])) } + /// Searches the character range `range` with `find`, which answers in bytes + /// relative to the range, and reports the hit as a character index. #[inline] fn _find(&self, args: FindArgs, find: F) -> Option where F: Fn(&Wtf8, &Wtf8) -> Option, { let (sub, range) = args.get_value(self.len()); - self.as_wtf8().py_find(sub.as_wtf8(), range, find) + let (start, haystack) = self.char_range_bytes(range)?; + let found = find(haystack, sub.as_wtf8())?; + Some(self.byte_to_char_index(start + found)) } #[pymethod] fn find(&self, args: FindArgs) -> isize { - self._find(args, |r, s| Some(Self::_to_char_idx(r, r.find(s)?))) - .map_or(-1, |v| v as isize) + self._find(args, Wtf8::find).map_or(-1, |v| v as isize) } #[pymethod] fn rfind(&self, args: FindArgs) -> isize { - self._find(args, |r, s| Some(Self::_to_char_idx(r, r.rfind(s)?))) - .map_or(-1, |v| v as isize) + self._find(args, Wtf8::rfind).map_or(-1, |v| v as isize) } #[pymethod] fn index(&self, args: FindArgs, vm: &VirtualMachine) -> PyResult { - self._find(args, |r, s| Some(Self::_to_char_idx(r, r.find(s)?))) + self._find(args, Wtf8::find) .ok_or_else(|| vm.new_value_error("substring not found")) } #[pymethod] fn rindex(&self, args: FindArgs, vm: &VirtualMachine) -> PyResult { - self._find(args, |r, s| Some(Self::_to_char_idx(r, r.rfind(s)?))) + self._find(args, Wtf8::rfind) .ok_or_else(|| vm.new_value_error("substring not found")) } @@ -1268,16 +1305,28 @@ impl PyStr { #[pymethod] fn count(&self, args: FindArgs) -> usize { let (needle, range) = args.get_value(self.len()); - self.as_wtf8() - .py_count(needle.as_wtf8(), range, |h, n| h.find_iter(n).count()) + let chars = range.len(); + self.char_range_bytes(range).map_or(0, |(_, haystack)| { + if needle.is_empty() { + // An empty needle sits between every pair of characters and at + // both ends, so it occurs once more than the range holds + // characters. Counting it in the bytes would answer in encoded + // positions instead. + chars + 1 + } else { + haystack.find_iter(needle.as_wtf8()).count() + } + }) } #[pymethod] - fn zfill(&self, width: isize) -> Wtf8Buf { - unsafe { - // SAFETY: this is safe-guaranteed because the original self.as_wtf8() is valid wtf8 - Wtf8Buf::from_bytes_unchecked(self.as_wtf8().py_zfill(width)) - } + fn zfill(&self, width: isize, vm: &VirtualMachine) -> PyResult { + let filled = self + .as_wtf8() + .py_zfill(width) + .ok_or_else(|| vm.new_memory_error(""))?; + // SAFETY: this is safe-guaranteed because the original self.as_wtf8() is valid wtf8 + Ok(unsafe { Wtf8Buf::from_bytes_unchecked(filled) }) } #[inline] @@ -1285,7 +1334,7 @@ impl PyStr { &self, width: isize, fillchar: OptionalArg, - pad: fn(&Wtf8, usize, CodePoint, usize) -> Wtf8Buf, + pad: fn(&Wtf8, usize, CodePoint, usize) -> Option, vm: &VirtualMachine, ) -> PyResult { let fillchar = fillchar.map_or(Ok(' '.into()), |ref s| { @@ -1293,11 +1342,11 @@ impl PyStr { vm.new_type_error("The fill character must be exactly one character long") }) })?; - Ok(if self.len() as isize >= width { - self.as_wtf8().to_owned() - } else { - pad(self.as_wtf8(), width as usize, fillchar, self.len()) - }) + if self.len() as isize >= width { + return Ok(self.as_wtf8().to_owned()); + } + pad(self.as_wtf8(), width as usize, fillchar, self.len()) + .ok_or_else(|| vm.new_memory_error("")) } #[pymethod] @@ -1556,6 +1605,11 @@ impl Comparable for PyStr { return Ok(res.into()); } let other = class_or_notimplemented!(Self, other); + // Equality does not need the ordering, and answers two strings of + // different length without reading either. + if let Some(res) = op.eval_eq(|| zelf.as_wtf8() == other.as_wtf8()) { + return Ok(res.into()); + } Ok(op.eval_ord(zelf.as_wtf8().cmp(other.as_wtf8())).into()) } } @@ -1800,6 +1854,31 @@ pub(crate) fn init(ctx: &'static Context) { PyStrIterator::extend_class(ctx, ctx.types.str_iterator_type); } +impl PyStr { + /// The code points at `indices`, in that order, as a new string. + /// + /// Each index is resolved through the string's own index table, so the + /// cost is one lookup per collected character rather than a walk to the + /// furthest one. The iterator's length is the result's character count, + /// which is why it has to be exact. + fn gather_chars(&self, indices: impl ExactSizeIterator) -> Self { + let char_len = indices.len(); + // Not ascii, so the code points are at least two bytes each. + let mut out = Wtf8Buf::with_capacity(2 * char_len); + let s = self.as_wtf8(); + for index in indices { + out.push( + s[self.data.char_index_to_byte(index)..] + .code_points() + .next() + .expect("index is below the character count"), + ); + } + // SAFETY: char_len is accurate + unsafe { Self::new_with_char_len(out, char_len) } + } +} + impl SliceableSequenceOp for PyStr { type Item = CodePoint; type Sliced = Self; @@ -1809,125 +1888,56 @@ impl SliceableSequenceOp for PyStr { } fn do_slice(&self, range: Range) -> Self::Sliced { - match self.as_str_kind() { - PyKindStr::Ascii(s) => s[range].into(), - PyKindStr::Utf8(s) => { - let char_len = range.len(); - let out = rustpython_common::str::get_chars(s, range); - // SAFETY: char_len is accurate - unsafe { Self::new_with_char_len(out, char_len) } - } - PyKindStr::Wtf8(w) => { - let char_len = range.len(); - let out = rustpython_common::str::get_codepoints(w, range); - // SAFETY: char_len is accurate - unsafe { Self::new_with_char_len(out, char_len) } - } + if let PyKindStr::Ascii(s) = self.as_str_kind() { + return s[range].into(); } + // Both ends resolve through the string's own index, so the slice is a + // byte reslice rather than a walk to `range.start` and another to + // `range.end`. + let char_len = range.len(); + let bytes = self.data.char_range_to_bytes(range); + let out = &self.as_wtf8()[bytes]; + // SAFETY: char_len is accurate + unsafe { Self::new_with_char_len(out.to_owned(), char_len) } } fn do_slice_reverse(&self, range: Range) -> Self::Sliced { - match self.as_str_kind() { - PyKindStr::Ascii(s) => { - let mut out = s[range].to_owned(); - out.as_mut_slice().reverse(); - out.into() - } - PyKindStr::Utf8(s) => { - let char_len = range.len(); - let mut out = String::with_capacity(2 * char_len); - out.extend( - s.chars() - .rev() - .skip(self.char_len() - range.end) - .take(range.len()), - ); - // SAFETY: char_len is accurate - unsafe { Self::new_with_char_len(out, range.len()) } - } - PyKindStr::Wtf8(w) => { - let char_len = range.len(); - let mut out = Wtf8Buf::with_capacity(2 * char_len); - out.extend( - w.code_points() - .rev() - .skip(self.char_len() - range.end) - .take(range.len()), - ); - // SAFETY: char_len is accurate - unsafe { Self::new_with_char_len(out, char_len) } - } + if let PyKindStr::Ascii(s) = self.as_str_kind() { + let mut out = s[range].to_owned(); + out.as_mut_slice().reverse(); + return out.into(); } + let char_len = range.len(); + let bytes = self.data.char_range_to_bytes(range); + let mut out = Wtf8Buf::with_capacity(bytes.len()); + out.extend(self.as_wtf8()[bytes].code_points().rev()); + // SAFETY: char_len is accurate + unsafe { Self::new_with_char_len(out, char_len) } } fn do_stepped_slice(&self, range: Range, step: usize) -> Self::Sliced { - match self.as_str_kind() { - PyKindStr::Ascii(s) => s[range] + if let PyKindStr::Ascii(s) = self.as_str_kind() { + return s[range] .as_slice() .iter() .copied() .step_by(step) .collect::() - .into(), - PyKindStr::Utf8(s) => { - let char_len = (range.len() / step) + 1; - let mut out = String::with_capacity(2 * char_len); - out.extend(s.chars().skip(range.start).take(range.len()).step_by(step)); - // SAFETY: char_len is accurate - unsafe { Self::new_with_char_len(out, char_len) } - } - PyKindStr::Wtf8(w) => { - let char_len = (range.len() / step) + 1; - let mut out = Wtf8Buf::with_capacity(2 * char_len); - out.extend( - w.code_points() - .skip(range.start) - .take(range.len()) - .step_by(step), - ); - // SAFETY: char_len is accurate - unsafe { Self::new_with_char_len(out, char_len) } - } + .into(); } + self.gather_chars(range.step_by(step)) } fn do_stepped_slice_reverse(&self, range: Range, step: usize) -> Self::Sliced { - match self.as_str_kind() { - PyKindStr::Ascii(s) => s[range] + if let PyKindStr::Ascii(s) = self.as_str_kind() { + return s[range] .chars() .rev() .step_by(step) .collect::() - .into(), - PyKindStr::Utf8(s) => { - let char_len = (range.len() / step) + 1; - // not ascii, so the codepoints have to be at least 2 bytes each - let mut out = String::with_capacity(2 * char_len); - out.extend( - s.chars() - .rev() - .skip(self.char_len() - range.end) - .take(range.len()) - .step_by(step), - ); - // SAFETY: char_len is accurate - unsafe { Self::new_with_char_len(out, char_len) } - } - PyKindStr::Wtf8(w) => { - let char_len = (range.len() / step) + 1; - // not ascii, so the codepoints have to be at least 2 bytes each - let mut out = Wtf8Buf::with_capacity(2 * char_len); - out.extend( - w.code_points() - .rev() - .skip(self.char_len() - range.end) - .take(range.len()) - .step_by(step), - ); - // SAFETY: char_len is accurate - unsafe { Self::new_with_char_len(out, char_len) } - } + .into(); } + self.gather_chars(range.rev().step_by(step)) } fn empty() -> Self::Sliced { @@ -2206,6 +2216,12 @@ impl AnyStrContainer for String { Self::with_capacity(capacity) } + fn try_with_capacity(capacity: usize) -> Option { + let mut s = Self::new(); + s.try_reserve_exact(capacity).ok()?; + Some(s) + } + fn push_str(&mut self, other: &str) { Self::push_str(self, other) } @@ -2319,6 +2335,12 @@ impl AnyStrContainer for Wtf8Buf { Self::with_capacity(capacity) } + fn try_with_capacity(capacity: usize) -> Option { + let mut s = Self::new(); + s.try_reserve_exact(capacity).ok()?; + Some(s) + } + fn push_str(&mut self, other: &Wtf8) { self.push_wtf8(other) } @@ -2439,6 +2461,12 @@ impl AnyStrContainer for AsciiString { Self::with_capacity(capacity) } + fn try_with_capacity(capacity: usize) -> Option { + let mut v = Vec::new(); + v.try_reserve_exact(capacity).ok()?; + Some(Self::from(v)) + } + fn push_str(&mut self, other: &AsciiStr) { Self::push_str(self, other) } diff --git a/crates/vm/src/builtins/template.rs b/crates/vm/src/builtins/template.rs index 30812b4f171..94c4d653df3 100644 --- a/crates/vm/src/builtins/template.rs +++ b/crates/vm/src/builtins/template.rs @@ -186,7 +186,11 @@ impl PyTemplate { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } diff --git a/crates/vm/src/builtins/tuple.rs b/crates/vm/src/builtins/tuple.rs index d48639b2c11..d510e35326f 100644 --- a/crates/vm/src/builtins/tuple.rs +++ b/crates/vm/src/builtins/tuple.rs @@ -23,12 +23,60 @@ use crate::{ vm::VirtualMachine, }; use alloc::fmt; -use core::cell::Cell; +use core::cell::{Cell, UnsafeCell}; use core::ptr::NonNull; #[pyclass(module = false, name = "tuple", traverse = "manual")] pub struct PyTuple { - elements: Box<[R]>, + elements: TupleElements, +} + +/// Tuple storage is immutable after publication, but marshal must publish a +/// tuple in its reference table before recursively reading its children. +/// This mirrors CPython's `PyTuple_New` followed by `PyTuple_SET_ITEM`. +struct TupleElements(UnsafeCell>); + +unsafe impl Send for TupleElements {} +unsafe impl Sync for TupleElements {} + +impl TupleElements { + const fn new(elements: Box<[R]>) -> Self { + Self(UnsafeCell::new(elements)) + } + + fn as_slice(&self) -> &[R] { + // SAFETY: initialization writes happen only while the tuple is owned by + // the synchronous marshal decoder; afterwards the storage is immutable. + unsafe { &*self.0.get() } + } + + fn get_mut(&mut self) -> &mut Box<[R]> { + self.0.get_mut() + } + + /// # Safety + /// The tuple must still be in its private initialization phase, and each + /// placeholder index must be replaced at most once before it is observable. + unsafe fn set_initializing(&self, index: usize, value: R) { + unsafe { (*self.0.get())[index] = value }; + } +} + +impl core::ops::Deref for TupleElements { + type Target = [R]; + + fn deref(&self) -> &Self::Target { + self.as_slice() + } +} + +impl<'a, R> IntoIterator for &'a TupleElements { + type Item = &'a R; + type IntoIter = core::slice::Iter<'a, R>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } } impl fmt::Debug for PyTuple { @@ -42,11 +90,11 @@ impl fmt::Debug for PyTuple { // Note: Only impl for PyTuple (the default) unsafe impl Traverse for PyTuple { fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) { - self.elements.traverse(traverse_fn); + self.elements.as_slice().traverse(traverse_fn); } fn clear(&mut self, out: &mut Vec) { - let elements = core::mem::take(&mut self.elements); + let elements = core::mem::take(self.elements.get_mut()); out.extend(elements.into_vec()); } } @@ -206,7 +254,7 @@ impl Constructor for PyTuple { fn py_new(_cls: &Py, elements: Self::Args, _vm: &VirtualMachine) -> PyResult { Ok(Self { - elements: elements.into_boxed_slice(), + elements: TupleElements::new(elements.into_boxed_slice()), }) } } @@ -245,19 +293,19 @@ impl<'a, R> core::iter::IntoIterator for &'a Py> { impl PyTuple { #[must_use] - pub const fn as_slice(&self) -> &[R] { + pub fn as_slice(&self) -> &[R] { &self.elements } #[inline] #[must_use] - pub const fn len(&self) -> usize { + pub fn len(&self) -> usize { self.elements.len() } #[inline] #[must_use] - pub const fn is_empty(&self) -> bool { + pub fn is_empty(&self) -> bool { self.elements.is_empty() } @@ -274,7 +322,13 @@ impl PyTuple { ctx.empty_tuple.clone() } else { let elements = elements.into_boxed_slice(); - PyRef::new_ref(Self { elements }, ctx.types.tuple_type.to_owned(), None) + PyRef::new_ref( + Self { + elements: TupleElements::new(elements), + }, + ctx.types.tuple_type.to_owned(), + None, + ) } } @@ -283,7 +337,16 @@ impl PyTuple { /// Calling this function implies trying micro optimization for non-zero-sized tuple. #[must_use] pub const fn new_unchecked(elements: Box<[PyObjectRef]>) -> Self { - Self { elements } + Self { + elements: TupleElements::new(elements), + } + } + + /// # Safety + /// This tuple must be a marshal placeholder which has not escaped the + /// decoder, and `index` must not have been replaced previously. + pub(crate) unsafe fn set_marshal_item(&self, index: usize, value: PyObjectRef) { + unsafe { self.elements.set_initializing(index, value) }; } fn repeat(zelf: PyRef, value: isize, vm: &VirtualMachine) -> PyResult> { @@ -298,7 +361,10 @@ impl PyTuple { } else { let v = zelf.elements.mul(vm, value)?; let elements = v.into_boxed_slice(); - Self { elements }.into_ref(&vm.ctx) + Self { + elements: TupleElements::new(elements), + } + .into_ref(&vm.ctx) }) } @@ -341,7 +407,10 @@ impl PyTuple { .chain(other.as_slice()) .cloned() .collect::>(); - Self { elements }.into_ref(&vm.ctx) + Self { + elements: TupleElements::new(elements), + } + .into_ref(&vm.ctx) } }); PyArithmeticValue::from_option(added.ok()) @@ -360,7 +429,7 @@ impl PyTuple { #[inline] #[must_use] - pub const fn __len__(&self) -> usize { + pub fn __len__(&self) -> usize { self.elements.len() } @@ -425,13 +494,17 @@ impl PyTuple { let tup_arg = if zelf.class().is(vm.ctx.types.tuple_type) { zelf } else { - Self::new_ref(zelf.elements.clone().into_vec(), &vm.ctx) + Self::new_ref(zelf.elements.as_slice().to_vec(), &vm.ctx) }; (tup_arg,) } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -536,7 +609,7 @@ impl Representable for PyTuple { let s = if zelf.len() == 1 { wtf8_concat!("(", zelf.elements[0].repr(vm)?.as_wtf8(), ",)") } else { - collection_repr(None, "(", ")", zelf.elements.iter(), vm)? + collection_repr(None, "(", ")", "()", zelf.elements.iter(), vm)? }; vm.ctx.new_str(s) } else { diff --git a/crates/vm/src/builtins/type.rs b/crates/vm/src/builtins/type.rs index 1776270751e..46e2e4ebfc6 100644 --- a/crates/vm/src/builtins/type.rs +++ b/crates/vm/src/builtins/type.rs @@ -294,6 +294,16 @@ pub struct HeapTypeExt { pub slots: Option>>, pub type_data: PyRwLock>, pub specialization_cache: TypeSpecializationCache, + /// The interpreter this type was created in, or `None` for the types the + /// shared context builds before any interpreter exists. + pub interpreter_id: Option, +} + +impl HeapTypeExt { + /// The interpreter a type created right now belongs to. + fn creating_interpreter_id() -> Option { + crate::vm::thread::try_with_current_vm(|vm| vm.state.interpreter_id) + } } pub struct TypeSpecializationCache { @@ -553,6 +563,22 @@ impl PyType { self.modified_inner(); } + /// Whether the interpreter with `interpreter_id` can see this type. + /// + /// Interpreters share the context, so a subclass of a shared type is + /// recorded on an object every interpreter reaches. Only the interpreter + /// that created it can name it, so only that one lists it. + pub fn is_visible_to_interpreter(&self, interpreter_id: i64) -> bool { + match self + .heaptype_ext + .as_ref() + .and_then(|ext| ext.interpreter_id) + { + Some(owner) => owner == interpreter_id, + None => true, + } + } + pub fn new_simple_heap( name: &str, base: &Py, @@ -589,6 +615,7 @@ impl PyType { slots: None, type_data: PyRwLock::new(None), specialization_cache: TypeSpecializationCache::new(), + interpreter_id: HeapTypeExt::creating_interpreter_id(), }; let base = bases[0].clone(); @@ -794,8 +821,6 @@ impl PyType { slots.basicsize = base.slots.basicsize; } - Self::inherit_readonly_slots(&mut slots, &base); - // Normalize: any type with HAS_WEAKREF gets MANAGED_WEAKREF if slots.flags.has_feature(PyTypeFlags::HAS_WEAKREF) { slots.flags |= PyTypeFlags::MANAGED_WEAKREF; @@ -864,8 +889,6 @@ impl PyType { slots.basicsize = base.slots.basicsize; } - Self::inherit_readonly_slots(&mut slots, &base); - // Normalize: any type with HAS_WEAKREF gets MANAGED_WEAKREF if slots.flags.has_feature(PyTypeFlags::HAS_WEAKREF) { slots.flags |= PyTypeFlags::MANAGED_WEAKREF; @@ -991,18 +1014,9 @@ impl PyType { } } - /// Inherit readonly slots from base type at creation time. - /// These slots are not AtomicCell and must be set before the type is used. - fn inherit_readonly_slots(slots: &mut PyTypeSlots, base: &Self) { - if slots.as_buffer.is_none() { - slots.as_buffer = base.slots.as_buffer; - } - } - /// Inherit slots from base type. inherit_slots pub(crate) fn inherit_slots(&self, base: &Self) { // Use SLOT_DEFS to iterate all slots - // Note: as_buffer is handled in inherit_readonly_slots (not AtomicCell) for def in SLOT_DEFS { def.accessor.copyslot_if_none(self, base); } @@ -1948,13 +1962,18 @@ impl PyType { } #[pymethod] - fn __subclasses__(&self) -> PyList { + fn __subclasses__(&self, vm: &VirtualMachine) -> PyList { let mut subclasses = self.subclasses.write(); subclasses.retain(|x| x.upgrade().is_some()); + let interpreter_id = vm.state.interpreter_id; PyList::from( subclasses .iter() - .map(|x| x.upgrade().unwrap()) + .filter_map(|x| x.upgrade()) + .filter(|obj| { + obj.downcast_ref::() + .is_none_or(|typ| typ.is_visible_to_interpreter(interpreter_id)) + }) .collect::>(), ) } @@ -2368,6 +2387,7 @@ impl Constructor for PyType { slots: heaptype_slots.clone(), type_data: PyRwLock::new(None), specialization_cache: TypeSpecializationCache::new(), + interpreter_id: HeapTypeExt::creating_interpreter_id(), }; (slots, heaptype_ext) }; diff --git a/crates/vm/src/builtins/union.rs b/crates/vm/src/builtins/union.rs index cb6dd0d6559..c1be5c8ec9a 100644 --- a/crates/vm/src/builtins/union.rs +++ b/crates/vm/src/builtins/union.rs @@ -234,7 +234,7 @@ pub(crate) fn or_op(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) } fn make_parameters(args: &Py, vm: &VirtualMachine) -> PyResult { - let parameters = genericalias::make_parameters(args, vm); + let parameters = genericalias::make_parameters(args, vm)?; let result = dedup_and_flatten_args(¶meters, vm)?; Ok(result.args) } diff --git a/crates/vm/src/builtins/weakref.rs b/crates/vm/src/builtins/weakref.rs index 9e88ffaa2e6..e0f012f169c 100644 --- a/crates/vm/src/builtins/weakref.rs +++ b/crates/vm/src/builtins/weakref.rs @@ -92,7 +92,11 @@ impl PyWeak { } #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/byte.rs b/crates/vm/src/byte.rs index 933ddead4b9..0e90f296ac9 100644 --- a/crates/vm/src/byte.rs +++ b/crates/vm/src/byte.rs @@ -2,11 +2,16 @@ use num_traits::ToPrimitive; -use crate::{AsObject, PyObject, PyResult, VirtualMachine}; +use crate::{ + AsObject, PyObject, PyResult, VirtualMachine, + protocol::{BufferFlags, PyBuffer}, +}; +// PyBytes_FromObject pub fn bytes_from_object(vm: &VirtualMachine, obj: &PyObject) -> PyResult> { - if let Ok(elements) = obj.try_bytes_like(vm, |bytes| bytes.to_vec()) { - return Ok(elements); + if obj.check_buffer() { + let buffer = PyBuffer::from_object(vm, obj, BufferFlags::FULL_RO)?; + return Ok(buffer.contiguous_or_collect(|bytes| bytes.to_vec())); } if !obj.fast_isinstance(vm.ctx.types.str_type) diff --git a/crates/vm/src/bytes_inner.rs b/crates/vm/src/bytes_inner.rs index 6c76808c5ec..65a9dc0a01c 100644 --- a/crates/vm/src/bytes_inner.rs +++ b/crates/vm/src/bytes_inner.rs @@ -1,6 +1,7 @@ // spell-checker:ignore unchunked use crate::{ - AsObject, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, VirtualMachine, + AsObject, PyObject, PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, + VirtualMachine, anystr::{self, AnyStr, AnyStrContainer, AnyStrWrapper}, builtins::{ PyBaseExceptionRef, PyByteArray, PyBytes, PyBytesRef, PyInt, PyIntRef, PyStr, PyStrRef, @@ -12,7 +13,7 @@ use crate::{ common::wtf8::is_py_ascii_whitespace, function::{ArgIterable, Either, OptionalArg, OptionalOption, PyComparisonValue}, literal::escape::Escape, - protocol::PyBuffer, + protocol::{BufferFlags, PyBuffer}, sequence::{SequenceExt, SequenceMutExt}, types::PyComparisonOp, }; @@ -35,9 +36,10 @@ impl From> for PyBytesInner { } } +/// "y*": any bytes-like object, and nothing else. impl<'a> TryFromBorrowedObject<'a> for PyBytesInner { fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { - bytes_from_object(vm, obj).map(Self::from) + obj.try_bytes_like(vm, <[u8]>::to_vec).map(Self::from) } } @@ -76,7 +78,7 @@ impl ByteInnerNewOptions { } else { size as usize }; - Ok(vec![0; size].into()) + Ok(vm.new_zeroed_bytes(size)?.into()) } fn handle_object_fallback(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -137,10 +139,50 @@ impl ByteInnerNewOptions { } } +/// What is searched for: a bytes-like object, or a single byte given as an +/// integer. parse_args_finds_byte +pub enum ByteInnerSub { + Buffer(PyBytesInner), + Byte(PyIntRef), +} + +impl TryFromObject for ByteInnerSub { + fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { + if obj.check_buffer() { + return PyBytesInner::try_from_object(vm, obj).map(Self::Buffer); + } + match obj.try_index_opt(vm) { + Some(int) => int.map(Self::Byte), + None => Err(vm.new_type_error(format!( + "argument should be integer or bytes-like object, not '{}'", + obj.class().name() + ))), + } + } +} + +impl ByteInnerSub { + /// The needle of a containment test, which is an integer if it is one at + /// all and a bytes-like object otherwise. bytes_contains + pub fn from_contains_arg(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { + match obj.try_index_opt(vm) { + Some(int) => int.map(Self::Byte), + None => PyBytesInner::try_from_object(vm, obj).map(Self::Buffer), + } + } + + fn into_vec(self, vm: &VirtualMachine) -> PyResult> { + Ok(match self { + Self::Buffer(buffer) => buffer.elements, + Self::Byte(int) => vec![int.as_bigint().byte_or(vm)?], + }) + } +} + #[derive(FromArgs)] pub struct ByteInnerFindOptions { #[pyarg(positional)] - sub: Either, + sub: ByteInnerSub, #[pyarg(positional, default)] start: Option, #[pyarg(positional, default)] @@ -153,10 +195,7 @@ impl ByteInnerFindOptions { len: usize, vm: &VirtualMachine, ) -> PyResult<(Vec, core::ops::Range)> { - let sub = match self.sub { - Either::A(v) => v.elements.to_vec(), - Either::B(int) => vec![int.as_bigint().byte_or(vm)?], - }; + let sub = self.sub.into_vec(vm)?; let range = anystr::adjust_indices(self.start, self.end, len); Ok((sub, range)) } @@ -203,14 +242,11 @@ impl ByteInnerTranslateOptions { let table = self.table.map_or_else( || Ok((0..=u8::MAX).collect::>()), |v| { - let bytes = v - .try_into_value::(vm) - .ok() - .filter(|v| v.elements.len() == 256) - .ok_or_else(|| { - vm.new_value_error("translation table must be 256 characters long") - })?; - Ok(bytes.elements.to_vec()) + let bytes: PyBytesInner = v.try_into_value(vm)?; + if bytes.elements.len() != 256 { + return Err(vm.new_value_error("translation table must be 256 characters long")); + } + Ok(bytes.elements) }, )?; @@ -345,7 +381,12 @@ impl PyBytesInner { // but not memoryview, and not equal if compare with unicode str(PyStr) PyComparisonValue::from_option( other - .try_bytes_like(vm, |other| op.eval_ord(self.elements.as_slice().cmp(other))) + .try_bytes_like(vm, |other| { + // Equality does not need the ordering, and answers two + // buffers of different length without reading either. + op.eval_eq(|| self.elements.as_slice() == other) + .unwrap_or_else(|| op.eval_ord(self.elements.as_slice().cmp(other))) + }) .ok(), ) } @@ -358,10 +399,10 @@ impl PyBytesInner { self.elements.py_add(other) } - pub fn contains(&self, needle: Either, vm: &VirtualMachine) -> PyResult { + pub fn contains(&self, needle: ByteInnerSub, vm: &VirtualMachine) -> PyResult { Ok(match needle { - Either::A(byte) => self.elements.contains_str(byte.elements.as_slice()), - Either::B(int) => self.elements.contains(&int.as_bigint().byte_or(vm)?), + ByteInnerSub::Buffer(sub) => self.elements.contains_str(sub.elements.as_slice()), + ByteInnerSub::Byte(int) => self.elements.contains(&int.as_bigint().byte_or(vm)?), }) } @@ -517,7 +558,8 @@ impl PyBytesInner { pub fn fromhex_object(string: PyObjectRef, vm: &VirtualMachine) -> PyResult> { if let Some(s) = string.downcast_ref::() { Self::fromhex(s.as_bytes(), vm) - } else if let Ok(buffer) = PyBuffer::try_from_borrowed_object(vm, &string) { + } else if string.check_buffer() { + let buffer = PyBuffer::from_object(vm, &string, BufferFlags::SIMPLE)?; let borrowed = buffer .as_contiguous() .ok_or_else(|| vm.new_buffer_error("fromhex() requires a contiguous buffer"))?; @@ -534,16 +576,15 @@ impl PyBytesInner { fn _pad( &self, options: ByteInnerPaddingOptions, - pad: fn(&[u8], usize, u8, usize) -> Vec, + pad: PadFn, vm: &VirtualMachine, ) -> PyResult> { let (width, fillchar) = options.get_value("center", vm)?; let len = self.len(); - Ok(if len as isize >= width { - Vec::from(&self.elements[..]) - } else { - pad(&self.elements, width as usize, fillchar, len) - }) + if len as isize >= width { + return Ok(Vec::from(&self.elements[..])); + } + pad(&self.elements, width as usize, fillchar, len).ok_or_else(|| vm.new_memory_error("")) } pub fn center( @@ -779,8 +820,10 @@ impl PyBytesInner { self.elements.py_bytes_splitlines(options, into_wrapper) } - pub fn zfill(&self, width: isize) -> Vec { - self.elements.py_zfill(width) + pub fn zfill(&self, width: isize, vm: &VirtualMachine) -> PyResult> { + self.elements + .py_zfill(width) + .ok_or_else(|| vm.new_memory_error("")) } // len(self)>=1, from="", len(to)>=1, max_count>=1 @@ -969,7 +1012,7 @@ impl PyBytesInner { } pub fn concat(&self, other: &PyObject, vm: &VirtualMachine) -> PyResult> { - let buffer = PyBuffer::try_from_borrowed_object(vm, other)?; + let buffer = PyBuffer::from_object(vm, other, BufferFlags::SIMPLE)?; let borrowed = buffer.as_contiguous(); if let Some(other) = borrowed { let mut v = Vec::with_capacity(self.elements.len() + other.len()); @@ -1035,11 +1078,21 @@ impl AnyStrContainer<[u8]> for Vec { Self::with_capacity(capacity) } + fn try_with_capacity(capacity: usize) -> Option { + let mut v = Self::new(); + v.try_reserve_exact(capacity).ok()?; + Some(v) + } + fn push_str(&mut self, other: &[u8]) { self.extend(other) } } +/// A padding function from `AnyStr`, returning `None` for a width whose result +/// cannot be allocated. +type PadFn = fn(&[u8], usize, u8, usize) -> Option>; + const ASCII_WHITESPACES: [u8; 6] = [0x20, 0x09, 0x0a, 0x0c, 0x0d, 0x0b]; impl anystr::AnyChar for u8 { diff --git a/crates/vm/src/cformat.rs b/crates/vm/src/cformat.rs index 3bba0e5f8e7..7d47da39928 100644 --- a/crates/vm/src/cformat.rs +++ b/crates/vm/src/cformat.rs @@ -22,7 +22,7 @@ use crate::{ wtf8::{CodePoint, Wtf8, Wtf8Buf}, }, function::ArgIntoFloat, - protocol::PyBuffer, + protocol::{BufferFlags, PyBuffer}, stdlib::builtins, }; @@ -39,24 +39,29 @@ fn spec_format_bytes( let b = builtins::ascii(obj, vm)?.as_bytes().to_vec(); Ok(b) } + // format_obj CFormatConversion::Str | CFormatConversion::Bytes => { - if let Ok(buffer) = PyBuffer::try_from_borrowed_object(vm, &obj) { - Ok(buffer.contiguous_or_collect(|bytes| spec.format_bytes(bytes))) - } else { - let bytes = vm - .get_special_method(&obj, identifier!(vm, __bytes__))? - .ok_or_else(|| { - let msg = format!( - "%b requires a bytes-like object, or an object that \ - implements __bytes__, not '{}'", - obj.class().name() - ); - vm.new_type_error(msg) - })? - .invoke((), vm)?; + if let Some(bytes) = obj.downcast_ref::() { + return Ok(spec.format_bytes(bytes.as_bytes())); + } + if let Some(bytearray) = obj.downcast_ref::() { + return Ok(spec.format_bytes(&bytearray.borrow_buf())); + } + if let Some(method) = vm.get_special_method(&obj, identifier!(vm, __bytes__))? { + let bytes = method.invoke((), vm)?; let bytes = PyBytes::try_from_borrowed_object(vm, &bytes)?; - Ok(spec.format_bytes(bytes.as_bytes())) + return Ok(spec.format_bytes(bytes.as_bytes())); } + if obj.check_buffer() { + let buffer = PyBuffer::from_object(vm, &obj, BufferFlags::FULL_RO)?; + return Ok(buffer.contiguous_or_collect(|bytes| spec.format_bytes(bytes))); + } + let msg = format!( + "%b requires a bytes-like object, or an object that \ + implements __bytes__, not '{}'", + obj.class().name() + ); + Err(vm.new_type_error(msg)) } }, CFormatType::Number(number_type) => match number_type { diff --git a/crates/vm/src/codecs.rs b/crates/vm/src/codecs.rs index c06caefef51..073fd102f38 100644 --- a/crates/vm/src/codecs.rs +++ b/crates/vm/src/codecs.rs @@ -802,7 +802,7 @@ impl DecodeContext for PyDecodeContext<'_> { } else { vm.ctx.new_bytes(self.data.to_vec()) }; - vm.new_unicode_decode_error_real( + vm.new_unicode_decode_error( vm.ctx.new_str(self.encoding), data, byte_range.start, diff --git a/crates/vm/src/coroutine.rs b/crates/vm/src/coroutine.rs index 43a28320e00..61431ea82e3 100644 --- a/crates/vm/src/coroutine.rs +++ b/crates/vm/src/coroutine.rs @@ -51,6 +51,21 @@ unsafe impl Traverse for Coro { } } +/// An exclusive claim on a generator's frame, released when dropped. +/// +/// Only the holder may look at the frame or resume it. Resuming decides from +/// the frame state whether the sent value goes on the value stack, so a state +/// read taken before the claim can be answered by a frame that another thread +/// then advances: resuming it leaves the stack short of what the code after +/// the yield pops. +struct RunningGuard<'a>(&'a Coro); + +impl Drop for RunningGuard<'_> { + fn drop(&mut self) { + self.0.running.store(false); + } +} + fn gen_name(jen: &PyObject, vm: &VirtualMachine) -> &'static str { let typ = jen.class(); if typ.is(vm.ctx.types.coroutine_type) { @@ -88,10 +103,10 @@ impl Coro { } } - fn maybe_close(&self, res: &PyResult, entered_frame: bool) { - if !entered_frame { - return; - } + /// Retire the generator if the frame it just ran came to an end. The claim + /// is still held, so a thread waiting for it cannot resume a frame that has + /// already finished. + fn maybe_close(&self, res: &PyResult, _claim: &RunningGuard<'_>) { match res { Ok(ExecutionResult::Return(_)) | Err(_) => { self.closed.store(true); @@ -109,45 +124,44 @@ impl Coro { } } - fn run_with_context( + /// Take the frame for this thread, or report that another thread holds it. + /// + /// What the resume depends on -- whether the generator is closed, and + /// whether it has started -- has to be read from here onwards. + fn claim(&self, jen: &PyObject, vm: &VirtualMachine) -> PyResult> { + if self.running.compare_exchange(false, true).is_err() { + return Err(vm.new_value_error(format!("{} already executing", gen_name(jen, vm)))); + } + Ok(RunningGuard(self)) + } + + fn run_claimed( &self, - jen: &PyObject, + _claim: &RunningGuard<'_>, vm: &VirtualMachine, func: F, - ) -> (PyResult, bool) + ) -> PyResult where F: FnOnce(&Py) -> PyResult, { - if self.running.compare_exchange(false, true).is_err() { - return ( - Err(vm.new_value_error(format!("{} already executing", gen_name(jen, vm)))), - false, - ); - } - - // SAFETY: running.compare_exchange guarantees exclusive access + // SAFETY: the claim guarantees exclusive access let gen_exc = unsafe { self.exception.swap(None) }; let exception_ptr = &self.exception as *const PyAtomicRef>; - let result = vm.resume_gen_frame(&self.frame, gen_exc, |f| { + vm.resume_gen_frame(&self.frame, gen_exc, |f| { let result = func(f); - // SAFETY: exclusive access guaranteed by running flag + // SAFETY: exclusive access guaranteed by the claim let _old = unsafe { (*exception_ptr).swap(vm.current_exception()) }; result - }); - - self.running.store(false); - (result, true) + }) } fn finalize_send_result( &self, result: PyResult, - entered_frame: bool, jen: &PyObject, vm: &VirtualMachine, ) -> PyResult { - self.maybe_close(&result, entered_frame); match result { Ok(exec_res) => Ok(exec_res.into_iter_return(vm)), Err(e) => { @@ -177,16 +191,20 @@ impl Coro { if self.closed.load() { return Ok(PyIterReturn::StopIteration(None)); } - if self.running.load() { - return Err(vm.new_value_error(format!("{} already executing", gen_name(jen, vm)))); + let claim = self.claim(jen, vm)?; + // The generator can have run to its end in the meantime. + if self.closed.load() { + return Ok(PyIterReturn::StopIteration(None)); } let value = if self.frame.lasti() > 0 { Some(vm.ctx.none()) } else { None }; - let (result, entered_frame) = self.run_with_context(jen, vm, |f| f.resume(value, vm)); - self.finalize_send_result(result, entered_frame, jen, vm) + let result = self.run_claimed(&claim, vm, |f| f.resume(value, vm)); + self.maybe_close(&result, &claim); + drop(claim); + self.finalize_send_result(result, jen, vm) } pub fn send( @@ -198,8 +216,10 @@ impl Coro { if self.closed.load() { return Ok(PyIterReturn::StopIteration(None)); } - if self.running.load() { - return Err(vm.new_value_error(format!("{} already executing", gen_name(jen, vm)))); + let claim = self.claim(jen, vm)?; + // The generator can have run to its end in the meantime. + if self.closed.load() { + return Ok(PyIterReturn::StopIteration(None)); } let value = if self.frame.lasti() > 0 { Some(value) @@ -211,8 +231,10 @@ impl Coro { } else { None }; - let (result, entered_frame) = self.run_with_context(jen, vm, |f| f.resume(value, vm)); - self.finalize_send_result(result, entered_frame, jen, vm) + let result = self.run_claimed(&claim, vm, |f| f.resume(value, vm)); + self.maybe_close(&result, &claim); + drop(claim); + self.finalize_send_result(result, jen, vm) } pub fn throw( @@ -237,13 +259,25 @@ impl Coro { // Validate exception type before entering generator context. // Invalid types propagate to caller without closing the generator. crate::exceptions::ExceptionCtor::try_from_object(vm, exc_type.clone())?; - let (result, entered_frame) = - self.run_with_context(jen, vm, |f| f.gen_throw(vm, exc_type, exc_val, exc_tb)); - self.maybe_close(&result, entered_frame); + let claim = self.claim(jen, vm)?; + // The generator can have run to its end in the meantime. Normalizing + // runs the exception's constructor, so let the claim go first. + if self.closed.load() { + drop(claim); + return Err(vm.normalize_exception(exc_type, exc_val, exc_tb)?); + } + let result = self.run_claimed(&claim, vm, |f| f.gen_throw(vm, exc_type, exc_val, exc_tb)); + self.maybe_close(&result, &claim); + drop(claim); Ok(result?.into_iter_return(vm)) } pub fn close(&self, jen: &PyObject, vm: &VirtualMachine) -> PyResult { + if self.closed.load() { + return Ok(vm.ctx.none()); + } + let claim = self.claim(jen, vm)?; + // The generator can have run to its end in the meantime. if self.closed.load() { return Ok(vm.ctx.none()); } @@ -252,7 +286,7 @@ impl Coro { self.closed.store(true); return Ok(vm.ctx.none()); } - let (result, entered_frame) = self.run_with_context(jen, vm, |f| { + let result = self.run_claimed(&claim, vm, |f| { f.gen_throw( vm, vm.ctx.exceptions.generator_exit.to_owned().into(), @@ -260,16 +294,11 @@ impl Coro { vm.ctx.none(), ) }); - if !entered_frame { - return match result { - Err(err) => Err(err), - Ok(_) => unreachable!("run_with_context preflight returned without an error"), - }; - } self.closed.store(true); // Release frame locals and stack to free references held by the // closed generator, matching gen_send_ex2 with close_on_completion. self.clear_frame_locals_on_close(); + drop(claim); match result { Ok(ExecutionResult::Yield(_)) => { Err(vm.new_runtime_error(format!("{} ignored GeneratorExit", gen_name(jen, vm)))) diff --git a/crates/vm/src/datastack.rs b/crates/vm/src/datastack.rs index 101369fba57..ec4c22ae808 100644 --- a/crates/vm/src/datastack.rs +++ b/crates/vm/src/datastack.rs @@ -61,6 +61,9 @@ pub struct DataStack { top: *mut u8, /// End of usable space in the current chunk. limit: *mut u8, + /// Most recently popped full-frame allocation whose localsplus slots were + /// cleared before the pop. An exact LIFO reuse can skip zero-filling them. + reusable_frame: Option<(*mut u8, usize)>, } impl DataStack { @@ -73,7 +76,12 @@ impl DataStack { // Skip one ALIGN-sized slot in the root chunk so that `pop()` never // frees it (`push_chunk` convention). let top = unsafe { top.add(ALIGN) }; - Self { chunk, top, limit } + Self { + chunk, + top, + limit, + reusable_frame: None, + } } /// Check if the current chunk has at least `size` bytes available. @@ -91,6 +99,26 @@ impl DataStack { /// (LIFO order). #[inline(always)] pub fn push(&mut self, size: usize) -> *mut u8 { + self.reusable_frame = None; + self.push_inner(size) + } + + /// Allocate a full interpreter frame and report whether it exactly reuses + /// a just-cleared frame block. + #[inline(always)] + pub fn push_frame(&mut self, size: usize) -> (*mut u8, bool) { + let reusable_frame = self.reusable_frame.take(); + let ptr = self.push_inner(size); + // Exact sizes, not aligned ones: the caller reads "reused" as "every + // slot of this frame was cleared by the last one", and two frames whose + // sizes differ by less than ALIGN share an aligned size while the + // larger one's tail slots were never touched, let alone cleared. + let reused = reusable_frame.is_some_and(|(base, old_size)| base == ptr && old_size == size); + (ptr, reused) + } + + #[inline(always)] + fn push_inner(&mut self, size: usize) -> *mut u8 { let aligned_size = (size + ALIGN - 1) & !(ALIGN - 1); unsafe { if self.top.add(aligned_size) <= self.limit { @@ -138,6 +166,24 @@ impl DataStack { /// and all allocations made after it must already have been popped. #[inline(always)] pub unsafe fn pop(&mut self, base: *mut u8) { + self.reusable_frame = None; + unsafe { self.pop_inner(base) }; + } + + /// Pop a full frame whose localsplus slots have already been cleared. + /// + /// # Safety + /// `base` and `size` must describe the most recent allocation returned by + /// `push_frame`, every later allocation must already be popped, and all + /// localsplus slots in the frame must have been cleared. + #[inline(always)] + pub unsafe fn pop_frame(&mut self, base: *mut u8, size: usize) { + unsafe { self.pop_inner(base) }; + self.reusable_frame = Some((base, size)); + } + + #[inline(always)] + unsafe fn pop_inner(&mut self, base: *mut u8) { debug_assert!(!base.is_null()); if self.is_in_current_chunk(base) { // Common case: base is within the current chunk. diff --git a/crates/vm/src/dict_inner.rs b/crates/vm/src/dict_inner.rs index 3e75e6f27a6..76d2c50f0cb 100644 --- a/crates/vm/src/dict_inner.rs +++ b/crates/vm/src/dict_inner.rs @@ -20,7 +20,7 @@ use alloc::fmt; use core::mem::size_of; use core::ops::ControlFlow; use core::sync::atomic::{ - AtomicU32, AtomicU64, + AtomicU32, Ordering::{AcqRel, Acquire, Relaxed, Release}, }; use num_traits::ToPrimitive; @@ -39,7 +39,6 @@ type EntryIndex = usize; pub(crate) struct Dict { inner: PyRwLock>, - version: AtomicU64, /// Keys-version stamp, assigned lazily by `assign_keys_version` and /// reset to 0 whenever the key set changes. Value-only updates keep it. /// @@ -202,7 +201,6 @@ impl Clone for Dict { fn clone(&self) -> Self { Self { inner: PyRwLock::new(self.inner.read().clone()), - version: AtomicU64::new(0), keys_version: AtomicU32::new(0), } } @@ -217,7 +215,6 @@ impl Default for Dict { indices: vec![IndexEntry::FREE; 8], entries: Vec::new(), }), - version: AtomicU64::new(0), keys_version: AtomicU32::new(0), } } @@ -240,6 +237,10 @@ pub struct DictSize { filled: usize, } +/// The dict was resized under an iterator holding an older [`DictSize`]. +#[derive(Debug)] +pub(crate) struct DictChanged; + struct GenIndexes { idx: HashIndex, perturb: HashValue, @@ -309,7 +310,7 @@ impl DictInner { key: PyObjectRef, value: T, index_entry: IndexEntry, - ) { + ) -> usize { let entry = DictEntry { hash: hash_value, key, @@ -330,6 +331,9 @@ impl DictInner { self.resize(new_size) } } + // A resize keeps entry positions and rewrites only the index-index, so + // this stays the entry's index afterwards. + entry_index } const fn size(&self) -> DictSize { @@ -362,16 +366,6 @@ impl DictInner { type PopInnerResult = ControlFlow>>; impl Dict { - /// Monotonically increasing version counter for mutation tracking. - pub(crate) fn version(&self) -> u64 { - self.version.load(Acquire) - } - - /// Bump the version counter after any mutation. - fn bump_version(&self) { - self.version.fetch_add(1, Release); - } - /// Current keys-version stamp, or 0 if none has been assigned since the /// last key-set change. Equal nonzero stamps guarantee an unchanged key /// set (values may differ). @@ -463,7 +457,44 @@ impl Dict { K: DictKey + ?Sized, { let hash = key.key_hash(vm)?; - let _removed = loop { + self.insert_known_hash(vm, key, hash, value) + } + + /// Store a key whose hash the caller already knows. + /// + /// `hash` must equal `key.key_hash(vm)`; a wrong one lands the entry in a + /// bucket no lookup probes, silently losing the key. Only pass a hash from + /// [`Self::keys_with_hashes`] on a container holding this same key. + pub(crate) fn insert_known_hash( + &self, + vm: &VirtualMachine, + key: &K, + hash: HashValue, + value: T, + ) -> PyResult<()> + where + K: DictKey + ?Sized, + { + self.insert_known_hash_indexed(vm, key, hash, value)?; + Ok(()) + } + + /// [`Self::insert_known_hash`], also reporting the entry index it stored to. + /// + /// The index doubles as a `hint` for [`Self::get_hint`] / + /// [`Self::insert_with_hint`], so a caller that wants one gets it from the + /// store itself instead of probing the dict a second time. + fn insert_known_hash_indexed( + &self, + vm: &VirtualMachine, + key: &K, + hash: HashValue, + value: T, + ) -> PyResult + where + K: DictKey + ?Sized, + { + let (stored_index, _removed) = loop { let (entry_index, index_index) = self.lookup(vm, key, hash, None)?; let mut inner = self.write(); if let Some(index) = entry_index.index() { @@ -482,9 +513,8 @@ impl Dict { )] if entry.index == index_index { let removed = core::mem::replace(&mut entry.value, value); - self.bump_version(); // defer dec RC - break Some(removed); + break (index, Some(removed)); } else { // stuff shifted around, let's try again } @@ -498,12 +528,17 @@ impl Dict { continue; } self.invalidate_keys_version(); - inner.unchecked_push(index_index, hash, key.to_pyobject(vm), value, entry_index); - self.bump_version(); - break None; + let stored = inner.unchecked_push( + index_index, + hash, + key.to_pyobject(vm), + value, + entry_index, + ); + break (stored, None); } }; - Ok(()) + Ok(stored_index) } pub(crate) fn contains( @@ -512,7 +547,18 @@ impl Dict { key: &K, ) -> PyResult { let key_hash = key.key_hash(vm)?; - let (entry, _) = self.lookup(vm, key, key_hash, None)?; + self.contains_known_hash(vm, key, key_hash) + } + + /// [`Self::contains`] with a known hash. Same contract as + /// [`Self::insert_known_hash`]. + pub(crate) fn contains_known_hash( + &self, + vm: &VirtualMachine, + key: &K, + hash: HashValue, + ) -> PyResult { + let (entry, _) = self.lookup(vm, key, hash, None)?; Ok(entry.index().is_some()) } @@ -587,7 +633,6 @@ impl Dict { match inner.entries.get_mut(hint) { Some(Some(entry)) if key.key_is(&entry.key) => { let removed = core::mem::replace(&mut entry.value, value); - self.bump_version(); drop(inner); // defer dec RC until after the lock is released drop(removed); @@ -596,8 +641,9 @@ impl Dict { _ => value, } }; - self.insert(vm, key, value)?; - self.hint_for_key(vm, key) + let hash = key.key_hash(vm)?; + let stored = self.insert_known_hash_indexed(vm, key, hash, value)?; + Ok(u16::try_from(stored).ok()) } /// Fast path lookup using a cached entry index (`hint`). @@ -627,6 +673,22 @@ impl Dict { } } + /// Read an entry directly when a cached keys-version still describes the + /// dictionary layout. The version is rechecked while holding the read lock + /// so the entry index and value are observed from the same key-set state. + #[inline] + pub(crate) fn get_index_if_keys_version(&self, version: u32, index: usize) -> Option { + let inner = self.read(); + if self.keys_version.load(Acquire) != version { + return None; + } + inner + .entries + .get(index) + .and_then(Option::as_ref) + .map(|entry| entry.value.clone()) + } + fn _get_inner( &self, vm: &VirtualMachine, @@ -634,7 +696,12 @@ impl Dict { hash: HashValue, ) -> PyResult> { let ret = loop { - let (entry, index_index) = self.lookup(vm, key, hash, None)?; + let (entry, index_index) = + match self.lookup_extract(vm, key, hash, None, |entry| entry.value.clone())? { + // Read under the probe's own guard: nothing to re-check. + (_, Some(value)) => break Some(value), + (lookup, None) => lookup, + }; if let Some(index) = entry.index() { let inner = self.read(); if let Some(entry) = inner.get_entry_checked(index, index_index) { @@ -672,7 +739,6 @@ impl Dict { inner.indices.resize(8, IndexEntry::FREE); inner.used = 0; inner.filled = 0; - self.bump_version(); // defer dec rc core::mem::take(&mut inner.entries) }; @@ -697,6 +763,21 @@ impl Dict { self.remove_if_exists(vm, key).map(|opt| opt.is_some()) } + /// [`Self::delete_if_exists`] with a known hash. Same contract as + /// [`Self::insert_known_hash`]. + pub(crate) fn delete_if_exists_known_hash( + &self, + vm: &VirtualMachine, + key: &K, + hash: HashValue, + ) -> PyResult + where + K: DictKey + ?Sized, + { + self.remove_if_known_hash(vm, key, hash, |_| Ok(true)) + .map(|opt| opt.is_some()) + } + pub(crate) fn delete_if(&self, vm: &VirtualMachine, key: &K, pred: F) -> PyResult where K: DictKey + ?Sized, @@ -725,6 +806,22 @@ impl Dict { F: Fn(&T) -> PyResult, { let hash = key.key_hash(vm)?; + self.remove_if_known_hash(vm, key, hash, pred) + } + + /// [`Self::remove_if`] with a known hash. Same contract as + /// [`Self::insert_known_hash`]. + fn remove_if_known_hash( + &self, + vm: &VirtualMachine, + key: &K, + hash: HashValue, + pred: F, + ) -> PyResult> + where + K: DictKey + ?Sized, + F: Fn(&T) -> PyResult, + { let removed = loop { let lookup = self.lookup(vm, key, hash, None)?; match self.pop_inner_if(lookup, &pred)? { @@ -742,6 +839,18 @@ impl Dict { value: T, ) -> PyResult<()> { let hash = key.key_hash(vm)?; + self.delete_or_insert_known_hash(vm, key, hash, value) + } + + /// [`Self::delete_or_insert`] with a known hash. Same contract as + /// [`Self::insert_known_hash`]. + pub(crate) fn delete_or_insert_known_hash( + &self, + vm: &VirtualMachine, + key: &PyObject, + hash: HashValue, + value: T, + ) -> PyResult<()> { let _removed = loop { let lookup = self.lookup(vm, key, hash, None)?; let (entry, index_index) = lookup; @@ -758,7 +867,6 @@ impl Dict { } self.invalidate_keys_version(); inner.unchecked_push(index_index, hash, key.to_owned(), value, entry); - self.bump_version(); break None; }; Ok(()) @@ -795,7 +903,6 @@ impl Dict { value.clone(), index_entry, ); - self.bump_version(); return Ok(value); } } @@ -833,7 +940,6 @@ impl Dict { let ret = (key_obj.clone(), value.clone()); self.invalidate_keys_version(); inner.unchecked_push(index_index, hash, key_obj, value, index_entry); - self.bump_version(); return Ok(ret); } } @@ -846,6 +952,58 @@ impl Dict { self.read().size() } + /// Step to the first live entry at or after `position`, verifying the size + /// against `old` under the same read guard. + /// + /// `project` runs under that guard, so it must not run Python or take + /// another dict lock; it is there so an iterator clones only the field it + /// keeps rather than both the key and the value. + pub(crate) fn next_entry_checked( + &self, + mut position: EntryIndex, + old: &DictSize, + project: impl FnOnce(&PyObjectRef, &T) -> R, + ) -> Result, DictChanged> { + let inner = self.read(); + if inner.size() != *old { + return Err(DictChanged); + } + loop { + let Some(entry) = inner.entries.get(position) else { + return Ok(None); + }; + position += 1; + if let Some(entry) = entry { + return Ok(Some((position, project(&entry.key, &entry.value)))); + } + } + } + + /// [`Self::next_entry_checked`] in reverse. + pub(crate) fn prev_entry_checked( + &self, + mut position: EntryIndex, + old: &DictSize, + project: impl FnOnce(&PyObjectRef, &T) -> R, + ) -> Result, DictChanged> { + let inner = self.read(); + if inner.size() != *old { + return Err(DictChanged); + } + loop { + let Some(entry) = inner.entries.get(position) else { + return Ok(None); + }; + if let Some(entry) = entry { + return Ok(Some((position, project(&entry.key, &entry.value)))); + } + if position == 0 { + return Ok(None); + } + position -= 1; + } + } + pub(crate) fn next_entry(&self, mut position: EntryIndex) -> Option<(usize, PyObjectRef, T)> { let inner = self.read(); loop { @@ -888,6 +1046,16 @@ impl Dict { .collect() } + /// All keys paired with the hash stored in their entry, for feeding + /// [`Self::insert_known_hash`] without re-calling `__hash__`. + pub(crate) fn keys_with_hashes(&self) -> Vec<(PyObjectRef, HashValue)> { + self.read() + .entries + .iter() + .filter_map(|v| v.as_ref().map(|v| (v.key.clone(), v.hash))) + .collect() + } + pub(crate) fn values(&self) -> Vec { self.read() .entries @@ -922,8 +1090,30 @@ impl Dict { vm: &VirtualMachine, key: &K, hash_value: HashValue, - mut lock: Option>>, + lock: Option>>, ) -> PyResult { + let (ret, _) = self.lookup_extract(vm, key, hash_value, lock, |_| ())?; + Ok(ret) + } + + /// [`Self::lookup`], additionally reading the matched entry when the probe + /// settles it by key identity. + /// + /// That is the common case, and it is decided while the read guard is still + /// held — so a caller that only wants the entry's value gets it here instead + /// of taking the lock a second time to re-find what the probe already had. + /// `extract` therefore runs under the guard and must not run Python. It is + /// not called when the key had to be compared with `key_eq`, which does run + /// Python and so releases the guard first. + #[cfg_attr(feature = "flame-it", flame("Dict"))] + fn lookup_extract( + &self, + vm: &VirtualMachine, + key: &K, + hash_value: HashValue, + mut lock: Option>>, + extract: impl Fn(&DictEntry) -> R, + ) -> PyResult<(LookupResult, Option)> { let mut idxs = None; let mut free_slot = None; let ret = 'outer: loop { @@ -953,7 +1143,7 @@ impl Dict { Some(free) => (IndexEntry::DUMMY, free), None => (IndexEntry::FREE, index_index), }; - return Ok(idxs); + return Ok((idxs, None)); } idx => { let entry = unsafe { @@ -969,7 +1159,7 @@ impl Dict { reason = "Keeping the empty `else` block here for documentation" )] if key.key_is(&entry.key) { - break 'outer ret; + return Ok((ret, Some(extract(entry)))); } else if entry.hash == hash_value { break (entry.key.clone(), ret); } else { @@ -994,7 +1184,7 @@ impl Dict { // warn!("Perturb value: {}", i); }; - Ok(ret) + Ok((ret, None)) } // returns Err(()) if changed since lookup @@ -1035,7 +1225,6 @@ impl Dict { } = IndexEntry::DUMMY; inner.used -= 1; let removed = slot.take(); - self.bump_version(); Ok(ControlFlow::Break(removed)) } @@ -1070,7 +1259,6 @@ impl Dict { // entry.index always refers valid index inner.indices.get_unchecked_mut(entry.index) } = IndexEntry::DUMMY; - self.bump_version(); Some((entry.key, entry.value)) } diff --git a/crates/vm/src/exception_group.rs b/crates/vm/src/exception_group.rs index c6d18cc6594..11c13912b76 100644 --- a/crates/vm/src/exception_group.rs +++ b/crates/vm/src/exception_group.rs @@ -60,7 +60,7 @@ pub(super) mod types { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } @@ -255,20 +255,11 @@ pub(super) mod types { ))); } - // Validate exceptions is a sequence (not set or None) + // Validate exceptions is a sequence let exceptions_arg = &args[1]; - - // Check for set/frozenset (not a sequence - unordered) - if exceptions_arg.fast_isinstance(vm.ctx.types.set_type) - || exceptions_arg.fast_isinstance(vm.ctx.types.frozenset_type) - { - return Err(vm.new_type_error("second argument (exceptions) must be a sequence")); - } - - // Check for None - if exceptions_arg.is(&vm.ctx.none) { - return Err(vm.new_type_error("second argument (exceptions) must be a sequence")); - } + exceptions_arg.try_sequence(vm).map_err(|_| { + vm.new_type_error("second argument (exceptions) must be a sequence") + })?; let exceptions: Vec = exceptions_arg.try_to_value(vm).map_err(|_| { vm.new_type_error("second argument (exceptions) must be a sequence") diff --git a/crates/vm/src/exceptions.rs b/crates/vm/src/exceptions.rs index 9c42df966ea..0a1c2cb75ee 100644 --- a/crates/vm/src/exceptions.rs +++ b/crates/vm/src/exceptions.rs @@ -988,6 +988,9 @@ impl ExceptionZoo { extend_exception!(PyImportError, ctx, excs.import_error, { "msg" => ctx.new_readonly_getset("msg", excs.import_error, make_arg_getter(0)), + "name" => ctx.none(), + "path" => ctx.none(), + "name_from" => ctx.none(), }); extend_exception!(PyModuleNotFoundError, ctx, excs.module_not_found_error); @@ -1908,10 +1911,11 @@ pub(super) mod types { #[pymethod] fn __reduce__(exc: PyBaseExceptionRef, vm: &VirtualMachine) -> PyTupleRef { let obj = exc.as_object().to_owned(); - let mut result: Vec = vec![ - obj.class().to_owned().into(), - vm.new_tuple((exc.get_arg(0).unwrap(),)).into(), - ]; + let args: PyObjectRef = match exc.get_arg(0) { + Some(arg) => vm.new_tuple((arg,)).into(), + None => exc.args().into(), + }; + let mut result: Vec = vec![obj.class().to_owned().into(), args]; if let Some(dict) = obj.dict().filter(|x| !x.is_empty()) { result.push(dict.into()); @@ -1938,10 +1942,21 @@ pub(super) mod types { ))); } - let dict = crate::builtins::object::object_get_dict(zelf.clone(), vm)?; - dict.set_item("name", vm.unwrap_or_none(name), vm)?; - dict.set_item("path", vm.unwrap_or_none(path), vm)?; - dict.set_item("name_from", vm.unwrap_or_none(name_from), vm)?; + if let Some(name) = name { + zelf.set_attr("name", name, vm)?; + } else if let Some(dict) = zelf.dict() { + dict.del_item("name", vm).ok(); + } + if let Some(path) = path { + zelf.set_attr("path", path, vm)?; + } else if let Some(dict) = zelf.dict() { + dict.del_item("path", vm).ok(); + } + if let Some(name_from) = name_from { + zelf.set_attr("name_from", name_from, vm)?; + } else if let Some(dict) = zelf.dict() { + dict.del_item("name_from", vm).ok(); + } PyBaseException::slot_init(zelf, args, vm) } diff --git a/crates/vm/src/frame.rs b/crates/vm/src/frame.rs index ebb158c8f71..d102b9a6d8e 100644 --- a/crates/vm/src/frame.rs +++ b/crates/vm/src/frame.rs @@ -387,33 +387,6 @@ impl LocalsPlus { Some(base) } - /// Create a new heap-backed LocalsPlus that is a clone of the - /// fastlocals portion of this one. Stack slots are NOT copied - /// (stack_top = 0, stacksize = 0). - /// - /// # Safety - /// The caller must ensure that `self.fastlocals()` is a valid slice - /// (backing storage is alive, not concurrently mutated). - pub(crate) unsafe fn snapshot_to_heap(&self) -> Self { - let n = self.nlocalsplus as usize; - let src = self.fastlocals(); - let mut data = vec![0usize; n]; - // Clone each Option into the heap buffer. - for (i, slot) in src.iter().enumerate() { - if let Some(obj) = slot { - let cloned: Option = Some(obj.clone()); - // SAFETY: Option has the same layout as usize. - data[i] = unsafe { core::mem::transmute_copy(&cloned) }; - core::mem::forget(cloned); - } - } - Self { - data: LocalsPlusData::Heap(data.into_boxed_slice()), - nlocalsplus: self.nlocalsplus, - stack_top: 0, - } - } - /// Update fastlocals in `self` from `src`. For each slot, drops the old /// value and clones the new one. `self` must be heap-backed. /// @@ -562,6 +535,19 @@ impl LocalsPlus { unsafe { core::mem::transmute::>(raw) } } + /// Give every borrowed stack ref its own reference. + /// + /// A borrowed ref is only sound while whatever it points at is guaranteed + /// to outlive it, which stops holding where the frame itself outlives the + /// running block — at a yield, where the stack is saved with the frame. + fn promote_stack(&mut self) { + for idx in 0..self.stack_top as usize { + if let Some(stack_ref) = self.stack_index_mut(idx) { + stack_ref.promote(); + } + } + } + /// Immutable view of the active stack as `Option` slice. #[inline(always)] fn stack_as_slice(&self) -> &[Option] { @@ -813,6 +799,12 @@ pub(crate) struct FrameColdData { pub retained_back: PyMutex>, pub pending_stack_pops: PyAtomic, pub pending_unwind_from_stack: PyAtomic, + /// Thread that is still running the frame this one was materialized from, + /// or 0 once that frame has returned (and for every frame object that was + /// not materialized from a running frame). Only a thread id, never a + /// pointer: reading it can never chase freed memory, so it stays usable + /// as the gate for frames that belong to another thread. + pub attached_tid: atomic::AtomicU64, } impl Default for FrameColdData { @@ -828,6 +820,7 @@ impl Default for FrameColdData { retained_back: PyMutex::new(None), pending_stack_pops: Default::default(), pending_unwind_from_stack: Default::default(), + attached_tid: atomic::AtomicU64::new(0), } } } @@ -899,6 +892,7 @@ impl InterpreterFrame { /// For stack-allocated frames (future), the pointers remain valid for the /// frame's lifetime on the native stack. #[allow(clippy::too_many_arguments)] + #[inline(always)] pub(crate) fn new( code: &Py, globals: &Py, @@ -968,9 +962,10 @@ impl InterpreterFrame { /// Returns a mutable reference whose lifetime is bounded by the data /// stack's LIFO discipline. The caller must call /// `release_datastack_frame()` (unsafe) when done, then - /// `vm.datastack_pop(base)`. The reference must not be used after + /// `vm.datastack_pop_frame(base, size)`. The reference must not be used after /// `release_datastack_frame` returns. #[allow(clippy::too_many_arguments)] + #[inline(always)] pub(crate) fn new_on_datastack<'a>( code: &Py, globals: &Py, @@ -987,7 +982,7 @@ impl InterpreterFrame { .expect("LocalsPlus capacity overflow"); let total_bytes = datastack_iframe_total_bytes(nlocalsplus, stacksize); - let base = vm.datastack_push(total_bytes); + let (base, reused_cleared_frame) = vm.datastack_push_frame(total_bytes); // InterpreterFrame lives at the start of the allocation. let iframe_ptr = base as *mut Self; @@ -995,8 +990,10 @@ impl InterpreterFrame { let localsplus_data_ptr = unsafe { base.add(datastack_iframe_localsplus_offset()) } as *mut usize; - // Zero-initialize localsplus data. - unsafe { core::ptr::write_bytes(localsplus_data_ptr, 0, capacity) }; + if !reused_cleared_frame { + // Fresh or differently shaped storage may contain old frame data. + unsafe { core::ptr::write_bytes(localsplus_data_ptr, 0, capacity) }; + } let nlocalsplus_u32 = u32::try_from(nlocalsplus).expect("nlocalsplus exceeds u32"); let localsplus = LocalsPlus { @@ -1038,11 +1035,15 @@ impl InterpreterFrame { /// After this call, the InterpreterFrame at `self` is logically dead — /// the caller must not use `self` again except to pass the returned /// base to `vm.datastack_pop()`. - pub(crate) unsafe fn release_datastack_frame(&mut self) -> Option<*mut u8> { + pub(crate) unsafe fn release_datastack_frame(&mut self) -> Option<(*mut u8, usize)> { let base = self.datastack_base; if base.is_null() { return None; } + let total_bytes = datastack_iframe_total_bytes( + self.localsplus.nlocalsplus as usize, + self.localsplus.stack_capacity(), + ); self.datastack_base = core::ptr::null_mut(); // Drop all localsplus values while the backing store is still valid. self.localsplus.drop_values(); @@ -1055,7 +1056,7 @@ impl InterpreterFrame { // SAFETY: `self` points to valid, initialized memory on the data // stack. After this call the memory is logically dead. unsafe { core::ptr::drop_in_place(self) }; - Some(base) + Some((base, total_bytes)) } /// Get the last instruction index. @@ -1093,6 +1094,62 @@ impl InterpreterFrame { self.materialize_slow(vm) } + /// Take a standalone copy of this frame, values included, for a thread + /// that does not own it. + /// + /// Nothing links the copy back to this frame: the owning thread will not + /// find it at `exit_iframe` and so never writes into it once the world + /// restarts. That is the whole point — a linked copy is a buffer the owner + /// rewrites slot by slot while the reader clones out of it. + /// + /// # Safety + /// Caller must hold the world stopped, so the owning thread is parked and + /// its fast locals are not moving while they are read. + #[cfg(feature = "threading")] + #[cold] + #[inline(never)] + pub(crate) unsafe fn materialize_detached(&self, vm: &VirtualMachine) -> FrameObjectRef { + // Deliberately not `materialize_chain`: that hands back an existing + // linked copy when the owning thread has already made one. + let fo = self.materialize_slow_chain(vm); + unsafe { + fo.iframe_mut() + .localsplus + .sync_fastlocals_from(&self.localsplus) + }; + fo + } + + /// Copy this frame and everything it was called from for a thread that + /// does not own them, linking `f_back` along the way, and return the copy + /// of this frame. The links are `retained_back`, so the chain keeps + /// resolving once the world restarts and the real frames return. + /// + /// # Safety + /// Caller must hold the world stopped, so the owning thread is parked and + /// the chain is not being popped while it is walked. + #[cfg(feature = "threading")] + #[cold] + #[inline(never)] + pub(crate) unsafe fn materialize_detached_chain(&self, vm: &VirtualMachine) -> FrameObjectRef { + let top = unsafe { self.materialize_detached(vm) }; + let mut child = top.clone(); + let mut cur = self.previous(); + while !cur.is_null() { + let caller = unsafe { &*cur }; + let caller_fo = unsafe { caller.materialize_detached(vm) }; + { + let mut guard = child.iframe().cold().retained_back.lock(); + if guard.is_none() { + *guard = Some(caller_fo.clone()); + } + } + child = caller_fo; + cur = caller.previous(); + } + top + } + /// Create a lightweight FrameObject with empty localsplus, suitable for /// f_back chain building (retained_back). Unlike `materialize`, this does /// NOT store into `temporary_refs` or set the `materialized` pointer, so @@ -1117,9 +1174,18 @@ impl InterpreterFrame { let builtins: PyObjectRef = self.builtins().to_owned(); let func_obj: Option = self.func_obj().map(|o| o.to_owned()); - // Copy localsplus from the stack frame so materialized frames have - // usable fastlocals (for locals(), f_locals, tracebacks, etc). - let localsplus = unsafe { self.localsplus.snapshot_to_heap() }; + // Empty localsplus, sized for the code object. While the source frame + // runs, every reader resolves it through `find_live_source_iframe`, and + // `exit_iframe` fills these slots from the live frame as it returns. + // Copying the values here instead would give each of them a second + // reference lasting as long as this FrameObject — a frame reached by + // one traceback entry would keep all of its locals alive. + let nlocalsplus = code.localspluskinds.len() as u32; + let localsplus = LocalsPlus { + data: LocalsPlusData::Heap(vec![0usize; nlocalsplus as usize].into_boxed_slice()), + nlocalsplus, + stack_top: 0, + }; // Copy the locals mapping if it exists. let locals = match self.locals.get() { @@ -1143,14 +1209,16 @@ impl InterpreterFrame { // that become dangling after their call returns. The f_back chain // is resolved through the TLS CURRENT_FRAME chain instead. previous: Radium::new(0), - // Materialized frame is a detached snapshot — always FrameObject-owned. - // If we copied Thread from the source iframe, frame.clear() would - // reject the frame with "cannot clear an executing frame". + // Always FrameObject-owned. If we copied Thread from the source + // iframe, frame.clear() would reject the frame with "cannot clear + // an executing frame"; `attached_tid` carries the "still running" + // half of that state instead, so the owner field does not have to. owner: atomic::AtomicI8::new(FrameOwner::FrameObject as i8), datastack_base: core::ptr::null_mut(), materialized: Radium::new(0), cold: OnceCell::from(Box::new(FrameColdData { escaped: atomic::AtomicBool::new(true), + attached_tid: atomic::AtomicU64::new(current_thread_ident()), ..FrameColdData::default() })), }; @@ -1177,8 +1245,8 @@ impl InterpreterFrame { self.materialized.store(fo_ptr, Relaxed); // Keep the FrameObject alive by storing it in temporary_refs. - // GC tracking is deferred to with_iframe cleanup, where the frame - // is no longer executing and temporary_refs is cleared — at that + // GC tracking is deferred to `exit_iframe`, where the frame is no + // longer executing and temporary_refs is cleared — at that // point the FrameObject is self-sustaining and GC can safely // traverse and collect it. self.cold() @@ -1288,6 +1356,22 @@ impl InterpreterFrame { pub(crate) fn cold_opt(&self) -> Option<&FrameColdData> { self.cold.get().map(|b| &**b) } + + /// Thread still running the frame this one was materialized from, or 0. + #[inline] + pub(crate) fn attached_tid(&self) -> u64 { + self.cold_opt() + .map_or(0, |c| c.attached_tid.load(atomic::Ordering::Acquire)) + } + + /// Mark the frame this one was materialized from as returned, so its + /// values may be read from here. + #[inline] + pub(crate) fn detach(&self) { + if let Some(cold) = self.cold_opt() { + cold.attached_tid.store(0, atomic::Ordering::Release); + } + } } /// Python-visible frame object. Currently always wraps an `InterpreterFrame`. @@ -1718,10 +1802,29 @@ impl FrameObject { self.iframe().lasti.store(val, Relaxed); } + /// Fast-local slots of the live source frame when this frame object's + /// frame is still running on this thread, and this frame object's own + /// slots otherwise. A running frame's slots live on the data stack; the + /// frame object's are empty until `exit_iframe` fills them. + /// + /// # Safety + /// Caller must ensure no concurrent mutable access: either the frame is + /// not executing (callers pass through `check_locals_access`), or this is + /// a trace callback on the thread that is executing it. + unsafe fn live_fastlocals(&self) -> &[Option] { + let live = self.find_live_source_iframe(); + if live.is_null() { + unsafe { self.iframe_ref().localsplus.fastlocals() } + } else { + unsafe { (*live).localsplus.fastlocals() } + } + } + fn has_active_hidden_locals(&self) -> bool { use rustpython_compiler_core::bytecode::{CO_FAST_CELL, CO_FAST_FREE, CO_FAST_HIDDEN}; let code = self.iframe().code(); - let fastlocals = unsafe { self.iframe_ref().localsplus.fastlocals() }; + // SAFETY: reached from `locals()` on the thread running this frame. + let fastlocals = unsafe { self.live_fastlocals() }; let is_optimized = code.flags.contains(bytecode::CodeFlags::OPTIMIZED); !is_optimized && code.localspluskinds.iter().enumerate().any(|(i, &kind)| { @@ -1753,15 +1856,7 @@ impl FrameObject { // SAFETY: Either the frame is not executing (caller checked owner), // or we're in a trace callback on the same thread that's executing. let code = self.iframe().code(); - // If this FrameObject has a live source iframe on the TLS chain, read - // its localsplus for up-to-date values (the materialized copy is a - // stale snapshot from materialize time). - let live = self.find_live_source_iframe(); - let fastlocals = if !live.is_null() { - unsafe { (*live).localsplus.fastlocals() } - } else { - unsafe { self.iframe_ref().localsplus.fastlocals() } - }; + let fastlocals = unsafe { self.live_fastlocals() }; // Iterate through all localsplus slots using localspluskinds let nlocalsplus = code.localspluskinds.len(); @@ -1865,6 +1960,15 @@ impl FrameObject { /// builtin, trace callbacks) is fine: the frame sits on the current /// thread's frame chain and is at a bytecode boundary. pub(crate) fn check_locals_access(&self, vm: &VirtualMachine) -> PyResult<()> { + // A frame object materialized from a running data stack frame is + // FrameObject-owned, so the owner test below cannot speak for it: the + // thread running that frame fills these slots when it returns. + let attached = self.iframe().attached_tid(); + if attached != 0 && attached != current_thread_ident() { + return Err(vm.new_runtime_error( + "cannot access frame locals while the frame is executing in another thread", + )); + } let owner = FrameOwner::from_i8(self.iframe().owner.load(atomic::Ordering::Acquire)); if owner != FrameOwner::Thread { return Ok(()); @@ -1948,13 +2052,7 @@ impl FrameObject { use rustpython_compiler_core::bytecode::{CO_FAST_CELL, CO_FAST_FREE}; // SAFETY: callers first pass through `check_locals_access`, so the // frame is not executing on another thread. - // Use live source iframe if available for up-to-date values. - let live = self.find_live_source_iframe(); - let fastlocals = if !live.is_null() { - unsafe { (*live).localsplus.fastlocals() } - } else { - unsafe { self.iframe_ref().localsplus.fastlocals() } - }; + let fastlocals = unsafe { self.live_fastlocals() }; let obj = fastlocals.get(i)?.as_ref()?; let kind = self .iframe() @@ -2297,6 +2395,21 @@ impl Py { } } +/// Identity of the calling thread, or 0 where there is only one thread to be. +/// 0 doubles as "no thread", which is what `attached_tid` wants for a build +/// that cannot have a frame running anywhere else. +#[inline] +fn current_thread_ident() -> u64 { + #[cfg(feature = "threading")] + { + crate::stdlib::_thread::get_ident() + } + #[cfg(not(feature = "threading"))] + { + 0 + } +} + /// Byte offset from the start of a datastack allocation to the LocalsPlus data, /// accounting for alignment padding after the InterpreterFrame header. #[inline] @@ -2457,11 +2570,11 @@ pub(crate) struct ExecutingFrame<'a> { } #[inline] -fn specialization_compact_int_value(i: &PyInt, vm: &VirtualMachine) -> Option { +fn specialization_compact_int_value(i: &PyInt) -> Option { // _PyLong_IsCompact(): a one-digit PyLong (base 2^30), // i.e. abs(value) <= 2^30 - 1. const CPYTHON_COMPACT_LONG_ABS_MAX: i64 = (1i64 << 30) - 1; - let v = i.try_to_primitive::(vm).ok()?; + let v = i.try_to_i64_fast()?; if (-CPYTHON_COMPACT_LONG_ABS_MAX..=CPYTHON_COMPACT_LONG_ABS_MAX).contains(&v) { Some(v as isize) } else { @@ -2472,7 +2585,7 @@ fn specialization_compact_int_value(i: &PyInt, vm: &VirtualMachine) -> Option Option { obj.downcast_ref_if_exact::(vm) - .and_then(|i| specialization_compact_int_value(i, vm)) + .and_then(|i| specialization_compact_int_value(i)) } #[inline] @@ -2611,7 +2724,10 @@ pub(crate) fn release_datastack_frame(frame: &Py, vm: &VirtualMachi ); // SAFETY: the frame is alive (held by `frame` and the escaped reference) // and untracked. - unsafe { crate::gc_state::gc_state().track_object(NonNull::from(frame_obj)) }; + unsafe { + crate::gc_state::gc_state() + .track_object(NonNull::from(frame_obj), crate::gc_state::current_owner()) + }; } type BinaryOpExtendGuard = fn(&PyObject, &PyObject, &VirtualMachine) -> bool; @@ -2976,8 +3092,9 @@ impl ExecutingFrame<'_> { // Advance lasti past the current instruction BEFORE firing the // line event. This ensures that f_lineno (which reads // locations[lasti - 1]) returns the line of the instruction - // being traced, not the previous one. - self.update_lasti(|i| *i += 1); + // being traced, not the previous one. Stored from `idx` rather + // than read-modify-written, which would re-load what was just read. + self.lasti.store(idx as u32 + 1, Relaxed); // Fire 'line' trace event when line number changes. // Only fire if this frame has a per-frame trace function set @@ -4286,9 +4403,10 @@ impl ExecutingFrame<'_> { Ok(None) } Instruction::LoadSmallInt { i: idx } => { - // Push small integer (-5..=256) directly without constant table lookup - let value = vm.ctx.new_int(idx.get(arg) as i32); - self.push_value(value.into()); + // Cached small integers live for the whole Context, so the value stack can + // borrow them without touching the refcount. + let value = vm.ctx.cached_int(idx.get(arg) as i32); + unsafe { self.push_borrowed(value.as_object()) }; Ok(None) } Instruction::LoadDeref { i } => { @@ -4791,8 +4909,12 @@ impl ExecutingFrame<'_> { } Instruction::RaiseVarargs { argc: kind } => self.execute_raise(vm, kind.get(arg)), Instruction::Resume { .. } | Instruction::ResumeCheck => { - // Lazy quickening: initialize adaptive counters on first execution - if !self.code.quickened.swap(true, atomic::Ordering::Relaxed) { + // Lazy quickening: initialize adaptive counters on first execution. + // Read before the swap so that the steady state — every call after + // the first — costs a load rather than a read-modify-write. + if !self.code.quickened.load(atomic::Ordering::Relaxed) + && !self.code.quickened.swap(true, atomic::Ordering::Relaxed) + { self.code.instructions.quicken(); atomic::fence(atomic::Ordering::Release); } @@ -5050,6 +5172,9 @@ impl ExecutingFrame<'_> { Ok(None) } Instruction::YieldValue { .. } => { + // The frame outlives this block from here on, so nothing it + // still holds may be a borrow of something else's slot. + self.localsplus.promote_stack(); debug_assert!( self.localsplus .stack_as_slice() @@ -5259,7 +5384,7 @@ impl ExecutingFrame<'_> { if type_version != 0 && owner.class().tp_version_tag.load(Acquire) == type_version - && owner.dict().is_none() + && !owner.has_instance_dict() && let Some(func) = self.try_read_cached_descriptor(cache_base, type_version) { let owner = self.pop_value(); @@ -5683,8 +5808,8 @@ impl ExecutingFrame<'_> { b.downcast_ref_if_exact::(vm), ) { let result = a_str.as_wtf8().py_add(b_str.as_wtf8()); - self.pop_value(); - self.pop_value(); + self.pop_stackref(); + self.pop_stackref(); self.push_value(result.to_pyobject(vm)); Ok(None) } else { @@ -5843,6 +5968,9 @@ impl ExecutingFrame<'_> { && func.func_version() == cached_version && cached_version != 0 { + if func.is_jitted() { + return self.execute_call_vectorcall(nargs, vm); + } let effective_nargs = nargs + u32::from(self_or_null_is_some); if !func.has_exact_argcount(effective_nargs) { return self.execute_call_vectorcall(nargs, vm); @@ -5905,6 +6033,9 @@ impl ExecutingFrame<'_> { && func.func_version() == cached_version && cached_version != 0 { + if func.is_jitted() { + return self.execute_call_vectorcall(nargs, vm); + } if !func.has_exact_argcount(nargs + 1) { return self.execute_call_vectorcall(nargs, vm); } @@ -6076,15 +6207,8 @@ impl ExecutingFrame<'_> { | PyMethodFlags::O | PyMethodFlags::KEYWORDS); if call_conv == PyMethodFlags::O && effective_nargs == 1 { - let nargs_usize = nargs as usize; - let pos_args: Vec = self.pop_multiple(nargs_usize).collect(); - let self_or_null = self.pop_value_opt(); - let callable = self.pop_value(); - let mut args_vec = Vec::with_capacity(effective_nargs as usize); - if let Some(self_val) = self_or_null { - args_vec.push(self_val); - } - args_vec.extend(pos_args); + let (callable, args_vec) = self.take_call_args(nargs as usize); + debug_assert_eq!(args_vec.len(), effective_nargs as usize); let result = callable.vectorcall(args_vec, effective_nargs as usize, None, vm)?; self.push_value(result); @@ -6110,15 +6234,8 @@ impl ExecutingFrame<'_> { | PyMethodFlags::O | PyMethodFlags::KEYWORDS); if call_conv == PyMethodFlags::FASTCALL { - let nargs_usize = nargs as usize; - let pos_args: Vec = self.pop_multiple(nargs_usize).collect(); - let self_or_null = self.pop_value_opt(); - let callable = self.pop_value(); - let mut args_vec = Vec::with_capacity(effective_nargs as usize); - if let Some(self_val) = self_or_null { - args_vec.push(self_val); - } - args_vec.extend(pos_args); + let (callable, args_vec) = self.take_call_args(nargs as usize); + debug_assert_eq!(args_vec.len(), effective_nargs as usize); let result = callable.vectorcall(args_vec, effective_nargs as usize, None, vm)?; self.push_value(result); @@ -6140,21 +6257,14 @@ impl ExecutingFrame<'_> { && func.func_version() == cached_version && cached_version != 0 { + if func.is_jitted() { + return self.execute_call_vectorcall(nargs, vm); + } if self.specialization_call_recursion_guard(vm) { return self.execute_call_vectorcall(nargs, vm); } - let nargs_usize = nargs as usize; - let pos_args: Vec = self.pop_multiple(nargs_usize).collect(); - let self_or_null = self.pop_value_opt(); - let callable = self.pop_value(); - let (args_vec, effective_nargs) = if let Some(self_val) = self_or_null { - let mut v = Vec::with_capacity(nargs_usize + 1); - v.push(self_val); - v.extend(pos_args); - (v, nargs_usize + 1) - } else { - (pos_args, nargs_usize) - }; + let (callable, args_vec) = self.take_call_args(nargs as usize); + let effective_nargs = args_vec.len(); let result = vectorcall_function(&callable, args_vec, effective_nargs, None, vm)?; self.push_value(result); @@ -6186,16 +6296,18 @@ impl ExecutingFrame<'_> { && func.func_version() == cached_version && cached_version != 0 { + if func.is_jitted() { + return self.execute_call_vectorcall(nargs, vm); + } if self.specialization_call_recursion_guard(vm) { return self.execute_call_vectorcall(nargs, vm); } let nargs_usize = nargs as usize; - let pos_args: Vec = self.pop_multiple(nargs_usize).collect(); - self.pop_value_opt(); // null (self_or_null) - self.pop_value(); // callable (bound method) let mut args_vec = Vec::with_capacity(nargs_usize + 1); args_vec.push(bound_self); - args_vec.extend(pos_args); + args_vec.extend(self.pop_multiple(nargs_usize)); + self.pop_value_opt(); // null (self_or_null) + self.pop_value(); // callable (bound method) let result = vectorcall_function( &bound_function, args_vec, @@ -6277,15 +6389,8 @@ impl ExecutingFrame<'_> { .is_some_and(|self_obj| self_obj.class().is(descr.objclass)) { let func = descr.method.func; - let positional_args: Vec = - self.pop_multiple(nargs as usize).collect(); - let self_or_null = self.pop_value_opt(); - self.pop_value(); // callable - let mut all_args = Vec::with_capacity(total_nargs as usize); - if let Some(self_val) = self_or_null { - all_args.push(self_val); - } - all_args.extend(positional_args); + let (_callable, all_args) = self.take_call_args(nargs as usize); + debug_assert_eq!(all_args.len(), total_nargs as usize); let args = FuncArgs { args: all_args, kwargs: Default::default(), @@ -6324,15 +6429,8 @@ impl ExecutingFrame<'_> { .is_some_and(|self_obj| self_obj.class().is(descr.objclass)) { let func = descr.method.func; - let positional_args: Vec = - self.pop_multiple(nargs as usize).collect(); - let self_or_null = self.pop_value_opt(); - self.pop_value(); // callable - let mut all_args = Vec::with_capacity(total_nargs as usize); - if let Some(self_val) = self_or_null { - all_args.push(self_val); - } - all_args.extend(positional_args); + let (_callable, all_args) = self.take_call_args(nargs as usize); + debug_assert_eq!(all_args.len(), total_nargs as usize); let args = FuncArgs { args: all_args, kwargs: Default::default(), @@ -6371,15 +6469,8 @@ impl ExecutingFrame<'_> { .is_some_and(|self_obj| self_obj.class().is(descr.objclass)) { let func = descr.method.func; - let positional_args: Vec = - self.pop_multiple(nargs as usize).collect(); - let self_or_null = self.pop_value_opt(); - self.pop_value(); // callable - let mut all_args = Vec::with_capacity(total_nargs as usize); - if let Some(self_val) = self_or_null { - all_args.push(self_val); - } - all_args.extend(positional_args); + let (_callable, all_args) = self.take_call_args(nargs as usize); + debug_assert_eq!(all_args.len(), total_nargs as usize); let args = FuncArgs { args: all_args, kwargs: Default::default(), @@ -6396,22 +6487,9 @@ impl ExecutingFrame<'_> { if let Some(cls) = callable.downcast_ref::() && cls.slots.vectorcall.load().is_some() { - let nargs_usize = nargs as usize; - let pos_args: Vec = self.pop_multiple(nargs_usize).collect(); - let self_or_null = self.pop_value_opt(); - let callable = self.pop_value(); - let self_is_some = self_or_null.is_some(); - let mut args_vec = Vec::with_capacity(nargs_usize + usize::from(self_is_some)); - if let Some(self_val) = self_or_null { - args_vec.push(self_val); - } - args_vec.extend(pos_args); - let result = callable.vectorcall( - args_vec, - nargs_usize + usize::from(self_is_some), - None, - vm, - )?; + let (callable, args_vec) = self.take_call_args(nargs as usize); + let effective_nargs = args_vec.len(); + let result = callable.vectorcall(args_vec, effective_nargs, None, vm)?; self.push_value(result); return Ok(None); } @@ -6496,15 +6574,8 @@ impl ExecutingFrame<'_> { .is_some_and(|self_obj| self_obj.class().is(descr.objclass)) { let func = descr.method.func; - let positional_args: Vec = - self.pop_multiple(nargs as usize).collect(); - let self_or_null = self.pop_value_opt(); - self.pop_value(); // callable - let mut all_args = Vec::with_capacity(total_nargs as usize); - if let Some(self_val) = self_or_null { - all_args.push(self_val); - } - all_args.extend(positional_args); + let (_callable, all_args) = self.take_call_args(nargs as usize); + debug_assert_eq!(all_args.len(), total_nargs as usize); let args = FuncArgs { args: all_args, kwargs: Default::default(), @@ -6533,15 +6604,8 @@ impl ExecutingFrame<'_> { | PyMethodFlags::O | PyMethodFlags::KEYWORDS); if call_conv == (PyMethodFlags::FASTCALL | PyMethodFlags::KEYWORDS) { - let nargs_usize = nargs as usize; - let pos_args: Vec = self.pop_multiple(nargs_usize).collect(); - let self_or_null = self.pop_value_opt(); - let callable = self.pop_value(); - let mut args_vec = Vec::with_capacity(effective_nargs as usize); - if let Some(self_val) = self_or_null { - args_vec.push(self_val); - } - args_vec.extend(pos_args); + let (callable, args_vec) = self.take_call_args(nargs as usize); + debug_assert_eq!(args_vec.len(), effective_nargs as usize); let result = callable.vectorcall(args_vec, effective_nargs as usize, None, vm)?; self.push_value(result); @@ -6565,22 +6629,13 @@ impl ExecutingFrame<'_> { { return self.execute_call_vectorcall(nargs, vm); } - let nargs_usize = nargs as usize; - let pos_args: Vec = self.pop_multiple(nargs_usize).collect(); - let self_or_null = self.pop_value_opt(); - let callable = self.pop_value(); - let mut args_vec = - Vec::with_capacity(nargs_usize + usize::from(self_or_null_is_some)); - if let Some(self_val) = self_or_null { - args_vec.push(self_val); - } - args_vec.extend(pos_args); - let result = callable.vectorcall( - args_vec, - nargs_usize + usize::from(self_or_null_is_some), - None, - vm, - )?; + let (callable, args_vec) = self.take_call_args(nargs as usize); + debug_assert_eq!( + args_vec.len(), + nargs as usize + usize::from(self_or_null_is_some) + ); + let effective_nargs = args_vec.len(); + let result = callable.vectorcall(args_vec, effective_nargs, None, vm)?; self.push_value(result); Ok(None) } @@ -6598,6 +6653,9 @@ impl ExecutingFrame<'_> { && func.func_version() == cached_version && cached_version != 0 { + if func.is_jitted() { + return self.execute_call_kw_vectorcall(nargs, vm); + } if self.specialization_call_recursion_guard(vm) { return self.execute_call_kw_vectorcall(nargs, vm); } @@ -6656,6 +6714,9 @@ impl ExecutingFrame<'_> { && func.func_version() == cached_version && cached_version != 0 { + if func.is_jitted() { + return self.execute_call_kw_vectorcall(nargs, vm); + } let nargs_usize = nargs as usize; let kwarg_names_obj = self.pop_value(); let kwarg_names_tuple = kwarg_names_obj @@ -6851,14 +6912,16 @@ impl ExecutingFrame<'_> { a.downcast_ref_if_exact::(vm), b.downcast_ref_if_exact::(vm), ) && let (Some(a_val), Some(b_val)) = ( - specialization_compact_int_value(a_int, vm), - specialization_compact_int_value(b_int, vm), + specialization_compact_int_value(a_int), + specialization_compact_int_value(b_int), ) { let op = self.compare_op_from_arg(arg); let result = op.eval_ord(a_val.cmp(&b_val)); - self.pop_value(); - self.pop_value(); - self.push_value(vm.ctx.new_bool(result).into()); + self.pop_stackref(); + self.pop_stackref(); + if !self.try_fused_compare_int_jump(result, vm) { + self.push_value(vm.ctx.new_bool(result).into()); + } Ok(None) } else { self.execute_compare(vm, arg) @@ -6894,10 +6957,13 @@ impl ExecutingFrame<'_> { b.downcast_ref_if_exact::(vm), ) { let op = self.compare_op_from_arg(arg); - if op != PyComparisonOp::Eq && op != PyComparisonOp::Ne { + // The same two shortcuts the unspecialized comparison takes: + // one object is equal to itself, and equality answers two + // strings of different length without reading either. + let Some(result) = op.eval_eq(|| a.is(b) || a_str.as_wtf8() == b_str.as_wtf8()) + else { return self.execute_compare(vm, arg); - } - let result = op.eval_ord(a_str.as_wtf8().cmp(b_str.as_wtf8())); + }; self.pop_value(); self.pop_value(); self.push_value(vm.ctx.new_bool(result).into()); @@ -7181,17 +7247,16 @@ impl ExecutingFrame<'_> { // Keep specialized opcode on guard miss (JUMP_TO_PREDICTED behavior). let cached_version = self.code.instructions.read_cache_u16(cache_base + 1); let cached_index = self.code.instructions.read_cache_u16(cache_base + 3); - if let Ok(current_version) = u16::try_from(self.globals.version()) - && cached_version == current_version + if cached_version != 0 + && let Some(x) = self + .globals + .get_item_by_index_and_keys_version(cached_version, cached_index) { - let name = self.code.names[(oparg >> 1) as usize]; - if let Some(x) = self.globals.get_item_opt_hint(name, cached_index, vm)? { - self.push_value(x); - if (oparg & 1) != 0 { - self.push_value_opt(None); - } - return Ok(None); + self.push_value(x); + if (oparg & 1) != 0 { + self.push_value_opt(None); } + return Ok(None); } let name = self.code.names[(oparg >> 1) as usize]; let x = self.load_global_or_builtin(name, vm)?; @@ -7207,20 +7272,19 @@ impl ExecutingFrame<'_> { let cached_globals_ver = self.code.instructions.read_cache_u16(cache_base + 1); let cached_builtins_ver = self.code.instructions.read_cache_u16(cache_base + 2); let cached_index = self.code.instructions.read_cache_u16(cache_base + 3); - if let Ok(current_globals_ver) = u16::try_from(self.globals.version()) + if cached_globals_ver != 0 + && cached_builtins_ver != 0 + && let Ok(current_globals_ver) = u16::try_from(self.globals.keys_version()) && cached_globals_ver == current_globals_ver && let Some(builtins_dict) = self.builtins.downcast_ref_if_exact::(vm) - && let Ok(current_builtins_ver) = u16::try_from(builtins_dict.version()) - && cached_builtins_ver == current_builtins_ver + && let Some(x) = builtins_dict + .get_item_by_index_and_keys_version(cached_builtins_ver, cached_index) { - let name = self.code.names[(oparg >> 1) as usize]; - if let Some(x) = builtins_dict.get_item_opt_hint(name, cached_index, vm)? { - self.push_value(x); - if (oparg & 1) != 0 { - self.push_value_opt(None); - } - return Ok(None); + self.push_value(x); + if (oparg & 1) != 0 { + self.push_value_opt(None); } + return Ok(None); } let name = self.code.names[(oparg >> 1) as usize]; let x = self.load_global_or_builtin(name, vm)?; @@ -7308,6 +7372,7 @@ impl ExecutingFrame<'_> { self.unwind_blocks(vm, UnwindReason::Returning { value }) } Instruction::InstrumentedYieldValue => { + self.localsplus.promote_stack(); debug_assert!( self.localsplus .stack_as_slice() @@ -8564,7 +8629,7 @@ impl ExecutingFrame<'_> { a_ref.downcast_ref_if_exact::(vm), b_ref.downcast_ref_if_exact::(vm), ) { - Ok(Self::int_add(a.as_bigint(), b.as_bigint(), vm)) + Ok(Self::int_add(a, b, vm)) } else if matches!(op, bytecode::BinaryOperator::Add) { vm._add(a_ref, b_ref) } else { @@ -8576,7 +8641,7 @@ impl ExecutingFrame<'_> { a_ref.downcast_ref_if_exact::(vm), b_ref.downcast_ref_if_exact::(vm), ) { - Ok(Self::int_sub(a.as_bigint(), b.as_bigint(), vm)) + Ok(Self::int_sub(a, b, vm)) } else if matches!(op, bytecode::BinaryOperator::Subtract) { vm._sub(a_ref, b_ref) } else { @@ -8588,7 +8653,7 @@ impl ExecutingFrame<'_> { a_ref.downcast_ref_if_exact::(vm), b_ref.downcast_ref_if_exact::(vm), ) { - Ok(Self::int_mul(a.as_bigint(), b.as_bigint(), vm)) + Ok(Self::int_mul(a, b, vm)) } else if matches!(op, bytecode::BinaryOperator::Multiply) { vm._mul(a_ref, b_ref) } else { @@ -8654,36 +8719,37 @@ impl ExecutingFrame<'_> { /// small-int cache is consulted identically. #[inline] fn int_fast_op( - a: &BigInt, - b: &BigInt, + a: &PyInt, + b: &PyInt, vm: &VirtualMachine, checked: fn(i64, i64) -> Option, fallback: impl FnOnce(&BigInt, &BigInt) -> BigInt, ) -> PyObjectRef { - use num_traits::ToPrimitive; - if let (Some(av), Some(bv)) = (a.to_i64(), b.to_i64()) + if let (Some(av), Some(bv)) = (a.try_to_i64_fast(), b.try_to_i64_fast()) && let Some(result) = checked(av, bv) { return vm.ctx.new_int(result).into(); } - vm.ctx.new_int(fallback(a, b)).into() + vm.ctx + .new_int(fallback(a.as_bigint(), b.as_bigint())) + .into() } /// Int addition with i64 fast path to avoid BigInt heap allocation. #[inline] - fn int_add(a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> PyObjectRef { + fn int_add(a: &PyInt, b: &PyInt, vm: &VirtualMachine) -> PyObjectRef { Self::int_fast_op(a, b, vm, i64::checked_add, |a, b| a + b) } /// Int subtraction with i64 fast path to avoid BigInt heap allocation. #[inline] - fn int_sub(a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> PyObjectRef { + fn int_sub(a: &PyInt, b: &PyInt, vm: &VirtualMachine) -> PyObjectRef { Self::int_fast_op(a, b, vm, i64::checked_sub, |a, b| a - b) } /// Int multiplication with i64 fast path to avoid BigInt heap allocation. #[inline] - fn int_mul(a: &BigInt, b: &BigInt, vm: &VirtualMachine) -> PyObjectRef { + fn int_mul(a: &PyInt, b: &PyInt, vm: &VirtualMachine) -> PyObjectRef { Self::int_fast_op(a, b, vm, i64::checked_mul, |a, b| a * b) } @@ -8981,13 +9047,19 @@ impl ExecutingFrame<'_> { attr_name: &'static PyStrInterned, vm: &VirtualMachine, ) -> PyResult> { - let Some(dict) = self.top_value().dict() else { - return Ok(None); - }; let stamp = self.code.instructions.read_cache_ptr(cache_base + 3); - if stamp != 0 && stamp == dict.keys_version() as usize { + // Take the stamp check first, on a borrowed dict: a hit is the whole + // fast path, and cloning the dict for it would cost more than the + // comparison it exists to make. + let stamped = self.top_value().with_instance_dict(|dict| { + dict.is_some_and(|d| stamp != 0 && stamp == d.keys_version() as usize) + }); + if stamped { return Ok(None); } + let Some(dict) = self.top_value().dict() else { + return Ok(None); + }; // Take the stamp before probing so it attests the probed key set. let stamp = dict.assign_keys_version(vm); if let Some(value) = dict.get_item_opt(attr_name, vm)? { @@ -9290,9 +9362,14 @@ impl ExecutingFrame<'_> { if has_data_descr { // Check for member descriptor (slot access) + // The slot offset only means anything on the layout the + // descriptor was defined for; the specialized instruction + // guards on the type version alone, so what descr_get() + // checks on every access has to be checked here instead. if let Some(ref descr) = cls_attr && let Some(member_descr) = descr.downcast_ref::() && let MemberGetter::Offset(offset) = member_descr.member.getter + && cls.fast_issubclass(&member_descr.common.typ) { unsafe { self.code @@ -9828,7 +9905,7 @@ impl ExecutingFrame<'_> { fn execute_binary_op_int( &mut self, vm: &VirtualMachine, - op: impl FnOnce(&BigInt, &BigInt, &VirtualMachine) -> PyObjectRef, + op: impl FnOnce(&PyInt, &PyInt, &VirtualMachine) -> PyObjectRef, deopt_op: bytecode::BinaryOperator, ) -> FrameResult { let b = self.top_value(); @@ -9837,9 +9914,9 @@ impl ExecutingFrame<'_> { a.downcast_ref_if_exact::(vm), b.downcast_ref_if_exact::(vm), ) { - let result = op(a_int.as_bigint(), b_int.as_bigint(), vm); - self.pop_value(); - self.pop_value(); + let result = op(a_int, b_int, vm); + self.pop_stackref(); + self.pop_stackref(); self.push_value(result); Ok(None) } else { @@ -9896,7 +9973,7 @@ impl ExecutingFrame<'_> { let callable = self.nth_value(nargs + 1); if let Some(func) = callable.downcast_ref_if_exact::(vm) { - if self.specialization_eval_frame_active(vm) { + if self.specialization_eval_frame_active(vm) || func.is_jitted() { unsafe { self.code.instructions.write_adaptive_counter( cache_base, @@ -9959,7 +10036,7 @@ impl ExecutingFrame<'_> { .function_obj() .downcast_ref_if_exact::(vm) { - if self.specialization_eval_frame_active(vm) { + if self.specialization_eval_frame_active(vm) || func.is_jitted() { unsafe { self.code.instructions.write_adaptive_counter( cache_base, @@ -10254,7 +10331,7 @@ impl ExecutingFrame<'_> { let callable = self.nth_value(nargs + 2); if let Some(func) = callable.downcast_ref_if_exact::(vm) { - if self.specialization_eval_frame_active(vm) { + if self.specialization_eval_frame_active(vm) || func.is_jitted() { unsafe { self.code.instructions.write_adaptive_counter( cache_base, @@ -10305,7 +10382,7 @@ impl ExecutingFrame<'_> { .function_obj() .downcast_ref_if_exact::(vm) { - if self.specialization_eval_frame_active(vm) { + if self.specialization_eval_frame_active(vm) || func.is_jitted() { unsafe { self.code.instructions.write_adaptive_counter( cache_base, @@ -10448,8 +10525,8 @@ impl ExecutingFrame<'_> { a.downcast_ref_if_exact::(vm), b.downcast_ref_if_exact::(vm), ) { - if specialization_compact_int_value(a_int, vm).is_some() - && specialization_compact_int_value(b_int, vm).is_some() + if specialization_compact_int_value(a_int).is_some() + && specialization_compact_int_value(b_int).is_some() { Some(Instruction::CompareOpInt) } else { @@ -10480,6 +10557,37 @@ impl ExecutingFrame<'_> { .into() } + /// Execute an immediately following conditional jump without materializing + /// the comparison result as a Python bool. This is the adaptive interpreter + /// equivalent of keeping the result virtual across the two-opcode trace. + #[inline] + fn try_fused_compare_int_jump(&mut self, result: bool, vm: &VirtualMachine) -> bool { + if self.specialization_eval_frame_active(vm) { + return false; + } + + let jump_idx = self.lasti() as usize + Instruction::CompareOpInt.cache_entries(); + if jump_idx >= self.code.instructions.len() { + return false; + } + + let jump_op = self.code.instructions.read_op(jump_idx); + let jump_on = match jump_op { + Instruction::PopJumpIfFalse { .. } => false, + Instruction::PopJumpIfTrue { .. } => true, + _ => return false, + }; + let jump_delta = self.code.instructions.read_arg(jump_idx).as_u32(); + let after_jump = jump_idx as u32 + 1 + jump_op.cache_entries() as u32; + let target = if result == jump_on { + after_jump + jump_delta + } else { + after_jump + }; + self.update_lasti(|i| *i = target); + true + } + /// Recover the BinaryOperator from the instruction arg byte. /// `replace_op` preserves the arg byte, so the original op remains accessible. fn binary_op_from_arg(&self, arg: bytecode::OpArg) -> bytecode::BinaryOperator { @@ -10680,11 +10788,10 @@ impl ExecutingFrame<'_> { } } - // Pop the callable and transfer ownership to the trampoline via - // the VM side channel, avoiding a per-frame mutex lock on - // temporary_refs. + // Pop the callable and transfer ownership to the trampoline. This one + // reference keeps every field borrowed by the callee frame alive. let callable = self.pop_value(); - unsafe { &mut *vm.pending_tailcall_refs.get() }.push(callable); + vm.set_pending_tailcall_owner(callable); vm.set_pending_tailcall(callee_iframe); } @@ -10734,13 +10841,13 @@ impl ExecutingFrame<'_> { *dst = Some(arg); } self.pop_value_opt(); // null (self_or_null) - let callable = self.pop_value(); // callable (bound method) + self.pop_value(); // callable (bound method) fastlocals[0] = Some(bound_self); - // Transfer ownership to the trampoline via the VM side channel. - let refs = unsafe { &mut *vm.pending_tailcall_refs.get() }; - refs.push(bound_function); - refs.push(callable); + // The function owns every field borrowed by the callee frame. + // bound_self is owned by fastlocals; the bound-method object itself is + // no longer needed and was dropped above, matching the recursive path. + vm.set_pending_tailcall_owner(bound_function); vm.set_pending_tailcall(callee_iframe); } @@ -10792,7 +10899,7 @@ impl ExecutingFrame<'_> { return; } let name = self.code.names[(oparg >> 1) as usize]; - let Ok(globals_version) = u16::try_from(self.globals.version()) else { + let Ok(globals_version @ 1..) = u16::try_from(self.globals.assign_keys_version(vm)) else { unsafe { self.code.instructions.write_adaptive_counter( cache_base, @@ -10820,7 +10927,7 @@ impl ExecutingFrame<'_> { if let Some(builtins_dict) = self.builtins.downcast_ref_if_exact::(vm) && let Ok(Some(builtins_hint)) = builtins_dict.hint_for_key(name, vm) - && let Ok(builtins_version) = u16::try_from(builtins_dict.version()) + && let Ok(builtins_version @ 1..) = u16::try_from(builtins_dict.assign_keys_version(vm)) { unsafe { self.code @@ -10997,9 +11104,12 @@ impl ExecutingFrame<'_> { if has_data_descr { // Check for member descriptor (slot access) + // As in the load specialization, the offset is only valid for + // instances of the type the descriptor belongs to. if let Some(ref descr) = cls_attr && let Some(member_descr) = descr.downcast_ref::() && let MemberGetter::Offset(offset) = member_descr.member.getter + && cls.fast_issubclass(&member_descr.common.typ) { unsafe { self.code @@ -11349,6 +11459,49 @@ impl ExecutingFrame<'_> { } } + /// Take a call's `[self_or_null, arg1, ..., argN]` off the stack as one + /// vectorcall argument list, along with the callable underneath them. + /// + /// The stack already holds the arguments in vectorcall order, so filling a + /// single vector by index costs one allocation — collecting the positional + /// arguments first and then pushing `self` in front of them costs two plus + /// a copy. + fn take_call_args(&mut self, nargs: usize) -> (PyObjectRef, Vec) { + let stack_len = self.localsplus.stack_len(); + debug_assert!( + stack_len >= nargs + 2, + "CALL stack underflow: need callable + self_or_null + {nargs} args, have {stack_len}" + ); + let callable_idx = stack_len - nargs - 2; + let self_or_null_idx = callable_idx + 1; + + let self_or_null = self + .localsplus + .stack_index_mut(self_or_null_idx) + .take() + .map(|sr| sr.to_pyobj()); + let mut args = Vec::with_capacity(nargs + usize::from(self_or_null.is_some())); + args.extend(self_or_null); + for stack_idx in self_or_null_idx + 1..stack_len { + let val = self + .localsplus + .stack_index_mut(stack_idx) + .take() + .unwrap() + .to_pyobj(); + args.push(val); + } + + let callable = self + .localsplus + .stack_index_mut(callable_idx) + .take() + .unwrap() + .to_pyobj(); + self.localsplus.stack_truncate(callable_idx); + (callable, args) + } + /// Pop multiple values from the stack. Panics if any slot is NULL. fn pop_multiple(&mut self, count: usize) -> impl ExactSizeIterator + '_ { let stack_len = self.localsplus.stack_len(); diff --git a/crates/vm/src/function/argument.rs b/crates/vm/src/function/argument.rs index aabe484c282..6bf4ae2107b 100644 --- a/crates/vm/src/function/argument.rs +++ b/crates/vm/src/function/argument.rs @@ -8,6 +8,7 @@ use crate::{ use core::ops::{Deref, DerefMut, RangeInclusive}; use indexmap::IndexMap; use itertools::Itertools; +use std::hash::DefaultHasher; pub trait IntoFuncArgs: Sized { fn into_args(self, vm: &VirtualMachine) -> FuncArgs; @@ -414,16 +415,24 @@ impl FromArgOptional for T { // issue #8228). `PyStr` is WTF-8 backed, and CPython only requires that a // keyword key be a `str`, not that it be valid UTF-8. #[derive(Clone, Debug)] -pub struct KwArgs(IndexMap); +pub struct KwArgs(KwArgsMap); + +/// The map behind [`KwArgs`]. +/// +/// The hasher is zero-sized rather than the randomly seeded default: a +/// `KwArgs` is built for every call, including the far more common +/// keyword-less one, and seeding reads a thread-local. Keyword names come +/// from the program text, so per-process hash randomization buys nothing. +pub type KwArgsMap = IndexMap>; impl Default for KwArgs { fn default() -> Self { - Self(IndexMap::new()) + Self(KwArgsMap::default()) } } impl Deref for KwArgs { - type Target = IndexMap; + type Target = KwArgsMap; fn deref(&self) -> &Self::Target { &self.0 @@ -447,7 +456,7 @@ where impl KwArgs { #[must_use] - pub const fn new(map: IndexMap) -> Self { + pub const fn new(map: KwArgsMap) -> Self { Self(map) } @@ -508,7 +517,7 @@ where T: TryFromObject, { fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result { - let mut kwargs = IndexMap::new(); + let mut kwargs = KwArgsMap::default(); for (name, value) in args.remaining_keywords() { kwargs.insert(name, value.try_into_value(vm)?); } diff --git a/crates/vm/src/function/buffer.rs b/crates/vm/src/function/buffer.rs index 213193bb9c8..dba97e9c77f 100644 --- a/crates/vm/src/function/buffer.rs +++ b/crates/vm/src/function/buffer.rs @@ -3,7 +3,7 @@ use crate::{ VirtualMachine, builtins::{PyStr, PyStrRef}, common::borrow::{BorrowedValue, BorrowedValueMut}, - protocol::PyBuffer, + protocol::{BufferFlags, PyBuffer}, }; // Python/getargs.c @@ -17,7 +17,7 @@ impl PyObject { where F: FnOnce(&[u8]) -> R, { - let buffer = PyBuffer::try_from_borrowed_object(vm, self)?; + let buffer = PyBuffer::from_object(vm, self, BufferFlags::SIMPLE)?; buffer .as_contiguous() .map(|x| f(&x)) @@ -28,7 +28,7 @@ impl PyObject { where F: FnOnce(&mut [u8]) -> R, { - let buffer = PyBuffer::try_from_borrowed_object(vm, self)?; + let buffer = PyBuffer::from_object(vm, self, BufferFlags::WRITABLE)?; buffer .as_contiguous_mut() .map(|mut x| f(&mut x)) @@ -49,11 +49,41 @@ impl ArgBytesLike { f(&self.borrow_buf()) } + /// The bytes to hand to an operation that may wait, and whatever keeps + /// them readable while it does. + /// + /// `borrow_buf` may answer with a lock that every other thread writing to + /// the same object waits on, and a thread waiting on a lock never reaches + /// a safepoint, so keeping one across a wait for a peer, a pipe or a + /// signal stops the world from being stopped at all. Bytes reached that + /// way are copied out first. Bytes that lock nothing -- an immutable + /// object's -- are borrowed where they lie, which is all CPython holds in + /// either case. + pub fn borrow_buf_unlocked(&self, vm: &VirtualMachine) -> PyResult> { + let borrowed = self.borrow_buf(); + if !borrowed.is_locked() { + return Ok(UnlockedBuf::Borrowed(borrowed)); + } + let mut copy = Vec::new(); + copy.try_reserve_exact(borrowed.len()) + .map_err(|_| vm.new_memory_error(""))?; + copy.extend_from_slice(&borrowed); + Ok(UnlockedBuf::Copied(copy)) + } + #[must_use] pub const fn len(&self) -> usize { self.0.desc.len } + /// The width of one item. Callers that read the buffer as bytes rather + /// than as whatever it holds have to ask, since a contiguous buffer of + /// wider items is contiguous all the same. + #[must_use] + pub const fn itemsize(&self) -> usize { + self.0.desc.itemsize + } + #[must_use] pub const fn is_empty(&self) -> bool { self.len() == 0 @@ -63,6 +93,16 @@ impl ArgBytesLike { pub fn as_object(&self) -> &PyObject { &self.0.obj } + + /// The object whose storage is borrowed while this buffer is read: a view + /// borrows the object it looks at, not itself. + #[must_use] + pub fn source_object(&self) -> &PyObject { + self.0 + .obj + .downcast_ref::() + .map_or(&self.0.obj, |view| view.viewed_object()) + } } impl From for PyBuffer { @@ -77,9 +117,9 @@ impl From for PyObjectRef { } } -impl<'a> TryFromBorrowedObject<'a> for ArgBytesLike { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { - let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?; +impl ArgBytesLike { + fn from_request(vm: &VirtualMachine, obj: &PyObject, flags: BufferFlags) -> PyResult { + let buffer = PyBuffer::from_object(vm, obj, flags)?; if buffer.desc.is_contiguous() { Ok(Self(buffer)) } else { @@ -88,6 +128,49 @@ impl<'a> TryFromBorrowedObject<'a> for ArgBytesLike { } } +impl<'a> TryFromBorrowedObject<'a> for ArgBytesLike { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { + Self::from_request(vm, obj, BufferFlags::SIMPLE) + } +} + +/// A bytes-like object asked for as `PyBUF_CONTIG_RO`, which is what a shape is +/// requested with rather than assumed. +#[derive(Debug, Traverse)] +pub struct ArgContiguousBytesLike(ArgBytesLike); + +impl core::ops::Deref for ArgContiguousBytesLike { + type Target = ArgBytesLike; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<'a> TryFromBorrowedObject<'a> for ArgContiguousBytesLike { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { + ArgBytesLike::from_request(vm, obj, BufferFlags::CONTIG_RO).map(Self) + } +} + +/// Bytes that stay readable across a wait, from [`ArgBytesLike::borrow_buf_unlocked`]. +#[derive(Debug)] +pub enum UnlockedBuf<'a> { + Borrowed(BorrowedValue<'a, [u8]>), + Copied(Vec), +} + +impl core::ops::Deref for UnlockedBuf<'_> { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + match self { + Self::Borrowed(b) => b, + Self::Copied(v) => v, + } + } +} + /// A memory buffer, read-write access. Like the `w*` format code for `PyArg_Parse` in CPython. #[derive(Debug, Traverse)] pub struct ArgMemoryBuffer(PyBuffer); @@ -114,6 +197,16 @@ impl ArgMemoryBuffer { pub const fn is_empty(&self) -> bool { self.len() == 0 } + + /// The object whose storage is borrowed while this buffer is written: a + /// view borrows the object it looks at, not itself. + #[must_use] + pub fn source_object(&self) -> &PyObject { + self.0 + .obj + .downcast_ref::() + .map_or(&self.0.obj, |view| view.viewed_object()) + } } impl From for PyBuffer { @@ -124,7 +217,15 @@ impl From for PyBuffer { impl<'a> TryFromBorrowedObject<'a> for ArgMemoryBuffer { fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { - let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?; + let buffer = PyBuffer::from_object(vm, obj, BufferFlags::WRITABLE).map_err(|exc| { + if obj.check_buffer() { + // An exporter that cannot serve the request leaves the argument + // simply the wrong kind of object, as `PyArg_Parse` reports it. + vm.new_type_error("buffer is not a read-write bytes-like object") + } else { + exc + } + })?; if !buffer.desc.is_contiguous() { Err(vm.new_buffer_error("non-contiguous buffer is not a bytes-like object")) } else if buffer.desc.readonly { diff --git a/crates/vm/src/function/fspath.rs b/crates/vm/src/function/fspath.rs index 50feef86dd0..f0ab2909059 100644 --- a/crates/vm/src/function/fspath.rs +++ b/crates/vm/src/function/fspath.rs @@ -3,7 +3,6 @@ use crate::{ builtins::{PyBytes, PyBytesRef, PyStrRef}, convert::{IntoPyException, ToPyObject}, function::PyStr, - protocol::PyBuffer, }; use alloc::borrow::Cow; use core::hint::cold_path; @@ -126,7 +125,7 @@ impl FsPath { pub fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> { rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| { - vm.new_unicode_decode_error_real( + vm.new_unicode_decode_error( vm.ctx.new_str("utf-8"), vm.ctx.new_bytes(b.to_vec()), e.valid_up_to(), @@ -147,16 +146,9 @@ impl ToPyObject for FsPath { } impl TryFromObject for FsPath { - // PyUnicode_FSDecoder in CPython + // PyUnicode_FSDecoder, which takes what PyOS_FSPath takes: str, bytes, or an + // object with __fspath__, and nothing that merely exports a buffer. fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { - let obj = match obj.try_to_value::(vm) { - Ok(buffer) => { - let mut bytes = vec![]; - buffer.append_to(&mut bytes); - vm.ctx.new_bytes(bytes).into() - } - Err(_) => obj, - }; Self::try_from_path_like(obj, true, vm) } } diff --git a/crates/vm/src/function/mod.rs b/crates/vm/src/function/mod.rs index 7eb87fea3ed..2b37c9fc8de 100644 --- a/crates/vm/src/function/mod.rs +++ b/crates/vm/src/function/mod.rs @@ -11,11 +11,13 @@ mod protocol; mod time; pub use argument::{ - ArgumentError, FromArgOptional, FromArgs, FuncArgs, IntoFuncArgs, KwArgs, OptionalArg, - OptionalOption, PosArgs, + ArgumentError, FromArgOptional, FromArgs, FuncArgs, IntoFuncArgs, KwArgs, KwArgsMap, + OptionalArg, OptionalOption, PosArgs, }; pub use arithmetic::{PyArithmeticValue, PyComparisonValue}; -pub use buffer::{ArgAsciiBuffer, ArgBytesLike, ArgMemoryBuffer, ArgStrOrBytesLike}; +pub use buffer::{ + ArgAsciiBuffer, ArgBytesLike, ArgContiguousBytesLike, ArgMemoryBuffer, ArgStrOrBytesLike, +}; pub use builtin::{IntoPyNativeFn, PyNativeFn, static_func, static_raw_func}; pub use either::Either; pub use fspath::FsPath; diff --git a/crates/vm/src/function/protocol.rs b/crates/vm/src/function/protocol.rs index 25ef62b458d..d503fabaca8 100644 --- a/crates/vm/src/function/protocol.rs +++ b/crates/vm/src/function/protocol.rs @@ -86,6 +86,11 @@ unsafe impl Traverse for ArgIterable { } impl ArgIterable { + #[must_use] + pub(crate) fn as_object(&self) -> &PyObject { + &self.iterable + } + /// Returns an iterator over this sequence of objects. /// /// This operation may fail if an exception is raised while invoking the diff --git a/crates/vm/src/gc_state.rs b/crates/vm/src/gc_state.rs index 9744d4ae992..e5eb3758950 100644 --- a/crates/vm/src/gc_state.rs +++ b/crates/vm/src/gc_state.rs @@ -4,11 +4,10 @@ use crate::common::linked_list::LinkedList; use crate::common::lock::{PyMutex, PyRwLock}; -use crate::object::{GC_PERMANENT, GC_UNTRACKED, GcLink}; +use crate::object::{GC_NO_OWNER, GC_PERMANENT, GC_UNTRACKED, GcLink, GcOwner}; use crate::{AsObject, PyObject, PyObjectRef}; use core::ptr::NonNull; -use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; -use std::collections::HashSet; +use core::sync::atomic::{AtomicBool, AtomicU16, AtomicU32, AtomicUsize, Ordering}; fn elapsed_secs( #[cfg(target_arch = "wasm32")] _start: (), @@ -56,10 +55,12 @@ pub struct GcStats { pub duration: f64, } -/// A single GC generation with intrusive linked list +/// One generation's collection policy and statistics, per interpreter. +/// +/// The objects themselves live in the process-wide lists on [`GcState`], so the +/// occupancy count sits there; what an interpreter owns is when to collect and +/// what its own collections have done. pub struct GcGeneration { - /// Number of objects in this generation - count: AtomicUsize, /// Threshold for triggering collection threshold: AtomicU32, /// Collection statistics @@ -70,7 +71,6 @@ impl GcGeneration { #[must_use] pub const fn new(threshold: u32) -> Self { Self { - count: AtomicUsize::new(0), threshold: AtomicU32::new(threshold), stats: PyMutex::new(GcStats { collections: 0, @@ -82,16 +82,14 @@ impl GcGeneration { } } - pub fn count(&self) -> usize { - self.count.load(Ordering::SeqCst) - } - + /// Relaxed: this is policy read once per allocation, and a collection + /// racing `gc.set_threshold()` may use either value. pub fn threshold(&self) -> u32 { - self.threshold.load(Ordering::SeqCst) + self.threshold.load(Ordering::Relaxed) } pub fn set_threshold(&self, value: u32) { - self.threshold.store(value, Ordering::SeqCst); + self.threshold.store(value, Ordering::Relaxed); } pub fn stats(&self) -> GcStats { @@ -131,11 +129,70 @@ impl GcGeneration { } } +/// Drop one from a generation's occupancy. +/// +/// A collection resets the counts of the generations it emptied, but it only +/// empties its own interpreter's objects; another interpreter's stay behind with +/// the count already zeroed, and untracking one of those must not wrap. +fn release_count(count: &AtomicUsize) { + if count.load(Ordering::Relaxed) > 0 { + count.fetch_sub(1, Ordering::Relaxed); + } +} + +/// Whether `owner`'s collections act on `obj`. +/// +/// Objects with no owner — everything the shared context allocates, and anything +/// allocated with no interpreter current — belong to all of them. +fn is_owned_by(obj: &PyObject, owner: GcOwner) -> bool { + let obj_owner = obj.gc_owner(); + obj_owner == owner || obj_owner == GC_NO_OWNER +} + /// Wrapper for NonNull to impl Hash/Eq for use in temporary collection sets. /// Only used within collect_inner, never shared across threads. #[derive(Clone, Copy, PartialEq, Eq, Hash)] struct GcPtr(NonNull); +/// Hashing for the tables a collection keys by an object's address. +/// +/// The default hasher is SipHash, which buys resistance against a caller +/// choosing keys that collide. Nothing chooses these keys: they are addresses +/// this process handed out, and the tables live and die inside one collection. +/// What a collection needs from them is speed -- it hashes every tracked +/// object and every edge between them -- so this runs the address through a +/// handful of multiplies and shifts instead. The shifts are what earns the +/// speed: a table picks its bucket from the low bits, and an address arrives +/// with its low bits zeroed by alignment, so entropy has to be carried +/// downward or every object lands in the same few buckets. +#[derive(Default)] +struct GcPtrHasher(u64); + +impl core::hash::Hasher for GcPtrHasher { + fn finish(&self) -> u64 { + self.0 + } + + fn write_usize(&mut self, value: usize) { + let mut z = (value as u64).wrapping_add(0x9E37_79B9_7F4A_7C15); + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + self.0 = z ^ (z >> 31); + } + + fn write(&mut self, bytes: &[u8]) { + // Addresses reach this hasher through `write_usize`; a key hashed any + // other way still has to land somewhere sensible. + for &byte in bytes { + self.0 = (self.0 ^ u64::from(byte)).wrapping_mul(0x0100_0000_01B3); + } + } +} + +type GcBuildHasher = core::hash::BuildHasherDefault; +type GcSet = std::collections::HashSet; +type GcMap = std::collections::HashMap; + /// RAII barrier that parks every other thread for the pointer-reading phases /// of a collection and lets them run again before finalizers execute. /// @@ -146,41 +203,83 @@ struct GcPtr(NonNull); /// well-defined while all other threads are parked at a safepoint. Restarting /// happens explicitly once the snapshot has pinned every object; `Drop` is a /// backstop that also restarts on the early-return paths. +/// +/// A collection acts on one interpreter's objects, but its candidates include +/// the ones no interpreter owns, which every interpreter can reference and so +/// incref. Reading a refcount that another interpreter is changing is what +/// makes an object look unreachable when it is not, so every live interpreter +/// is stopped, not just the collecting one. Stopping in `runtime` id order +/// keeps exclusion acquisition ordered; the `collecting` mutex additionally +/// serializes collections process-wide, so no second collector can take these +/// exclusions in another order. #[cfg(feature = "threading")] struct CollectStopTheWorld { - vm: *const crate::VirtualMachine, - stopped: bool, + /// Stopped interpreter states, in stop order. Held as strong references so + /// an interpreter cannot be dropped between stop and restart, and kept past + /// the restart so that releasing the last one — which frees that + /// interpreter's objects, and so removes them from these lists — happens + /// after the collection has let go of the generation locks. + stopped: Vec>, + /// Keeps interpreters from registering between the snapshot below and the + /// restart. One registered in that window would be missing from `stopped`, + /// so its bootstrap would keep running — and mutating the shared generation + /// lists — while this collection reads them. + admission: Option>, + restarted: bool, } #[cfg(feature = "threading")] impl CollectStopTheWorld { - /// Request stop-the-world when the current thread has an attached VM. - /// Falls back to no barrier when no VM is attached (the tracked-object - /// reads then run without other threads only if the caller guarantees it). + /// Request stop-the-world on every live interpreter when the current thread + /// has an attached VM. Falls back to no barrier when no VM is attached (the + /// tracked-object reads then run without other threads only if the caller + /// guarantees it). fn new() -> Self { - let vm = crate::vm::thread::try_with_current_vm(|vm| { - vm.state.stop_the_world.stop_the_world(vm); - vm as *const crate::VirtualMachine - }); - match vm { - Some(vm) => Self { vm, stopped: true }, - None => Self { - vm: core::ptr::null(), - stopped: false, - }, + // No attached VM means no interpreter is running Python on this thread; + // keep the historical no-barrier fallback. + if !crate::vm::thread::current_vm_is_set() { + return Self { + stopped: Vec::new(), + admission: None, + restarted: true, + }; + } + + // Accumulate into a live `Self` rather than a bare Vec: if a later + // `stop_the_world` unwinds, dropping this guard restarts the + // interpreters already stopped, instead of leaving their threads parked + // and their exclusion held forever. + let mut guard = Self { + stopped: Vec::new(), + admission: Some(crate::vm::runtime::lock_admission_for_stop()), + restarted: false, + }; + for state in crate::vm::runtime::live_interpreter_states() { + state.stop_the_world.stop_the_world(&state); + guard.stopped.push(state); } + guard } /// Restart the world. Idempotent. fn restart(&mut self) { - if self.stopped { - // SAFETY: the current thread stays attached to this VM for the - // whole collection — the VM is never popped from the thread's VM - // stack while collecting — so the pointer is valid here. - let vm = unsafe { &*self.vm }; - vm.state.stop_the_world.start_the_world(vm); - self.stopped = false; + if self.restarted { + return; + } + self.restarted = true; + // Reverse of the stop order. The references stay until this guard is + // dropped; see the field comment. + for state in self.stopped.iter().rev() { + state.stop_the_world.start_the_world(state); } + // Nothing is parked any more, so registration may resume. + self.admission = None; + } + + /// Whether this collection actually stopped the world. + #[cfg(all(unix, debug_assertions))] + fn is_stopped(&self) -> bool { + !self.stopped.is_empty() } } @@ -191,29 +290,35 @@ impl Drop for CollectStopTheWorld { } } -/// Global GC state +/// The process-wide object lists every interpreter's collections walk. +/// +/// Interpreter-owned policy and results live in [`GcInterpreterState`]; what is +/// here is shared because the lists are: an object is untracked from +/// `default_dealloc`, where no interpreter is in scope, so it has to be findable +/// without one. pub struct GcState { - /// 3 generations (0 = youngest, 2 = oldest) - pub generations: [GcGeneration; 3], - /// Permanent generation (frozen objects) - pub permanent: GcGeneration, - /// GC enabled flag - pub enabled: AtomicBool, /// Per-generation intrusive linked lists for object tracking. /// Objects start in gen0, survivors are promoted to gen1, then gen2. generation_lists: [PyRwLock>; 3], /// Frozen/permanent objects (excluded from normal GC) permanent_list: PyRwLock>, - /// Debug flags - pub debug: AtomicU32, - /// gc.garbage list (uncollectable objects with __del__) - pub garbage: PyMutex>, - /// gc.callbacks list - pub callbacks: PyMutex>, + /// Number of tracked objects per generation, across all interpreters. + /// + /// Advisory: they drive the collection threshold and `gc.get_count()`, and + /// the generation locks — not these counters — order the list changes they + /// describe. Every access is therefore relaxed, which keeps the tracking and + /// untracking of every object off the barrier path. + counts: [AtomicUsize; 3], + /// Number of frozen objects. Advisory, like `counts`. + permanent_count: AtomicUsize, /// Mutex for collection (prevents concurrent collections) collecting: PyMutex<()>, - /// Allocation counter for gen0 - alloc_count: AtomicUsize, + /// Next `gc_owner` tag to hand to an interpreter. + next_owner: AtomicU16, + /// Tags of interpreters that are gone. Their objects outlived them, so a + /// collection adopts them — tags them `GC_NO_OWNER` again — as it walks, + /// rather than leaving them for a collector that will never come. + retired: PyMutex>, } // SAFETY: All fields are either inherently Send/Sync (atomics, RwLock, Mutex) or protected by PyMutex. @@ -233,103 +338,70 @@ impl GcState { #[must_use] pub const fn new() -> Self { Self { - generations: [ - GcGeneration::new(2000), // young - GcGeneration::new(10), // old[0] - GcGeneration::new(0), // old[1] - ], - permanent: GcGeneration::new(0), - enabled: AtomicBool::new(true), generation_lists: [ PyRwLock::new(LinkedList::new()), PyRwLock::new(LinkedList::new()), PyRwLock::new(LinkedList::new()), ], permanent_list: PyRwLock::new(LinkedList::new()), - debug: AtomicU32::new(0), - garbage: PyMutex::new(Vec::new()), - callbacks: PyMutex::new(Vec::new()), + counts: [ + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + ], + permanent_count: AtomicUsize::new(0), collecting: PyMutex::new(()), - alloc_count: AtomicUsize::new(0), + next_owner: AtomicU16::new(GC_NO_OWNER + 1), + retired: PyMutex::new(Vec::new()), } } - /// Check if GC is enabled - pub fn is_enabled(&self) -> bool { - self.enabled.load(Ordering::SeqCst) - } - - /// Enable GC - pub fn enable(&self) { - self.enabled.store(true, Ordering::SeqCst); - } - - /// Disable GC - pub fn disable(&self) { - self.enabled.store(false, Ordering::SeqCst); - } - - /// Get debug flags - pub fn get_debug(&self) -> GcDebugFlags { - GcDebugFlags::from_bits_truncate(self.debug.load(Ordering::SeqCst)) - } - - /// Set debug flags - pub fn set_debug(&self, flags: GcDebugFlags) { - self.debug.store(flags.bits(), Ordering::SeqCst); - } - - /// Get thresholds for all generations - pub fn get_threshold(&self) -> (u32, u32, u32) { - ( - self.generations[0].threshold(), - self.generations[1].threshold(), - self.generations[2].threshold(), - ) + /// Reserve a tag for a new interpreter. Tags are never reused; exhausting + /// the tag space falls back to `GC_NO_OWNER`, which costs isolation but + /// stays correct, rather than aliasing a live interpreter. + fn alloc_owner(&self) -> GcOwner { + self.next_owner + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |next| { + next.checked_add(1) + }) + .unwrap_or(GC_NO_OWNER) } - /// Set thresholds - pub fn set_threshold(&self, t0: u32, t1: Option, t2: Option) { - self.generations[0].set_threshold(t0); - if let Some(t1) = t1 { - self.generations[1].set_threshold(t1); - } - if let Some(t2) = t2 { - self.generations[2].set_threshold(t2); + /// Record that `owner`'s interpreter is gone, so the next collection adopts + /// whatever it left behind. Retagging the objects here would mean walking + /// every list under an interpreter drop, which happens while a collection + /// holds the collecting lock. + fn retire_owner(&self, owner: GcOwner) { + if owner == GC_NO_OWNER { + return; } + self.retired.lock().push(owner); } - /// Get counts for all generations + /// Get counts for all generations. Tracked objects are shared, so these are + /// process-wide even though the thresholds they are compared against are + /// per interpreter. pub fn get_count(&self) -> (usize, usize, usize) { ( - self.generations[0].count(), - self.generations[1].count(), - self.generations[2].count(), + self.counts[0].load(Ordering::Relaxed), + self.counts[1].load(Ordering::Relaxed), + self.counts[2].load(Ordering::Relaxed), ) } - /// Get statistics for all generations - pub fn get_stats(&self) -> [GcStats; 3] { - [ - self.generations[0].stats(), - self.generations[1].stats(), - self.generations[2].stats(), - ] - } - - /// Track a new object (add to gen0). + /// Track a new object (add to gen0) as owned by `owner`. /// O(1) — intrusive linked list push_front, no hashing. /// /// # Safety /// obj must be a valid pointer to a PyObject - pub unsafe fn track_object(&self, obj: NonNull) { + pub unsafe fn track_object(&self, obj: NonNull, owner: GcOwner) { let obj_ref = unsafe { obj.as_ref() }; obj_ref.set_gc_tracked(); obj_ref.set_gc_generation(0); + obj_ref.set_gc_owner(owner); self.generation_lists[0].write().push_front(obj); - self.generations[0].count.fetch_add(1, Ordering::SeqCst); - self.alloc_count.fetch_add(1, Ordering::SeqCst); + self.counts[0].fetch_add(1, Ordering::Relaxed); } /// Untrack an object (remove from GC lists). @@ -348,10 +420,10 @@ impl GcState { ( &self.generation_lists[obj_gen as usize] as &PyRwLock>, - &self.generations[obj_gen as usize].count, + &self.counts[obj_gen as usize], ) } else if obj_gen == GC_PERMANENT { - (&self.permanent_list, &self.permanent.count) + (&self.permanent_list, &self.permanent_count) } else { return; // GC_UNTRACKED or unknown — already untracked }; @@ -363,7 +435,7 @@ impl GcState { continue; // Retry with the updated generation } if unsafe { list.remove(obj) }.is_some() { - count.fetch_sub(1, Ordering::SeqCst); + release_count(count); obj_ref.clear_gc_tracked(); obj_ref.set_gc_generation(GC_UNTRACKED); } else { @@ -381,14 +453,18 @@ impl GcState { } } - /// Get tracked objects (for gc.get_objects) - /// If generation is None, returns all tracked objects. - /// If generation is Some(n), returns objects in generation n only. - pub fn get_objects(&self, generation: Option) -> Vec { + /// Get the objects `owner` tracks (for gc.get_objects), plus the ones no + /// interpreter owns. + /// If generation is None, returns all such objects. + /// If generation is Some(n), returns those in generation n only. + pub fn get_objects(&self, generation: Option, owner: GcOwner) -> Vec { fn collect_from_list( list: &LinkedList, + owner: GcOwner, ) -> impl Iterator + '_ { - list.iter().filter_map(|obj| obj.try_to_owned()) + list.iter() + .filter(move |obj| is_owned_by(obj, owner)) + .filter_map(|obj| obj.try_to_owned()) } match generation { @@ -396,14 +472,14 @@ impl GcState { // Return all tracked objects from all generations + permanent let mut result = Vec::new(); for gen_list in &self.generation_lists { - result.extend(collect_from_list(&gen_list.read())); + result.extend(collect_from_list(&gen_list.read(), owner)); } - result.extend(collect_from_list(&self.permanent_list.read())); + result.extend(collect_from_list(&self.permanent_list.read(), owner)); result } Some(g) if (0..=2).contains(&g) => { let guard = self.generation_lists[g as usize].read(); - collect_from_list(&guard).collect() + collect_from_list(&guard, owner).collect() } _ => Vec::new(), } @@ -412,14 +488,14 @@ impl GcState { /// Check if automatic GC should run and run it if needed. /// Called after object allocation. /// Returns true if GC was run, false otherwise. - pub fn maybe_collect(&self) -> bool { - if !self.is_enabled() { + fn maybe_collect(&self, gc: &GcInterpreterState) -> bool { + if !gc.is_enabled() { return false; } // Check gen0 threshold - let count0 = self.generations[0].count.load(Ordering::SeqCst) as u32; - let threshold0 = self.generations[0].threshold(); + let count0 = self.counts[0].load(Ordering::Relaxed) as u32; + let threshold0 = gc.generations[0].threshold(); if threshold0 > 0 && count0 >= threshold0 { #[cfg(feature = "threading")] { @@ -435,7 +511,7 @@ impl GcState { // thread whose frames could be read mid-mutation, so collect inline. #[cfg(not(feature = "threading"))] { - self.collect(0); + self.collect_inner(gc, 0, false); return true; } } @@ -443,18 +519,13 @@ impl GcState { false } - /// Perform garbage collection on the given generation - pub fn collect(&self, generation: usize) -> CollectResult { - self.collect_inner(generation, false) - } - - /// Force collection even if GC is disabled (for manual gc.collect() calls) - pub fn collect_force(&self, generation: usize) -> CollectResult { - self.collect_inner(generation, true) - } - - fn collect_inner(&self, generation: usize, force: bool) -> CollectResult { - if !force && !self.is_enabled() { + fn collect_inner( + &self, + gc: &GcInterpreterState, + generation: usize, + force: bool, + ) -> CollectResult { + if !force && !gc.is_enabled() { return CollectResult::default(); } @@ -473,7 +544,7 @@ impl GcState { core::sync::atomic::fence(Ordering::SeqCst); let generation = generation.min(2); - let debug = self.get_debug(); + let debug = gc.get_debug(); // Clear the method cache to release strong references that // might prevent cycle collection (_PyType_ClearCache). @@ -511,26 +582,67 @@ impl GcState { .map(|i| self.generation_lists[i].read()) .collect(); - let mut collecting: HashSet = HashSet::new(); + // Only this interpreter's objects, plus the ones no interpreter owns. + // Another interpreter's objects stay out of the candidate set, so they + // act as external roots: anything they reference survives this pass. + let owner = gc.owner; + // Sorted so that the test below, which every scanned object pays for, + // stays logarithmic in the number of interpreters that have been + // dropped instead of linear. + let retired = { + let mut retired = self.retired.lock().clone(); + retired.sort_unstable(); + retired + }; + // The candidates and their reference counts go in one table, not a set + // beside a map: every edge in the heap is looked up here, and the two + // held the same keys, so a second table only bought a second hash of + // the same address. `candidate_ptrs` keeps them in a walkable order, + // since the counts are written while the candidates are read. + let mut gc_refs: GcMap = GcMap::default(); + let mut candidate_ptrs: Vec = Vec::new(); for gen_list in &gen_locks { for obj in gen_list.iter() { - if obj.strong_count() > 0 { - collecting.insert(GcPtr(NonNull::from(obj))); + if retired.binary_search(&obj.gc_owner()).is_ok() { + obj.set_gc_owner(GC_NO_OWNER); + } + let strong_count = obj.strong_count(); + let ptr = GcPtr(NonNull::from(obj)); + if strong_count > 0 + && is_owned_by(obj, owner) + && gc_refs.insert(ptr, strong_count).is_none() + { + candidate_ptrs.push(ptr); + } + } + } + + // A full collection is the only one that sees every generation, so it + // is where adoption finishes and the tags stop being tracked. + if generation == 2 && !retired.is_empty() { + for obj in self.permanent_list.read().iter() { + if retired.binary_search(&obj.gc_owner()).is_ok() { + obj.set_gc_owner(GC_NO_OWNER); } } + // Only the tags this scan saw: one retired while it ran still has + // objects nobody has adopted. + self.retired + .lock() + .retain(|tag| retired.binary_search(tag).is_err()); } - if collecting.is_empty() { + if candidate_ptrs.is_empty() { // Reset counts for generations whose objects were promoted away. // For gen2 (oldest), survivors stay in-place so don't reset gen2 count. let reset_end = if generation >= 2 { 2 } else { generation + 1 }; for i in 0..reset_end { - self.generations[i].count.store(0, Ordering::SeqCst); + self.counts[i].store(0, Ordering::Relaxed); } let duration = elapsed_secs(start_time); - self.generations[generation].update_stats(0, 0, 0, duration); + gc.generations[generation].update_stats(0, 0, 0, duration); return CollectResult { collected: 0, uncollectable: 0, @@ -539,26 +651,10 @@ impl GcState { }; } - let candidates = collecting.len(); + let candidates = candidate_ptrs.len(); if debug.contains(GcDebugFlags::STATS) { - eprintln!( - "gc: collecting {} objects from generations 0..={}", - collecting.len(), - generation - ); - } - - // Step 2: Build gc_refs map (copy reference counts) - let mut gc_refs: std::collections::HashMap = std::collections::HashMap::new(); - - #[expect( - clippy::iter_over_hash_type, - reason = "Iteration order doesn't matter here" - )] - for &ptr in &collecting { - let obj = unsafe { ptr.0.as_ref() }; - gc_refs.insert(ptr, obj.strong_count()); + eprintln!("gc: collecting {candidates} objects from generations 0..={generation}"); } // Step 3: Subtract internal references @@ -567,32 +663,31 @@ impl GcState { // of each object's children. Without this, a dict whose write lock is // held during one traversal but not the other can yield inconsistent // results, causing live objects to be incorrectly collected. - let mut referents_map: std::collections::HashMap>> = - std::collections::HashMap::new(); + // + // Every object's referents go in one buffer, with each object holding + // the range that is its own: a vector each would be an allocation per + // tracked object, and the collection wants them all at once anyway. + let mut referent_ptrs: Vec> = Vec::new(); + let mut referent_ranges: GcMap = GcMap::default(); - #[expect( - clippy::iter_over_hash_type, - reason = "Iteration order doesn't matter here" - )] - for &ptr in &collecting { + for &ptr in &candidate_ptrs { let obj = unsafe { ptr.0.as_ref() }; if obj.strong_count() == 0 { continue; } - let referent_ptrs = unsafe { obj.gc_get_referent_ptrs() }; - referents_map.insert(ptr, referent_ptrs.clone()); - for child_ptr in referent_ptrs { - let gc_ptr = GcPtr(child_ptr); - if collecting.contains(&gc_ptr) - && let Some(refs) = gc_refs.get_mut(&gc_ptr) - { + let start = referent_ptrs.len(); + unsafe { obj.gc_extend_referent_ptrs(&mut referent_ptrs) }; + let end = referent_ptrs.len(); + for &child_ptr in &referent_ptrs[start..end] { + if let Some(refs) = gc_refs.get_mut(&GcPtr(child_ptr)) { *refs = refs.saturating_sub(1); } } + referent_ranges.insert(ptr, (start, end)); } // Step 4: Find reachable objects (gc_refs > 0) and traverse from them - let mut reachable: HashSet = HashSet::new(); + let mut reachable: GcSet = GcSet::default(); let mut worklist: Vec = Vec::new(); #[expect( @@ -609,16 +704,21 @@ impl GcState { while let Some(ptr) = worklist.pop() { let obj = unsafe { ptr.0.as_ref() }; if obj.is_gc_tracked() { - // Reuse the pre-computed referent pointers from step 3. - // For objects that were skipped in step 3 (strong_count was 0), - // compute them now as a fallback. - let referent_ptrs = referents_map - .get(&ptr) - .cloned() - .unwrap_or_else(|| unsafe { obj.gc_get_referent_ptrs() }); - for child_ptr in referent_ptrs { + // Reuse the pre-computed referent pointers from step 3, in + // place: copying them out again costs a second pass over every + // edge in the heap. Objects skipped in step 3 (strong_count was + // 0) have none stored and are traversed here instead. + let computed; + let children: &[NonNull] = match referent_ranges.get(&ptr) { + Some(&(start, end)) => &referent_ptrs[start..end], + None => { + computed = unsafe { obj.gc_get_referent_ptrs() }; + &computed + } + }; + for &child_ptr in children { let gc_ptr = GcPtr(child_ptr); - if collecting.contains(&gc_ptr) && reachable.insert(gc_ptr) { + if gc_refs.contains_key(&gc_ptr) && reachable.insert(gc_ptr) { worklist.push(gc_ptr); } } @@ -626,7 +726,11 @@ impl GcState { } // Step 5: Find unreachable objects - let unreachable: Vec = collecting.difference(&reachable).copied().collect(); + let unreachable: Vec = candidate_ptrs + .iter() + .filter(|ptr| !reachable.contains(ptr)) + .copied() + .collect(); // With the world stopped, every frame on any thread's call stack is a // live root that is externally referenced and must have been @@ -638,8 +742,8 @@ impl GcState { // because stack-allocated frames update only CURRENT_FRAME (via // set_current_frame_nosave), not top_frame. #[cfg(all(unix, feature = "threading", debug_assertions))] - if stw.stopped { - let unreachable_set: HashSet = unreachable.iter().copied().collect(); + if stw.is_stopped() { + let unreachable_set: GcSet = unreachable.iter().copied().collect(); let mut cur = crate::vm::thread::get_current_frame(); while !cur.is_null() { let iframe = unsafe { &*cur }; @@ -701,12 +805,12 @@ impl GcState { self.promote_survivors(generation, &survivor_refs); let reset_end = if generation >= 2 { 2 } else { generation + 1 }; for i in 0..reset_end { - self.generations[i].count.store(0, Ordering::SeqCst); + self.counts[i].store(0, Ordering::Relaxed); } let duration = elapsed_secs(start_time); - self.generations[generation].update_stats(0, 0, candidates, duration); + gc.generations[generation].update_stats(0, 0, candidates, duration); return CollectResult { collected: 0, uncollectable: 0, @@ -724,12 +828,12 @@ impl GcState { self.promote_survivors(generation, &survivor_refs); let reset_end = if generation >= 2 { 2 } else { generation + 1 }; for i in 0..reset_end { - self.generations[i].count.store(0, Ordering::SeqCst); + self.counts[i].store(0, Ordering::Relaxed); } let duration = elapsed_secs(start_time); - self.generations[generation].update_stats(0, 0, candidates, duration); + gc.generations[generation].update_stats(0, 0, candidates, duration); return CollectResult { collected: 0, uncollectable: 0, @@ -739,7 +843,7 @@ impl GcState { } // 6b: Record initial strong counts (for resurrection detection) - let initial_counts: std::collections::HashMap = unreachable_refs + let initial_counts: GcMap = unreachable_refs .iter() .map(|obj| { let ptr = GcPtr(core::ptr::NonNull::from(obj.as_ref())); @@ -770,8 +874,8 @@ impl GcState { } // Detect resurrection - let mut resurrected_set: HashSet = HashSet::new(); - let unreachable_set: HashSet = unreachable.iter().copied().collect(); + let mut resurrected_set: GcSet = GcSet::default(); + let unreachable_set: GcSet = unreachable.iter().copied().collect(); for obj in &unreachable_refs { let ptr = GcPtr(core::ptr::NonNull::from(obj.as_ref())); @@ -811,7 +915,7 @@ impl GcState { // Compute collected count (exclude instance dicts in truly_dead) let collected = { - let dead_ptrs: HashSet = truly_dead + let dead_ptrs: GcSet = truly_dead .iter() .map(|obj| obj.as_ref() as *const PyObject as usize) .collect(); @@ -849,7 +953,7 @@ impl GcState { } if debug.contains(GcDebugFlags::SAVEALL) { - let mut garbage_guard = self.garbage.lock(); + let mut garbage_guard = gc.garbage.lock(); for obj_ref in &truly_dead { garbage_guard.push(obj_ref.clone()); } @@ -869,10 +973,9 @@ impl GcState { // never be observable through the generation lists, or another // thread could obtain a strong reference via gc.get_objects() // and access the cleared payload. - let mut late_resurrected: HashSet = HashSet::new(); + let mut late_resurrected: GcSet = GcSet::default(); if !save_all { - let mut expected_counts: std::collections::HashMap = - std::collections::HashMap::new(); + let mut expected_counts: GcMap = GcMap::default(); for obj_ref in &truly_dead { let obj = obj_ref.as_ref(); if obj.is_gc_tracked() { @@ -886,8 +989,7 @@ impl GcState { // the dead set; any surplus in strong_count means another thread // grabbed a reference before untracking (late resurrection) and // the object must not be cleared. - let mut referents: std::collections::HashMap>> = - std::collections::HashMap::new(); + let mut referents: GcMap>> = GcMap::default(); for obj_ref in &truly_dead { let referent_ptrs = unsafe { obj_ref.gc_get_referent_ptrs() }; for child_ptr in &referent_ptrs { @@ -926,7 +1028,10 @@ impl GcState { reason = "Iteration order doesn't matter here" )] for &ptr in &late_resurrected { - unsafe { self.track_object(ptr.0) }; + // Re-tracking a resurrected object: it keeps the owner it + // was allocated under. + let owner = unsafe { ptr.0.as_ref() }.gc_owner(); + unsafe { self.track_object(ptr.0, owner) }; } } rustpython_common::refcount::with_deferred_drops(|| { @@ -950,12 +1055,12 @@ impl GcState { // For gen2 (oldest), survivors stay in-place so don't reset gen2 count. let reset_end = if generation >= 2 { 2 } else { generation + 1 }; for i in 0..reset_end { - self.generations[i].count.store(0, Ordering::SeqCst); + self.counts[i].store(0, Ordering::Relaxed); } let duration = elapsed_secs(start_time); - self.generations[generation].update_stats(collected, 0, candidates, duration); + gc.generations[generation].update_stats(collected, 0, candidates, duration); CollectResult { collected, @@ -998,14 +1103,10 @@ impl GcState { } if unsafe { src.remove(ptr) }.is_some() { - self.generations[src_gen] - .count - .fetch_sub(1, Ordering::SeqCst); + release_count(&self.counts[src_gen]); dst.push_front(ptr); - self.generations[next_gen] - .count - .fetch_add(1, Ordering::SeqCst); + self.counts[next_gen].fetch_add(1, Ordering::Relaxed); obj.set_gc_generation(next_gen as u8); } @@ -1015,45 +1116,66 @@ impl GcState { /// Get count of frozen objects pub fn get_freeze_count(&self) -> usize { - self.permanent.count() + self.permanent_count.load(Ordering::Relaxed) } - /// Freeze all tracked objects (move to permanent generation). + /// Freeze the objects `owner` could collect (move them to the permanent + /// generation). /// Lock order: generation_lists[i] → permanent_list (consistent with unfreeze). - pub fn freeze(&self) { + fn freeze(&self, owner: GcOwner) { let mut count = 0usize; for (gen_idx, gen_list) in self.generation_lists.iter().enumerate() { let mut list = gen_list.write(); let mut perm = self.permanent_list.write(); - while let Some(ptr) = list.pop_front() { + let moving: Vec<_> = list + .iter() + .filter(|obj| is_owned_by(obj, owner)) + .map(NonNull::from) + .collect(); + for ptr in moving { + if unsafe { list.remove(ptr) }.is_none() { + continue; + } perm.push_front(ptr); unsafe { ptr.as_ref().set_gc_generation(GC_PERMANENT) }; count += 1; + release_count(&self.counts[gen_idx]); } - self.generations[gen_idx].count.store(0, Ordering::SeqCst); } - self.permanent.count.fetch_add(count, Ordering::SeqCst); + self.permanent_count.fetch_add(count, Ordering::Relaxed); } - /// Unfreeze all objects (move from permanent to gen2). + /// Unfreeze the objects `owner` froze (move them from permanent to gen2). /// Lock order: generation_lists[2] → permanent_list (consistent with freeze). - pub fn unfreeze(&self) { + fn unfreeze(&self, owner: GcOwner) { let mut count = 0usize; { let mut gen2 = self.generation_lists[2].write(); let mut perm_list = self.permanent_list.write(); - while let Some(ptr) = perm_list.pop_front() { + let moving: Vec<_> = perm_list + .iter() + .filter(|obj| is_owned_by(obj, owner)) + .map(NonNull::from) + .collect(); + for ptr in moving { + if unsafe { perm_list.remove(ptr) }.is_none() { + continue; + } gen2.push_front(ptr); unsafe { ptr.as_ref().set_gc_generation(2) }; count += 1; } - self.permanent.count.store(0, Ordering::SeqCst); + let _ = self.permanent_count.fetch_update( + Ordering::Relaxed, + Ordering::Relaxed, + |permanent| Some(permanent.saturating_sub(count)), + ); } - self.generations[2].count.fetch_add(count, Ordering::SeqCst); + self.counts[2].fetch_add(count, Ordering::Relaxed); } /// Reset all locks to unlocked state after fork(). @@ -1070,13 +1192,7 @@ impl GcState { unsafe { reinit_mutex_after_fork(&self.collecting); - reinit_mutex_after_fork(&self.garbage); - reinit_mutex_after_fork(&self.callbacks); - - for generation in &self.generations { - generation.reinit_stats_after_fork(); - } - self.permanent.reinit_stats_after_fork(); + reinit_mutex_after_fork(&self.retired); for rw in &self.generation_lists { reinit_rwlock_after_fork(rw); @@ -1086,11 +1202,196 @@ impl GcState { } } +/// Per-interpreter garbage collector state (≈ `PyInterpreterState.gc`). +/// +/// The generation lists are process-wide (see [`GcState`]); what an interpreter +/// owns is the policy applied to them and the results — which objects its +/// collections consider, whether they run automatically, and where uncollectable +/// objects end up. +pub struct GcInterpreterState { + /// Tag written into every object this interpreter tracks. + owner: GcOwner, + /// Per-generation thresholds and statistics. + pub generations: [GcGeneration; 3], + /// GC enabled flag + enabled: AtomicBool, + /// Debug flags + debug: AtomicU32, + /// Uncollectable objects saved by this interpreter's collections, drained + /// into `py_garbage` by `gc.collect()`. + pub garbage: PyMutex>, + /// `gc.garbage` + pub py_garbage: crate::builtins::PyListRef, + /// `gc.callbacks` + pub py_callbacks: crate::builtins::PyListRef, +} + +impl GcInterpreterState { + pub fn new(ctx: &crate::vm::Context) -> Self { + Self { + owner: gc_state().alloc_owner(), + generations: [ + GcGeneration::new(2000), // young + GcGeneration::new(10), // old[0] + GcGeneration::new(0), // old[1] + ], + enabled: AtomicBool::new(true), + debug: AtomicU32::new(0), + garbage: PyMutex::new(Vec::new()), + py_garbage: ctx.new_list(Vec::new()), + py_callbacks: ctx.new_list(Vec::new()), + } + } + + /// Check if GC is enabled. + /// + /// Relaxed, like [`GcGeneration::threshold`]: it is read once per + /// allocation, and an allocation racing `gc.disable()` may use either value. + pub fn is_enabled(&self) -> bool { + self.enabled.load(Ordering::Relaxed) + } + + /// Enable GC + pub fn enable(&self) { + self.enabled.store(true, Ordering::Relaxed); + } + + /// Disable GC + pub fn disable(&self) { + self.enabled.store(false, Ordering::Relaxed); + } + + /// Get debug flags + pub fn get_debug(&self) -> GcDebugFlags { + GcDebugFlags::from_bits_truncate(self.debug.load(Ordering::SeqCst)) + } + + /// Set debug flags + pub fn set_debug(&self, flags: GcDebugFlags) { + self.debug.store(flags.bits(), Ordering::SeqCst); + } + + /// Get thresholds for all generations + pub fn get_threshold(&self) -> (u32, u32, u32) { + ( + self.generations[0].threshold(), + self.generations[1].threshold(), + self.generations[2].threshold(), + ) + } + + /// Set thresholds + pub fn set_threshold(&self, t0: u32, t1: Option, t2: Option) { + self.generations[0].set_threshold(t0); + if let Some(t1) = t1 { + self.generations[1].set_threshold(t1); + } + if let Some(t2) = t2 { + self.generations[2].set_threshold(t2); + } + } + + /// Get statistics for all generations + pub fn get_stats(&self) -> [GcStats; 3] { + [ + self.generations[0].stats(), + self.generations[1].stats(), + self.generations[2].stats(), + ] + } + + /// Perform garbage collection on the given generation + pub fn collect(&self, generation: usize) -> CollectResult { + gc_state().collect_inner(self, generation, false) + } + + /// Force collection even if GC is disabled (for manual gc.collect() calls) + pub fn collect_force(&self, generation: usize) -> CollectResult { + gc_state().collect_inner(self, generation, true) + } + + /// The tracked objects this interpreter can reach (for gc.get_objects). + pub fn get_objects(&self, generation: Option) -> Vec { + gc_state().get_objects(generation, self.owner) + } + + /// Move the objects this interpreter could collect into the permanent + /// generation. + pub fn freeze(&self) { + gc_state().freeze(self.owner); + } + + /// Move them back out of it. + pub fn unfreeze(&self) { + gc_state().unfreeze(self.owner); + } + + /// Reset this interpreter's GC locks to unlocked state after fork(). + /// + /// # Safety + /// Must only be called after fork() in the child process when no other + /// threads exist. The calling thread must NOT hold any of these locks. + #[cfg(all(unix, feature = "threading"))] + pub unsafe fn reinit_after_fork(&self) { + unsafe { + crate::common::lock::reinit_mutex_after_fork(&self.garbage); + for generation in &self.generations { + generation.reinit_stats_after_fork(); + } + } + } +} + +impl Drop for GcInterpreterState { + fn drop(&mut self) { + // Objects this interpreter tracked can outlive it (another interpreter + // may still hold one). Clearing the tag hands them to every collection + // instead of stranding them. The tag itself is not handed back: it stays + // retired so that a later interpreter cannot inherit these objects. + gc_state().retire_owner(self.owner); + } +} + +/// The tag `track_object` should write for the interpreter running now. +#[must_use] +pub fn current_owner() -> GcOwner { + // SAFETY: the pointee is owned by the `PyGlobalState` of the VM on top of + // this thread's VM stack, which outlives the section this call runs in. + crate::vm::thread::current_gc_state().map_or(GC_NO_OWNER, |gc| unsafe { gc.as_ref() }.owner) +} + +/// Track a freshly allocated object under the interpreter running now, and let +/// it collect if the allocation pushed gen0 past its threshold. +/// +/// # Safety +/// obj must be a valid pointer to a PyObject that is not already tracked. +pub(crate) unsafe fn track_new_object(obj: NonNull) { + let state = gc_state(); + let Some(gc) = crate::vm::thread::current_gc_state() else { + // No interpreter is running: the shared context builds its own objects + // this way. They are left unowned, so every interpreter collects them. + unsafe { state.track_object(obj, GC_NO_OWNER) }; + return; + }; + // SAFETY: as in `current_owner`. + let gc = unsafe { gc.as_ref() }; + unsafe { state.track_object(obj, gc.owner) }; + state.maybe_collect(gc); +} + /// Get a reference to the GC state. /// /// In threading mode this is a true global (OnceLock). /// In non-threading mode this is thread-local, because PyRwLock/PyMutex /// use Cell-based locks that are not Sync. +/// +/// Every interpreter's tracked objects live in these lists, because untracking +/// happens in `default_dealloc`, where no interpreter is in scope to route to. +/// What a collection *acts on* is still one interpreter's own objects, selected +/// by the `gc_owner` tag; [`GcInterpreterState`] holds the rest of the state +/// that goes with that. The counts here, and so `gc.get_count()` and +/// `gc.get_freeze_count()`, stay process-wide: they measure how full these +/// lists are. pub fn gc_state() -> &'static GcState { rustpython_common::static_cell! { static GC_STATE: GcState; @@ -1102,18 +1403,21 @@ pub fn gc_state() -> &'static GcState { mod tests { use super::*; + fn interpreter_state() -> GcInterpreterState { + GcInterpreterState::new(crate::vm::Context::genesis()) + } + #[test] fn gc_state_default() { - let state = GcState::new(); + let state = interpreter_state(); assert!(state.is_enabled()); assert_eq!(state.get_debug(), GcDebugFlags::empty()); assert_eq!(state.get_threshold(), (2000, 10, 0)); - assert_eq!(state.get_count(), (0, 0, 0)); } #[test] fn gc_enable_disable() { - let state = GcState::new(); + let state = interpreter_state(); assert!(state.is_enabled()); state.disable(); assert!(!state.is_enabled()); @@ -1123,18 +1427,29 @@ mod tests { #[test] fn gc_threshold() { - let state = GcState::new(); + let state = interpreter_state(); state.set_threshold(100, Some(20), Some(30)); assert_eq!(state.get_threshold(), (100, 20, 30)); } #[test] fn gc_debug_flags() { - let state = GcState::new(); + let state = interpreter_state(); state.set_debug(GcDebugFlags::STATS | GcDebugFlags::COLLECTABLE); assert_eq!( state.get_debug(), GcDebugFlags::STATS | GcDebugFlags::COLLECTABLE ); } + + /// Live interpreters never share an owner tag, or their collections would + /// reach each other's objects. + #[test] + fn gc_owner_tags_are_distinct_while_live() { + let first = interpreter_state(); + let second = interpreter_state(); + assert_ne!(first.owner, second.owner); + assert_ne!(first.owner, GC_NO_OWNER); + assert_ne!(second.owner, GC_NO_OWNER); + } } diff --git a/crates/vm/src/lib.rs b/crates/vm/src/lib.rs index df67c979739..a15e30c34a3 100644 --- a/crates/vm/src/lib.rs +++ b/crates/vm/src/lib.rs @@ -109,7 +109,11 @@ pub use self::object::{ AsObject, Py, PyAtomicRef, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult, PyStackRef, PyWeakRef, }; -pub use self::vm::{Context, Interpreter, InterpreterBuilder, Settings, VirtualMachine}; +pub use self::vm::runtime; +pub use self::vm::{ + Context, Interpreter, InterpreterBuilder, InterpreterInfo, InterpreterWhence, + MAIN_INTERPRETER_ID, Settings, VirtualMachine, +}; pub use rustpython_common as common; pub use rustpython_compiler_core::{bytecode, frozen}; diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index 0ee7a062ee7..bdacb7c5b83 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -303,6 +303,18 @@ bitflags::bitflags! { /// GC generation constants pub(crate) const GC_UNTRACKED: u8 = 0xFF; pub(crate) const GC_PERMANENT: u8 = 3; +/// Width of an interpreter's `gc_owner` tag. +/// +/// Sized to the padding the header alignment already forces, so the tag costs +/// no space on either pointer width. Running out of tags is not an error: an +/// interpreter that gets none uses [`GC_NO_OWNER`] and its objects stay +/// collectable by every interpreter, which is how they behaved before tagging. +pub(crate) type GcOwner = u16; + +/// `gc_owner` of an object that belongs to no single interpreter: everything +/// the shared context allocates, and anything allocated with no interpreter +/// current. Every interpreter collects these. +pub(crate) const GC_NO_OWNER: GcOwner = 0; /// Link implementation for GC intrusive linked list tracking pub(crate) struct GcLink; @@ -389,6 +401,10 @@ pub(super) struct PyInner { /// GC generation index (0-2=gen, GC_PERMANENT=permanent, GC_UNTRACKED=not tracked). /// Uses PyAtomic for interior mutability (writes happen through &self under list locks). pub(super) gc_generation: PyAtomic, + /// Interpreter that tracked this object, or `GC_NO_OWNER`. Written by + /// `track_object`; read to scope a collection to one interpreter. + /// Sits in what would otherwise be padding, so it costs no space. + pub(super) gc_owner: PyAtomic, /// Intrusive linked list pointers for GC generational tracking pub(super) gc_pointers: Pointers, @@ -398,6 +414,11 @@ pub(super) struct PyInner { } pub(crate) const SIZEOF_PYOBJECT_HEAD: usize = core::mem::size_of::>(); +// ref_count, vtable, gc_pointers (two) and typ are one word each; the gc bits, +// generation and owner share the word of padding their alignment forces. Adding +// to that group is free only while this holds. +const _: () = assert!(SIZEOF_PYOBJECT_HEAD == 6 * core::mem::size_of::()); + impl PyInner { /// Read type flags and member_count via raw pointers to avoid Stacked Borrows /// violations during bootstrap, where type objects have self-referential typ pointers. @@ -1052,6 +1073,16 @@ impl InstanceDict { self.d.read().clone() } + /// Run `f` on the dict without cloning it. + /// + /// For callers that only need to look at the dict — a predicate, a version + /// stamp — this drops the refcount round-trip [`Self::get`] pays. `f` runs + /// under the read guard, so it must not run Python or take this lock again. + #[inline] + pub(crate) fn with(&self, f: impl FnOnce(Option<&Py>) -> R) -> R { + f(self.d.read().as_deref()) + } + #[inline] pub(crate) fn set(&self, d: Option) { self.replace(d); @@ -1216,6 +1247,7 @@ impl PyInner { vtable: PyObjVTable::of::(), gc_bits: Radium::new(0), gc_generation: Radium::new(GC_UNTRACKED), + gc_owner: Radium::new(GC_NO_OWNER), gc_pointers: Pointers::new(), typ: PyAtomicRef::from(typ), payload, @@ -1228,6 +1260,7 @@ impl PyInner { vtable: PyObjVTable::of::(), gc_bits: Radium::new(0), gc_generation: Radium::new(GC_UNTRACKED), + gc_owner: Radium::new(GC_NO_OWNER), gc_pointers: Pointers::new(), typ: PyAtomicRef::from(typ), payload, @@ -1615,6 +1648,28 @@ impl PyObject { self.instance_dict().and_then(|d| d.get()) } + /// Whether this object currently has an instance dict, without cloning it. + /// + /// `false` both for an object with no dict slot and for one whose slot is + /// still empty, which is what `dict().is_none()` reports. + #[inline(always)] + pub fn has_instance_dict(&self) -> bool { + self.instance_dict() + .is_some_and(|d| d.with(|dict| dict.is_some())) + } + + /// Run `f` on the instance dict without cloning it; see [`InstanceDict::with`]. + #[inline(always)] + pub(crate) fn with_instance_dict( + &self, + f: impl FnOnce(Option<&Py>) -> R, + ) -> R { + match self.instance_dict() { + Some(d) => d.with(f), + None => f(None), + } + } + /// Set the dict field. Returns `Err(dict)` if this object does not have a dict field /// in the first place. pub fn set_dict(&self, dict: Option) -> Result<(), Option> { @@ -1734,6 +1789,20 @@ impl PyObject { self.0.gc_generation.store(generation, Ordering::Relaxed); } + /// The interpreter whose collections consider this object. + #[inline] + pub(crate) fn gc_owner(&self) -> GcOwner { + self.0.gc_owner.load(Ordering::Relaxed) + } + + /// Set the owning interpreter. Written by `track_object` before the object + /// enters a generation list, and reset to `GC_NO_OWNER` when the owning + /// interpreter goes away. + #[inline] + pub(crate) fn set_gc_owner(&self, owner: GcOwner) { + self.0.gc_owner.store(owner, Ordering::Relaxed); + } + /// _PyObject_GC_TRACK #[inline] pub(crate) fn set_gc_tracked(&self) { @@ -1888,11 +1957,20 @@ impl PyObject { /// and its contents haven't been modified. pub unsafe fn gc_get_referent_ptrs(&self) -> Vec> { let mut result = Vec::new(); + unsafe { self.gc_extend_referent_ptrs(&mut result) }; + result + } + + /// Append this object's referents to `out`, for a caller that holds many + /// objects' referents in one buffer rather than one buffer each. + /// + /// # Safety + /// Same as [`Self::gc_get_referent_ptrs`]. + pub unsafe fn gc_extend_referent_ptrs(&self, out: &mut Vec>) { // Traverse the entire object including dict and slots self.0.traverse(&mut |child: &Self| { - result.push(NonNull::from(child)); + out.push(NonNull::from(child)); }); - result } /// Pop edges from this object for cycle breaking. @@ -2392,12 +2470,11 @@ impl PyRef { if (::HAS_TRAVERSE || has_dict || is_heaptype) && !T::NEW_REF_UNTRACKED { - let gc = crate::gc_state::gc_state(); + // Tracks under the interpreter running now and collects if this + // allocation pushed gen0 past its threshold. unsafe { - gc.track_object(ptr.cast()); + crate::gc_state::track_new_object(ptr.cast()); } - // Check if automatic GC should run - gc.maybe_collect(); } Self { ptr } @@ -2645,6 +2722,7 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { vtable: PyObjVTable::of::(), gc_bits: Radium::new(0), gc_generation: Radium::new(GC_UNTRACKED), + gc_owner: Radium::new(GC_NO_OWNER), gc_pointers: Pointers::new(), payload: type_payload, }, @@ -2660,6 +2738,7 @@ pub(crate) fn init_type_hierarchy() -> (PyTypeRef, PyTypeRef, PyTypeRef) { vtable: PyObjVTable::of::(), gc_bits: Radium::new(0), gc_generation: Radium::new(GC_UNTRACKED), + gc_owner: Radium::new(GC_NO_OWNER), gc_pointers: Pointers::new(), payload: object_payload, }, diff --git a/crates/vm/src/object/ext.rs b/crates/vm/src/object/ext.rs index 69ee0e3c510..186fa8e8a84 100644 --- a/crates/vm/src/object/ext.rs +++ b/crates/vm/src/object/ext.rs @@ -269,13 +269,16 @@ cfg_select! { _ => {} } -impl fmt::Debug for PyAtomicRef { +impl fmt::Debug for PyAtomicRef { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "PyAtomicRef(")?; + // The stored pointer is a `Py` — the full object, header included — + // as `Deref`, `load_raw` and `swap` all read it. Formatting it as a + // bare payload would skip the header and print misaligned bytes. unsafe { self.inner .load(Ordering::Relaxed) - .cast::() + .cast::() .as_ref() .fmt(f) }?; diff --git a/crates/vm/src/object/mod.rs b/crates/vm/src/object/mod.rs index b06957e1bc6..becfcabb1d4 100644 --- a/crates/vm/src/object/mod.rs +++ b/crates/vm/src/object/mod.rs @@ -9,5 +9,5 @@ pub use self::core::*; pub use self::ext::*; pub use self::payload::*; pub(crate) use core::SIZEOF_PYOBJECT_HEAD; -pub(crate) use core::{GC_PERMANENT, GC_UNTRACKED, GcLink}; +pub(crate) use core::{GC_NO_OWNER, GC_PERMANENT, GC_UNTRACKED, GcLink, GcOwner}; pub use traverse::{MaybeTraverse, Traverse, TraverseFn}; diff --git a/crates/vm/src/protocol/buffer.rs b/crates/vm/src/protocol/buffer.rs index d79c5e9933d..050c568b7ac 100644 --- a/crates/vm/src/protocol/buffer.rs +++ b/crates/vm/src/protocol/buffer.rs @@ -6,14 +6,101 @@ use crate::{ common::{ borrow::{BorrowedValue, BorrowedValueMut}, lock::{MapImmutable, PyMutex, PyMutexGuard}, + rc::PyRc, }, object::PyObjectPayload, sliceable::SequenceIndexOp, }; use alloc::borrow::Cow; +use bitflags::bitflags; use core::{fmt::Debug, ops::Range}; +use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; +bitflags! { + /// Capabilities a consumer asks a buffer exporter for, the `flags` argument of + /// `bf_getbuffer` and of `__buffer__` (`PyBUF_*`). + /// + /// The composite requests are supersets of the simpler ones, so + /// [`contains`](Self::contains) answers the `REQ_*` questions an exporter asks: + /// `flags.contains(BufferFlags::C_CONTIGUOUS)` is `REQ_C_CONTIGUOUS(flags)`. + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + pub struct BufferFlags: u32 { + const WRITABLE = 0x0001; + const FORMAT = 0x0004; + const ND = 0x0008; + const STRIDES = 0x0010 | Self::ND.bits(); + const C_CONTIGUOUS = 0x0020 | Self::STRIDES.bits(); + const F_CONTIGUOUS = 0x0040 | Self::STRIDES.bits(); + const ANY_CONTIGUOUS = 0x0080 | Self::STRIDES.bits(); + const INDIRECT = 0x0100 | Self::STRIDES.bits(); + } +} + +impl BufferFlags { + /// `PyBUF_SIMPLE`: a plain read-only block of bytes. + pub const SIMPLE: Self = Self::empty(); + /// `PyBUF_CONTIG` + pub const CONTIG: Self = Self::ND.union(Self::WRITABLE); + /// `PyBUF_CONTIG_RO` + pub const CONTIG_RO: Self = Self::ND; + /// `PyBUF_STRIDED` + pub const STRIDED: Self = Self::STRIDES.union(Self::WRITABLE); + /// `PyBUF_STRIDED_RO` + pub const STRIDED_RO: Self = Self::STRIDES; + /// `PyBUF_RECORDS` + pub const RECORDS: Self = Self::STRIDED.union(Self::FORMAT); + /// `PyBUF_RECORDS_RO` + pub const RECORDS_RO: Self = Self::STRIDED_RO.union(Self::FORMAT); + /// `PyBUF_FULL`: everything an exporter can describe, writable. + pub const FULL: Self = Self::INDIRECT.union(Self::WRITABLE).union(Self::FORMAT); + /// `PyBUF_FULL_RO`: everything an exporter can describe, read-only. + pub const FULL_RO: Self = Self::INDIRECT.union(Self::FORMAT); + + /// `PyBUF_READ`. Belongs to `PyMemoryView_FromMemory`, not to `bf_getbuffer`. + const MEMORY_READ: Self = Self::from_bits_retain(0x100); + /// `PyBUF_WRITE`. Belongs to `PyMemoryView_FromMemory`, not to `bf_getbuffer`. + const MEMORY_WRITE: Self = Self::from_bits_retain(0x200); + + /// Whether this request is really a `PyMemoryView_FromMemory` access mode, + /// which no exporter can serve. + #[must_use] + pub const fn is_memory_access_mode(self) -> bool { + self.bits() == Self::MEMORY_READ.bits() || self.bits() == Self::MEMORY_WRITE.bits() + } + + /// Whether the consumer demands a writable buffer. + #[must_use] + pub const fn is_writable(self) -> bool { + self.intersects(Self::WRITABLE) + } + + /// The argument checks `PyBuffer_FillInfo` performs, for exporters that hand + /// out a flat block of bytes. + pub fn fill_info_check(self, readonly: bool, vm: &VirtualMachine) -> PyResult<()> { + if self == Self::SIMPLE { + return Ok(()); + } + if self.is_memory_access_mode() { + return Err(vm.new_system_error("bad argument to internal function")); + } + self.check_writable(readonly, "Object is not writable.", vm) + } + + /// Reject a writable request against a read-only export. + pub fn check_writable( + self, + readonly: bool, + message: &str, + vm: &VirtualMachine, + ) -> PyResult<()> { + if self.is_writable() && readonly { + return Err(vm.new_buffer_error(message.to_owned())); + } + Ok(()) + } +} + pub struct BufferMethods { pub obj_bytes: fn(&PyBuffer) -> BorrowedValue<'_, [u8]>, pub obj_bytes_mut: fn(&PyBuffer) -> BorrowedValueMut<'_, [u8]>, @@ -32,13 +119,46 @@ impl Debug for BufferMethods { } } -#[derive(Debug, Clone, Traverse)] +/// One acquisition from an exporter: the state a single `bf_getbuffer` set up, +/// shared by every handle taken from it. _PyManagedBufferObject +#[derive(Debug)] +struct BufferExport { + /// Handles and raw shares that have not been given up yet. mbuf->exports + shares: AtomicCell, + /// Whether the exporter's release has already run. + /// _Py_MANAGED_BUFFER_RELEASED + released: AtomicCell, +} + +#[derive(Debug, Traverse)] pub struct PyBuffer { pub obj: PyObjectRef, #[pytraverse(skip)] pub desc: BufferDescriptor, #[pytraverse(skip)] methods: &'static BufferMethods, + #[pytraverse(skip)] + export: PyRc, + /// Whether this handle still holds its share of `export`. + #[pytraverse(skip)] + owns_share: AtomicCell, +} + +/// Cloning takes another share of the same acquisition rather than asking the +/// exporter for a new one, and the exporter's release waits for the last share. +/// mbuf_add_view +impl Clone for PyBuffer { + fn clone(&self) -> Self { + debug_assert!(!self.export.released.load()); + self.export.shares.fetch_add(1); + Self { + obj: self.obj.clone(), + desc: self.desc.clone(), + methods: self.methods, + export: self.export.clone(), + owns_share: AtomicCell::new(true), + } + } } impl PyBuffer { @@ -47,8 +167,17 @@ impl PyBuffer { #[cfg(debug_assertions)] let desc = desc.validate(); - let zelf = Self { obj, desc, methods }; - zelf.retain(); + let zelf = Self { + obj, + desc, + methods, + export: PyRc::new(BufferExport { + shares: AtomicCell::new(1), + released: AtomicCell::new(false), + }), + owns_share: AtomicCell::new(true), + }; + (zelf.methods.retain)(&zelf); zelf } @@ -78,14 +207,16 @@ impl PyBuffer { /// assume the buffer is contiguous #[must_use] pub unsafe fn contiguous_unchecked(&self) -> BorrowedValue<'_, [u8]> { - self.obj_bytes() + let range = self.desc.contiguous_range(); + BorrowedValue::map(self.obj_bytes(), |x| &x[range]) } /// # Safety /// assume the buffer is contiguous and writable #[must_use] pub unsafe fn contiguous_mut_unchecked(&self) -> BorrowedValueMut<'_, [u8]> { - self.obj_bytes_mut() + let range = self.desc.contiguous_range(); + BorrowedValueMut::map(self.obj_bytes_mut(), |x| &mut x[range]) } pub fn append_to(&self, buf: &mut Vec) { @@ -113,6 +244,18 @@ impl PyBuffer { f(v) } + /// A copy of these bytes in C order, keeping shape and format. The copy + /// borrows nothing from the exporter, so it can be read while the exporter is + /// borrowed for writing. + #[must_use] + pub fn to_contiguous(&self, vm: &VirtualMachine) -> Self { + let mut data = vec![]; + self.append_to(&mut data); + VecBuffer::from(data) + .into_ref(&vm.ctx) + .into_pybuffer_with_descriptor(self.desc.contiguous()) + } + #[must_use] pub fn obj_as(&self) -> &Py { unsafe { self.obj.downcast_unchecked_ref() } @@ -128,31 +271,87 @@ impl PyBuffer { (self.methods.obj_bytes_mut)(self) } + /// Give up this handle's share of the acquisition. PyBuffer_Release + /// + /// Idempotent: a handle that has already been released owns nothing, so + /// dropping it afterwards does nothing, like a `Py_buffer` whose `obj` was + /// cleared. + /// + /// This can run arbitrary Python through `__release_buffer__`, so no borrow + /// of the exporter may be held while a buffer is released or dropped. pub fn release(&self) { + if self.owns_share.swap(false) { + self.drop_share(); + } + } + + /// Take a share of this acquisition that no handle owns. An exporter that + /// forwards a consumer's export onto a buffer it holds itself keeps the + /// acquisition alive this way. memory_getbuf + pub(crate) fn retain_share(&self) { + self.export.shares.fetch_add(1); + } + + /// Give back a share taken by [`Self::retain_share`]. memory_releasebuf + pub(crate) fn release_share(&self) { + self.drop_share(); + } + + fn drop_share(&self) { + if self.export.shares.fetch_sub(1) == 1 { + self.finalize(); + } + } + + /// The exporter learns its export is gone, once per acquisition. mbuf_release + fn finalize(&self) { + // Latched before the hook runs, so a release re-entered from Python is + // inert. + if self.export.released.swap(true) { + return; + } + // slot_bf_releasebuffer: a Python-level `__release_buffer__` runs first, + // then the exporter's own release so export counts stay balanced. + if self.obj.class().slots.python_release_buffer.load() { + crate::builtins::memory::release_buffer_call_python(self); + } (self.methods.release)(self) } - pub fn retain(&self) { - (self.methods.retain)(self) + /// Undo an acquisition the exporter had already handed out but that could not + /// be served, without telling Python: `bf_releasebuffer` does not run when + /// `bf_getbuffer` fails. + pub(crate) fn abort_acquisition(self) { + debug_assert_eq!(self.export.shares.load(), 1); + self.owns_share.store(false); + self.export.released.store(true); + (self.methods.release)(&self); } - // drop PyBuffer without calling release - // after this function, the owner should use forget() - // or wrap PyBuffer in the ManuallyDrop to prevent drop() - pub(crate) unsafe fn drop_without_release(&mut self) { - // SAFETY: requirements forwarded from caller - unsafe { - core::ptr::drop_in_place(&mut self.obj); - core::ptr::drop_in_place(&mut self.desc); + /// A copy that owns no share: it reads the same memory, but releasing it is + /// inert and it never finalizes the acquisition. A `Py_buffer` whose `obj` is + /// NULL. + #[must_use] + pub fn detached(&self) -> Self { + Self { + obj: self.obj.clone(), + desc: self.desc.clone(), + methods: self.methods, + export: self.export.clone(), + owns_share: AtomicCell::new(false), } } } -impl<'a> TryFromBorrowedObject<'a> for PyBuffer { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { +impl PyBuffer { + /// Acquire a buffer from `obj`. PyObject_GetBuffer + pub fn from_object(vm: &VirtualMachine, obj: &PyObject, flags: BufferFlags) -> PyResult { + if flags.is_memory_access_mode() { + return Err(vm.new_system_error("bad argument to internal function")); + } let cls = obj.class(); - if let Some(f) = cls.slots.as_buffer { - return f(obj, vm); + if let Some(f) = cls.slots.as_buffer.load() { + return f(obj, flags, vm); } Err(vm.new_type_error(format!( "a bytes-like object is required, not '{}'", @@ -161,6 +360,26 @@ impl<'a> TryFromBorrowedObject<'a> for PyBuffer { } } +impl PyObject { + /// Whether this object's type exports the buffer protocol. PyObject_CheckBuffer + /// + /// A consumer that falls back to something else for non-buffer objects asks + /// this instead of attempting an acquisition, so that an error raised by + /// `__buffer__` is not mistaken for "not a buffer". + #[must_use] + pub fn check_buffer(&self) -> bool { + self.class().slots.as_buffer.load().is_some() + } +} + +/// The request a conversion makes when the consumer has no say in it: describe +/// the export as fully as possible, read-only. +impl<'a> TryFromBorrowedObject<'a> for PyBuffer { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &'a PyObject) -> PyResult { + Self::from_object(vm, obj, BufferFlags::FULL_RO) + } +} + impl Drop for PyBuffer { fn drop(&mut self) { self.release(); @@ -172,10 +391,20 @@ pub struct BufferDescriptor { /// product(shape) * itemsize /// bytes length, but not the length for obj_bytes() even is contiguous pub len: usize, + /// Byte position of the element at index `[0, .., 0]` within + /// [`PyBuffer::obj_bytes`], which always yields the exporter's whole memory. + /// `Py_buffer.buf` + /// + /// A view that walks backwards begins at the far end of its data, so this is + /// where addressing starts rather than a lower bound. A view with no elements + /// addresses nothing and may sit outside the exporter altogether, which is why + /// this is signed. + pub offset: isize, pub readonly: bool, pub itemsize: usize, pub format: Cow<'static, str>, - /// (shape, stride, suboffset) for each dimension + /// (shape, stride, suboffset) for each dimension. A non-zero suboffset means + /// the dimension is reached through a pointer; slicing never introduces one. pub dim_desc: Vec<(usize, isize, isize)>, // TODO: flags } @@ -185,6 +414,7 @@ impl BufferDescriptor { pub fn simple(bytes_len: usize, readonly: bool) -> Self { Self { len: bytes_len, + offset: 0, readonly, itemsize: 1, format: Cow::Borrowed("B"), @@ -201,6 +431,7 @@ impl BufferDescriptor { ) -> Self { Self { len: bytes_len, + offset: 0, readonly, itemsize, format, @@ -208,9 +439,48 @@ impl BufferDescriptor { } } + /// The descriptor an exporter hands to a consumer that asked for `flags`: + /// everything the request did not ask for is dropped. + /// + /// A `Py_buffer` drops a field by setting it to NULL and leaves the consumer to + /// reconstruct it. A descriptor has no NULL, so a dropped field is filled in + /// here with what that reconstruction would produce: `"B"` for a format, C-order + /// strides for strides, and a single dimension of `len / itemsize` items for a + /// shape. `itemsize` is never touched, so `calcsize(format)` and `itemsize` can + /// disagree on a projected descriptor — the format governs an element's width, + /// the item size governs the step — and `product(shape) * itemsize == len` + /// continues to hold. + #[must_use] + pub fn projected(&self, flags: BufferFlags) -> Self { + let mut desc = self.clone(); + if !flags.contains(BufferFlags::FORMAT) { + desc.format = Cow::Borrowed("B"); + } + if !flags.contains(BufferFlags::ND) { + // A request this flat is refused unless the layout is C-contiguous, so + // one dimension addresses the same bytes. + let shape = desc.len.checked_div(desc.itemsize).unwrap_or(0); + desc.dim_desc = vec![(shape, desc.itemsize as isize, 0)]; + } else if !flags.contains(BufferFlags::STRIDES) { + // Shape survives but strides do not, which means C order. + let mut stride = desc.itemsize as isize; + for (shape, dim_stride, suboffset) in desc.dim_desc.iter_mut().rev() { + *dim_stride = stride; + *suboffset = 0; + stride *= *shape as isize; + } + } + desc + } + #[cfg(debug_assertions)] #[must_use] pub fn validate(self) -> Self { + // Only a view with nothing to address is allowed to start outside the + // exporter. + if self.len != 0 { + debug_assert!(self.offset >= 0); + } // ndim=0 is valid for scalar types (e.g., ctypes Structure) if self.ndim() == 0 { // Empty structures (len=0) can have itemsize=0 @@ -239,6 +509,7 @@ impl BufferDescriptor { self.dim_desc.len() } + /// Whether the elements are laid out in row-major order. _IsCContiguous #[must_use] pub fn is_contiguous(&self) -> bool { if self.len == 0 { @@ -254,11 +525,76 @@ impl BufferDescriptor { true } + /// Whether the elements are laid out in column-major order. A view whose + /// dimensions are all but one of length 1 is laid out both ways at once. + /// _IsFortranContiguous + #[must_use] + pub fn is_fortran_contiguous(&self) -> bool { + if self.len == 0 { + return true; + } + let mut sd = self.itemsize; + for (shape, stride, _) in self.dim_desc.iter().copied() { + if shape > 1 && stride != sd as isize { + return false; + } + sd *= shape; + } + true + } + + /// The byte range this view occupies in [`PyBuffer::obj_bytes`], for a + /// contiguous view. + /// + /// A view with no bytes maps to the empty range at zero: its offset is + /// wherever slicing left it and need not be a position that exists. + #[must_use] + pub fn contiguous_range(&self) -> Range { + if self.len == 0 { + return 0..0; + } + debug_assert!(self.offset >= 0); + let start = self.offset as usize; + start..start + self.len + } + + /// The same shape, format and item size, laid out in C order from byte zero. + #[must_use] + pub fn contiguous(&self) -> Self { + let itemsize = self.itemsize; + let mut dim_desc = self.dim_desc.clone(); + if let Some((_, stride, suboffset)) = dim_desc.last_mut() { + *stride = itemsize as isize; + *suboffset = 0; + } + for i in (1..dim_desc.len()).rev() { + dim_desc[i - 1].1 = dim_desc[i].1 * dim_desc[i].0 as isize; + dim_desc[i - 1].2 = 0; + } + Self { + len: self.len, + offset: 0, + readonly: self.readonly, + itemsize: self.itemsize, + format: self.format.clone(), + dim_desc, + } + } + + /// Whether any dimension is reached through a pointer rather than by + /// stepping, the layout `PyBUF_INDIRECT` describes. + #[must_use] + pub fn has_suboffsets(&self) -> bool { + self.dim_desc + .iter() + .any(|(_, _, suboffset)| *suboffset != 0) + } + /// this function do not check the bound /// panic if indices.len() != ndim #[must_use] pub fn fast_position(&self, indices: &[usize]) -> isize { - let mut pos = 0; + let mut pos = self.offset; for (i, (_, stride, suboffset)) in indices .iter() .copied() @@ -271,7 +607,7 @@ impl BufferDescriptor { /// panic if indices.len() != ndim pub fn position(&self, indices: &[isize], vm: &VirtualMachine) -> PyResult { - let mut pos = 0; + let mut pos = self.offset; for (i, (shape, stride, suboffset)) in indices .iter() .copied() @@ -289,14 +625,60 @@ impl BufferDescriptor { where F: FnMut(Range), { + // A view with no bytes reaches nothing, and its offset need not be a + // position that exists, so it yields no segment at all. + if self.len == 0 { + return; + } if self.ndim() == 0 { - f(0..self.itemsize as isize); + f(self.offset..self.offset + self.itemsize as isize); return; } if try_contiguous && self.is_last_dim_contiguous() { - self._for_each_segment::<_, true>(0, 0, &mut f); + self._for_each_segment::<_, true>(self.offset, 0, &mut f); } else { - self._for_each_segment::<_, false>(0, 0, &mut f); + self._for_each_segment::<_, false>(self.offset, 0, &mut f); + } + } + + /// Visit each item's byte range with the *first* dimension varying + /// fastest, which is the order a Fortran-ordered copy is written in. + /// `for_each_segment` visits in the opposite order and can hand over whole + /// rows at once; here every item is its own range, since consecutive items + /// in this order are a row apart. + pub fn for_each_segment_fortran(&self, mut f: F) + where + F: FnMut(Range), + { + if self.len == 0 { + return; + } + if self.ndim() == 0 { + f(self.offset..self.offset + self.itemsize as isize); + return; + } + let mut indices = vec![0usize; self.ndim()]; + loop { + let pos = self.offset + + indices + .iter() + .zip_eq(self.dim_desc.iter()) + .map(|(&i, &(_, stride, suboffset))| i as isize * stride + suboffset) + .sum::(); + f(pos..pos + self.itemsize as isize); + + let mut dim = 0; + loop { + indices[dim] += 1; + if indices[dim] < self.dim_desc[dim].0 { + break; + } + indices[dim] = 0; + dim += 1; + if dim == self.ndim() { + return; + } + } } } @@ -328,14 +710,24 @@ impl BufferDescriptor { where F: FnMut(Range, Range) -> bool, { + if self.len == 0 { + return; + } if self.ndim() == 0 { - f(0..self.itemsize as isize, 0..other.itemsize as isize); + f( + self.offset..self.offset + self.itemsize as isize, + other.offset..other.offset + other.itemsize as isize, + ); return; } - if try_contiguous && self.is_last_dim_contiguous() { - self._zip_eq::<_, true>(other, 0, 0, 0, &mut f); + // last_dim_is_contiguous: the whole-run path walks both sides at once, so + // both have to be laid out that way. + let run_at_once = + try_contiguous && self.is_last_dim_contiguous() && other.is_last_dim_contiguous(); + if run_at_once { + self._zip_eq::<_, true>(other, self.offset, other.offset, 0, &mut f); } else { - self._zip_eq::<_, false>(other, 0, 0, 0, &mut f); + self._zip_eq::<_, false>(other, self.offset, other.offset, 0, &mut f); } } diff --git a/crates/vm/src/protocol/iter.rs b/crates/vm/src/protocol/iter.rs index 2f51287b181..1aa0bcd5b13 100644 --- a/crates/vm/src/protocol/iter.rs +++ b/crates/vm/src/protocol/iter.rs @@ -16,7 +16,11 @@ where unsafe impl> Traverse for PyIter { fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { - self.0.borrow().traverse(tracer_fn); + // Report the iterator itself, not its referents: an owner holding a + // `PyIter` owns the iterator object, and reporting what the iterator + // points at instead leaves the iterator's own reference unaccounted + // for, so a cycle running through it is never collected. + tracer_fn(self.0.borrow()); } } diff --git a/crates/vm/src/protocol/mod.rs b/crates/vm/src/protocol/mod.rs index 411aa4dfad3..4061e06458a 100644 --- a/crates/vm/src/protocol/mod.rs +++ b/crates/vm/src/protocol/mod.rs @@ -6,7 +6,9 @@ mod number; mod object; mod sequence; -pub use buffer::{BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, VecBuffer}; +pub use buffer::{ + BufferDescriptor, BufferFlags, BufferMethods, BufferResizeGuard, PyBuffer, VecBuffer, +}; pub use callable::PyCallable; pub(crate) use callable::TraceEvent; pub use iter::{PyIter, PyIterIter, PyIterReturn}; diff --git a/crates/vm/src/protocol/object.rs b/crates/vm/src/protocol/object.rs index 37007422404..993f3442aa3 100644 --- a/crates/vm/src/protocol/object.rs +++ b/crates/vm/src/protocol/object.rs @@ -7,7 +7,7 @@ use crate::{ PyType, PyTypeRef, PyUtf8Str, int::check_int_to_str_digits, pystr::AsPyStr, }, common::{hash::PyHash, str::to_ascii}, - convert::{ToPyObject, ToPyResult}, + convert::ToPyObject, dict_inner::DictKey, function::{Either, FuncArgs, PyArithmeticValue, PySetterValue}, object::PyPayload, @@ -230,7 +230,6 @@ impl PyObject { dict: Option, vm: &VirtualMachine, ) -> PyResult> { - let name = name_str.as_wtf8(); let obj_cls = self.class(); let cls_attr_name = vm.ctx.interned_str(name_str); let cls_attr = match cls_attr_name.and_then(|name| obj_cls.get_attr(name)) { @@ -251,7 +250,9 @@ impl PyObject { let dict = dict.or_else(|| self.dict()); let attr = if let Some(dict) = dict { - dict.get_item_opt(name, vm)? + // `Py` rather than its `&Wtf8`: the key type carries the + // cached hash and compares interned keys by pointer. + dict.get_item_opt(name_str, vm)? } else { None }; @@ -694,7 +695,7 @@ impl PyObject { pub fn hash(&self, vm: &VirtualMachine) -> PyResult { if let Some(hash) = self.class().slots.hash.load() { - return hash(self, vm); + return vm.with_recursion("while hashing", || hash(self, vm)); } Err(vm.new_type_error(format!("unhashable type: '{}'", self.class().name()))) @@ -741,8 +742,8 @@ impl PyObject { } else { if self.class().fast_issubclass(vm.ctx.types.type_type) { if self.is(vm.ctx.types.type_type) { - return PyGenericAlias::from_args(self.class().to_owned(), needle, vm) - .to_pyresult(vm); + let alias = PyGenericAlias::from_args(self.class().to_owned(), needle, vm)?; + return Ok(alias.to_pyobject(vm)); } if let Some(class_getitem) = diff --git a/crates/vm/src/py_serde.rs b/crates/vm/src/py_serde.rs index 50ea4422b16..0e8b70781cd 100644 --- a/crates/vm/src/py_serde.rs +++ b/crates/vm/src/py_serde.rs @@ -63,7 +63,10 @@ impl serde::Serialize for PyObjectSerializer<'_> { seq.end() }; if let Some(s) = self.pyobject.downcast_ref::() { - serializer.serialize_str(s.as_ref()) + serializer.serialize_str( + s.to_str() + .ok_or_else(|| serde::ser::Error::custom("str contains surrogates"))?, + ) } else if self.pyobject.fast_isinstance(self.vm.ctx.types.float_type) { serializer.serialize_f64(float::get_value(self.pyobject)) } else if self.pyobject.fast_isinstance(self.vm.ctx.types.bool_type) { diff --git a/crates/vm/src/sequence.rs b/crates/vm/src/sequence.rs index 4e6ed97f21c..1e126d087ea 100644 --- a/crates/vm/src/sequence.rs +++ b/crates/vm/src/sequence.rs @@ -104,7 +104,12 @@ where return Err(vm.new_memory_error("")); } - let mut v = Vec::with_capacity(n * self.as_ref().len()); + let total = n + .checked_mul(self.as_ref().len()) + .ok_or_else(|| vm.new_memory_error(""))?; + let mut v = Vec::new(); + v.try_reserve_exact(total) + .map_err(|_| vm.new_memory_error(""))?; for _ in 0..n { v.extend_from_slice(self.as_ref()); } diff --git a/crates/vm/src/sliceable.rs b/crates/vm/src/sliceable.rs index b0f4c7808ff..ef78614efd5 100644 --- a/crates/vm/src/sliceable.rs +++ b/crates/vm/src/sliceable.rs @@ -419,6 +419,50 @@ impl SaturatedSlice { (range, self.step, slice_len) } + // PySlice_AdjustIndices, keeping the adjusted start rather than a range. + /// The index the slice begins at, clamped into `0..=len` for a positive step + /// and into `-1..=len-1` for a negative one, together with its length. + /// + /// Unlike [`Self::adjust_indices`] this stays meaningful for an empty slice, + /// where it is still the position a strided view moves to. + #[must_use] + pub fn adjust_indices_start(&self, len: usize) -> (isize, usize) { + let len = len as isize; + let clamp = |i: isize| { + if i < 0 { + let i = i.saturating_add(len); + if i < 0 { + if self.step.is_negative() { -1 } else { 0 } + } else { + i + } + } else if i >= len { + if self.step.is_negative() { + len - 1 + } else { + len + } + } else { + i + } + }; + let start = clamp(self.start); + let stop = clamp(self.stop); + let step = self.step.unsigned_abs(); + let slice_len = if self.step.is_negative() { + if stop < start { + (start - stop - 1) as usize / step + 1 + } else { + 0 + } + } else if start < stop { + (stop - start - 1) as usize / step + 1 + } else { + 0 + }; + (start, slice_len) + } + #[must_use] pub fn iter(&self, len: usize) -> SaturatedSliceIter { SaturatedSliceIter::new(self, len) diff --git a/crates/vm/src/stdlib/_ast/pyast.rs b/crates/vm/src/stdlib/_ast/pyast.rs index eb97eec8024..ebce1a788d2 100644 --- a/crates/vm/src/stdlib/_ast/pyast.rs +++ b/crates/vm/src/stdlib/_ast/pyast.rs @@ -1718,12 +1718,16 @@ fn populate_field_types(vm: &VirtualMachine, module: &Py) { FieldType::ListOf(name) => { let elem = resolve_node(name); let args = PyTuple::new_ref(vec![elem], &vm.ctx); - PyGenericAlias::new(list_type.clone(), args, false, vm).to_pyobject(vm) + PyGenericAlias::new(list_type.clone(), args, false, vm) + .expect("static field types are not nested, so no recursion is possible") + .to_pyobject(vm) } FieldType::ListOfBuiltin(name) => { let elem = resolve_builtin(name); let args = PyTuple::new_ref(vec![elem], &vm.ctx); - PyGenericAlias::new(list_type.clone(), args, false, vm).to_pyobject(vm) + PyGenericAlias::new(list_type.clone(), args, false, vm) + .expect("static field types are not nested, so no recursion is possible") + .to_pyobject(vm) } FieldType::Optional(name) => { let base = resolve_node(name); diff --git a/crates/vm/src/stdlib/_ast/python.rs b/crates/vm/src/stdlib/_ast/python.rs index db92f20db17..b6f7948293d 100644 --- a/crates/vm/src/stdlib/_ast/python.rs +++ b/crates/vm/src/stdlib/_ast/python.rs @@ -10,13 +10,11 @@ pub(crate) mod _ast { AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyDictRef, PySet, PyStr, PyTupleRef, PyType, PyTypeRef}, class::{PyClassImpl, StaticType}, - common::wtf8::Wtf8Buf, function::{ArgIterable, FuncArgs, KwArgs, PyMethodDef, PyMethodFlags}, stdlib::_ast::repr, types::{Constructor, Initializer}, warn, }; - use indexmap::IndexMap; #[pyattr] #[pyclass(module = "_ast", name = "AST")] #[derive(Debug, PyPayload)] @@ -295,7 +293,7 @@ pub(crate) mod _ast { .map_err(|_| vm.new_type_error("keywords must be strings"))?; Ok((key.as_wtf8().to_owned(), value)) }) - .collect::>>()?; + .collect::>>()?; let result = type_obj.call(FuncArgs::new(vec![], KwArgs::new(kwargs)), vm)?; Ok(result) } diff --git a/crates/vm/src/stdlib/_codecs.rs b/crates/vm/src/stdlib/_codecs.rs index 69d9e0e4fde..8f6ea5f1900 100644 --- a/crates/vm/src/stdlib/_codecs.rs +++ b/crates/vm/src/stdlib/_codecs.rs @@ -382,6 +382,23 @@ mod _codecs_windows { use crate::{builtins::PyStrRef, builtins::PyUtf8StrRef, function::ArgBytesLike}; use rustpython_host_env::windows as host_windows; + fn string_from_utf16( + encoding: &str, + data: &[u8], + wide: &[u16], + vm: &VirtualMachine, + ) -> PyResult { + String::from_utf16(wide).map_err(|err| { + vm.new_unicode_decode_error( + vm.ctx.new_str(encoding), + vm.ctx.new_bytes(data.to_vec()), + 0, + data.len(), + vm.ctx.new_str(format!("{encoding}_decode failed: {err}")), + ) + }) + } + #[derive(FromArgs)] struct MbcsEncodeArgs { #[pyarg(positional)] @@ -399,9 +416,7 @@ mod _codecs_windows { Some(s) => s, None => { // String contains surrogates - not encodable with mbcs - return Err(vm.new_unicode_encode_error( - "'mbcs' codec can't encode character: surrogates not allowed", - )); + return encode_code_page_errors(host_windows::CP_ACP, &args.s, errors, "mbcs", vm); } }; let char_len = args.s.char_len(); @@ -433,9 +448,7 @@ mod _codecs_windows { .map_err(|err| vm.new_os_error(format!("mbcs_encode failed: {err}")))?; if errors == "strict" && used_default_char { - return Err(vm.new_unicode_encode_error( - "'mbcs' codec can't encode characters: invalid character", - )); + return encode_code_page_errors(host_windows::CP_ACP, &args.s, errors, "mbcs", vm); } buffer.truncate(result); @@ -484,8 +497,7 @@ mod _codecs_windows { ) .map_err(|err| vm.new_os_error(format!("mbcs_decode failed: {err}")))?; buffer.truncate(result); - let s = String::from_utf16(&buffer) - .map_err(|e| vm.new_unicode_decode_error(format!("mbcs_decode failed: {e}")))?; + let s = string_from_utf16("mbcs", data.as_ref(), &buffer, vm)?; return Ok((s, len)); } @@ -500,8 +512,7 @@ mod _codecs_windows { ) .map_err(|err| vm.new_os_error(format!("mbcs_decode failed: {err}")))?; buffer.truncate(result); - let s = String::from_utf16(&buffer) - .map_err(|e| vm.new_unicode_decode_error(format!("mbcs_decode failed: {e}")))?; + let s = string_from_utf16("mbcs", data.as_ref(), &buffer, vm)?; Ok((s, len)) } @@ -523,9 +534,7 @@ mod _codecs_windows { Some(s) => s, None => { // String contains surrogates - not encodable with oem - return Err(vm.new_unicode_encode_error( - "'oem' codec can't encode character: surrogates not allowed", - )); + return encode_code_page_errors(host_windows::CP_OEMCP, &args.s, errors, "oem", vm); } }; let char_len = args.s.char_len(); @@ -557,9 +566,7 @@ mod _codecs_windows { .map_err(|err| vm.new_os_error(format!("oem_encode failed: {err}")))?; if errors == "strict" && used_default_char { - return Err(vm.new_unicode_encode_error( - "'oem' codec can't encode characters: invalid character", - )); + return encode_code_page_errors(host_windows::CP_OEMCP, &args.s, errors, "oem", vm); } buffer.truncate(result); @@ -609,8 +616,7 @@ mod _codecs_windows { ) .map_err(|err| vm.new_os_error(format!("oem_decode failed: {err}")))?; buffer.truncate(result); - let s = String::from_utf16(&buffer) - .map_err(|e| vm.new_unicode_decode_error(format!("oem_decode failed: {e}")))?; + let s = string_from_utf16("oem", data.as_ref(), &buffer, vm)?; return Ok((s, len)); } @@ -625,8 +631,7 @@ mod _codecs_windows { ) .map_err(|err| vm.new_os_error(format!("oem_decode failed: {err}")))?; buffer.truncate(result); - let s = String::from_utf16(&buffer) - .map_err(|e| vm.new_unicode_decode_error(format!("oem_decode failed: {e}")))?; + let s = string_from_utf16("oem", data.as_ref(), &buffer, vm)?; Ok((s, len)) } @@ -791,19 +796,18 @@ mod _codecs_windows { // Convert code point to UTF-16 let mut wchars = [0u16; 2]; - let wchar_len; let is_surrogate = (0xD800..=0xDFFF).contains(&ch); - if is_surrogate { - wchar_len = 0; // Can't encode surrogates normally + let wchar_len = if is_surrogate { + 0 // Can't encode surrogates normally } else if ch < 0x10000 { wchars[0] = ch as u16; - wchar_len = 1; + 1 } else { wchars[0] = ((ch - 0x10000) >> 10) as u16 + 0xD800; wchars[1] = ((ch - 0x10000) & 0x3FF) as u16 + 0xDC00; - wchar_len = 2; - } + 2 + }; if !is_surrogate { let mut buf = [0u8; 8]; @@ -1025,7 +1029,7 @@ mod _codecs_windows { } } let object = vm.ctx.new_bytes(data.to_vec()); - return Err(vm.new_unicode_decode_error_real( + return Err(vm.new_unicode_decode_error( encoding_str, object, fail_pos, @@ -1116,7 +1120,7 @@ mod _codecs_windows { } "strict" => { let object = vm.ctx.new_bytes(data.to_vec()); - return Err(vm.new_unicode_decode_error_real( + return Err(vm.new_unicode_decode_error( encoding_str, object, pos, @@ -1127,7 +1131,7 @@ mod _codecs_windows { _ => { // Custom error handler let object = vm.ctx.new_bytes(data.to_vec()); - let exc = vm.new_unicode_decode_error_real( + let exc = vm.new_unicode_decode_error( encoding_str.clone(), object, pos, diff --git a/crates/vm/src/stdlib/_collections.rs b/crates/vm/src/stdlib/_collections.rs index c7cce5c735a..b48c0e670ac 100644 --- a/crates/vm/src/stdlib/_collections.rs +++ b/crates/vm/src/stdlib/_collections.rs @@ -13,6 +13,7 @@ mod _collections { convert::ToPyObject, function::{FuncArgs, KwArgs, OptionalArg, PyComparisonValue}, iter::PyExactSizeIterator, + object::{Traverse, TraverseFn}, protocol::{PyIterReturn, PyMappingMethods, PyNumberMethods, PySequenceMethods}, recursion::ReprGuard, sequence::{MutObjectSequenceOp, OptionalRangeArgs}, @@ -22,13 +23,19 @@ mod _collections { Initializer, IterNext, Iterable, PyComparisonOp, Representable, SelfIter, }, utils::collection_repr, + vm::MAX_MEMORY_SIZE, }; use alloc::collections::VecDeque; - use core::cmp::max; + use core::{cmp::max, mem::size_of}; use crossbeam_utils::atomic::AtomicCell; #[pyattr] - #[pyclass(module = "collections", name = "deque", unhashable = true)] + #[pyclass( + module = "collections", + name = "deque", + unhashable = true, + traverse = "manual" + )] #[derive(Debug, Default, PyPayload)] struct PyDeque { deque: PyRwLock>, @@ -36,6 +43,21 @@ mod _collections { state: AtomicCell, // incremented whenever the indices move } + // SAFETY: Traverse visits each owned Python reference at most once. + unsafe impl Traverse for PyDeque { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + if let Some(deque) = self.deque.try_read_recursive() { + for obj in deque.iter() { + obj.traverse(tracer_fn); + } + } + } + + fn clear(&mut self, out: &mut Vec) { + out.extend(self.deque.get_mut().drain(..)); + } + } + type PyDequeRef = PyRef; #[derive(FromArgs)] @@ -318,6 +340,10 @@ mod _collections { let deque = self.borrow_deque(); let n = vm.check_repeat_or_overflow_error(deque.len(), n)?; let mul_len = n * deque.len(); + let result_len = self.maxlen.map_or(mul_len, |maxlen| mul_len.min(maxlen)); + if n > 1 && result_len.saturating_mul(size_of::()) >= MAX_MEMORY_SIZE { + return Err(vm.new_memory_error("")); + } let iter = deque.iter().cycle().take(mul_len); let skipped = self .maxlen @@ -400,7 +426,7 @@ mod _collections { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -576,9 +602,10 @@ mod _collections { let closing_part = zelf .maxlen .map_or_else(|| "]".to_owned(), |maxlen| format!("], maxlen={maxlen}")); + let empty = format!("{class_name}([{closing_part})"); if zelf.__len__() == 0 { - return Ok(vm.ctx.new_str(format!("{class_name}([{closing_part})"))); + return Ok(vm.ctx.new_str(empty)); } if let Some(_guard) = ReprGuard::enter(vm, zelf.as_object()) { @@ -586,6 +613,7 @@ mod _collections { Some(&class_name), "[", &closing_part, + &empty, deque.iter(), vm, )?)) @@ -753,7 +781,8 @@ mod _collections { module = "collections", name = "defaultdict", base = PyDict, - unhashable = true + unhashable = true, + traverse = "manual" )] #[derive(Debug, Default)] struct PyDefaultDict { @@ -761,6 +790,21 @@ mod _collections { default_factory: PyRwLock>, } + // SAFETY: Traverse visits each owned Python reference at most once. + unsafe impl Traverse for PyDefaultDict { + fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) { + self.dict.traverse(tracer_fn); + self.default_factory.traverse(tracer_fn); + } + + fn clear(&mut self, out: &mut Vec) { + Traverse::clear(&mut self.dict, out); + if let Some(factory) = self.default_factory.get_mut().take() { + out.push(factory); + } + } + } + #[pyclass( with(AsMapping, AsNumber, Constructor, Initializer, Representable), flags(BASETYPE, MAPPING, HAS_DICT) diff --git a/crates/vm/src/stdlib/_ctypes.rs b/crates/vm/src/stdlib/_ctypes.rs index adf047ec750..e4857d0ee06 100644 --- a/crates/vm/src/stdlib/_ctypes.rs +++ b/crates/vm/src/stdlib/_ctypes.rs @@ -141,7 +141,6 @@ pub(crate) mod _ctypes { ffi_value_from_type_code(code.encode_utf8(&mut buf), bytes) } super::CArgValue::Int(v) => FfiValue::I32(*v), - super::CArgValue::Double(v) => FfiValue::F64(*v), super::CArgValue::Pointer(v) => FfiValue::Pointer(*v), // 'V' aggregates format via the object-address default arm below. super::CArgValue::Aggregate { .. } => FfiValue::Pointer(0), diff --git a/crates/vm/src/stdlib/_ctypes/array.rs b/crates/vm/src/stdlib/_ctypes/array.rs index a99fabc812d..d4674f33b07 100644 --- a/crates/vm/src/stdlib/_ctypes/array.rs +++ b/crates/vm/src/stdlib/_ctypes/array.rs @@ -511,7 +511,11 @@ impl AsMapping for PyCArray { )] impl PyCArray { #[pyclassmethod] - fn __class_getitem__(cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine) -> PyGenericAlias { + fn __class_getitem__( + cls: PyTypeRef, + args: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } @@ -990,13 +994,19 @@ impl PyCArray { let (range, step, slice_len) = sat_slice.adjust_indices(length); // other_len = PySequence_Length(value); - let items: Vec = vm.extract_elements_with(&value, Ok)?; - let other_len = items.len(); + // Size the operand before consuming it so an unbounded iterable is + // rejected without being materialized. + let other_len = value + .sequence_unchecked() + .length(vm) + .map_err(|_| vm.new_value_error("Can only assign sequence of same size"))?; if other_len != slice_len { return Err(vm.new_value_error("Can only assign sequence of same size")); } + let items: Vec = vm.extract_elements_with(&value, Ok)?; + // Use SaturatedSliceIter for correct index iteration (handles negative step) let iter = SaturatedSliceIter::from_adjust_indices(range, step, slice_len); @@ -1051,6 +1061,7 @@ impl AsBuffer for PyCArray { dim_desc.reverse(); BufferDescriptor { + offset: 0, len: buffer_len, readonly: false, itemsize, diff --git a/crates/vm/src/stdlib/_ctypes/base.rs b/crates/vm/src/stdlib/_ctypes/base.rs index 1cc84750cb5..6067fc61bf0 100644 --- a/crates/vm/src/stdlib/_ctypes/base.rs +++ b/crates/vm/src/stdlib/_ctypes/base.rs @@ -624,7 +624,9 @@ impl PyCData { // Get buffer pointer - the memory is owned by source let ptr = { - let bytes = buffer.obj_bytes(); + // Contiguity is checked above, so this is the view's own bytes rather + // than the whole exporter's. + let bytes = unsafe { buffer.contiguous_unchecked() }; bytes.as_ptr().wrapping_add(offset) }; @@ -1939,7 +1941,7 @@ fn struct_union_paramfunc(obj: &PyObject, stg_info: &StgInfo, _vm: &VirtualMachi /// A foreign-call argument in a form the unified `call` entry point accepts: a /// simple-typed scalar (its ctypes code plus a native-endian bytes snapshot), -/// an untyped int/float, or an address. Any object whose memory an address +/// an untyped int, or an address. Any object whose memory an address /// refers to is kept alive by the enclosing `Argument`/`CArgObject`, not here. #[derive(Debug, Clone)] pub enum CArgValue { @@ -1947,8 +1949,6 @@ pub enum CArgValue { Typed { code: char, bytes: Vec }, /// Untyped Python int (ConvParam default: C int). Int(i32), - /// Untyped Python float (ConvParam default: C double). - Double(f64), /// Address-valued argument (pointer decay, byref, buffer copies, NULL = 0). Pointer(usize), /// By-value aggregate: its call layout plus a snapshot of its bytes. @@ -1985,7 +1985,6 @@ impl CArgValue { buffer: bytes, }, Self::Int(value) => CallArg::Int(*value), - Self::Double(value) => CallArg::Double(*value), Self::Pointer(value) => CallArg::Pointer(*value), Self::Aggregate { layout, bytes } => CallArg::Aggregate { layout, diff --git a/crates/vm/src/stdlib/_ctypes/function.rs b/crates/vm/src/stdlib/_ctypes/function.rs index 676ee5be8eb..afbe0ae76ea 100644 --- a/crates/vm/src/stdlib/_ctypes/function.rs +++ b/crates/vm/src/stdlib/_ctypes/function.rs @@ -9,7 +9,7 @@ use super::{ }; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - builtins::{PyBytes, PyDict, PyStr, PyTuple, PyType, PyTypeRef}, + builtins::{PyBytes, PyDict, PyInt, PyStr, PyTuple, PyType, PyTypeRef}, class::StaticType, function::FuncArgs, protocol::{BufferDescriptor, PyBuffer, PyNumberMethods}, @@ -171,7 +171,10 @@ fn conv_param(value: &PyObject, vm: &VirtualMachine) -> PyResult { } // 10. Python int -> i32 (default integer type) - if let Ok(int_val) = value.try_int(vm) { + // PyLong_Check: only an int (or a subclass) converts. Going through + // `__int__` would accept a float and pass its truncated value where the + // callee expects a pointer. + if let Some(int_val) = value.downcast_ref::() { let val = int_val.as_bigint().to_i32().unwrap_or(0); return Ok(Argument { keep: None, @@ -179,15 +182,7 @@ fn conv_param(value: &PyObject, vm: &VirtualMachine) -> PyResult { }); } - // 11. Python float -> f64 - if let Ok(float_val) = value.try_float(vm) { - return Ok(Argument { - keep: None, - value: CArgValue::Double(float_val.to_f64()), - }); - } - - // 12. Check _as_parameter_ attribute + // 11. Check _as_parameter_ attribute if let Ok(as_param) = value.get_attr("_as_parameter_", vm) { return conv_param(&as_param, vm); } @@ -939,6 +934,23 @@ struct CallInfo { ret: RetSpec, } +fn extract_arg_types(argtypes: &PyObject, vm: &VirtualMachine) -> PyResult> { + let error = || vm.new_type_error("_argtypes_ must be a sequence of types"); + let sequence = argtypes.try_sequence(vm).map_err(|_| error())?; + let length = sequence.length(vm).map_err(|_| error())?; + let mut types = Vec::new(); + types + .try_reserve(length) + .map_err(|_| vm.new_memory_error(""))?; + + for index in 0..length { + let item = sequence.get_item(index as isize, vm).map_err(|_| error())?; + types.push(item.downcast::().map_err(|_| error())?); + } + + Ok(types) +} + /// Determine how to retrieve the return value from restype, reproducing the /// prior `ffi_return_type` + `is_pointer_return` dispatch. fn compute_ret_spec( @@ -1007,13 +1019,7 @@ fn extract_call_info(zelf: &Py, vm: &VirtualMachine) -> PyResult> = if let Some(argtypes_obj) = zelf.argtypes.read().as_ref() { if !vm.is_none(argtypes_obj) { - Some( - argtypes_obj - .try_to_value::>(vm)? - .into_iter() - .filter_map(|obj| obj.downcast::().ok()) - .collect(), - ) + Some(extract_arg_types(argtypes_obj, vm)?) } else { None // argtypes is None -> use ConvParam } @@ -1023,13 +1029,7 @@ fn extract_call_info(zelf: &Py, vm: &VirtualMachine) -> PyResult>(vm)? - .into_iter() - .filter_map(|obj| obj.downcast::().ok()) - .collect(), - ) + Some(extract_arg_types(&class_argtypes, vm)?) } else { None // No argtypes -> use ConvParam }; @@ -1655,6 +1655,7 @@ impl AsBuffer for PyCFuncPtr { (Cow::Borrowed(pointer_format()), pointer_size()) }; let desc = BufferDescriptor { + offset: 0, len: itemsize, readonly: false, itemsize, @@ -1944,14 +1945,7 @@ impl PyCThunk { vm: &VirtualMachine, ) -> PyResult { let arg_type_vec: Vec = match arg_types { - Some(args) if !vm.is_none(&args) => args - .try_to_value::>(vm)? - .into_iter() - .map(|item| { - item.downcast::() - .map_err(|_| vm.new_type_error("_argtypes_ must be a sequence of types")) - }) - .collect::>>()?, + Some(args) if !vm.is_none(&args) => extract_arg_types(&args, vm)?, _ => Vec::new(), }; diff --git a/crates/vm/src/stdlib/_ctypes/pointer.rs b/crates/vm/src/stdlib/_ctypes/pointer.rs index f522e6dfb7e..bcc39fd5745 100644 --- a/crates/vm/src/stdlib/_ctypes/pointer.rs +++ b/crates/vm/src/stdlib/_ctypes/pointer.rs @@ -668,7 +668,7 @@ impl PyCPointer { let ptr_val = if vm.is_none(value) { 0usize } else if let Ok(int_val) = value.try_index(vm) { - int_val.as_bigint().to_usize().unwrap_or(0) + super::simple::bigint_to_i128_wrapping(int_val.as_bigint()) as usize } else { return Err(vm.new_type_error("bytes/string or integer address expected")); }; @@ -684,12 +684,13 @@ impl PyCPointer { // Use write_unaligned for safety on strict-alignment architectures if let Ok(int_val) = value.try_int(vm) { let i = int_val.as_bigint(); + let wrapped = super::simple::bigint_to_i128_wrapping(i); let bytes; let write_value = match size { - 1 => AddressWriteValue::U8(i.to_u8().expect("int too large")), - 2 => AddressWriteValue::I16(i.to_i16().expect("int too large")), - 4 => AddressWriteValue::I32(i.to_i32().expect("int too large")), - 8 => AddressWriteValue::I64(i.to_i64().expect("int too large")), + 1 => AddressWriteValue::U8(wrapped as u8), + 2 => AddressWriteValue::I16(wrapped as i16), + 4 => AddressWriteValue::I32(wrapped as i32), + 8 => AddressWriteValue::I64(wrapped as i64), _ => { bytes = i.to_signed_bytes_le(); AddressWriteValue::Bytes(&bytes) @@ -711,7 +712,8 @@ impl PyCPointer { } // Try bytes - if let Ok(bytes) = value.try_bytes_like(vm, |b| b.to_vec()) { + if value.check_buffer() { + let bytes = value.try_bytes_like(vm, |b| b.to_vec())?; rustpython_host_env::ctypes::write_value_to_address( addr, size, @@ -775,6 +777,7 @@ impl AsBuffer for PyCPointer { let itemsize = stg_info.size; // Pointer types are scalars with ndim=0, shape=() let desc = BufferDescriptor { + offset: 0, len: itemsize, readonly: false, itemsize, diff --git a/crates/vm/src/stdlib/_ctypes/simple.rs b/crates/vm/src/stdlib/_ctypes/simple.rs index c947e56010a..9699cef984b 100644 --- a/crates/vm/src/stdlib/_ctypes/simple.rs +++ b/crates/vm/src/stdlib/_ctypes/simple.rs @@ -72,6 +72,17 @@ fn new_simple_type( Ok(PyCSimple(PyCData::from_bytes(zeroed_bytes(size), None))) } +pub(super) fn bigint_to_i128_wrapping(value: &malachite_bigint::BigInt) -> i128 { + let bytes = value.to_signed_bytes_le(); + let fill = bytes + .last() + .map_or(0, |byte| if *byte & 0x80 == 0 { 0 } else { u8::MAX }); + let mut wrapped = [fill; 16]; + let len = bytes.len().min(wrapped.len()); + wrapped[..len].copy_from_slice(&bytes[..len]); + i128::from_le_bytes(wrapped) +} + fn set_primitive(_type_: &str, value: &PyObject, vm: &VirtualMachine) -> PyResult { match _type_ { "c" => { @@ -756,7 +767,7 @@ fn value_to_bytes_endian( "b" => { // c_byte - signed char (1 byte) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -764,7 +775,7 @@ fn value_to_bytes_endian( "B" => { // c_ubyte - unsigned char (1 byte) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -772,7 +783,7 @@ fn value_to_bytes_endian( "h" => { // c_short (2 bytes) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -780,7 +791,7 @@ fn value_to_bytes_endian( "H" => { // c_ushort (2 bytes) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -788,7 +799,7 @@ fn value_to_bytes_endian( "i" => { // c_int (4 bytes) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -796,7 +807,7 @@ fn value_to_bytes_endian( "I" => { // c_uint (4 bytes) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -804,7 +815,7 @@ fn value_to_bytes_endian( "l" => { // c_long (platform dependent) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -812,7 +823,7 @@ fn value_to_bytes_endian( "L" => { // c_ulong (platform dependent) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -820,7 +831,7 @@ fn value_to_bytes_endian( "q" => { // c_longlong (8 bytes) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -828,7 +839,7 @@ fn value_to_bytes_endian( "Q" => { // c_ulonglong (8 bytes) if let Ok(int_val) = value.try_index(vm) { - SimpleStorageValue::Signed(int_val.as_bigint().to_i128().expect("int too large")) + SimpleStorageValue::Signed(bigint_to_i128_wrapping(int_val.as_bigint())) } else { SimpleStorageValue::Zero } @@ -889,10 +900,7 @@ fn value_to_bytes_endian( "P" => { // c_void_p - pointer type (platform pointer size) if let Ok(int_val) = value.try_index(vm) { - let v = int_val - .as_bigint() - .to_usize() - .expect("int too large for pointer"); + let v = bigint_to_i128_wrapping(int_val.as_bigint()) as usize; SimpleStorageValue::Pointer(v) } else { SimpleStorageValue::Zero @@ -902,10 +910,7 @@ fn value_to_bytes_endian( // c_char_p - pointer to char (stores pointer value from int) // PyBytes case is handled in slot_new/set_value with make_z_buffer() if let Ok(int_val) = value.try_index(vm) { - let v = int_val - .as_bigint() - .to_usize() - .expect("int too large for pointer"); + let v = bigint_to_i128_wrapping(int_val.as_bigint()) as usize; SimpleStorageValue::Pointer(v) } else { SimpleStorageValue::Zero @@ -915,10 +920,7 @@ fn value_to_bytes_endian( // c_wchar_p - pointer to wchar_t (stores pointer value from int) // PyStr case is handled in slot_new/set_value with make_wchar_buffer() if let Ok(int_val) = value.try_index(vm) { - let v = int_val - .as_bigint() - .to_usize() - .expect("int too large for pointer"); + let v = bigint_to_i128_wrapping(int_val.as_bigint()) as usize; SimpleStorageValue::Pointer(v) } else { SimpleStorageValue::Zero @@ -1281,6 +1283,7 @@ impl AsBuffer for PyCSimple { let itemsize = stg_info.size; // Simple types are scalars with ndim=0, shape=() let desc = BufferDescriptor { + offset: 0, len: itemsize, readonly: false, itemsize, diff --git a/crates/vm/src/stdlib/_ctypes/structure.rs b/crates/vm/src/stdlib/_ctypes/structure.rs index 34f53f52d60..12ddf8b5dee 100644 --- a/crates/vm/src/stdlib/_ctypes/structure.rs +++ b/crates/vm/src/stdlib/_ctypes/structure.rs @@ -1,6 +1,5 @@ use super::base::{CDATA_BUFFER_METHODS, PyCData, PyCField, StgInfo, StgInfoFlags}; use crate::builtins::{PyList, PyStr, PyTuple, PyType, PyTypeRef, PyUtf8Str}; -use crate::common::wtf8::Wtf8Buf; use crate::convert::ToPyObject; use crate::function::{FuncArgs, OptionalArg, PySetterValue}; use crate::protocol::{BufferDescriptor, PyBuffer, PyNumberMethods}; @@ -713,7 +712,7 @@ impl PyCStructure { self_obj: &Py, type_obj: &Py, args: &[PyObjectRef], - kwargs: &indexmap::IndexMap, + kwargs: &crate::function::KwArgsMap, index: usize, vm: &VirtualMachine, ) -> PyResult { @@ -823,6 +822,7 @@ impl AsBuffer for PyCStructure { let buf = PyBuffer::new( zelf.to_owned().into(), BufferDescriptor { + offset: 0, len: buffer_len, readonly: false, itemsize: buffer_len, diff --git a/crates/vm/src/stdlib/_ctypes/union.rs b/crates/vm/src/stdlib/_ctypes/union.rs index 727ad0118ad..326e1fbd704 100644 --- a/crates/vm/src/stdlib/_ctypes/union.rs +++ b/crates/vm/src/stdlib/_ctypes/union.rs @@ -1,7 +1,6 @@ use super::base::{CDATA_BUFFER_METHODS, StgInfoFlags}; use super::{PyCData, PyCField, StgInfo}; use crate::builtins::{PyList, PyStr, PyTuple, PyType, PyTypeRef, PyUtf8Str}; -use crate::common::wtf8::Wtf8Buf; use crate::convert::ToPyObject; use crate::function::{ArgBytesLike, FuncArgs, OptionalArg, PySetterValue}; use crate::protocol::{BufferDescriptor, PyBuffer}; @@ -582,7 +581,7 @@ impl PyCUnion { self_obj: &Py, type_obj: &Py, args: &[PyObjectRef], - kwargs: &indexmap::IndexMap, + kwargs: &crate::function::KwArgsMap, index: usize, vm: &VirtualMachine, ) -> PyResult { @@ -686,6 +685,7 @@ impl AsBuffer for PyCUnion { let buf = PyBuffer::new( zelf.to_owned().into(), BufferDescriptor { + offset: 0, len: buffer_len, readonly: false, itemsize: buffer_len, diff --git a/crates/vm/src/stdlib/_functools.rs b/crates/vm/src/stdlib/_functools.rs index 94b9565e79d..944a2e8abdb 100644 --- a/crates/vm/src/stdlib/_functools.rs +++ b/crates/vm/src/stdlib/_functools.rs @@ -15,7 +15,6 @@ mod _functools { recursion::ReprGuard, types::{Callable, Constructor, GetDescriptor, Representable}, }; - use indexmap::IndexMap; use rustpython_common::wtf8::Wtf8Buf; #[derive(FromArgs)] @@ -302,7 +301,7 @@ mod _functools { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -432,7 +431,7 @@ mod _functools { combined_args.extend(new_args_iter.cloned()); // Merge keywords from self.keywords and args.kwargs - let mut final_kwargs = IndexMap::new(); + let mut final_kwargs = crate::function::KwArgsMap::default(); // Add keywords from self.keywords for (key, value) in &*keywords { diff --git a/crates/vm/src/stdlib/_imp.rs b/crates/vm/src/stdlib/_imp.rs index 322eaedd7d0..838012a1d0a 100644 --- a/crates/vm/src/stdlib/_imp.rs +++ b/crates/vm/src/stdlib/_imp.rs @@ -178,7 +178,6 @@ mod _imp { use crate::{ PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{PyBytesRef, PyCode, PyMemoryView, PyModule, PyStrRef, PyUtf8StrRef}, - convert::TryFromBorrowedObject, function::OptionalArg, import, version, }; @@ -264,21 +263,26 @@ mod _imp { if let OptionalArg::Present(data) = data && !vm.is_none(&data) { - let buf = crate::protocol::PyBuffer::try_from_borrowed_object(vm, &data)?; - let contiguous = buf.as_contiguous().ok_or_else(|| { - vm.new_buffer_error("get_frozen_object() requires a contiguous buffer") - })?; let invalid_err = || { vm.new_import_error( format!("Frozen object named '{}' is invalid", name.as_str()), name.clone().into_wtf8(), ) }; - let bag = crate::builtins::code::PyVmBag(vm); - let code = - rustpython_compiler_core::marshal::deserialize_code(&mut &contiguous[..], bag) - .map_err(|_| invalid_err())?; - return Ok(PyCode::new_ref_with_bag(vm, code)); + // A non-buffer is a TypeError, not invalid frozen data. The request + // is the one marshal.loads() makes, so that what passes here is + // exactly what it accepts. + crate::protocol::PyBuffer::from_object( + vm, + &data, + crate::protocol::BufferFlags::SIMPLE, + )?; + // The data is a marshalled code object: a whole marshal value, which + // deserialize_code() does not read — it takes the code body alone, + // without the type byte the writer puts in front of it. + let loads = vm.import("marshal", 0)?.get_attr("loads", vm)?; + let code = loads.call((data,), vm).map_err(|_| invalid_err())?; + return code.downcast::().map_err(|_| invalid_err()); } import::make_frozen(vm, name.as_str()) } @@ -317,17 +321,21 @@ mod _imp { .collect() } + #[derive(FromArgs)] + struct FindFrozenArgs { + #[pyarg(positional)] + name: PyUtf8StrRef, + #[pyarg(named, default = false)] + withdata: bool, + } + #[allow(clippy::type_complexity)] #[pyfunction] fn find_frozen( - name: PyUtf8StrRef, - withdata: OptionalArg, + args: FindFrozenArgs, vm: &VirtualMachine, ) -> PyResult>, bool, Option)>> { - if withdata.into_option().is_some() { - // this is keyword-only argument in CPython - unimplemented!(); - } + let FindFrozenArgs { name, withdata } = args; let name_str = name.as_str(); let info = match super::find_frozen(name_str, vm) { @@ -338,6 +346,18 @@ mod _imp { Err(e) => return Err(e.to_pyexception(name_str, vm)), }; + // The data is what get_frozen_object() takes back, i.e. marshalled code. + // Frozen modules are stored in their own encoding, so it has to be + // re-serialized rather than handed out as a view of the stored bytes. + let data = if withdata { + let code = PyCode::new_ref_from_frozen(vm, info.code); + let dumps = vm.import("marshal", 0)?.get_attr("dumps", vm)?; + let bytes = dumps.call((code,), vm)?; + Some(PyMemoryView::from_object(&bytes, vm)?.into_ref(&vm.ctx)) + } else { + None + }; + // When origname is empty (e.g. __hello_only__), return None. // Otherwise return the resolved alias name. let origname_str = super::resolve_frozen_alias(name_str); @@ -346,7 +366,7 @@ mod _imp { } else { Some(vm.ctx.new_utf8_str(origname_str).into()) }; - Ok(Some((None, info.package, origname))) + Ok(Some((data, info.package, origname))) } #[pyfunction] diff --git a/crates/vm/src/stdlib/_io.rs b/crates/vm/src/stdlib/_io.rs index ab1be4297ec..1b4439007b2 100644 --- a/crates/vm/src/stdlib/_io.rs +++ b/crates/vm/src/stdlib/_io.rs @@ -135,8 +135,8 @@ mod _io { convert::ToPyObject, exceptions::nul_char_error, function::{ - ArgBytesLike, ArgIterable, ArgMemoryBuffer, ArgSize, Either, FsPath, FuncArgs, - IntoFuncArgs, OptionalArg, OptionalOption, PySetterValue, + ArgBytesLike, ArgContiguousBytesLike, ArgIterable, ArgMemoryBuffer, ArgSize, Either, + FsPath, FuncArgs, IntoFuncArgs, OptionalArg, OptionalOption, PySetterValue, }, protocol::{ BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn, VecBuffer, @@ -1258,7 +1258,7 @@ mod _io { let current_size = self.readahead() as usize; - let mut out = vec![0u8; n]; + let mut out = vm.new_zeroed_bytes(n)?; let mut remaining = n; let mut written = 0; if current_size > 0 { @@ -1673,7 +1673,7 @@ mod _io { check_writable(&raw, vm)?; } - data.buffer = vec![0; buffer_size]; + data.buffer = vm.new_zeroed_bytes(buffer_size)?; if Self::READABLE { data.reset_read(); @@ -1938,7 +1938,7 @@ mod _io { if data.writable() { data.flush_rewind(vm)?; } - let mut v = vec![0; n]; + let mut v = vm.new_zeroed_bytes(n)?; data.reset_read(); let r = data .raw_read(Either::A(Some(&mut v)), 0..n, vm)? @@ -3364,14 +3364,17 @@ mod _io { *snapshot = Some((cookie.dec_flags, input_chunk.clone())); let decoded = vm.call_method(decoder, "decode", (input_chunk, cookie.need_eof))?; let decoded = check_decoded(decoded, vm)?; - let pos_is_valid = decoded - .as_wtf8() - .is_code_point_boundary(cookie.bytes_to_skip as usize); + // The position is stored both as a count of characters and as + // an offset in bytes, so both have to land inside what was + // just decoded: everything read back from here indexes it. + let num_to_skip = cookie.num_to_skip(); + let pos_is_valid = num_to_skip.chars <= decoded.char_len() + && decoded.as_wtf8().is_code_point_boundary(num_to_skip.bytes); textio.set_decoded_chars(Some(decoded)); if !pos_is_valid { return Err(vm.new_os_error("can't restore logical file position")); } - textio.decoded_chars_used = cookie.num_to_skip(); + textio.decoded_chars_used = num_to_skip; } else { textio.snapshot = Some((cookie.dec_flags, PyBytes::from(vec![]).into_ref(&vm.ctx))) } @@ -4078,7 +4081,10 @@ mod _io { vm.new_runtime_error(format!("reentrant call inside {type_name}.__repr__")) ); }; - let Some(data) = zelf.data.lock() else { + // Detach while blocked, like `lock_opt`: another thread can be + // stopped holding this mutex, and blocking on it while attached + // would leave no safepoint for that stop to complete at. + let Some(data) = zelf.data.lock_wrapped(|do_lock| vm.allow_threads(do_lock)) else { // Reentrant call return Ok(vm.ctx.new_str(Wtf8Buf::from(format!("<{type_name}>")))); }; @@ -4187,6 +4193,37 @@ mod _io { } } + impl SeenNewline { + fn observe(&mut self, text: &Wtf8) { + let bytes = text.as_bytes(); + let mut matches = memchr::memchr2_iter(b'\r', b'\n', bytes); + while !self.is_all() { + let Some(i) = matches.next() else { break }; + match bytes[i] { + b'\n' => self.insert(Self::LF), + _ if bytes.get(i + 1) == Some(&b'\n') => { + matches.next(); + self.insert(Self::CRLF); + } + _ => self.insert(Self::CR), + } + } + } + + fn to_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { + match self.bits() { + 1 => "\n".to_pyobject(vm), + 2 => "\r".to_pyobject(vm), + 3 => ("\r", "\n").to_pyobject(vm), + 4 => "\r\n".to_pyobject(vm), + 5 => ("\n", "\r\n").to_pyobject(vm), + 6 => ("\r", "\r\n").to_pyobject(vm), + 7 => ("\r", "\n", "\r\n").to_pyobject(vm), + _ => vm.ctx.none(), + } + } + } + impl DefaultConstructor for IncrementalNewlineDecoder {} #[derive(FromArgs)] @@ -4278,16 +4315,7 @@ mod _io { #[pygetset] fn newlines(&self, vm: &VirtualMachine) -> PyResult { let data = self.lock(vm)?; - Ok(match data.seennl.bits() { - 1 => "\n".to_pyobject(vm), - 2 => "\r".to_pyobject(vm), - 3 => ("\r", "\n").to_pyobject(vm), - 4 => "\r\n".to_pyobject(vm), - 5 => ("\n", "\r\n").to_pyobject(vm), - 6 => ("\r", "\r\n").to_pyobject(vm), - 7 => ("\r", "\n", "\r\n").to_pyobject(vm), - _ => vm.ctx.none(), - }) + Ok(data.seennl.to_pyobject(vm)) } } @@ -4334,20 +4362,7 @@ mod _io { self.seennl.insert(SeenNewline::LF); } } else if !self.translate { - let output = output.as_bytes(); - let mut matches = memchr::memchr2_iter(b'\r', b'\n', output); - while !self.seennl.is_all() { - let Some(i) = matches.next() else { break }; - match output[i] { - b'\n' => self.seennl.insert(SeenNewline::LF), - // if c isn't \n, it can only be \r - _ if output.get(i + 1) == Some(&b'\n') => { - matches.next(); - self.seennl.insert(SeenNewline::CRLF); - } - _ => self.seennl.insert(SeenNewline::CR), - } - } + self.seennl.observe(&output); } else { let bytes = output.as_bytes(); let mut matches = memchr::memchr2_iter(b'\r', b'\n', bytes); @@ -4390,6 +4405,7 @@ mod _io { _base: _TextIOBase, buffer: PyRwLock, newline: AtomicCell, + seennl: AtomicCell, closed: AtomicCell, } @@ -4410,6 +4426,7 @@ mod _io { _base: Default::default(), buffer: PyRwLock::new(BufferedIO::new(Cursor::new(Vec::new()))), newline: AtomicCell::new(Newlines::Lf), + seennl: AtomicCell::new(SeenNewline::empty()), closed: AtomicCell::new(false), }) } @@ -4428,11 +4445,16 @@ mod _io { OptionalArg::Present(None) => Newlines::Universal, OptionalArg::Present(Some(newline)) => newline, }; - let raw_bytes = object.flatten().map_or_else(Vec::new, |v| { + let object = object.flatten(); + let raw_bytes = object.as_ref().map_or_else(Vec::new, |v| { Self::translate_newlines(v.as_wtf8(), newline).into_bytes() }); *zelf.buffer.write() = BufferedIO::new(Cursor::new(raw_bytes)); zelf.newline.store(newline); + zelf.seennl.store(SeenNewline::empty()); + if let Some(object) = object { + zelf.observe_newlines(object.as_wtf8(), newline); + } Ok(()) } } @@ -4457,6 +4479,14 @@ mod _io { } } + fn observe_newlines(&self, data: &Wtf8, newline: Newlines) { + if matches!(newline, Newlines::Universal | Newlines::Passthrough) { + let mut seennl = self.seennl.load(); + seennl.observe(data); + self.seennl.store(seennl); + } + } + fn text(bytes: &[u8]) -> &Wtf8 { // SAFETY: StringIO is populated only from PyStr values, which are valid WTF-8. unsafe { Wtf8::from_bytes_unchecked(bytes) } @@ -4509,6 +4539,15 @@ mod _io { self.closed.load() } + #[pygetset] + fn newlines(&self, vm: &VirtualMachine) -> PyResult { + if self.closed.load() { + Err(io_closed_error(vm)) + } else { + Ok(self.seennl.load().to_pyobject(vm)) + } + } + #[pymethod] fn close(&self) { self.closed.store(true); @@ -4517,8 +4556,11 @@ mod _io { // write string to underlying vector #[pymethod] fn write(&self, data: PyStrRef, vm: &VirtualMachine) -> PyResult { - let bytes = Self::translate_newlines(data.as_wtf8(), self.newline.load()).into_bytes(); - self.buffer(vm)? + let newline = self.newline.load(); + let bytes = Self::translate_newlines(data.as_wtf8(), newline).into_bytes(); + let mut buffer = self.buffer(vm)?; + self.observe_newlines(data.as_wtf8(), newline); + buffer .write(&bytes) .ok_or_else(|| vm.new_type_error("Error Writing String"))?; Ok(data.char_len() as u64) @@ -4678,6 +4720,11 @@ mod _io { .map_err(|err| os_err(vm, err))?; drop(buffer); zelf.newline.store(newline); + let mut seennl = SeenNewline::empty(); + if matches!(newline, Newlines::Universal | Newlines::Passthrough) { + seennl.observe(content.as_wtf8()); + } + zelf.seennl.store(seennl); // Set __dict__ if provided if !vm.is_none(dict) { @@ -4782,8 +4829,12 @@ mod _io { } #[pymethod] - fn write(&self, data: ArgBytesLike, vm: &VirtualMachine) -> PyResult { + fn write(&self, data: ArgContiguousBytesLike, vm: &VirtualMachine) -> PyResult { let mut buffer = self.try_resizable(vm)?; + // Acquiring the buffer can run `__buffer__`, which may have closed us. + if self.closed.load() { + return Err(io_closed_error(vm)); + } data.with_ref(|b| buffer.write(b)) .ok_or_else(|| vm.new_type_error("Error Writing Bytes")) } @@ -4806,8 +4857,20 @@ mod _io { } #[pymethod] - fn readinto(&self, obj: ArgMemoryBuffer, vm: &VirtualMachine) -> PyResult { - let mut buf = self.buffer(vm)?; + fn readinto(zelf: &Py, obj: ArgMemoryBuffer, vm: &VirtualMachine) -> PyResult { + // Reading locks this object, and a destination that views it locks + // it too, so such a destination is filled after the read is done. + if obj.source_object().is(zelf.as_object()) { + let mut data = vm.new_zeroed_bytes(obj.len())?; + let ret = zelf + .buffer(vm)? + .cursor + .read(&mut data) + .map_err(|_| vm.new_value_error("Error readinto from Take"))?; + obj.borrow_buf_mut()[..ret].copy_from_slice(&data[..ret]); + return Ok(ret); + } + let mut buf = zelf.buffer(vm)?; let ret = buf .cursor .read(&mut obj.borrow_buf_mut()) @@ -5760,7 +5823,7 @@ mod fileio { } let handle = zelf.get_fd(vm)?; let bytes = if let Some(read_byte) = read_byte.to_usize() { - let mut bytes = vec![0; read_byte]; + let mut bytes = vm.new_zeroed_bytes(read_byte)?; // Loop on EINTR (PEP 475) let n = loop { match vm.allow_threads(|| host_io::read_once(handle, &mut bytes)) { @@ -5804,6 +5867,26 @@ mod fileio { Ok(Some(bytes)) } + /// One `read()` into `buf`, retried on EINTR (PEP 475). `None` on EAGAIN. + fn read_once_into( + zelf: &Py, + handle: crt_fd::Borrowed<'_>, + buf: &mut [u8], + vm: &VirtualMachine, + ) -> PyResult> { + loop { + match vm.allow_threads(|| host_io::read_once(handle, buf)) { + Ok(n) => return Ok(Some(n)), + Err(e) if host_io::is_interrupted_error(&e) => { + vm.check_signals()?; + } + // Non-blocking mode: return None if EAGAIN + Err(e) if host_io::is_would_block_error(&e) => return Ok(None), + Err(e) => return Err(Self::io_error(zelf, e, vm)), + } + } + } + #[pymethod] fn readinto( zelf: &Py, @@ -5819,24 +5902,28 @@ mod fileio { let handle = zelf.get_fd(vm)?; - let mut buf = obj.borrow_buf_mut(); - // Loop on EINTR (PEP 475) - let ret = loop { - match vm.allow_threads(|| host_io::read_once(handle, &mut buf)) { - Ok(n) => break n, - Err(e) if host_io::is_interrupted_error(&e) => { - vm.check_signals()?; - continue; - } - // Non-blocking mode: return None if EAGAIN - Err(e) if host_io::is_would_block_error(&e) => { - return Ok(None); - } - Err(e) => return Err(Self::io_error(zelf, e, vm)), - } - }; - - Ok(Some(ret)) + if host_io::reads_without_waiting(handle) { + // The read answers from the file itself, so it returns without + // waiting on anyone; write where the caller asked directly. + // Seekability is not the question -- a pipe on Windows seeks. + let mut buf = obj.borrow_buf_mut(); + return Self::read_once_into(zelf, handle, &mut buf, vm); + } + + // A pipe, socket or terminal answers only when the other end + // writes, which may be never. Holding the export for the whole + // call is what keeps the target from being resized meanwhile, as a + // Py_buffer does; but reaching its bytes takes a lock that every + // other thread touching the same object waits on, and a thread + // waiting on a lock never reaches a safepoint, so holding that one + // across the wait stops the world from being stopped at all. Read + // aside and take the lock for the copy. + let mut scratch = vm.new_zeroed_bytes(obj.len())?; + let ret = Self::read_once_into(zelf, handle, &mut scratch, vm)?; + if let Some(n) = ret { + obj.borrow_buf_mut()[..n].copy_from_slice(&scratch[..n]); + } + Ok(ret) } #[pymethod] @@ -5854,9 +5941,14 @@ mod fileio { let handle = zelf.get_fd(vm)?; + // A pipe, socket or terminal takes the bytes only when the other + // end makes room, which may be never; see readinto above for what + // holding the source's lock across that wait costs. + let buf = obj.borrow_buf_unlocked(vm)?; + // Loop on EINTR (PEP 475) let len = loop { - match obj.with_ref(|b| vm.allow_threads(|| host_io::write_once(handle, b))) { + match vm.allow_threads(|| host_io::write_once(handle, &buf)) { Ok(n) => break n, Err(e) if host_io::is_interrupted_error(&e) => { vm.check_signals()?; diff --git a/crates/vm/src/stdlib/_signal.rs b/crates/vm/src/stdlib/_signal.rs index 5879f877676..5abfd327553 100644 --- a/crates/vm/src/stdlib/_signal.rs +++ b/crates/vm/src/stdlib/_signal.rs @@ -177,7 +177,9 @@ pub(crate) mod _signal { module: &Py, vm: &VirtualMachine, ) { - if vm.state.config.settings.install_signal_handlers { + // Process-global signal disposition is owned by the main interpreter only. + // Subinterpreters (PEP 734) must not reinstall SIGINT / probe handlers. + if vm.state.is_main_interpreter() && vm.state.config.settings.install_signal_handlers { let sig_dfl = vm.new_pyobj(SIG_DFL as u8); let sig_ign = vm.new_pyobj(SIG_IGN as u8); diff --git a/crates/vm/src/stdlib/_sre.rs b/crates/vm/src/stdlib/_sre.rs index 03382549b47..a9f98ca7015 100644 --- a/crates/vm/src/stdlib/_sre.rs +++ b/crates/vm/src/stdlib/_sre.rs @@ -3,8 +3,8 @@ pub(crate) use _sre::module_def; #[pymodule] mod _sre { use crate::{ - Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromBorrowedObject, - TryFromObject, VirtualMachine, atomic_func, + Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, + atomic_func, builtins::{ PyCallableIterator, PyDictRef, PyGenericAlias, PyInt, PyList, PyListRef, PyStr, PyStrRef, PyTuple, PyTupleRef, PyTypeRef, @@ -13,7 +13,7 @@ mod _sre { common::{ascii, hash::PyHash}, convert::ToPyObject, function::{ArgCallable, OptionalArg, PosArgs, PyComparisonValue}, - protocol::{PyBuffer, PyCallable, PyMappingMethods}, + protocol::{BufferFlags, PyBuffer, PyCallable, PyMappingMethods}, stdlib::sys, types::{AsMapping, Comparable, Hashable, Representable}, }; @@ -22,7 +22,7 @@ mod _sre { use itertools::Itertools; use num_traits::ToPrimitive; use rustpython_sre_engine::{ - Request, SearchIter, SreFlag, State, StrDrive, + Request, SearchIter, SreFlag, State, StrDrive, StringCursor, string::{lower_ascii, lower_unicode}, }; @@ -70,19 +70,146 @@ mod _sre { } } - impl SreStr for &Wtf8 { + /// A `str` subject with non-ASCII characters, driven through the string's + /// own character-index table. + /// + /// The `&Wtf8` drive answers `count` and `create_cursor` by decoding from + /// the start of the subject, so both are O(n) and a scan that restarts at + /// successive positions walks the subject once per position. `PyStr` + /// already caches its character length and can resolve a character index to + /// a byte offset in constant time, so this drive asks the string instead of + /// re-deriving: the table it builds on the first lookup is shared by every + /// later one, including by `Match` objects that outlive the scan and have + /// no cursor of their own to move relative to. + /// + /// Stepping is the `&Wtf8` drive's, unchanged -- the subject is the same + /// buffer, decoded the same way. Only the two operations that resolve a + /// position from scratch differ. + #[derive(Clone, Copy)] + struct Utf8Str<'a>(&'a Py); + + impl StrDrive for Utf8Str<'_> { + fn count(&self) -> usize { + self.0.char_len() + } + + fn create_cursor(&self, n: usize) -> StringCursor { + // `StringCursor`'s pointer is private to the engine, so the cursor + // is taken from the `&Wtf8` drive at the start of the suffix that + // begins at `n` -- an O(1) reslice -- rather than built here. + let suffix = &self.0.as_wtf8()[self.0.char_index_to_byte(n)..]; + let mut cursor = <&Wtf8 as StrDrive>::create_cursor(&suffix, 0); + cursor.position = n; + cursor + } + + fn adjust_cursor(&self, cursor: &mut StringCursor, n: usize) { + // Rebuilding is O(1), so it is never the slower branch and the + // `&Wtf8` drive's walk-or-restart choice does not apply. + *cursor = self.create_cursor(n); + } + + fn advance(cursor: &mut StringCursor) -> u32 { + <&Wtf8 as StrDrive>::advance(cursor) + } + + fn peek(cursor: &StringCursor) -> u32 { + <&Wtf8 as StrDrive>::peek(cursor) + } + + fn skip(cursor: &mut StringCursor, n: usize) { + <&Wtf8 as StrDrive>::skip(cursor, n) + } + + fn back_advance(cursor: &mut StringCursor) -> u32 { + <&Wtf8 as StrDrive>::back_advance(cursor) + } + + fn back_peek(cursor: &StringCursor) -> u32 { + <&Wtf8 as StrDrive>::back_peek(cursor) + } + + fn back_skip(cursor: &mut StringCursor, n: usize) { + <&Wtf8 as StrDrive>::back_skip(cursor, n) + } + } + + impl SreStr for Utf8Str<'_> { fn slice(&self, start: usize, end: usize, vm: &VirtualMachine) -> PyObjectRef { + let end = self.0.char_index_to_byte(end); + let start = self.0.char_index_to_byte(start).min(end); vm.ctx - .new_str( - self.code_points() - .take(end) - .skip(start) - .collect::(), - ) + .new_str(self.0.as_wtf8()[start..end].to_owned()) .into() } } + /// An all-ASCII `str` subject, driven over its bytes. + /// + /// For ASCII a character index *is* a byte index, so `&[u8]`'s cursor + /// arithmetic is already the right arithmetic: `count` is the byte length + /// and `create_cursor` is a pointer offset. The `&Wtf8` drive has to count + /// code points from the start of the subject to answer either, once per + /// `Request`, which makes a scan that restarts at successive positions -- + /// `finditer`, or `re` module functions called in a loop -- walk the + /// subject again on every call. + /// + /// Matching is unaffected: `StrDrive` carries no unicode semantics of its + /// own, because the engine keys every unicode decision on the compiled + /// pattern's opcode rather than on the subject type. Only `slice` differs + /// from the `&[u8]` impl, to hand back `str` instead of `bytes`. + #[derive(Clone, Copy)] + struct AsciiStr<'a>(&'a [u8]); + + impl StrDrive for AsciiStr<'_> { + fn count(&self) -> usize { + <&[u8] as StrDrive>::count(&self.0) + } + + fn create_cursor(&self, n: usize) -> StringCursor { + <&[u8] as StrDrive>::create_cursor(&self.0, n) + } + + fn adjust_cursor(&self, cursor: &mut StringCursor, n: usize) { + <&[u8] as StrDrive>::adjust_cursor(&self.0, cursor, n) + } + + fn advance(cursor: &mut StringCursor) -> u32 { + <&[u8] as StrDrive>::advance(cursor) + } + + fn peek(cursor: &StringCursor) -> u32 { + <&[u8] as StrDrive>::peek(cursor) + } + + fn skip(cursor: &mut StringCursor, n: usize) { + <&[u8] as StrDrive>::skip(cursor, n) + } + + fn back_advance(cursor: &mut StringCursor) -> u32 { + <&[u8] as StrDrive>::back_advance(cursor) + } + + fn back_peek(cursor: &StringCursor) -> u32 { + <&[u8] as StrDrive>::back_peek(cursor) + } + + fn back_skip(cursor: &mut StringCursor, n: usize) { + <&[u8] as StrDrive>::back_skip(cursor, n) + } + } + + impl SreStr for AsciiStr<'_> { + fn slice(&self, start: usize, end: usize, vm: &VirtualMachine) -> PyObjectRef { + let end = end.min(self.0.len()); + let start = start.min(end); + // The subject is ASCII, so any span of it is valid UTF-8 and the + // span is a reslice rather than a walk from the subject's start. + let s = str::from_utf8(&self.0[start..end]).expect("ascii subject"); + vm.ctx.new_str(s).into() + } + } + #[pyfunction] fn compile( pattern: PyObjectRef, @@ -200,32 +327,66 @@ mod _sre { } macro_rules! with_sre_str { - ($pattern:expr, $string:expr, $vm:expr, $f:expr) => { + ($pattern:expr, $string:expr, $vm:expr, $f:expr) => {{ + // Bind once: the branches only borrow the subject, and callers pass + // a temporary (`&x.clone()`) that would otherwise be rebuilt per arm. + let subject = $string; if $pattern.isbytes { - Pattern::with_bytes($string, $vm, $f) + Pattern::with_bytes(subject, $vm, $f) + } else if Pattern::is_ascii_str(subject) { + Pattern::with_ascii_str(subject, $vm, $f) } else { - Pattern::with_str($string, $vm, $f) + Pattern::with_utf8_str(subject, $vm, $f) } - }; + }}; } #[pyclass(with(Hashable, Comparable, Representable), flags(HAS_WEAKREF))] impl Pattern { + fn downcast_str<'a>(string: &'a PyObject, vm: &VirtualMachine) -> PyResult<&'a Py> { + string.downcast_ref::().ok_or_else(|| { + vm.new_type_error(format!("expected string got '{}'", string.class())) + }) + } + fn with_str(string: &PyObject, vm: &VirtualMachine, f: F) -> PyResult where F: FnOnce(&Wtf8) -> PyResult, { - let string = string.downcast_ref::().ok_or_else(|| { - vm.new_type_error(format!("expected string got '{}'", string.class())) - })?; - f(string.as_wtf8()) + f(Self::downcast_str(string, vm)?.as_wtf8()) + } + + /// Whether a `str` subject can take the [`AsciiStr`] drive. + /// + /// `PyStr` already knows: `StrKind` is decided when the string is + /// built, so this is a field load rather than a scan. A non-`str` + /// argument answers `false` and is reported by [`Self::with_utf8_str`]. + fn is_ascii_str(string: &PyObject) -> bool { + string + .downcast_ref::() + .is_some_and(|s| s.kind().is_ascii()) + } + + fn with_ascii_str(string: &PyObject, vm: &VirtualMachine, f: F) -> PyResult + where + F: FnOnce(AsciiStr<'_>) -> PyResult, + { + let string = Self::downcast_str(string, vm)?; + f(AsciiStr(string.as_wtf8().as_bytes())) + } + + fn with_utf8_str(string: &PyObject, vm: &VirtualMachine, f: F) -> PyResult + where + F: FnOnce(Utf8Str<'_>) -> PyResult, + { + f(Utf8Str(Self::downcast_str(string, vm)?)) } fn with_bytes(string: &PyObject, vm: &VirtualMachine, f: F) -> PyResult where F: FnOnce(&[u8]) -> PyResult, { - PyBuffer::try_from_borrowed_object(vm, string)?.contiguous_or_collect(f) + PyBuffer::from_object(vm, string, BufferFlags::SIMPLE)?.contiguous_or_collect(f) } #[pymethod(name = "match")] @@ -498,7 +659,7 @@ mod _sre { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -597,7 +758,7 @@ mod _sre { regs: Vec<(isize, isize)>, } - #[pyclass(with(AsMapping, Representable))] + #[pyclass(with(AsMapping, Representable), flags(DISALLOW_INSTANTIATION))] impl Match { pub(crate) fn new(state: &mut State, pattern: PyRef, string: PyObjectRef) -> Self { let string_position = state.cursor.position; @@ -844,7 +1005,7 @@ mod _sre { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } diff --git a/crates/vm/src/stdlib/_symtable.rs b/crates/vm/src/stdlib/_symtable.rs index c4a4a7a2051..eb0ecaa87d7 100644 --- a/crates/vm/src/stdlib/_symtable.rs +++ b/crates/vm/src/stdlib/_symtable.rs @@ -185,8 +185,9 @@ mod _symtable { fn symbols(&self, vm: &VirtualMachine) -> PyDictRef { let dict = vm.ctx.new_dict(); for (name, symbol) in &self.symtable.symbols { - dict.set_item(name, vm.new_pyobj(symbol.flags.bits()), vm) - .unwrap(); + let packed_flags = + i32::from(symbol.flags.bits()) | (symbol.scope.as_i32() << SCOPE_OFFSET); + dict.set_item(name, vm.new_pyobj(packed_flags), vm).unwrap(); } dict } diff --git a/crates/vm/src/stdlib/_thread.rs b/crates/vm/src/stdlib/_thread.rs index 70304e63980..79dce3d21ce 100644 --- a/crates/vm/src/stdlib/_thread.rs +++ b/crates/vm/src/stdlib/_thread.rs @@ -605,14 +605,35 @@ pub(crate) mod _thread { vm.state.thread_count.fetch_sub(1); } + /// Default stack size for Python threads in **debug builds only**, where + /// Rust stack frames are substantially larger than in release. Rust's + /// `std::thread::Builder` otherwise defaults to 2 MB, which is too small + /// for the call chains the Python stdlib runs on helper threads in debug + /// (e.g. the SSL test server, see #7941). Release builds keep the prior + /// behavior — leave the stack size unset and let Rust's std default apply + /// — to avoid oversized virtual stack mappings when many threads spawn. + #[cfg(debug_assertions)] + const DEFAULT_THREAD_STACK_SIZE: usize = 8 * 1024 * 1024; + + /// Configure a `thread::Builder` with the stack size to use for a new + /// Python thread. Uses the value set via `threading.stack_size(N)` when + /// the user has provided one (non-zero). Otherwise, debug builds fall + /// back to [`DEFAULT_THREAD_STACK_SIZE`] and release builds leave the + /// builder unmodified (Rust's std default applies). fn apply_thread_stack_size( thread_builder: thread::Builder, vm: &VirtualMachine, ) -> thread::Builder { let configured = vm.state.stacksize.load(); if configured != 0 { - thread_builder.stack_size(configured) - } else { + return thread_builder.stack_size(configured); + } + #[cfg(debug_assertions)] + { + thread_builder.stack_size(DEFAULT_THREAD_STACK_SIZE) + } + #[cfg(not(debug_assertions))] + { thread_builder } } @@ -775,9 +796,8 @@ pub(crate) mod _thread { } #[pyfunction] - fn _is_main_interpreter() -> bool { - // RustPython only has one interpreter - true + fn _is_main_interpreter(vm: &VirtualMachine) -> bool { + vm.state.is_main_interpreter() } /// Initialize the main thread ident. Should be called once at interpreter startup. @@ -1173,8 +1193,8 @@ pub(crate) mod _thread { { use core::sync::atomic::Ordering; let current_ident = get_ident(); - vm.state.stop_the_world.stop_the_world(vm); - scopeguard::defer! { vm.state.stop_the_world.start_the_world(vm); } + vm.state.stop_the_world.stop_the_world(&vm.state); + scopeguard::defer! { vm.state.stop_the_world.start_the_world(&vm.state); } let registry = vm.state.thread_frames.lock(); registry .iter() @@ -1187,9 +1207,9 @@ pub(crate) mod _thread { // fall back to top_iframe (may be a stack-allocated frame). let top = slot.top_frame.load(Ordering::Relaxed); if let Some(p) = core::ptr::NonNull::new(top) { - let py = unsafe { - &*Py::::from_payload_ptr(p.as_ptr()) - }; + // SAFETY: world stopped -> the owning thread is parked + // with this frame on its chain, so it is alive. + let py = unsafe { p.as_ref() }; Some((*id, py.to_owned())) } else { // Stack-allocated frame: materialize from top_iframe. @@ -1197,26 +1217,10 @@ pub(crate) mod _thread { let iframe_ptr = slot.top_iframe.load(Ordering::Relaxed) as *const crate::frame::InterpreterFrame; if !iframe_ptr.is_null() { - // Materialize the entire frame chain and link - // retained_back so f_back works after STW ends. - let mut cur = iframe_ptr; - let mut child_fo: Option> = - None; - while !cur.is_null() { - let iframe = unsafe { &*cur }; - let fo = iframe.materialize(vm).to_owned(); - if let Some(child) = child_fo.take() { - let mut guard = child.iframe().cold().retained_back.lock(); - if guard.is_none() { - *guard = Some(fo.clone()); - } - } - child_fo = Some(fo); - cur = iframe.previous(); - } let iframe = unsafe { &*iframe_ptr }; - let fo = iframe.materialize(vm); - Some((*id, fo.to_owned())) + // SAFETY: world stopped -> owning thread parked. + let fo = unsafe { iframe.materialize_detached_chain(vm) }; + Some((*id, fo)) } else { None } @@ -1229,8 +1233,8 @@ pub(crate) mod _thread { { use core::sync::atomic::Ordering; let current_ident = get_ident(); - vm.state.stop_the_world.stop_the_world(vm); - scopeguard::defer! { vm.state.stop_the_world.start_the_world(vm); } + vm.state.stop_the_world.stop_the_world(&vm.state); + scopeguard::defer! { vm.state.stop_the_world.start_the_world(&vm.state); } let registry = vm.state.thread_frames.lock(); registry .iter() @@ -1246,24 +1250,10 @@ pub(crate) mod _thread { let iframe_ptr = slot.top_iframe.load(Ordering::Relaxed) as *const crate::frame::InterpreterFrame; if !iframe_ptr.is_null() { - let mut cur = iframe_ptr; - let mut child_fo: Option> = - None; - while !cur.is_null() { - let iframe = unsafe { &*cur }; - let fo = iframe.materialize(vm).to_owned(); - if let Some(child) = child_fo.take() { - let mut guard = child.iframe().cold().retained_back.lock(); - if guard.is_none() { - *guard = Some(fo.clone()); - } - } - child_fo = Some(fo); - cur = iframe.previous(); - } let iframe = unsafe { &*iframe_ptr }; - let fo = iframe.materialize(vm); - Some((*id, fo.to_owned())) + // SAFETY: world stopped -> owning thread parked. + let fo = unsafe { iframe.materialize_detached_chain(vm) }; + Some((*id, fo)) } else { // Fall back to frames stack for FrameObject-only path let frames = slot.frames.lock(); @@ -1365,6 +1355,19 @@ pub(crate) mod _thread { } } + /// Take a thread handle's completion mutex, detaching first. + /// + /// A joiner holds this mutex across its `allow_threads` wait, so it can + /// still hold it when stop-the-world stops it. An attached thread that + /// blocked on it would never reach a safepoint, so the stop could never + /// complete and the holder would never be resumed to release it. + fn lock_done<'a>( + lock: &'a parking_lot::Mutex, + vm: &VirtualMachine, + ) -> parking_lot::MutexGuard<'a, bool> { + vm.allow_threads(|| lock.lock()) + } + /// Reset a parking_lot::Mutex to unlocked state after fork. #[cfg(all(unix, feature = "host_env"))] fn reinit_parking_lot_mutex(mutex: &parking_lot::Mutex) { @@ -1447,7 +1450,7 @@ pub(crate) mod _thread { // Wait for thread completion using Condvar (supports timeout) // Loop to handle spurious wakeups let (lock, cvar) = &**done_event; - let mut done = lock.lock(); + let mut done = lock_done(lock, vm); // ThreadHandle_join semantics: self-join/finalizing checks // apply only while target thread has not reported it is exiting yet. @@ -1507,7 +1510,7 @@ pub(crate) mod _thread { drop(inner_guard); // Wait on done_event let (lock, cvar) = &**done_event; - let mut done = lock.lock(); + let mut done = lock_done(lock, vm); while !*done { vm.allow_threads(|| cvar.wait(&mut done)); } @@ -1569,7 +1572,7 @@ pub(crate) mod _thread { remove_from_shutdown_handles(vm, inner, done_event); let (lock, cvar) = &**done_event; - *lock.lock() = true; + *lock_done(lock, vm) = true; cvar.notify_all(); Ok(()) } @@ -1639,7 +1642,7 @@ pub(crate) mod _thread { // before returning True. let done = { let (lock, _) = &*self.done_event; - *lock.lock() + *lock_done(lock, vm) }; if !done { return Ok(false); @@ -1835,7 +1838,7 @@ pub(crate) mod _thread { // Starting a handle always resets the completion event. { let (done_lock, _) = &*handle.done_event; - *done_lock.lock() = false; + *lock_done(done_lock, vm) = false; } // Add non-daemon threads to shutdown registry so _shutdown() will wait for them @@ -1917,7 +1920,7 @@ pub(crate) mod _thread { // This must be LAST to ensure all cleanup is complete before join() returns { let (lock, cvar) = &*done_event_for_cleanup; - *lock.lock() = true; + *lock_done(lock, vm) = true; cvar.notify_all(); } } @@ -1952,7 +1955,7 @@ pub(crate) mod _thread { } { let (done_lock, done_cvar) = &*handle.done_event; - *done_lock.lock() = true; + *lock_done(done_lock, vm) = true; done_cvar.notify_all(); } if !daemon { @@ -1996,4 +1999,55 @@ pub(crate) mod _thread { Ok(handle_clone) } + + #[cfg(test)] + mod tests { + #[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] + use super::*; + #[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] + use crate::Interpreter; + + /// Regression test for #7941: a Python thread started without an + /// explicit `threading.stack_size()` must not run on Rust's 2 MiB + /// std default in debug builds, where the call chains the stdlib + /// runs on helper threads (e.g. the SSL test server) overflowed it. + #[test] + #[cfg(all(debug_assertions, any(target_os = "linux", target_os = "macos")))] + fn default_python_thread_stack_size_debug() { + Interpreter::without_stdlib(Default::default()).enter(|vm| { + assert_eq!(vm.state.stacksize.load(), 0); + let builder = apply_thread_stack_size(thread::Builder::new(), vm); + let stack_size = builder + .spawn(current_thread_stack_size) + .expect("failed to spawn thread") + .join() + .expect("thread panicked"); + assert!( + stack_size >= DEFAULT_THREAD_STACK_SIZE, + "Python thread stack size is {stack_size} bytes, expected at least {DEFAULT_THREAD_STACK_SIZE}" + ); + }); + } + + #[cfg(all(debug_assertions, target_os = "linux"))] + fn current_thread_stack_size() -> usize { + use libc::{ + pthread_attr_destroy, pthread_attr_getstacksize, pthread_attr_t, + pthread_getattr_np, pthread_self, + }; + let mut attr: pthread_attr_t = unsafe { core::mem::zeroed() }; + unsafe { + assert_eq!(pthread_getattr_np(pthread_self(), &mut attr), 0); + let mut size = 0; + assert_eq!(pthread_attr_getstacksize(&attr, &mut size), 0); + pthread_attr_destroy(&mut attr); + size + } + } + + #[cfg(all(debug_assertions, target_os = "macos"))] + fn current_thread_stack_size() -> usize { + unsafe { libc::pthread_get_stacksize_np(libc::pthread_self()) } + } + } } diff --git a/crates/vm/src/stdlib/_typing.rs b/crates/vm/src/stdlib/_typing.rs index 0214c3cc544..0b19d8e3c32 100644 --- a/crates/vm/src/stdlib/_typing.rs +++ b/crates/vm/src/stdlib/_typing.rs @@ -39,8 +39,17 @@ pub(crate) mod decl { }; #[pyfunction] - pub(crate) fn _idfunc(args: FuncArgs, _vm: &VirtualMachine) -> PyObjectRef { - args.args[0].clone() + pub(crate) fn _idfunc(args: FuncArgs, vm: &VirtualMachine) -> PyResult { + if !args.kwargs.is_empty() { + return Err(vm.new_type_error("_typing._idfunc() takes no keyword arguments")); + } + if args.args.len() != 1 { + return Err(vm.new_type_error(format!( + "_typing._idfunc() takes exactly one argument ({} given)", + args.args.len() + ))); + } + Ok(args.args[0].clone()) } #[pyfunction(name = "override")] @@ -288,7 +297,7 @@ pub(crate) mod decl { PyTuple::new_ref(vec![args], &vm.ctx) }; let origin: PyObjectRef = zelf.as_object().to_owned(); - Ok(PyGenericAlias::new(origin, args_tuple, false, vm).into_pyobject(vm)) + Ok(PyGenericAlias::new(origin, args_tuple, false, vm)?.into_pyobject(vm)) } #[pymethod] diff --git a/crates/vm/src/stdlib/_winapi.rs b/crates/vm/src/stdlib/_winapi.rs index 0d54530d4b2..34e7b897e12 100644 --- a/crates/vm/src/stdlib/_winapi.rs +++ b/crates/vm/src/stdlib/_winapi.rs @@ -8,7 +8,7 @@ mod _winapi { use crate::{ Py, PyObjectRef, PyPayload, PyResult, TryFromObject, VirtualMachine, builtins::PyStrRef, - common::lock::PyMutex, + common::lock::{PyMutex, PyMutexGuard}, convert::ToPyException, function::{ArgMapping, ArgSequence, OptionalArg}, types::Constructor, @@ -566,9 +566,18 @@ mod _winapi { .map_err(|e| e.to_pyexception(vm)) } + /// Take `inner`, detaching while blocked. + /// + /// `GetOverlappedResult` holds this mutex across its `allow_threads` + /// wait, so a stopped thread can still be holding it. Blocking on it + /// while attached would leave no safepoint for that stop to complete at. + fn lock_inner(&self, vm: &VirtualMachine) -> PyMutexGuard<'_, host_overlapped::Operation> { + vm.allow_threads(|| self.inner.lock()) + } + #[pymethod] fn GetOverlappedResult(&self, wait: bool, vm: &VirtualMachine) -> PyResult<(u32, u32)> { - let mut inner = self.inner.lock(); + let mut inner = self.lock_inner(vm); vm.allow_threads(|| inner.get_result(wait)) .map(|result| (result.transferred, result.error)) .map_err(|e| e.to_pyexception(vm)) @@ -576,7 +585,7 @@ mod _winapi { #[pymethod] fn getbuffer(&self, vm: &VirtualMachine) -> PyResult> { - let inner = self.inner.lock(); + let inner = self.lock_inner(vm); if !inner.is_completed() { return Err(vm.new_value_error( "can't get read buffer before GetOverlappedResult() signals the operation completed", @@ -589,13 +598,13 @@ mod _winapi { #[pymethod] fn cancel(&self, vm: &VirtualMachine) -> PyResult<()> { - let mut inner = self.inner.lock(); + let mut inner = self.lock_inner(vm); inner.cancel().map_err(|e| e.to_pyexception(vm)) } #[pygetset] - fn event(&self) -> isize { - let inner = self.inner.lock(); + fn event(&self, vm: &VirtualMachine) -> isize { + let inner = self.lock_inner(vm); inner.event() as isize } } diff --git a/crates/vm/src/stdlib/atexit.rs b/crates/vm/src/stdlib/atexit.rs index 891f8e5437b..0260b0f115d 100644 --- a/crates/vm/src/stdlib/atexit.rs +++ b/crates/vm/src/stdlib/atexit.rs @@ -3,7 +3,9 @@ pub(crate) use atexit::module_def; #[pymodule] mod atexit { - use crate::{AsObject, PyObjectRef, PyResult, VirtualMachine, function::FuncArgs}; + use crate::{ + AsObject, PyObjectRef, PyResult, VirtualMachine, common::rc::PyRc, function::FuncArgs, + }; #[pyfunction] fn register(func: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyObjectRef { @@ -11,7 +13,7 @@ mod atexit { vm.state .atexit_funcs .lock() - .insert(0, Box::new((func.clone(), args))); + .insert(0, PyRc::new((func.clone(), args))); func } @@ -29,24 +31,26 @@ mod atexit { funcs.len() as isize - 1 }; while i >= 0 { - let (cb, entry_ptr) = { + let entry = { let funcs = vm.state.atexit_funcs.lock(); if i as usize >= funcs.len() { i = funcs.len() as isize; i -= 1; continue; } - let entry = &funcs[i as usize]; - (entry.0.clone(), &**entry as *const (PyObjectRef, FuncArgs)) + // Keep the entry alive for as long as it is being compared, so + // it cannot be dropped and have its address handed to a + // callback registered from within __eq__. + funcs[i as usize].clone() }; // Lock released: __eq__ can safely call atexit functions - let eq = vm.bool_eq(&func, &cb)?; + let eq = vm.bool_eq(&func, &entry.0)?; if eq { // The entry may have moved during __eq__. Search backward by identity. let mut funcs = vm.state.atexit_funcs.lock(); let mut j = (funcs.len() as isize - 1).min(i); while j >= 0 { - if core::ptr::eq(&**funcs.get(j as usize).unwrap(), entry_ptr) { + if PyRc::ptr_eq(funcs.get(j as usize).unwrap(), &entry) { funcs.remove(j as usize); i = j; break; @@ -70,7 +74,7 @@ mod atexit { let funcs: Vec<_> = core::mem::take(&mut *vm.state.atexit_funcs.lock()); // Callbacks stored in LIFO order, iterate forward for entry in funcs { - let (func, args) = *entry; + let (func, args) = PyRc::try_unwrap(entry).unwrap_or_else(|e| (*e).clone()); if let Err(e) = func.call(args, vm) { let exit = e.fast_isinstance(vm.ctx.exceptions.system_exit); let msg = func diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index 35f404f0f3b..19e33110a5b 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -21,9 +21,9 @@ mod builtins { bytecode, common::hash::PyHash, function::{ - ArgBytesLike, ArgCallable, ArgIndex, ArgIntoBool, ArgIterable, ArgMapping, - ArgPrimitiveIndex, ArgStrOrBytesLike, Either, FsPath, FuncArgs, KwArgs, OptionalArg, - OptionalOption, PosArgs, + ArgCallable, ArgIndex, ArgIntoBool, ArgIterable, ArgMapping, ArgPrimitiveIndex, + ArgStrOrBytesLike, Either, FsPath, FuncArgs, KwArgs, OptionalArg, OptionalOption, + PosArgs, }, protocol::{PyIter, PyIterReturn}, py_io, @@ -341,6 +341,7 @@ mod builtins { }; match &source { ArgStrOrBytesLike::Str(source) => { + let source = source.try_as_utf8(vm)?.as_str(); if source.as_bytes().contains(&0) { return Err(vm.new_exception_msg( vm.ctx.exceptions.syntax_error.to_owned(), @@ -548,13 +549,14 @@ mod builtins { Either::A(either) => { let source = match &either { ArgStrOrBytesLike::Str(source) => { + let source = source.try_as_utf8(vm)?.as_str(); if source.as_bytes().contains(&0) { return Err(vm.new_exception_msg( vm.ctx.exceptions.syntax_error.to_owned(), "source code string cannot contain null bytes".into(), )); } - let source = source.expect_str().trim_start_matches([' ', '\t']); + let source = source.trim_start_matches([' ', '\t']); audit_compile_source(vm, source.as_bytes(), "")?; source.to_owned() } @@ -597,6 +599,7 @@ mod builtins { } let source = match &either { ArgStrOrBytesLike::Str(source) => { + let source = source.try_as_utf8(vm)?.as_str(); if source.as_bytes().contains(&0) { return Err(vm.new_exception_msg( vm.ctx.exceptions.syntax_error.to_owned(), @@ -604,7 +607,7 @@ mod builtins { )); } audit_compile_source(vm, source.as_bytes(), "")?; - source.expect_str().to_owned() + source.to_owned() } ArgStrOrBytesLike::Buf(source) => { let source: &[u8] = &source.borrow_buf(); @@ -994,18 +997,10 @@ mod builtins { } #[pyfunction] - fn ord(string: Either, vm: &VirtualMachine) -> PyResult { - match string { - Either::A(bytes) => bytes.with_ref(|bytes| { - let bytes_len = bytes.len(); - if bytes_len != 1 { - return Err(vm.new_type_error(format!( - "ord() expected a character, but string of length {bytes_len} found" - ))); - } - Ok(u32::from(bytes[0])) - }), - Either::B(string) => match string.as_wtf8().code_points().exactly_one() { + // builtin_ord + fn ord(c: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let bytes = if let Some(string) = c.downcast_ref::() { + return match string.as_wtf8().code_points().exactly_one() { Ok(character) => Ok(character.to_u32()), Err(_) => { let string_len = string.char_len(); @@ -1013,8 +1008,24 @@ mod builtins { "ord() expected a character, but string of length {string_len} found" ))) } - }, + }; + } else if let Some(bytes) = c.downcast_ref::() { + bytes.as_bytes().to_vec() + } else if let Some(bytearray) = c.downcast_ref::() { + bytearray.borrow_buf().to_vec() + } else { + return Err(vm.new_type_error(format!( + "ord() expected string of length 1, but {} found", + c.class().name() + ))); + }; + let bytes_len = bytes.len(); + if bytes_len != 1 { + return Err(vm.new_type_error(format!( + "ord() expected a character, but string of length {bytes_len} found" + ))); } + Ok(u32::from(bytes[0])) } #[derive(FromArgs)] diff --git a/crates/vm/src/stdlib/gc.rs b/crates/vm/src/stdlib/gc.rs index b0007b4c867..af861862edb 100644 --- a/crates/vm/src/stdlib/gc.rs +++ b/crates/vm/src/stdlib/gc.rs @@ -23,20 +23,20 @@ mod gc { /// Enable automatic garbage collection. #[pyfunction] - fn enable() { - gc_state::gc_state().enable(); + fn enable(vm: &VirtualMachine) { + vm.state.gc.enable(); } /// Disable automatic garbage collection. #[pyfunction] - fn disable() { - gc_state::gc_state().disable(); + fn disable(vm: &VirtualMachine) { + vm.state.gc.disable(); } /// Return true if automatic gc is enabled. #[pyfunction] - fn isenabled() -> bool { - gc_state::gc_state().is_enabled() + fn isenabled(vm: &VirtualMachine) -> bool { + vm.state.gc.is_enabled() } /// Run a garbage collection. Returns the number of unreachable objects found. @@ -58,15 +58,14 @@ mod gc { invoke_callbacks(vm, "start", generation_num as usize, &Default::default()); // Manual gc.collect() should run even if GC is disabled - let gc = gc_state::gc_state(); + let gc = &vm.state.gc; let result = gc.collect_force(generation_num as usize); - // Move objects from gc_state.garbage to vm.ctx.gc_garbage (for DEBUG_SAVEALL) + // Publish what the collection saved as gc.garbage (for DEBUG_SAVEALL) { let mut state_garbage = gc.garbage.lock(); if !state_garbage.is_empty() { - let py_garbage = &vm.ctx.gc_garbage; - let mut garbage_vec = py_garbage.borrow_vec_mut(); + let mut garbage_vec = gc.py_garbage.borrow_vec_mut(); for obj in state_garbage.drain(..) { garbage_vec.push(obj); } @@ -82,7 +81,7 @@ mod gc { /// Return the current collection thresholds as a tuple. #[pyfunction] fn get_threshold(vm: &VirtualMachine) -> PyObjectRef { - let (t0, t1, t2) = gc_state::gc_state().get_threshold(); + let (t0, t1, t2) = vm.state.gc.get_threshold(); vm.ctx .new_tuple(vec![ vm.ctx.new_int(t0).into(), @@ -94,8 +93,13 @@ mod gc { /// Set the collection thresholds. #[pyfunction] - fn set_threshold(threshold0: u32, threshold1: OptionalArg, threshold2: OptionalArg) { - gc_state::gc_state().set_threshold( + fn set_threshold( + threshold0: u32, + threshold1: OptionalArg, + threshold2: OptionalArg, + vm: &VirtualMachine, + ) { + vm.state.gc.set_threshold( threshold0, threshold1.into_option(), threshold2.into_option(), @@ -117,20 +121,22 @@ mod gc { /// Return the current debugging flags. #[pyfunction] - fn get_debug() -> u32 { - gc_state::gc_state().get_debug().bits() + fn get_debug(vm: &VirtualMachine) -> u32 { + vm.state.gc.get_debug().bits() } /// Set the debugging flags. #[pyfunction] - fn set_debug(flags: u32) { - gc_state::gc_state().set_debug(gc_state::GcDebugFlags::from_bits_truncate(flags)); + fn set_debug(flags: u32, vm: &VirtualMachine) { + vm.state + .gc + .set_debug(gc_state::GcDebugFlags::from_bits_truncate(flags)); } /// Return a list of per-generation gc stats. #[pyfunction] fn get_stats(vm: &VirtualMachine) -> PyResult { - let stats = gc_state::gc_state().get_stats(); + let stats = vm.state.gc.get_stats(); let mut result = Vec::with_capacity(3); for stat in &stats { @@ -165,7 +171,7 @@ mod gc { { return Err(vm.new_value_error(format!("generation must be in range(0, 3), not {g}"))); } - let objects = gc_state::gc_state().get_objects(generation_opt); + let objects = vm.state.gc.get_objects(generation_opt); Ok(vm.ctx.new_list(objects)) } @@ -208,7 +214,7 @@ mod gc { let mut result = Vec::new(); // Scan all tracked objects across all generations - let all_objects = gc_state::gc_state().get_objects(None); + let all_objects = vm.state.gc.get_objects(None); for obj in all_objects { let obj_ptr = obj.as_ref() as *const crate::PyObject as usize; if stack_frames.contains(&obj_ptr) { @@ -241,14 +247,14 @@ mod gc { /// Freeze all objects tracked by gc. #[pyfunction] - fn freeze() { - gc_state::gc_state().freeze(); + fn freeze(vm: &VirtualMachine) { + vm.state.gc.freeze(); } /// Unfreeze all objects in the permanent generation. #[pyfunction] - fn unfreeze() { - gc_state::gc_state().unfreeze(); + fn unfreeze(vm: &VirtualMachine) { + vm.state.gc.unfreeze(); } /// Return the number of objects in the permanent generation. @@ -260,13 +266,13 @@ mod gc { /// gc.garbage - list of uncollectable objects #[pyattr] fn garbage(vm: &VirtualMachine) -> PyListRef { - vm.ctx.gc_garbage.clone() + vm.state.gc.py_garbage.clone() } /// gc.callbacks - list of callbacks to be invoked #[pyattr] fn callbacks(vm: &VirtualMachine) -> PyListRef { - vm.ctx.gc_callbacks.clone() + vm.state.gc.py_callbacks.clone() } /// Helper function to invoke GC callbacks @@ -276,7 +282,7 @@ mod gc { generation: usize, result: &gc_state::CollectResult, ) { - let callbacks_list = &vm.ctx.gc_callbacks; + let callbacks_list = &vm.state.gc.py_callbacks; let callbacks: Vec = callbacks_list.borrow_vec().to_vec(); if callbacks.is_empty() { return; diff --git a/crates/vm/src/stdlib/itertools.rs b/crates/vm/src/stdlib/itertools.rs index 041620298e7..6eb268d94c1 100644 --- a/crates/vm/src/stdlib/itertools.rs +++ b/crates/vm/src/stdlib/itertools.rs @@ -4,13 +4,12 @@ pub(crate) use decl::module_def; mod decl { use crate::{ AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, PyWeakRef, VirtualMachine, - builtins::{PyGenericAlias, PyInt, PyIntRef, PyList, PyTuple, PyType, PyTypeRef, int}, - common::{ - lock::{PyMutex, PyRwLock, PyRwLockWriteGuard}, - rc::PyRc, + builtins::{ + PyGenericAlias, PyInt, PyIntRef, PyList, PyTuple, PyTupleRef, PyType, PyTypeRef, int, }, + common::lock::{PyMutex, PyRwLock, PyRwLockWriteGuard}, convert::ToPyObject, - function::{ArgCallable, FuncArgs, OptionalArg, OptionalOption, PosArgs}, + function::{FuncArgs, OptionalArg, OptionalOption, PosArgs}, protocol::{PyIter, PyIterReturn, PyNumber}, raise_if_stop, stdlib::sys, @@ -26,7 +25,7 @@ mod decl { use num_traits::{Signed, ToPrimitive}; #[pyattr] - #[pyclass(name = "chain")] + #[pyclass(name = "chain", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsChain { source: PyRwLock>, @@ -64,7 +63,7 @@ mod decl { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } } @@ -119,7 +118,7 @@ mod decl { } #[pyattr] - #[pyclass(name = "compress")] + #[pyclass(name = "compress", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsCompress { data: PyIter, @@ -166,7 +165,7 @@ mod decl { } #[pyattr] - #[pyclass(name = "count")] + #[pyclass(name = "count", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsCount { cur: PyRwLock, @@ -237,11 +236,12 @@ mod decl { } #[pyattr] - #[pyclass(name = "cycle")] + #[pyclass(name = "cycle", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsCycle { iter: PyIter, saved: PyRwLock>, + #[pytraverse(skip)] index: AtomicCell, } @@ -273,11 +273,15 @@ mod decl { return Ok(PyIterReturn::StopIteration(None)); } - let last_index = zelf.index.fetch_add(1); - - if last_index >= saved.len() - 1 { - zelf.index.store(0); - } + // Advance and wrap in a single atomic step. A separate + // fetch_add followed by a reset lets a second thread observe + // an index past the end of `saved`. + let last_index = match zelf.index.fetch_update(|index| { + let next = index + 1; + Some(if next < saved.len() { next } else { 0 }) + }) { + Ok(index) | Err(index) => index, + }; saved[last_index].clone() }; @@ -287,10 +291,11 @@ mod decl { } #[pyattr] - #[pyclass(name = "repeat")] + #[pyclass(name = "repeat", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsRepeat { object: PyObjectRef, + #[pytraverse(skip)] times: Option>, } @@ -365,7 +370,7 @@ mod decl { } #[pyattr] - #[pyclass(name = "starmap")] + #[pyclass(name = "starmap", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsStarmap { function: PyObjectRef, @@ -412,11 +417,12 @@ mod decl { } #[pyattr] - #[pyclass(name = "takewhile")] + #[pyclass(name = "takewhile", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsTakewhile { predicate: PyObjectRef, iterable: PyIter, + #[pytraverse(skip)] stop_flag: AtomicCell, } @@ -474,18 +480,19 @@ mod decl { } #[pyattr] - #[pyclass(name = "dropwhile")] + #[pyclass(name = "dropwhile", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsDropwhile { - predicate: ArgCallable, + predicate: PyObjectRef, iterable: PyIter, + #[pytraverse(skip)] start_flag: AtomicCell, } #[derive(FromArgs)] struct DropwhileNewArgs { #[pyarg(positional)] - predicate: ArgCallable, + predicate: PyObjectRef, #[pyarg(positional)] iterable: PyIter, } @@ -522,8 +529,7 @@ mod decl { if !zelf.start_flag.load() { loop { let obj = raise_if_stop!(iterable.next(vm)?); - let pred = predicate.clone(); - let pred_value = pred.invoke((obj.clone(),), vm)?; + let pred_value = predicate.call((obj.clone(),), vm)?; if !pred_value.try_to_bool(vm)? { zelf.start_flag.store(true); return Ok(PyIterReturn::Return(obj)); @@ -534,11 +540,13 @@ mod decl { } } - #[derive(Default)] + #[derive(Default, Traverse)] struct GroupByState { current_value: Option, current_key: Option, + #[pytraverse(skip)] next_group: bool, + #[pytraverse(skip)] grouper: Option>, } @@ -562,7 +570,7 @@ mod decl { } #[pyattr] - #[pyclass(name = "groupby")] + #[pyclass(name = "groupby", traverse)] #[derive(PyPayload)] struct PyItertoolsGroupBy { iterable: PyIter, @@ -662,7 +670,7 @@ mod decl { } #[pyattr] - #[pyclass(name = "_grouper")] + #[pyclass(name = "_grouper", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsGrouper { groupby: PyRef, @@ -704,13 +712,17 @@ mod decl { } #[pyattr] - #[pyclass(name = "islice")] + #[pyclass(name = "islice", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsIslice { iterable: PyIter, + #[pytraverse(skip)] cur: AtomicCell, + #[pytraverse(skip)] next: AtomicCell, + #[pytraverse(skip)] stop: Option, + #[pytraverse(skip)] step: usize, } @@ -829,7 +841,7 @@ mod decl { } #[pyattr] - #[pyclass(name = "filterfalse")] + #[pyclass(name = "filterfalse", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsFilterFalse { predicate: PyObjectRef, @@ -888,7 +900,7 @@ mod decl { } #[pyattr] - #[pyclass(name = "accumulate")] + #[pyclass(name = "accumulate", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsAccumulate { iterable: PyIter, @@ -949,20 +961,25 @@ mod decl { } } - #[derive(Debug)] + #[pyattr] + #[pyclass(name = "_tee_dataobject", traverse)] + #[derive(Debug, PyPayload)] struct PyItertoolsTeeData { iterable: PyIter, values: PyMutex>, + #[pytraverse(skip)] running: AtomicBool, } + #[pyclass(flags(DISALLOW_INSTANTIATION))] impl PyItertoolsTeeData { - fn new(iterable: PyIter, _vm: &VirtualMachine) -> PyRc { - PyRc::new(Self { + fn new(iterable: PyIter, vm: &VirtualMachine) -> PyRef { + Self { iterable, values: PyMutex::new(vec![]), running: AtomicBool::new(false), - }) + } + .into_ref(&vm.ctx) } fn get_item(&self, vm: &VirtualMachine, index: usize) -> PyResult { @@ -975,13 +992,15 @@ mod decl { return Ok(PyIterReturn::Return(values[index].clone())); } } - // Prevent concurrent/reentrant calls to iterable.next() + // Prevent concurrent/reentrant calls to iterable.next(). The claim + // covers caching the value as well: released any earlier, a second + // tee at the same index fetches a value of its own and one of the + // two is dropped without ever reaching a caller. if self.running.swap(true, Ordering::Acquire) { return Err(vm.new_runtime_error("cannot re-enter the tee iterator")); } - let result = self.iterable.next(vm); - self.running.store(false, Ordering::Release); - let obj = raise_if_stop!(result?); + scopeguard::defer! { self.running.store(false, Ordering::Release) } + let obj = raise_if_stop!(self.iterable.next(vm)?); let Some(mut values) = self.values.try_lock() else { return Err(vm.new_runtime_error("cannot re-enter the tee iterator")); }; @@ -993,59 +1012,44 @@ mod decl { } #[pyattr] - #[pyclass(name = "tee")] + #[pyclass(name = "_tee", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsTee { - tee_data: PyRc, + tee_data: PyRef, + #[pytraverse(skip)] index: AtomicCell, - } - - #[derive(FromArgs)] - struct TeeNewArgs { - #[pyarg(positional)] - iterable: PyIter, - #[pyarg(positional, optional)] - n: OptionalArg, + #[pytraverse(skip)] + advancing: AtomicBool, } impl Constructor for PyItertoolsTee { - type Args = TeeNewArgs; - - // TODO: make tee() a function, rename this class to itertools._tee and make - // teedata a python class - fn slot_new(_cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let TeeNewArgs { iterable, n } = args.bind(vm)?; - let n = n.unwrap_or(2); - - let copyable = if iterable.class().has_attr(identifier!(vm, __copy__)) { - vm.call_special_method(iterable.as_object(), identifier!(vm, __copy__), ())? - } else { - Self::from_iter(iterable, vm)? - }; + type Args = PyIter; - let mut tee_vec: Vec = Vec::with_capacity(n); - for _ in 0..n { - tee_vec.push(vm.call_special_method(©able, identifier!(vm, __copy__), ())?); + fn py_new(_cls: &Py, iterator: Self::Args, vm: &VirtualMachine) -> PyResult { + // An iterator that is already a tee shares its buffer rather than + // getting one of its own. + if let Some(tee) = iterator.as_object().downcast_ref::() { + return Ok(tee.__copy__()); } - - Ok(PyTuple::new_ref(tee_vec, &vm.ctx).into()) - } - - fn py_new(_cls: &Py, _args: Self::Args, _vm: &VirtualMachine) -> PyResult { - unimplemented!("use slot_new") + Ok(Self { + tee_data: PyItertoolsTeeData::new(iterator, vm), + index: AtomicCell::new(0), + advancing: AtomicBool::new(false), + }) } } - #[pyclass(with(IterNext, Iterable, Constructor))] + #[pyclass(with(IterNext, Iterable, Constructor), flags(HAS_WEAKREF))] impl PyItertoolsTee { fn from_iter(iterator: PyIter, vm: &VirtualMachine) -> PyResult { let class = Self::class(&vm.ctx); - if iterator.class().is(Self::class(&vm.ctx)) { + if iterator.class().is(class) { return vm.call_special_method(&iterator, identifier!(vm, __copy__), ()); } Ok(Self { tee_data: PyItertoolsTeeData::new(iterator, vm), index: AtomicCell::new(0), + advancing: AtomicBool::new(false), } .into_ref_with_type(vm, class.to_owned())? .into()) @@ -1054,27 +1058,65 @@ mod decl { #[pymethod] fn __copy__(&self) -> Self { Self { - tee_data: PyRc::clone(&self.tee_data), + tee_data: self.tee_data.clone(), index: AtomicCell::new(self.index.load()), + advancing: AtomicBool::new(false), } } } + + #[pyfunction] + fn tee(iterable: PyIter, n: OptionalArg, vm: &VirtualMachine) -> PyResult { + let n = n.unwrap_or(2); + if n < 0 { + return Err(vm.new_value_error("n must be >= 0")); + } + let n = n as usize; + + // Only an iterator that cannot copy itself needs a tee to buffer it. + let copyable = if iterable.class().has_attr(identifier!(vm, __copy__)) { + iterable.into() + } else { + PyItertoolsTee::from_iter(iterable, vm)? + }; + + let mut tee_vec: Vec = Vec::new(); + tee_vec + .try_reserve_exact(n) + .map_err(|_| vm.new_memory_error(""))?; + for _ in 0..n { + tee_vec.push(vm.call_special_method(©able, identifier!(vm, __copy__), ())?); + } + + Ok(PyTuple::new_ref(tee_vec, &vm.ctx)) + } impl SelfIter for PyItertoolsTee {} impl IterNext for PyItertoolsTee { fn next(zelf: &Py, vm: &VirtualMachine) -> PyResult { - let value = raise_if_stop!(zelf.tee_data.get_item(vm, zelf.index.load())?); - zelf.index.fetch_add(1); + // Reading the index and moving it on is one step: two callers that + // read the same index hand out the same value twice and leave the + // buffer to be filled out of order. + if zelf.advancing.swap(true, Ordering::Acquire) { + return Err(vm.new_runtime_error("cannot re-enter the tee iterator")); + } + scopeguard::defer! { zelf.advancing.store(false, Ordering::Release) } + let index = zelf.index.load(); + let value = raise_if_stop!(zelf.tee_data.get_item(vm, index)?); + zelf.index.store(index + 1); Ok(PyIterReturn::Return(value)) } } #[pyattr] - #[pyclass(name = "product")] + #[pyclass(name = "product", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsProduct { pools: Vec>, + #[pytraverse(skip)] idxs: PyRwLock>, + #[pytraverse(skip)] cur: AtomicCell, + #[pytraverse(skip)] stop: AtomicCell, } @@ -1170,13 +1212,16 @@ mod decl { } #[pyattr] - #[pyclass(name = "combinations")] + #[pyclass(name = "combinations", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsCombinations { pool: Vec, + #[pytraverse(skip)] indices: PyRwLock>, result: PyRwLock>>, + #[pytraverse(skip)] r: AtomicCell, + #[pytraverse(skip)] exhausted: AtomicCell, } @@ -1202,13 +1247,21 @@ mod decl { if r.is_negative() { return Err(vm.new_value_error("r must be non-negative")); } - let r = r.to_usize().unwrap(); + let r = r.to_isize().ok_or_else(|| { + vm.new_overflow_error("Python int too large to convert to C ssize_t") + })? as usize; let n = pool.len(); + let mut indices = Vec::new(); + indices + .try_reserve_exact(r) + .map_err(|_| vm.new_memory_error(""))?; + indices.extend(0..r); + Ok(Self { pool, - indices: PyRwLock::new((0..r).collect()), + indices: PyRwLock::new(indices), result: PyRwLock::new(None), r: AtomicCell::new(r), exhausted: AtomicCell::new(r > n), @@ -1281,12 +1334,15 @@ mod decl { } #[pyattr] - #[pyclass(name = "combinations_with_replacement")] + #[pyclass(name = "combinations_with_replacement", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsCombinationsWithReplacement { pool: Vec, + #[pytraverse(skip)] indices: PyRwLock>, + #[pytraverse(skip)] r: AtomicCell, + #[pytraverse(skip)] exhausted: AtomicCell, } @@ -1303,13 +1359,21 @@ mod decl { if r.is_negative() { return Err(vm.new_value_error("r must be non-negative")); } - let r = r.to_usize().unwrap(); + let r = r.to_isize().ok_or_else(|| { + vm.new_overflow_error("Python int too large to convert to C ssize_t") + })? as usize; let n = pool.len(); + let mut indices = Vec::new(); + indices + .try_reserve_exact(r) + .map_err(|_| vm.new_memory_error(""))?; + indices.resize(r, 0); + Ok(Self { pool, - indices: PyRwLock::new(vec![0; r]), + indices: PyRwLock::new(indices), r: AtomicCell::new(r), exhausted: AtomicCell::new(n == 0 && r > 0), }) @@ -1367,15 +1431,20 @@ mod decl { } #[pyattr] - #[pyclass(name = "permutations")] + #[pyclass(name = "permutations", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsPermutations { - pool: Vec, // Collected input iterable - indices: PyRwLock>, // One index per element in pool - cycles: PyRwLock>, // One rollover counter per element in the result + pool: Vec, // Collected input iterable + #[pytraverse(skip)] + indices: PyRwLock>, // One index per element in pool + #[pytraverse(skip)] + cycles: PyRwLock>, // One rollover counter per element in the result + #[pytraverse(skip)] result: PyRwLock>>, // Indexes of the most recently returned result - r: AtomicCell, // Size of result tuple - exhausted: AtomicCell, // Set when the iterator is exhausted + #[pytraverse(skip)] + r: AtomicCell, // Size of result tuple + #[pytraverse(skip)] + exhausted: AtomicCell, // Set when the iterator is exhausted } #[derive(FromArgs)] @@ -1409,7 +1478,9 @@ mod decl { if val.is_negative() { return Err(vm.new_value_error("r must be non-negative")); } - val.to_usize().unwrap() + val.to_isize().ok_or_else(|| { + vm.new_overflow_error("Python int too large to convert to C ssize_t") + })? as usize } None => n, }; @@ -1525,7 +1596,7 @@ mod decl { } #[pyattr] - #[pyclass(name = "zip_longest")] + #[pyclass(name = "zip_longest", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsZipLongest { iterators: Vec, @@ -1563,7 +1634,7 @@ mod decl { } #[pyattr] - #[pyclass(name = "pairwise")] + #[pyclass(name = "pairwise", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsPairwise { iterator: PyIter, @@ -1612,12 +1683,15 @@ mod decl { } #[pyattr] - #[pyclass(name = "batched")] + #[pyclass(name = "batched", traverse)] #[derive(Debug, PyPayload)] struct PyItertoolsBatched { + #[pytraverse(skip)] exhausted: AtomicCell, iterable: PyIter, + #[pytraverse(skip)] n: AtomicCell, + #[pytraverse(skip)] strict: AtomicCell, } diff --git a/crates/vm/src/stdlib/marshal.rs b/crates/vm/src/stdlib/marshal.rs index ace3aff58f2..08a8f589a77 100644 --- a/crates/vm/src/stdlib/marshal.rs +++ b/crates/vm/src/stdlib/marshal.rs @@ -9,14 +9,15 @@ mod decl { use crate::{ PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::{ - PyBool, PyByteArray, PyBytes, PyCode, PyComplex, PyDict, PyEllipsis, PyFloat, - PyFrozenSet, PyInt, PyList, PyNone, PySet, PyStopIteration, PyStr, PyTuple, + PyBaseExceptionRef, PyBool, PyByteArray, PyBytes, PyCode, PyComplex, PyDict, + PyEllipsis, PyFloat, PyFrozenSet, PyInt, PyList, PyNone, PySet, PyStopIteration, PyStr, + PyTuple, }, convert::ToPyObject, function::{ArgBytesLike, OptionalArg}, object::{AsObject, PyPayload}, - protocol::PyBuffer, }; + use core::cell::RefCell; use malachite_bigint::BigInt; use num_traits::Zero; use rustpython_compiler_core::marshal::{self, DumpableValue}; @@ -115,9 +116,6 @@ mod decl { )?; } - if !allow_code { - check_no_code(&value, vm)?; - } check_exact_type(&value, vm)?; let mut buf = Vec::new(); let mut refs = if version >= 3 { @@ -125,12 +123,19 @@ mod decl { } else { None }; - write_object(&mut buf, &value, &mut refs, version, vm)?; + write_object(&mut buf, &value, &mut refs, version, allow_code, vm)?; Ok(PyBytes::from(buf)) } + struct WriterRefEntry { + idx: u32, + /// Set between `reserve` and `complete` for the object kinds whose + /// immutable representation cannot be rebuilt from a back-reference. + incomplete: bool, + } + struct WriterRefTable { - map: std::collections::HashMap, + map: std::collections::HashMap, next_idx: u32, } @@ -141,23 +146,35 @@ mod decl { next_idx: 0, } } - fn try_ref(&mut self, buf: &mut Vec, obj: &PyObjectRef) -> bool { + /// `w_ref`: write a back-reference to an object already in the table. + /// Reaching an entry that is still being written is a recursion the + /// reader could not rebuild, so it is an error rather than a `TYPE_REF`. + fn try_ref(&mut self, buf: &mut Vec, obj: &PyObjectRef) -> Result { use marshal::Write; - let id = obj.get_id(); - if let Some(&idx) = self.map.get(&id) { - buf.write_u8(b'r'); - buf.write_u32(idx); - true - } else { - false + let Some(entry) = self.map.get(&obj.get_id()) else { + return Ok(false); + }; + if entry.incomplete { + return Err(()); } + buf.write_u8(b'r'); + buf.write_u32(entry.idx); + Ok(true) } - fn reserve(&mut self, obj: &PyObjectRef) -> u32 { + fn reserve(&mut self, obj: &PyObjectRef, incomplete: bool) -> u32 { let idx = self.next_idx; - self.map.insert(obj.get_id(), idx); + self.map + .insert(obj.get_id(), WriterRefEntry { idx, incomplete }); self.next_idx += 1; idx } + /// `w_complete`: the object's contents are on the stream, so a later + /// occurrence may reference it. + fn complete(&mut self, obj: &PyObjectRef) { + if let Some(entry) = self.map.get_mut(&obj.get_id()) { + entry.incomplete = false; + } + } } fn write_object( @@ -165,6 +182,7 @@ mod decl { obj: &PyObjectRef, refs: &mut Option, version: i32, + allow_code: bool, vm: &VirtualMachine, ) -> PyResult<()> { write_object_depth( @@ -172,6 +190,7 @@ mod decl { obj, refs, version, + allow_code, vm, marshal::MAX_MARSHAL_STACK_DEPTH, ) @@ -182,6 +201,7 @@ mod decl { obj: &PyObjectRef, refs: &mut Option, version: i32, + allow_code: bool, vm: &VirtualMachine, depth: usize, ) -> PyResult<()> { @@ -197,16 +217,28 @@ mod decl { || obj.downcast_ref::().is_some(); // FLAG_REF: check if already written, otherwise reserve slot - if !is_singleton - && let Some(rt) = refs.as_mut() - && rt.try_ref(buf, obj) - { - return Ok(()); + if !is_singleton && let Some(rt) = refs.as_mut() { + match rt.try_ref(buf, obj) { + Ok(true) => return Ok(()), + Ok(false) => {} + Err(()) => { + return Err(vm.new_value_error(format!( + "cannot marshal recursion {} objects", + obj.class().name() + ))); + } + } } let type_pos = buf.len(); let use_ref = refs.is_some() && !is_singleton; + // A code or slice entry stays incomplete until its contents are + // written: the reader rebuilds both from their fields, so a + // back-reference issued while those fields are still being emitted + // would name an object that does not exist yet. + let requires_completion = obj.downcast_ref::().is_some() + || obj.downcast_ref::().is_some(); if use_ref { - refs.as_mut().unwrap().reserve(obj); + refs.as_mut().unwrap().reserve(obj, requires_completion); } if vm.is_none(obj) { @@ -290,20 +322,20 @@ mod decl { buf.write_u8(b'('); buf.write_u32(t.len() as u32); for elem in t.as_slice() { - write_object_depth(buf, elem, refs, version, vm, depth - 1)?; + write_object_depth(buf, elem, refs, version, allow_code, vm, depth - 1)?; } } else if let Some(l) = obj.downcast_ref::() { buf.write_u8(b'['); let items = l.borrow_vec(); buf.write_u32(items.len() as u32); for elem in items.iter() { - write_object_depth(buf, elem, refs, version, vm, depth - 1)?; + write_object_depth(buf, elem, refs, version, allow_code, vm, depth - 1)?; } } else if let Some(d) = obj.downcast_ref::() { buf.write_u8(b'{'); for (k, v) in d { - write_object_depth(buf, &k, refs, version, vm, depth - 1)?; - write_object_depth(buf, &v, refs, version, vm, depth - 1)?; + write_object_depth(buf, &k, refs, version, allow_code, vm, depth - 1)?; + write_object_depth(buf, &v, refs, version, allow_code, vm, depth - 1)?; } buf.write_u8(b'0'); // TYPE_NULL terminator } else if let Some(s) = obj.downcast_ref::() { @@ -311,18 +343,28 @@ mod decl { let elems = s.elements(); buf.write_u32(elems.len() as u32); for elem in &elems { - write_object_depth(buf, elem, refs, version, vm, depth - 1)?; + write_object_depth(buf, elem, refs, version, allow_code, vm, depth - 1)?; } } else if let Some(s) = obj.downcast_ref::() { buf.write_u8(b'>'); let elems = s.elements(); buf.write_u32(elems.len() as u32); for elem in &elems { - write_object_depth(buf, elem, refs, version, vm, depth - 1)?; + write_object_depth(buf, elem, refs, version, allow_code, vm, depth - 1)?; } } else if let Some(co) = obj.downcast_ref::() { + if !allow_code { + return Err(vm.new_value_error("marshalling code objects is disallowed")); + } buf.write_u8(b'c'); - marshal::serialize_code(buf, &co.code); + // `Literal` holds the exact object a constant was built from, so + // route `co_consts` back through the object writer: it reaches the + // values `BorrowedConstant` cannot describe and shares the one + // reference table the reader indexes against. + marshal::serialize_code_with(buf, &co.code, |buf, constant| { + let constant = PyObjectRef::from(constant.clone()); + write_object_depth(buf, &constant, refs, version, allow_code, vm, depth - 1) + })?; } else if let Some(sl) = obj.downcast_ref::() { if version < 5 { return Err(vm.new_value_error("unmarshallable object")); @@ -334,15 +376,17 @@ mod decl { sl.start.as_ref().unwrap_or(&none), refs, version, + allow_code, vm, depth - 1, )?; - write_object_depth(buf, &sl.stop, refs, version, vm, depth - 1)?; + write_object_depth(buf, &sl.stop, refs, version, allow_code, vm, depth - 1)?; write_object_depth( buf, sl.step.as_ref().unwrap_or(&none), refs, version, + allow_code, vm, depth - 1, )?; @@ -357,6 +401,9 @@ mod decl { if use_ref { buf[type_pos] |= marshal::FLAG_REF; + if requires_completion { + refs.as_mut().unwrap().complete(obj); + } } Ok(()) } @@ -386,79 +433,201 @@ mod decl { } #[derive(Copy, Clone)] - struct PyMarshalBag<'a>(&'a VirtualMachine); + struct PyMarshalBag<'a> { + vm: &'a VirtualMachine, + pending_error: &'a RefCell>, + allow_code: bool, + } + + impl<'a> PyMarshalBag<'a> { + fn new( + vm: &'a VirtualMachine, + pending_error: &'a RefCell>, + allow_code: bool, + ) -> Self { + Self { + vm, + pending_error, + allow_code, + } + } + + /// Room for a container the decoder publishes before it reads what + /// goes in it. The length is the input's to choose, so the room is + /// asked for rather than assumed: a length no allocator can serve is + /// a MemoryError, not an aborted process. + fn placeholder_elements( + &self, + len: usize, + ) -> Result, marshal::MarshalError> { + let mut elements = Vec::new(); + elements + .try_reserve_exact(len) + .map_err(|_| self.remember_python_error(self.vm.new_memory_error("")))?; + elements.resize(len, self.vm.ctx.none()); + Ok(elements) + } + + fn remember_python_error(&self, error: PyBaseExceptionRef) -> marshal::MarshalError { + let mut pending = self.pending_error.borrow_mut(); + if pending.is_none() { + *pending = Some(error); + } + marshal::MarshalError::BadType + } + } impl<'a> marshal::MarshalBag for PyMarshalBag<'a> { type Value = PyObjectRef; type ConstantBag = PyVmBag<'a>; fn make_bool(&self, value: bool) -> Self::Value { - self.0.ctx.new_bool(value).into() + self.vm.ctx.new_bool(value).into() } fn make_none(&self) -> Self::Value { - self.0.ctx.none() + self.vm.ctx.none() } fn make_ellipsis(&self) -> Self::Value { - self.0.ctx.ellipsis.clone().into() + self.vm.ctx.ellipsis.clone().into() } fn make_float(&self, value: f64) -> Self::Value { - self.0.ctx.new_float(value).into() + self.vm.ctx.new_float(value).into() } fn make_complex(&self, value: num_complex::Complex64) -> Self::Value { - self.0.ctx.new_complex(value).into() + self.vm.ctx.new_complex(value).into() } fn make_str(&self, value: &Wtf8) -> Self::Value { - self.0.ctx.new_str(value).into() + self.vm.ctx.new_str(value).into() + } + fn make_interned_str(&self, value: &Wtf8) -> Self::Value { + self.vm.ctx.intern_str(value).to_owned().into() } fn make_bytes(&self, value: &[u8]) -> Self::Value { - self.0.ctx.new_bytes(value.to_vec()).into() + self.vm.ctx.new_bytes(value.to_vec()).into() } fn make_int(&self, value: BigInt) -> Self::Value { - self.0.ctx.new_int(value).into() + self.vm.ctx.new_int(value).into() } fn make_tuple(&self, elements: impl Iterator) -> Self::Value { - self.0.ctx.new_tuple(elements.collect()).into() + self.vm.ctx.new_tuple(elements.collect()).into() } - fn make_code(&self, code: CodeObject) -> Self::Value { - crate::builtins::PyCode::new_ref_with_bag(self.0, code).into() + fn make_tuple_placeholder( + &self, + len: usize, + ) -> Result, marshal::MarshalError> { + let elements = self.placeholder_elements(len)?; + Ok(Some(PyTuple::new_ref(elements, &self.vm.ctx).into())) + } + fn set_tuple_item( + &self, + tuple: &Self::Value, + index: usize, + value: Self::Value, + ) -> Result<(), marshal::MarshalError> { + let tuple = tuple + .downcast_ref::() + .ok_or(marshal::MarshalError::BadType)?; + // SAFETY: compiler-core calls this only on a fresh placeholder, + // once per index, before returning it to Python code. + unsafe { tuple.set_marshal_item(index, value) }; + Ok(()) + } + fn make_code(&self, code: CodeObject) -> Result { + if !self.allow_code { + return Err(self.remember_python_error( + self.vm + .new_value_error("unmarshalling code objects is disallowed"), + )); + } + Ok(crate::builtins::PyCode::new_ref_with_bag(self.vm, code).into()) } fn make_stop_iter(&self) -> Result { - Ok(self.0.ctx.exceptions.stop_iteration.to_owned().into()) + Ok(self.vm.ctx.exceptions.stop_iteration.to_owned().into()) } fn make_list( &self, it: impl Iterator, ) -> Result { - Ok(self.0.ctx.new_list(it.collect()).into()) + Ok(self.vm.ctx.new_list(it.collect()).into()) + } + fn make_list_placeholder( + &self, + len: usize, + ) -> Result, marshal::MarshalError> { + let elements = self.placeholder_elements(len)?; + Ok(Some(self.vm.ctx.new_list(elements).into())) + } + fn set_list_item( + &self, + list: &Self::Value, + index: usize, + value: Self::Value, + ) -> Result<(), marshal::MarshalError> { + let list = list + .downcast_ref::() + .ok_or(marshal::MarshalError::BadType)?; + list.borrow_vec_mut()[index] = value; + Ok(()) } fn make_set( &self, it: impl Iterator, ) -> Result { - let set = PySet::default().into_ref(&self.0.ctx); + let set = PySet::default().into_ref(&self.vm.ctx); for elem in it { - set.add(elem, self.0).unwrap() + set.add(elem, self.vm) + .map_err(|error| self.remember_python_error(error))?; } Ok(set.into()) } + fn make_set_placeholder(&self) -> Option { + Some(PySet::default().into_ref(&self.vm.ctx).into()) + } + fn insert_set_item( + &self, + set: &Self::Value, + value: Self::Value, + ) -> Result<(), marshal::MarshalError> { + let set = set + .downcast_ref::() + .ok_or(marshal::MarshalError::BadType)?; + set.add(value, self.vm) + .map_err(|error| self.remember_python_error(error)) + } fn make_frozenset( &self, it: impl Iterator, ) -> Result { - Ok(PyFrozenSet::from_iter(self.0, it) - .unwrap() - .to_pyobject(self.0)) + PyFrozenSet::from_iter(self.vm, it) + .map(|set| set.to_pyobject(self.vm)) + .map_err(|error| self.remember_python_error(error)) } fn make_dict( &self, it: impl Iterator, ) -> Result { - let dict = self.0.ctx.new_dict(); + let dict = self.vm.ctx.new_dict(); for (k, v) in it { - dict.set_item(&*k, v, self.0).unwrap() + dict.set_item(&*k, v, self.vm) + .map_err(|error| self.remember_python_error(error))?; } Ok(dict.into()) } + fn make_dict_placeholder(&self) -> Option { + Some(self.vm.ctx.new_dict().into()) + } + fn insert_dict_item( + &self, + dict: &Self::Value, + key: Self::Value, + value: Self::Value, + ) -> Result<(), marshal::MarshalError> { + let dict = dict + .downcast_ref::() + .ok_or(marshal::MarshalError::BadType)?; + dict.set_item(&*key, value, self.vm) + .map_err(|error| self.remember_python_error(error)) + } fn make_slice( &self, start: Self::Value, @@ -466,7 +635,7 @@ mod decl { step: Self::Value, ) -> Result { use crate::builtins::PySlice; - let vm = self.0; + let vm = self.vm; Ok(PySlice { start: if vm.is_none(&start) { None @@ -480,37 +649,67 @@ mod decl { .into()) } fn constant_bag(self) -> Self::ConstantBag { - PyVmBag(self.0) + PyVmBag(self.vm) + } + /// `Literal` wraps any object, so a decoded `co_consts` entry is + /// already its own compiler-side constant — no placeholder is needed + /// and `make_code_with_constants` keeps the default. + fn constant_ref_from_value(&self, value: &Self::Value) -> Option { + Some(Literal::from(value.clone())) + } + fn bytes_from_value(&self, value: &Self::Value) -> Option> { + value + .downcast_ref::() + .map(|bytes| bytes.as_bytes().to_vec()) + } + fn str_from_value(&self, value: &Self::Value) -> Option { + value + .downcast_ref::() + .map(|str| str.to_string_lossy().into_owned()) + } + fn tuple_elements_from_value(&self, value: &Self::Value) -> Option> { + value + .downcast_ref::() + .map(|tuple| tuple.as_slice().to_vec()) + } + } + + fn deserialize_value( + rdr: &mut impl marshal::Read, + allow_code: bool, + vm: &VirtualMachine, + ) -> PyResult { + let pending_error = RefCell::new(None); + match marshal::deserialize_value(rdr, PyMarshalBag::new(vm, &pending_error, allow_code)) { + Ok(value) => Ok(value), + Err(error) => Err(pending_error.into_inner().unwrap_or_else(|| match error { + marshal::MarshalError::Eof => vm.new_eof_error("marshal data too short"), + error @ marshal::MarshalError::NullObject => vm.new_type_error(error.to_string()), + error @ (marshal::MarshalError::BadSize(_) + | marshal::MarshalError::UnknownType + | marshal::MarshalError::InvalidRef) => { + vm.new_value_error(format!("bad marshal data ({error})")) + } + _ => vm.new_value_error("bad marshal data"), + })), } } #[derive(FromArgs)] struct LoadsArgs { #[pyarg(any)] - data: PyBuffer, + // marshal_loads_impl takes `bytes: Py_buffer`, a y* argument. + data: ArgBytesLike, #[pyarg(named, default = true)] allow_code: bool, } #[pyfunction] fn loads(args: LoadsArgs, vm: &VirtualMachine) -> PyResult { - let LoadsArgs { - data: pybuffer, - allow_code, - } = args; - let buf = pybuffer.as_contiguous().ok_or_else(|| { - vm.new_buffer_error("Buffer provided to marshal.loads() is not contiguous") - })?; + let LoadsArgs { data, allow_code } = args; + let buf = data.borrow_buf(); - let result = - marshal::deserialize_value(&mut &buf[..], PyMarshalBag(vm)).map_err(|e| match e { - marshal::MarshalError::Eof => vm.new_eof_error("marshal data too short"), - _ => vm.new_value_error("bad marshal data"), - })?; - if !allow_code { - check_no_code(&result, vm)?; - } - Ok(result) + deserialize_value(&mut &buf[..], allow_code, vm) } #[derive(FromArgs)] @@ -530,61 +729,25 @@ mod decl { .try_into_value::(vm)?; let read_res = vm.call_method(&args.f, "read", ())?; let bytes = ArgBytesLike::try_from_object(vm, read_res)?; - let buf = bytes.borrow_buf(); - - let mut rdr: &[u8] = &buf; - let len_before = rdr.len(); - let result = - marshal::deserialize_value(&mut rdr, PyMarshalBag(vm)).map_err(|e| match e { - marshal::MarshalError::Eof => vm.new_exception_msg( - vm.ctx.exceptions.eof_error.to_owned(), - "marshal data too short".into(), - ), - _ => vm.new_value_error("bad marshal data"), - })?; - let consumed = len_before - rdr.len(); + + // The borrow ends here: seek() below is the caller's, and reaching the + // same buffer from it would deadlock on a borrow still held. + let (result, consumed) = { + let buf = bytes.borrow_buf(); + let mut rdr: &[u8] = &buf; + let len_before = rdr.len(); + let result = deserialize_value(&mut rdr, args.allow_code, vm)?; + (result, len_before - rdr.len()) + }; // Seek file to just after the consumed bytes let new_pos = tell_before + consumed as i64; vm.call_method(&args.f, "seek", (new_pos,))?; - if !args.allow_code { - check_no_code(&result, vm)?; - } Ok(result) } /// Reject subclasses of marshallable types (int, float, complex, tuple, etc.). - /// Recursively check that no code objects are present. - fn check_no_code(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - if obj.downcast_ref::().is_some() { - return Err(vm.new_value_error("unmarshalling code objects is disallowed")); - } - if let Some(tup) = obj.downcast_ref::() { - for elem in tup.as_slice() { - check_no_code(elem, vm)?; - } - } else if let Some(list) = obj.downcast_ref::() { - for elem in list.borrow_vec().iter() { - check_no_code(elem, vm)?; - } - } else if let Some(set) = obj.downcast_ref::() { - for elem in set.elements() { - check_no_code(&elem, vm)?; - } - } else if let Some(fset) = obj.downcast_ref::() { - for elem in fset.elements() { - check_no_code(&elem, vm)?; - } - } else if let Some(dict) = obj.downcast_ref::() { - for (k, v) in dict { - check_no_code(&k, vm)?; - check_no_code(&v, vm)?; - } - } - Ok(()) - } - fn check_exact_type(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { let cls = obj.class(); // bool is a subclass of int but is marshallable diff --git a/crates/vm/src/stdlib/nt.rs b/crates/vm/src/stdlib/nt.rs index 31a08195c58..26412e352ce 100644 --- a/crates/vm/src/stdlib/nt.rs +++ b/crates/vm/src/stdlib/nt.rs @@ -19,7 +19,7 @@ pub(crate) mod module { use libc::intptr_t; use rustpython_common::wtf8::Wtf8Buf; use rustpython_host_env::nt as host_nt; - use std::os::windows::ffi::OsStringExt; + use std::os::windows::ffi::{OsStrExt, OsStringExt}; use std::os::windows::io::AsRawHandle; #[pyattr] @@ -49,6 +49,26 @@ pub(crate) mod module { #[pyattr] const TMP_MAX: i32 = i32::MAX; + fn utf8_from_bytes<'a>(bytes: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a str> { + core::str::from_utf8(bytes).map_err(|err| { + let reason = match err.error_len() { + None => "unexpected end of data", + Some(_) => match bytes[err.valid_up_to()] { + 0xc2..=0xf4 => "invalid continuation byte", + _ => "invalid start byte", + }, + }; + vm.new_unicode_decode_error( + vm.ctx.new_str("utf-8"), + vm.ctx.new_bytes(bytes.to_vec()), + err.valid_up_to(), + err.error_len() + .map_or(bytes.len(), |len| err.valid_up_to() + len), + vm.ctx.new_str(reason), + ) + }) + } + #[pyattr] use host_nt::{ LOAD_LIBRARY_SEARCH_APPLICATION_DIR as _LOAD_LIBRARY_SEARCH_APPLICATION_DIR, @@ -214,11 +234,8 @@ pub(crate) mod module { fn _findfirstfile(path: OsPath, vm: &VirtualMachine) -> PyResult { let filename = host_nt::find_first_file_name(path.as_ref()) .map_err(|err| OSErrorBuilder::with_filename(&err, path.clone(), vm))?; - let filename_str = filename - .to_str() - .ok_or_else(|| vm.new_unicode_decode_error("filename contains invalid UTF-8"))?; - - Ok(vm.ctx.new_str(filename_str)) + let filename_wide: Vec<_> = filename.encode_wide().collect(); + Ok(vm.ctx.new_str(Wtf8Buf::from_wide(&filename_wide))) } #[derive(FromArgs)] @@ -689,17 +706,7 @@ pub(crate) mod module { (wide, false) } else if let Some(b) = path.downcast_ref::() { // On Windows, bytes must be valid UTF-8 - this raises UnicodeDecodeError if not - let s = core::str::from_utf8(b.as_bytes()).map_err(|e| { - vm.new_exception_msg( - vm.ctx.exceptions.unicode_decode_error.to_owned(), - format!( - "'utf-8' codec can't decode byte {:#x} in position {}: invalid start byte", - b.as_bytes().get(e.valid_up_to()).copied().unwrap_or(0), - e.valid_up_to() - ) - .into(), - ) - })?; + let s = utf8_from_bytes(b.as_bytes(), vm)?; let wide: Vec = s.encode_utf16().collect(); (wide, true) } else { @@ -720,16 +727,13 @@ pub(crate) mod module { // Return as bytes if input was bytes, preserving the original content if is_bytes { // Convert UTF-16 back to UTF-8 for bytes output - let drv = String::from_utf16(&wide[..drv_size]) - .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?; - let root = String::from_utf16(&wide[drv_size..drv_size + root_size]) - .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?; - let tail = String::from_utf16(&wide[drv_size + root_size..]) - .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?; + let drv = Wtf8Buf::from_wide(&wide[..drv_size]).into_bytes(); + let root = Wtf8Buf::from_wide(&wide[drv_size..drv_size + root_size]).into_bytes(); + let tail = Wtf8Buf::from_wide(&wide[drv_size + root_size..]).into_bytes(); Ok(vm.ctx.new_tuple(vec![ - vm.ctx.new_bytes(drv.into_bytes()).into(), - vm.ctx.new_bytes(root.into_bytes()).into(), - vm.ctx.new_bytes(tail.into_bytes()).into(), + vm.ctx.new_bytes(drv).into(), + vm.ctx.new_bytes(root).into(), + vm.ctx.new_bytes(tail).into(), ])) } else { // For str output, use WTF-8 to handle surrogates @@ -913,17 +917,7 @@ pub(crate) mod module { let wide: Vec = s.as_wtf8().encode_wide().collect(); (wide, false) } else if let Some(b) = path.downcast_ref::() { - let s = core::str::from_utf8(b.as_bytes()).map_err(|e| { - vm.new_exception_msg( - vm.ctx.exceptions.unicode_decode_error.to_owned(), - format!( - "'utf-8' codec can't decode byte {:#x} in position {}: invalid start byte", - b.as_bytes().get(e.valid_up_to()).copied().unwrap_or(0), - e.valid_up_to() - ) - .into(), - ) - })?; + let s = utf8_from_bytes(b.as_bytes(), vm)?; let wide: Vec = s.encode_utf16().collect(); (wide, true) } else { @@ -936,9 +930,8 @@ pub(crate) mod module { let normalized = normpath_wide(&wide); if is_bytes { - let s = String::from_utf16(&normalized) - .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?; - Ok(vm.ctx.new_bytes(s.into_bytes()).into()) + let bytes = Wtf8Buf::from_wide(&normalized).into_bytes(); + Ok(vm.ctx.new_bytes(bytes).into()) } else { let s = Wtf8Buf::from_wide(&normalized); Ok(vm.ctx.new_str(s).into()) diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs index a934c6d812f..dccc0ae7e47 100644 --- a/crates/vm/src/stdlib/os.rs +++ b/crates/vm/src/stdlib/os.rs @@ -131,7 +131,7 @@ pub(super) struct FollowSymlinks( #[cfg(not(windows))] fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> { rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| { - vm.new_unicode_decode_error_real( + vm.new_unicode_decode_error( vm.ctx.new_str("utf-8"), vm.ctx.new_bytes(b.to_vec()), e.valid_up_to(), @@ -206,7 +206,10 @@ pub(super) mod _os { ospath::{OsPath, OsPathOrFd, OutputMode, PathConverter}, protocol::PyIterReturn, recursion::ReprGuard, - types::{Destructor, IterNext, Iterable, PyStructSequence, Representable, SelfIter}, + types::{ + Destructor, IterNext, Iterable, PyStructSequence, PyStructSequenceData, Representable, + SelfIter, + }, vm::VirtualMachine, }; #[cfg(not(windows))] @@ -883,7 +886,7 @@ pub(super) mod _os { cls: PyTypeRef, args: PyObjectRef, vm: &VirtualMachine, - ) -> PyGenericAlias { + ) -> PyResult { PyGenericAlias::from_args(cls, args, vm) } @@ -1314,8 +1317,12 @@ pub(super) mod _os { impl PyStatResult { #[pyslot] fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let seq: PyObjectRef = args.bind(vm)?; - let result = crate::types::struct_sequence_new(cls.clone(), seq, vm)?; + let result = crate::types::struct_sequence_new( + cls.clone(), + args.bind(vm)?, + StatResultData::OPTIONAL_FIELD_NAMES, + vm, + )?; let tuple = result.downcast_ref::().unwrap(); let mut items: Vec = tuple.to_vec(); @@ -1964,8 +1971,12 @@ pub(super) mod _os { impl PyStatvfsResult { #[pyslot] fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let seq: PyObjectRef = args.bind(vm)?; - crate::types::struct_sequence_new(cls, seq, vm) + crate::types::struct_sequence_new( + cls, + args.bind(vm)?, + StatvfsResultData::OPTIONAL_FIELD_NAMES, + vm, + ) } } diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index c16da1ee703..9233af9fbe0 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -626,10 +626,17 @@ pub mod module { crate::stdlib::_imp::acquire_imp_lock_for_fork(vm); #[cfg(feature = "threading")] - vm.state.stop_the_world.stop_the_world(vm); + vm.state.stop_the_world.stop_the_world(&vm.state); } fn py_os_after_fork_child(vm: &VirtualMachine) { + // The interpreter registry is reachable from every thread, so repair it + // before anything enumerates interpreters. + #[cfg(all(unix, feature = "threading"))] + unsafe { + crate::vm::runtime::reinit_after_fork() + }; + #[cfg(feature = "threading")] vm.state.stop_the_world.reset_after_fork(); @@ -639,6 +646,12 @@ pub mod module { #[cfg(feature = "threading")] reinit_locks_after_fork(vm); + // The collector stops every interpreter, so interpreters other than the + // forking one must be repaired too; otherwise the child's first + // collection waits for threads that did not survive the fork. + #[cfg(all(unix, feature = "threading"))] + reinit_other_interpreters_after_fork(vm); + // Reinit per-object IO buffer locks on std streams. // BufferedReader/Writer/TextIOWrapper use PyThreadMutex which can be // held by dead parent threads, causing deadlocks on any IO in the child. @@ -719,17 +732,67 @@ pub mod module { // Codec registry RwLock vm.state.codec_registry.reinit_after_fork(); - // GC state (multiple Mutex + RwLock) + // GC state (multiple Mutex + RwLock), shared lists and this + // interpreter's own policy state. crate::gc_state::gc_state().reinit_after_fork(); + vm.state.gc.reinit_after_fork(); // Import lock (RawReentrantMutex) crate::stdlib::_imp::reinit_imp_lock_after_fork(); } } + /// Repair every live interpreter other than the forking one after `fork()`. + /// + /// Only the forking thread survives, so each other interpreter is left with + /// slots for threads that no longer exist (still ATTACHED if they were + /// running bytecode) and possibly locks or stop-the-world flags held by + /// them. Since a collection stops all interpreters, that state would hang + /// the child's first collection. + /// + /// # Safety + /// Must only be called after `fork()` in the child, when no other threads exist. + #[cfg(all(unix, feature = "threading"))] + fn reinit_other_interpreters_after_fork(vm: &VirtualMachine) { + use rustpython_common::lock::reinit_mutex_after_fork; + + for state in crate::vm::runtime::live_interpreter_states() { + if state.interpreter_id == vm.state.interpreter_id { + continue; + } + + unsafe { + reinit_mutex_after_fork(&state.before_forkers); + reinit_mutex_after_fork(&state.after_forkers_child); + reinit_mutex_after_fork(&state.after_forkers_parent); + reinit_mutex_after_fork(&state.atexit_funcs); + reinit_mutex_after_fork(&state.global_trace_func); + reinit_mutex_after_fork(&state.global_profile_func); + reinit_mutex_after_fork(&state.type_mutex); + reinit_mutex_after_fork(&state.monitoring); + reinit_mutex_after_fork(&state.thread_frames); + reinit_mutex_after_fork(&state.thread_handles); + reinit_mutex_after_fork(&state.shutdown_handles); + + state.codec_registry.reinit_after_fork(); + state.gc.reinit_after_fork(); + } + + state.stop_the_world.reset_after_fork(); + + // Every thread registered here belongs to the parent, including any + // slot the forking thread itself registered before the fork. + state.thread_frames.lock().clear(); + state.thread_handles.lock().clear(); + state.shutdown_handles.lock().clear(); + } + + crate::vm::thread::purge_other_interpreter_slots_after_fork(vm.state.interpreter_id); + } + fn py_os_after_fork_parent(vm: &VirtualMachine) { #[cfg(feature = "threading")] - vm.state.stop_the_world.start_the_world(vm); + vm.state.stop_the_world.start_the_world(&vm.state); #[cfg(feature = "threading")] crate::stdlib::_imp::release_imp_lock_after_fork_parent(); @@ -1277,8 +1340,20 @@ pub mod module { #[pyfunction] fn uname(vm: &VirtualMachine) -> PyResult<_os::UnameResultData> { - let info = rustpython_host_env::posix::uname_info() - .map_err(|err| vm.new_unicode_decode_error(err.to_string()))?; + let info = rustpython_host_env::posix::uname_info().map_err(|err| { + let start = err.error.valid_up_to(); + let end = err + .error + .error_len() + .map_or(err.bytes.len(), |len| start + len); + vm.new_unicode_decode_error( + vm.ctx.new_str("utf-8"), + vm.ctx.new_bytes(err.bytes), + start, + end, + vm.ctx.new_str(err.error.to_string()), + ) + })?; Ok(_os::UnameResultData { sysname: info.sysname, nodename: info.nodename, @@ -1333,14 +1408,13 @@ pub mod module { // cfg from nix #[cfg(not(any(target_os = "ios", target_os = "macos", target_os = "redox")))] #[pyfunction] - fn setgroups( - group_ids: crate::function::ArgIterable, - vm: &VirtualMachine, - ) -> PyResult<()> { - let gids = group_ids - .iter(vm)? - .map(|gid| gid.map(|gid| gid.0)) - .collect::, _>>()?; + fn setgroups(group_ids: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + group_ids + .try_sequence(vm) + .map_err(|_| vm.new_type_error("setgroups argument must be a sequence"))?; + let gids = vm.extract_elements_with(&group_ids, |gid| { + RawGid::try_from_object(vm, gid).map(|gid| gid.0) + })?; rustpython_host_env::posix::setgroups_raw(&gids).map_err(|err| err.into_pyexception(vm)) } @@ -1400,7 +1474,7 @@ pub mod module { #[pyarg(positional)] path: OsPath, #[pyarg(positional)] - args: crate::function::ArgIterable, + args: PyObjectRef, #[pyarg(positional)] env: Option, #[pyarg(named, default)] @@ -1439,6 +1513,19 @@ pub mod module { .into_cstring(vm) .map_err(|_| vm.new_value_error("path should not have nul bytes"))?; + let function_name = if spawnp { + "posix_spawnp" + } else { + "posix_spawn" + }; + if !self.args.fast_isinstance(vm.ctx.types.list_type) + && !self.args.fast_isinstance(vm.ctx.types.tuple_type) + { + return Err( + vm.new_type_error(format!("{function_name}: argv must be a tuple or list")) + ); + } + let mut file_actions = Vec::new(); if let Some(it) = self.file_actions { for action in it.iter(vm)? { @@ -1478,20 +1565,21 @@ pub mod module { } } - let setsigdef = self - .setsigdef - .map(|sigs| { - let sigs = sigs.iter(vm)?.collect::>>()?; - for &sig in &sigs { - if !rustpython_host_env::posix::validate_posix_spawn_signal(sig) { - return Err( - vm.new_value_error(format!("signal number {sig} out of range")) - ); - } + let collect_signals = |sigs: crate::function::ArgIterable| { + let mut collected = Vec::new(); + for sig in sigs.iter(vm)? { + let sig = sig?; + if !rustpython_host_env::posix::validate_posix_spawn_signal(sig) { + return Err(vm.new_value_error(format!("signal number {sig} out of range"))); } - Ok(sigs) - }) - .transpose()?; + if !collected.contains(&sig) { + collected.push(sig); + } + } + Ok(collected) + }; + + let setsigdef = self.setsigdef.map(&collect_signals).transpose()?; if let Some(_scheduler) = self.scheduler { // TODO: Implement scheduler parameter handling @@ -1507,29 +1595,12 @@ pub mod module { )); } - let setsigmask = self - .setsigmask - .map(|sigs| { - let sigs = sigs.iter(vm)?.collect::>>()?; - for &sig in &sigs { - if !rustpython_host_env::posix::validate_posix_spawn_signal(sig) { - return Err( - vm.new_value_error(format!("signal number {sig} out of range")) - ); - } - } - Ok(sigs) - }) - .transpose()?; + let setsigmask = self.setsigmask.map(collect_signals).transpose()?; - let args: Vec = self - .args - .iter(vm)? - .map(|res| { - CString::new(res?.into_bytes()) - .map_err(|_| vm.new_value_error("path should not have nul bytes")) - }) - .collect::>()?; + let args = vm.extract_elements_with(&self.args, |arg| { + CString::new(OsPath::try_from_object(vm, arg)?.into_bytes()) + .map_err(|_| vm.new_value_error("path should not have nul bytes")) + })?; let env = if let Some(env_dict) = self.env { envp_from_dict(env_dict, vm)? } else { @@ -1732,7 +1803,7 @@ pub mod module { return Err(vm.new_os_error("unable to determine login name")); }; login.to_str().map(|s| s.to_owned()).map_err(|e| { - vm.new_unicode_decode_error_real( + vm.new_unicode_decode_error( vm.ctx.new_str("utf-8"), vm.ctx.new_bytes(login.as_bytes().to_vec()), e.valid_up_to(), diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index c7cc2fd298a..5ee36b450d4 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -669,7 +669,8 @@ pub mod sys { "_multiarch" => ctx.new_str(multiarch()), "version" => PyVersionInfo::from_data(VersionInfoData::IMPLEMENTATION, vm), "hexversion" => ctx.new_int(version::VERSION_HEX_IMPL), - "supports_isolated_interpreters" => ctx.new_bool(false), + "supports_isolated_interpreters" => + ctx.new_bool(crate::vm::runtime::SUPPORTS_ISOLATED_INTERPRETERS), }) } @@ -888,8 +889,7 @@ pub mod sys { format!("Ignoring unimportable $PYTHONBREAKPOINT: \"{env_var}\"",), 0, vm, - ) - .unwrap(); + )?; Ok(vm.ctx.none()) }; diff --git a/crates/vm/src/stdlib/sys/monitoring.rs b/crates/vm/src/stdlib/sys/monitoring.rs index 7e47185bbe5..a4be3ba5a5a 100644 --- a/crates/vm/src/stdlib/sys/monitoring.rs +++ b/crates/vm/src/stdlib/sys/monitoring.rs @@ -529,7 +529,7 @@ fn update_events_mask(vm: &VirtualMachine, state: &MonitoringState) { // own local events), preventing e.g. INSTRUCTION from being applied to // unrelated code objects. // Re-instrument all frames on the current thread's stack, including - // stack-allocated iframes (with_iframe path) that have no FrameObject. + // data stack frames that have no FrameObject. { let mut cur = crate::vm::thread::get_current_frame(); while !cur.is_null() { diff --git a/crates/vm/src/stdlib/time.rs b/crates/vm/src/stdlib/time.rs index 3d777c24b89..a5daa9cd2ff 100644 --- a/crates/vm/src/stdlib/time.rs +++ b/crates/vm/src/stdlib/time.rs @@ -18,7 +18,7 @@ mod decl { AsObject, Py, PyObjectRef, PyResult, VirtualMachine, builtins::{PyStrRef, PyTypeRef}, function::{Either, FuncArgs, OptionalArg}, - types::{PyStructSequence, struct_sequence_new}, + types::{PyStructSequence, PyStructSequenceData, struct_sequence_new}, }; #[cfg(any(unix, windows))] use crate::{ @@ -811,8 +811,12 @@ mod decl { impl PyStructTime { #[pyslot] fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { - let (seq, _dict): (PyObjectRef, OptionalArg) = args.bind(vm)?; - struct_sequence_new(cls, seq, vm) + struct_sequence_new( + cls, + args.bind(vm)?, + StructTimeData::OPTIONAL_FIELD_NAMES, + vm, + ) } } diff --git a/crates/vm/src/stdlib/typevar.rs b/crates/vm/src/stdlib/typevar.rs index 3e2581406e8..b784d8799f6 100644 --- a/crates/vm/src/stdlib/typevar.rs +++ b/crates/vm/src/stdlib/typevar.rs @@ -923,11 +923,12 @@ pub(crate) mod typevar { impl Representable for ParamSpecArgs { #[inline(always)] fn repr_str(zelf: &crate::Py, vm: &VirtualMachine) -> PyResult { - // Check if origin is a ParamSpec - if let Ok(name) = zelf.__origin__.get_attr("__name__", vm) { - return Ok(format!("{name}.args", name = name.str(vm)?)); + // A ParamSpec origin is named; anything else is shown by its repr, + // which carries the recursion guard a Rust `{:?}` walk does not. + if let Some(param_spec) = zelf.__origin__.downcast_ref::() { + return Ok(format!("{}.args", param_spec.__name__().str_utf8(vm)?)); } - Ok(format!("{:?}.args", zelf.__origin__)) + Ok(format!("{}.args", zelf.__origin__.repr(vm)?)) } } @@ -986,11 +987,12 @@ pub(crate) mod typevar { impl Representable for ParamSpecKwargs { #[inline(always)] fn repr_str(zelf: &crate::Py, vm: &VirtualMachine) -> PyResult { - // Check if origin is a ParamSpec - if let Ok(name) = zelf.__origin__.get_attr("__name__", vm) { - return Ok(format!("{name}.kwargs", name = name.str(vm)?)); + // A ParamSpec origin is named; anything else is shown by its repr, + // which carries the recursion guard a Rust `{:?}` walk does not. + if let Some(param_spec) = zelf.__origin__.downcast_ref::() { + return Ok(format!("{}.kwargs", param_spec.__name__().str_utf8(vm)?)); } - Ok(format!("{:?}.kwargs", zelf.__origin__)) + Ok(format!("{}.kwargs", zelf.__origin__.repr(vm)?)) } } diff --git a/crates/vm/src/stdlib/winsound.rs b/crates/vm/src/stdlib/winsound.rs index 95032ad8970..091a3f801aa 100644 --- a/crates/vm/src/stdlib/winsound.rs +++ b/crates/vm/src/stdlib/winsound.rs @@ -6,10 +6,10 @@ pub(crate) use winsound::module_def; #[pymodule] mod winsound { use crate::builtins::{PyBaseExceptionRef, PyBytes, PyStr}; - use crate::convert::{IntoPyException, ToPyException, TryFromBorrowedObject}; + use crate::convert::{IntoPyException, ToPyException}; use crate::exceptions; use crate::host_env::windows::ToWideString; - use crate::protocol::PyBuffer; + use crate::protocol::{BufferFlags, PyBuffer}; use crate::{AsObject, PyObjectRef, PyResult, VirtualMachine}; use rustpython_host_env::winsound::{PlaySoundError, PlaySoundSource, play_sound}; @@ -90,7 +90,7 @@ mod winsound { } if flags & SND_MEMORY != 0 { - let buffer = PyBuffer::try_from_borrowed_object(vm, &sound)?; + let buffer = PyBuffer::from_object(vm, &sound, BufferFlags::SIMPLE)?; let buf = buffer .as_contiguous() .ok_or_else(|| vm.new_type_error("a bytes-like object is required, not 'str'"))?; diff --git a/crates/vm/src/types/mod.rs b/crates/vm/src/types/mod.rs index b17a737545f..11c3a4dc51e 100644 --- a/crates/vm/src/types/mod.rs +++ b/crates/vm/src/types/mod.rs @@ -5,5 +5,7 @@ mod zoo; pub use slot::*; pub use slot_defs::{SLOT_DEFS, SlotAccessor, SlotDef}; -pub use structseq::{PyStructSequence, PyStructSequenceData, struct_sequence_new}; +pub use structseq::{ + PyStructSequence, PyStructSequenceData, StructSequenceNewArgs, struct_sequence_new, +}; pub(crate) use zoo::TypeZoo; diff --git a/crates/vm/src/types/slot.rs b/crates/vm/src/types/slot.rs index d834406cf80..c0b9142c780 100644 --- a/crates/vm/src/types/slot.rs +++ b/crates/vm/src/types/slot.rs @@ -9,7 +9,7 @@ use crate::{ convert::ToPyObject, function::{Either, FromArgs, FuncArgs, PyComparisonValue, PyMethodDef, PySetterValue}, protocol::{ - PyBuffer, PyIterReturn, PyMapping, PyMappingMethods, PyMappingSlots, PyNumber, + BufferFlags, PyBuffer, PyIterReturn, PyMapping, PyMappingMethods, PyMappingSlots, PyNumber, PyNumberMethods, PyNumberSlots, PySequence, PySequenceMethods, PySequenceSlots, }, types::slot_defs::{SlotAccessor, find_slot_defs_by_name}, @@ -149,7 +149,12 @@ pub struct PyTypeSlots { pub setattro: AtomicCell>, // Functions to access object as input/output buffer - pub as_buffer: Option, + pub as_buffer: AtomicCell>, + /// bf_releasebuffer: releasing an export of this type is observable, so the + /// type exposes `__release_buffer__`. + pub has_release_buffer: AtomicCell, + /// True when a Python-level `__release_buffer__` must be invoked on release. + pub python_release_buffer: AtomicCell, // Assigned meaning in release 2.1 // rich comparisons @@ -296,7 +301,8 @@ pub(crate) type StringifyFunc = fn(&PyObject, &VirtualMachine) -> PyResult, &VirtualMachine) -> PyResult; pub(crate) type SetattroFunc = fn(&PyObject, &Py, PySetterValue, &VirtualMachine) -> PyResult<()>; -pub(crate) type AsBufferFunc = fn(&PyObject, &VirtualMachine) -> PyResult; +/// bf_getbuffer +pub(crate) type AsBufferFunc = fn(&PyObject, BufferFlags, &VirtualMachine) -> PyResult; pub(crate) type RichCompareFunc = fn( &PyObject, &PyObject, @@ -329,6 +335,15 @@ pub(crate) type MapSubscriptFunc = fn(PyMapping<'_>, &PyObject, &VirtualMachine) pub(crate) type MapAssSubscriptFunc = fn(PyMapping<'_>, &PyObject, Option, &VirtualMachine) -> PyResult<()>; +// slot_bf_getbuffer +pub(crate) fn python_as_buffer( + obj: &PyObject, + flags: BufferFlags, + vm: &VirtualMachine, +) -> PyResult { + crate::builtins::memory::buffer_from_python_getbuffer(obj, flags, vm) +} + // slot_sq_length pub(crate) fn len_wrapper(obj: &PyObject, vm: &VirtualMachine) -> PyResult { let ret = vm.call_special_method(obj, identifier!(vm, __len__), ())?; @@ -512,7 +527,11 @@ pub fn hash_not_implemented(zelf: &PyObject, vm: &VirtualMachine) -> PyResult PyResult { - vm.call_special_method(zelf, identifier!(vm, __call__), args) + // `__call__` can name the object being called, and dispatching it pushes no + // Python frame, so nothing else counts the nesting. + vm.with_recursion("while calling a Python object", || { + vm.call_special_method(zelf, identifier!(vm, __call__), args) + }) } fn getattro_wrapper(zelf: &PyObject, name: &Py, vm: &VirtualMachine) -> PyResult { @@ -601,7 +620,11 @@ fn descr_get_wrapper( cls: Option, vm: &VirtualMachine, ) -> PyResult { - vm.call_special_method(&zelf, identifier!(vm, __get__), (obj, cls)) + // A descriptor whose `__get__` is the descriptor itself resolves it by + // fetching `__get__` again, and none of that pushes a Python frame. + vm.with_recursion("while calling a Python object", || { + vm.call_special_method(&zelf, identifier!(vm, __get__), (obj, cls)) + }) } fn descr_set_wrapper( @@ -1579,6 +1602,58 @@ impl PyType { } } + // === Buffer protocol === + SlotAccessor::BfGetBuffer => { + if ADD { + match self.lookup_slot_in_mro(name, ctx, |sf| { + if let SlotFunc::GetBuffer(f) = sf { + Some(*f) + } else { + None + } + }) { + SlotLookupResult::NativeSlot(func) => { + self.slots.as_buffer.store(Some(func)); + } + SlotLookupResult::PythonMethod => { + self.slots.as_buffer.store(Some(python_as_buffer)); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } + } + } else { + accessor.inherit_from_mro(self); + } + } + SlotAccessor::BfReleaseBuffer => { + // Which of the two implementations `__release_buffer__` resolves to + // decides whether buffer release has to call back into Python. + if ADD { + match self.lookup_slot_in_mro(name, ctx, |sf| { + if matches!(sf, SlotFunc::ReleaseBuffer) { + Some(()) + } else { + None + } + }) { + SlotLookupResult::NativeSlot(()) => { + self.slots.python_release_buffer.store(false); + self.slots.has_release_buffer.store(true); + } + SlotLookupResult::PythonMethod => { + self.slots.python_release_buffer.store(true); + self.slots.has_release_buffer.store(true); + } + SlotLookupResult::NotFound => { + accessor.inherit_from_mro(self); + } + } + } else { + accessor.inherit_from_mro(self); + } + } + // Reserved slots - no-op _ => {} } @@ -1995,6 +2070,29 @@ impl PyComparisonOp { self.map_eq(|| a.borrow().is(b.borrow())) } + /// The answer to this comparison for two operands that `equal` reports as + /// equal or not, or `None` for an ordering operator, which equality alone + /// cannot settle -- `equal` is not called in that case. + /// + /// This is what lets a type answer `==` and `!=` with an equality test + /// rather than with an ordering: the two agree on the answer, but equality + /// can settle a length mismatch without looking at the contents at all. + /// + /// The two neighbouring helpers answer different questions: [`Self::map_eq`] + /// answers only where its predicate holds, so a caller still handles the + /// other side, and [`Self::eq_only`] declares the comparison + /// `NotImplemented` for an ordering operator. This one leaves the ordering + /// operators to the caller, which is what a type with a real ordering + /// needs. + #[inline] + pub fn eval_eq(self, equal: impl FnOnce() -> bool) -> Option { + match self { + Self::Eq => Some(equal()), + Self::Ne => Some(!equal()), + _ => None, + } + } + /// Returns `Some(true)` when self is `Eq` and `f()` returns true. Returns `Some(false)` when self /// is `Ne` and `f()` returns true. Otherwise returns `None`. #[inline] @@ -2047,14 +2145,29 @@ pub trait SetAttr: PyPayload { #[pyclass] pub trait AsBuffer: PyPayload { - // TODO: `flags` parameter + /// bf_releasebuffer: set when releasing an export of this type is observable, + /// i.e. the exporter counts exports. Such types expose `__release_buffer__`. + const RELEASE_BUFFER: bool = false; + #[inline] #[pyslot] - fn slot_as_buffer(zelf: &PyObject, vm: &VirtualMachine) -> PyResult { + fn slot_as_buffer( + zelf: &PyObject, + flags: BufferFlags, + vm: &VirtualMachine, + ) -> PyResult { let zelf = zelf .downcast_ref() .ok_or_else(|| vm.new_type_error("unexpected payload for as_buffer"))?; - Self::as_buffer(zelf, vm) + let buffer = Self::as_buffer(zelf, vm)?; + if let Err(exc) = flags.check_writable(buffer.desc.readonly, "Object is not writable.", vm) + { + // An acquisition that cannot be served never happened, so the + // exporter's release is undone without running the Python hook. + buffer.abort_acquisition(); + return Err(exc); + } + Ok(buffer) } fn as_buffer(zelf: &Py, vm: &VirtualMachine) -> PyResult; diff --git a/crates/vm/src/types/slot_defs.rs b/crates/vm/src/types/slot_defs.rs index 69c7bb61045..300ee319907 100644 --- a/crates/vm/src/types/slot_defs.rs +++ b/crates/vm/src/types/slot_defs.rs @@ -71,7 +71,7 @@ pub struct SlotDef { #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] pub enum SlotAccessor { - // Buffer protocol (1-2) - Reserved, not used in RustPython + // Buffer protocol (1-2) BfGetBuffer = 1, BfReleaseBuffer = 2, @@ -173,9 +173,7 @@ impl SlotAccessor { pub fn is_reserved(&self) -> bool { matches!( self, - Self::BfGetBuffer - | Self::BfReleaseBuffer - | Self::TpAlloc + Self::TpAlloc | Self::TpBase | Self::TpBases | Self::TpClear @@ -411,6 +409,10 @@ impl SlotAccessor { ) } + // Buffer protocol + Self::BfGetBuffer => matches!(slot_func, SlotFunc::GetBuffer(_)), + Self::BfReleaseBuffer => matches!(slot_func, SlotFunc::ReleaseBuffer), + // New and reserved slots Self::TpNew => false, _ => false, // Reserved slots @@ -539,6 +541,18 @@ impl SlotAccessor { Self::MpSubscript => inherit_mapping!(subscript), Self::MpAssSubscript => inherit_mapping!(ass_subscript), + // Buffer protocol + Self::BfGetBuffer => { + let inherited = mro.iter().find_map(|cls| cls.slots.as_buffer.load()); + typ.slots.as_buffer.store(inherited); + } + Self::BfReleaseBuffer => { + let has_release = mro.iter().any(|cls| cls.slots.has_release_buffer.load()); + typ.slots.has_release_buffer.store(has_release); + let py_release = mro.iter().any(|cls| cls.slots.python_release_buffer.load()); + typ.slots.python_release_buffer.store(py_release); + } + // Reserved slots - no-op _ => {} } @@ -677,6 +691,25 @@ impl SlotAccessor { Self::MpSubscript => copy_mapping!(subscript), Self::MpAssSubscript => copy_mapping!(ass_subscript), + // Buffer protocol + Self::BfGetBuffer => { + if typ.slots.as_buffer.load().is_none() + && let Some(base_val) = base.slots.as_buffer.load() + { + typ.slots.as_buffer.store(Some(base_val)); + } + } + Self::BfReleaseBuffer => { + if !typ.slots.has_release_buffer.load() && base.slots.has_release_buffer.load() { + typ.slots.has_release_buffer.store(true); + } + if !typ.slots.python_release_buffer.load() + && base.slots.python_release_buffer.load() + { + typ.slots.python_release_buffer.store(true); + } + } + // Reserved slots - no-op _ => {} } @@ -816,6 +849,16 @@ impl SlotAccessor { .load() .map(SlotFunc::MapSetSubscript), + // Buffer protocol + Self::BfGetBuffer => slots.as_buffer.load().map(SlotFunc::GetBuffer), + Self::BfReleaseBuffer => { + if slots.has_release_buffer.load() || slots.python_release_buffer.load() { + Some(SlotFunc::ReleaseBuffer) + } else { + None + } + } + // Reserved slots _ => None, } @@ -973,6 +1016,19 @@ pub const SLOT_DEFS_COUNT: usize = SLOT_DEFS.len(); /// All slot definitions pub static SLOT_DEFS: &[SlotDef] = &[ + // Buffer protocol (bf_*) + SlotDef { + name: "__buffer__", + accessor: SlotAccessor::BfGetBuffer, + op: None, + doc: "Return a buffer object that exposes the underlying memory of the object.", + }, + SlotDef { + name: "__release_buffer__", + accessor: SlotAccessor::BfReleaseBuffer, + op: None, + doc: "Release the buffer object that exposes the underlying memory of the object.", + }, // Type slots (tp_*) SlotDef { name: "__init__", diff --git a/crates/vm/src/types/structseq.rs b/crates/vm/src/types/structseq.rs index 703cc79c193..7f8099e7efb 100644 --- a/crates/vm/src/types/structseq.rs +++ b/crates/vm/src/types/structseq.rs @@ -2,9 +2,11 @@ use crate::common::lock::LazyLock; use crate::common::wtf8::Wtf8; use crate::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, atomic_func, - builtins::{PyBaseExceptionRef, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef}, + builtins::{ + PyBaseExceptionRef, PyDict, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, + }, class::{PyClassImpl, StaticType}, - function::{Either, FuncArgs, PyComparisonValue, PyMethodDef, PyMethodFlags}, + function::{Either, FuncArgs, OptionalArg, PyComparisonValue, PyMethodDef, PyMethodFlags}, iter::PyExactSizeIterator, protocol::{PyMappingMethods, PySequenceMethods}, sliceable::{SequenceIndex, SliceableSequenceOp}, @@ -21,12 +23,35 @@ const DEFAULT_STRUCTSEQ_REDUCE: PyMethodDef = PyMethodDef::new_const( None, ); +/// The arguments every struct sequence constructor takes. +#[derive(FromArgs)] +pub struct StructSequenceNewArgs { + #[pyarg(any)] + pub sequence: PyObjectRef, + #[pyarg(any, optional)] + pub dict: OptionalArg, +} + /// Create a new struct sequence instance from a sequence. /// +/// `dict` supplies the hidden fields — the ones past `n_sequence_fields`, named +/// by `hidden_field_names` in order — that the sequence itself did not cover. It +/// may not name a field the sequence already supplied, nor one that does not +/// exist. +/// /// The class must have `n_sequence_fields` and `n_fields` attributes set /// (done automatically by `PyStructSequence::extend_pyclass`). -pub fn struct_sequence_new(cls: PyTypeRef, seq: PyObjectRef, vm: &VirtualMachine) -> PyResult { +pub fn struct_sequence_new( + cls: PyTypeRef, + args: StructSequenceNewArgs, + hidden_field_names: &[&str], + vm: &VirtualMachine, +) -> PyResult { // = structseq_new + let StructSequenceNewArgs { + sequence: seq, + dict, + } = args; #[cold] fn length_error( @@ -60,6 +85,16 @@ pub fn struct_sequence_new(cls: PyTypeRef, seq: PyObjectRef, vm: &VirtualMachine .ok_or_else(|| vm.new_type_error("missing n_fields attribute"))? .try_into_value(vm)?; + let dict = match dict { + OptionalArg::Missing => None, + OptionalArg::Present(dict) => Some(dict.downcast::().map_err(|_| { + vm.new_type_error(format!( + "{}() takes a dict as second arg, if any", + cls.slot_name() + )) + })?), + }; + let seq: Vec = seq.try_into_value(vm)?; let len = seq.len(); @@ -67,10 +102,30 @@ pub fn struct_sequence_new(cls: PyTypeRef, seq: PyObjectRef, vm: &VirtualMachine return Err(length_error(&cls.slot_name(), min_len, max_len, len, vm)); } - // Copy items and pad with None + // Copy items and pad the hidden fields the sequence did not cover with None. let mut items = seq; items.resize_with(max_len, || vm.ctx.none()); + // Fill those padded slots from `dict`. Every key has to land in one of them: + // a key naming a field the sequence already supplied, or no field at all, + // would otherwise be silently dropped. + if let Some(dict) = dict.filter(|dict| !dict.is_empty()) { + let mut found = 0; + let names = hidden_field_names.get(len - min_len..).unwrap_or(&[]); + for (item, name) in items[len..].iter_mut().zip(names) { + if let Some(value) = dict.get_item_opt(*name, vm)? { + *item = value; + found += 1; + } + } + if found != dict.__len__() { + return Err(vm.new_type_error(format!( + "{}() got duplicate or unexpected field name(s)", + cls.slot_name() + ))); + } + } + PyTuple::new_unchecked(items.into_boxed_slice()) .into_ref_with_type(vm, cls) .map(Into::into) @@ -193,6 +248,11 @@ pub trait PyStructSequence: StaticType + PyClassImpl + Sized + 'static { /// The Data struct that provides field definitions. type Data: PyStructSequenceData; + #[pyslot] + fn slot_new(cls: PyTypeRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult { + struct_sequence_new(cls, args.bind(vm)?, Self::Data::OPTIONAL_FIELD_NAMES, vm) + } + /// Convert a Data struct into a PyStructSequence instance. fn from_data(data: Self::Data, vm: &VirtualMachine) -> PyTupleRef { let tuple = diff --git a/crates/vm/src/utils.rs b/crates/vm/src/utils.rs index 80402480cfd..8a28a32f663 100644 --- a/crates/vm/src/utils.rs +++ b/crates/vm/src/utils.rs @@ -33,6 +33,7 @@ pub(crate) fn collection_repr<'a, I>( class_name: Option<&str>, prefix: &str, suffix: &str, + empty: &str, iter: I, vm: &VirtualMachine, ) -> PyResult @@ -47,10 +48,9 @@ where repr.push_str(prefix); { let mut parts_iter = iter.map(|o| o.repr(vm)); - let first = parts_iter - .next() - .transpose()? - .expect("this is not called for empty collection"); + let Some(first) = parts_iter.next().transpose()? else { + return Ok(Wtf8Buf::from(empty)); + }; repr.push_wtf8(first.as_wtf8()); for part in parts_iter { repr.push_str(", "); diff --git a/crates/vm/src/vm/context.rs b/crates/vm/src/vm/context.rs index 9a545663576..5deaffb3f6a 100644 --- a/crates/vm/src/vm/context.rs +++ b/crates/vm/src/vm/context.rs @@ -54,10 +54,7 @@ pub struct Context { pub(crate) string_pool: StringPool, pub(crate) slot_new_wrapper: PyMethodDef, pub names: ConstName, - // GC module state (callbacks and garbage lists) - pub gc_callbacks: PyListRef, - pub gc_garbage: PyListRef, } macro_rules! declare_const_name { @@ -106,6 +103,7 @@ declare_const_name! { __await__, __bases__, __bool__, + __buffer__, __build_class__, __builtins__, __bytes__, @@ -208,6 +206,7 @@ declare_const_name! { __rdivmod__, __reduce__, __reduce_ex__, + __release_buffer__, __repr__, __reversed__, __rfloordiv__, @@ -363,8 +362,6 @@ impl Context { let empty_bytes = create_object(PyBytes::from(Vec::new()), types.bytes_type); // GC callbacks and garbage lists - let gc_callbacks = PyRef::new_ref(PyList::default(), types.list_type.to_owned(), None); - let gc_garbage = PyRef::new_ref(PyList::default(), types.list_type.to_owned(), None); Self { true_value, @@ -387,9 +384,6 @@ impl Context { string_pool, slot_new_wrapper, names, - - gc_callbacks, - gc_garbage, } } @@ -442,6 +436,14 @@ impl Context { PyInt::from(i).into_ref(self) } + /// Borrow a cached small integer whose lifetime is tied to this context. + #[inline(always)] + pub(crate) fn cached_int(&self, i: i32) -> &PyIntRef { + debug_assert!(Self::INT_CACHE_POOL_RANGE.contains(&i)); + let inner_idx = (i - Self::INT_CACHE_POOL_MIN) as usize; + &self.int_cache_pool[inner_idx] + } + #[inline] pub fn new_bigint(&self, i: &BigInt) -> PyIntRef { if let Some(i) = i.to_i32() diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index f456e8587ea..c538eb32ec8 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -1,6 +1,11 @@ #[cfg(feature = "threading")] use super::StopTheWorldState; -use super::{Context, PyConfig, PyGlobalState, VirtualMachine, setting::Settings, thread}; +use super::{ + Context, PyConfig, PyGlobalState, VirtualMachine, + runtime::{self, InterpreterWhence}, + setting::Settings, + thread, +}; use crate::{ PyResult, builtins, common::rc::PyRc, frozen::FrozenModule, getpath, py_freeze, stdlib::atexit, vm::PyBaseExceptionRef, @@ -36,18 +41,34 @@ pub struct InterpreterBuilder { init_hooks: Vec, } -/// Private helper to initialize a VM with settings, context, and custom initialization. -fn initialize_main_vm( +/// Options for constructing a main or sub-interpreter VM. +struct InitializeVmOpts<'a> { settings: Settings, ctx: PyRc, module_defs: Vec<&'static builtins::PyModuleDef>, frozen_modules: Vec<(&'static str, FrozenModule)>, init_hooks: Vec, - init: F, -) -> (VirtualMachine, PyRc) + is_main: bool, + whence: InterpreterWhence, + /// When `Some`, reuse parent module_defs/frozen/config seeds for a subinterpreter. + parent_state: Option<&'a PyGlobalState>, +} + +/// Shared constructor for main and sub-interpreters. +fn initialize_vm(opts: InitializeVmOpts<'_>, init: F) -> (VirtualMachine, PyRc) where F: FnOnce(&mut VirtualMachine), { + let InitializeVmOpts { + settings, + ctx, + module_defs, + frozen_modules, + init_hooks, + is_main, + whence, + parent_state, + } = opts; use crate::codecs::CodecsRegistry; use crate::common::hash::HashSecret; use crate::common::lock::PyMutex; @@ -55,55 +76,85 @@ where use core::sync::atomic::{AtomicBool, AtomicU64}; use crossbeam_utils::atomic::AtomicCell; - let paths = getpath::init_path_config(&settings); - let config = PyConfig::new(settings, paths); + let (config, all_module_defs, frozen, hash_secret, int_max_str_digits) = + if let Some(parent) = parent_state { + // Subinterpreter: clone config and module tables from parent, fresh runtime state. + let int_max_str_digits = AtomicCell::new(parent.int_max_str_digits.load()); + ( + parent.config.clone(), + parent.module_defs.clone(), + parent.frozen.clone(), + parent.hash_secret, + int_max_str_digits, + ) + } else { + let paths = getpath::init_path_config(&settings); + let config = PyConfig::new(settings, paths); - // Build module_defs map from builtin modules + additional modules - let mut all_module_defs: BTreeMap<&'static str, &'static builtins::PyModuleDef> = - crate::stdlib::builtin_module_defs(&ctx) - .into_iter() - .chain(module_defs) - .map(|def| (def.name.as_str(), def)) - .collect(); + // Build module_defs map from builtin modules + additional modules + let mut all_module_defs: BTreeMap<&'static str, &'static builtins::PyModuleDef> = + crate::stdlib::builtin_module_defs(&ctx) + .into_iter() + .chain(module_defs) + .map(|def| (def.name.as_str(), def)) + .collect(); - // Register sysconfigdata under platform-specific name as well - if let Some(&sysconfigdata_def) = all_module_defs.get("_sysconfigdata") { - use std::sync::OnceLock; - static SYSCONFIGDATA_NAME: OnceLock<&'static str> = OnceLock::new(); - let leaked_name = *SYSCONFIGDATA_NAME.get_or_init(|| { - let name = crate::stdlib::sys::sysconfigdata_name(); - Box::leak(name.into_boxed_str()) - }); - all_module_defs.insert(leaked_name, sysconfigdata_def); - } + // Register sysconfigdata under platform-specific name as well + if let Some(&sysconfigdata_def) = all_module_defs.get("_sysconfigdata") { + use std::sync::OnceLock; + static SYSCONFIGDATA_NAME: OnceLock<&'static str> = OnceLock::new(); + let leaked_name = *SYSCONFIGDATA_NAME.get_or_init(|| { + let name = crate::stdlib::sys::sysconfigdata_name(); + Box::leak(name.into_boxed_str()) + }); + all_module_defs.insert(leaked_name, sysconfigdata_def); + } - // Create hash secret - let seed = match config.settings.hash_seed { - Some(seed) => seed, - None => super::process_hash_secret_seed(), - }; - let hash_secret = HashSecret::new(seed); + let seed = match config.settings.hash_seed { + Some(seed) => seed, + None => super::process_hash_secret_seed(), + }; + let hash_secret = HashSecret::new(seed); + + let int_max_str_digits = AtomicCell::new(match config.settings.int_max_str_digits { + -1 => 4300, + other => other, + } as usize); + + let mut frozen: std::collections::HashMap< + &'static str, + FrozenModule, + rapidhash::quality::RandomState, + > = core_frozen_inits().collect(); + frozen.extend(frozen_modules); + + ( + config, + all_module_defs, + frozen, + hash_secret, + int_max_str_digits, + ) + }; - // Create codec registry and warnings state + // Per-interpreter ephemeral state (must not be shared across interpreters). let codec_registry = CodecsRegistry::new(&ctx); let warnings = WarningsState::init_state(&ctx); - // Create int_max_str_digits - let int_max_str_digits = AtomicCell::new(match config.settings.int_max_str_digits { - -1 => 4300, - other => other, - } as usize); - - // Initialize frozen modules (core + user-provided) - let mut frozen: std::collections::HashMap< - &'static str, - FrozenModule, - rapidhash::quality::RandomState, - > = core_frozen_inits().collect(); - frozen.extend(frozen_modules); - - // Create PyGlobalState + let interpreter_id = runtime::alloc_interpreter_id(); + + // Process main OS thread identity is process-global; subinterpreters inherit + // it from the parent so `is_main_thread()` stays correct when running on the + // main OS thread under a subinterpreter. + #[cfg(feature = "threading")] + let main_thread_ident = AtomicCell::new(parent_state.map_or(0, |p| p.main_thread_ident.load())); + + // Create PyGlobalState (≈ PyInterpreterState) let global_state = PyRc::new(PyGlobalState { + gc: crate::gc_state::GcInterpreterState::new(&ctx), + interpreter_id, + whence, + is_main, config, module_defs: all_module_defs, frozen, @@ -124,7 +175,7 @@ where global_profile_func: PyMutex::default(), type_mutex: PyMutex::default(), #[cfg(feature = "threading")] - main_thread_ident: AtomicCell::new(0), + main_thread_ident, #[cfg(feature = "threading")] thread_frames: parking_lot::Mutex::new(std::collections::HashMap::new()), #[cfg(feature = "threading")] @@ -150,6 +201,13 @@ where // Call custom init function (can mutate vm.state) init(&mut vm); + // Register before `initialize()` runs any Python: it allocates GC-tracked + // objects, so a collection on another thread has to be able to stop this + // interpreter while that happens. It cannot be registered earlier — the + // hooks above take `PyRc::get_mut` on the state, which fails once the + // registry holds a weak reference to it. + runtime::register_interpreter(&vm.state); + // `initialize()` runs Python bytecode directly (e.g. importing `codecs` // and `encodings`) before any `enter_vm` scope exists, so attach this // thread for the duration so type cache reads see it as ATTACHED. @@ -271,12 +329,17 @@ impl InterpreterBuilder { /// This consumes the configuration and returns a fully initialized Interpreter. #[must_use] pub fn build(self) -> Interpreter { - let (vm, global_state) = initialize_main_vm( - self.settings, - self.ctx, - self.module_defs, - self.frozen_modules, - self.init_hooks, + let (vm, global_state) = initialize_vm( + InitializeVmOpts { + settings: self.settings, + ctx: self.ctx, + module_defs: self.module_defs, + frozen_modules: self.frozen_modules, + init_hooks: self.init_hooks, + is_main: true, + whence: InterpreterWhence::Runtime, + parent_state: None, + }, |_| {}, // No additional init needed ); Interpreter { global_state, vm } @@ -295,7 +358,13 @@ impl Default for InterpreterBuilder { } } -/// The general interface for the VM +/// One isolated Python interpreter in the process (≈ CPython `PyInterpreterState` + main tstate). +/// +/// Historically RustPython exposed a single process-level `Interpreter`. For PEP 734 +/// (multiple interpreters / subinterpreters) this type is now the owned handle for +/// **one** interpreter. Use [`Interpreter::create_subinterpreter`] to create additional +/// isolated interpreters that share the process-wide type context but not modules or +/// `PyGlobalState`. /// /// # Examples /// Runs a simple embedded hello world program. @@ -350,17 +419,113 @@ impl Interpreter { where F: FnOnce(&mut VirtualMachine), { - let (vm, global_state) = initialize_main_vm( - settings, - Context::genesis().clone(), - Vec::new(), // No module_defs - Vec::new(), // No frozen_modules - Vec::new(), // No init_hooks + let (vm, global_state) = initialize_vm( + InitializeVmOpts { + settings, + ctx: Context::genesis().clone(), + module_defs: Vec::new(), + frozen_modules: Vec::new(), + init_hooks: Vec::new(), + is_main: true, + whence: InterpreterWhence::Runtime, + parent_state: None, + }, init, ); Self { global_state, vm } } + /// Process-global interpreter id (main is [`super::MAIN_INTERPRETER_ID`]). + #[inline] + #[must_use] + pub fn id(&self) -> i64 { + self.global_state.interpreter_id + } + + /// Where this interpreter was created. + #[inline] + #[must_use] + pub fn whence(&self) -> InterpreterWhence { + self.global_state.whence + } + + /// Whether this is a top-level interpreter rather than a subinterpreter. + /// + /// Every top-level interpreter answers `true`; for *the* process main, use + /// [`Interpreter::is_process_main`]. + #[inline] + #[must_use] + pub fn is_main(&self) -> bool { + self.global_state.is_main + } + + /// Whether this is the PEP 734 process main interpreter (`get_main()`). + /// + /// Unlike [`Interpreter::is_main`], which is set for every top-level + /// interpreter, this is true for only the single first-registered main. + #[inline] + #[must_use] + pub fn is_process_main(&self) -> bool { + runtime::main_interpreter_id() == Some(self.id()) + } + + /// Create a subinterpreter and hand ownership to the runtime, returning its + /// id. The runtime keeps it alive until [`runtime::take_owned_interpreter`]. + /// + /// This is the shape `_interpreters.create()` will use: Python receives an + /// id, not an owned handle. + #[cfg(feature = "threading")] + #[must_use] + pub fn create_owned_subinterpreter(&self) -> i64 { + runtime::store_owned_interpreter(self.create_subinterpreter()) + } + + /// Create an isolated subinterpreter sharing this interpreter's type context + /// (`Context`) and module definitions, but with its own `sys.modules`, + /// builtins module instance, thread registry, and stop-the-world state. + /// + /// This is the Rust-side foundation for PEP 734 / `_interpreters.create()`. + /// It does not yet expose a Python module API. + /// + /// May be called while the parent is entered (matching CPython, where + /// `_interpreters.create()` runs under the main interpreter). When the + /// calling thread is currently attached to a VM, that attachment is + /// temporarily saved so the subinterpreter can bootstrap as an outermost + /// enter (correct thread-slot / stop-the-world state). + #[must_use] + pub fn create_subinterpreter(&self) -> Self { + // Suspend the caller's current VM attachment (if any) for the duration + // of subinterpreter initialization. Nested bootstrap would otherwise + // swap `CURRENT_THREAD_SLOT` to the new interpreter while leaving the + // outer interpreter's attach state inconsistent. Always restore, even + // if initialization panics. + #[cfg(feature = "threading")] + let _restore_parent = { + let saved = thread::current_vm_is_set().then(thread::save_current_thread); + scopeguard::guard(saved, |saved| { + if let Some(saved) = saved { + thread::restore_current_thread(saved); + } + }) + }; + + let (vm, global_state) = initialize_vm( + InitializeVmOpts { + // settings unused when parent_state is Some + settings: Settings::default(), + ctx: self.vm.ctx.clone(), + module_defs: Vec::new(), + frozen_modules: Vec::new(), + init_hooks: Vec::new(), + is_main: false, + whence: InterpreterWhence::Stdlib, + parent_state: Some(&self.global_state), + }, + |_| {}, + ); + Self { global_state, vm } + } + /// Run a function with the main virtual machine and return a PyResult of the result. /// /// To enter vm context multiple times or to avoid buffer/exception management, this function is preferred. @@ -456,7 +621,7 @@ impl Interpreter { vm.state.finalizing.store(true, Ordering::Release); // GC pass - collect cycles before module cleanup - crate::gc_state::gc_state().collect_force(2); + vm.state.gc.collect_force(2); // Module finalization: remove modules from sys.modules, GC collect // (while builtins is still available for __del__), then clear module dicts. @@ -582,8 +747,9 @@ fn core_frozen_inits() -> impl Iterator { mod tests { use super::*; use crate::{ - PyObjectRef, + AsObject, PyObjectRef, builtins::{PyStr, int}, + vm::{MAIN_INTERPRETER_ID, runtime}, }; use malachite_bigint::ToBigInt; @@ -608,4 +774,883 @@ mod tests { assert_eq!(value.as_wtf8(), "Hello Hello Hello Hello ") }) } + + /// Main interpreter is marked main with Runtime whence and is registered. + #[test] + fn main_interpreter_identity() { + let main = Interpreter::without_stdlib(Default::default()); + assert!(main.is_main()); + assert_eq!(main.whence(), InterpreterWhence::Runtime); + assert!( + runtime::list_interpreters() + .iter() + .any(|info| info.id == main.id() && info.whence == InterpreterWhence::Runtime) + ); + // When this is the sole sequential main in a quiet process, id is 0; + // under parallel tests the id is still unique and registered. + assert!(main.id() >= MAIN_INTERPRETER_ID); + } + + /// Subinterpreters get distinct ids, Stdlib whence, and appear in the registry. + #[test] + fn create_subinterpreter_registers_distinct_ids() { + let main = Interpreter::without_stdlib(Default::default()); + let sub1 = main.create_subinterpreter(); + let sub2 = main.create_subinterpreter(); + + assert!(main.is_main()); + assert!(!sub1.is_main()); + assert!(!sub2.is_main()); + assert_eq!(sub1.whence(), InterpreterWhence::Stdlib); + assert_eq!(sub2.whence(), InterpreterWhence::Stdlib); + assert_ne!(main.id(), sub1.id()); + assert_ne!(main.id(), sub2.id()); + assert_ne!(sub1.id(), sub2.id()); + + let ids: Vec = runtime::list_interpreters() + .into_iter() + .map(|i| i.id) + .collect(); + assert!(ids.contains(&main.id())); + assert!(ids.contains(&sub1.id())); + assert!(ids.contains(&sub2.id())); + } + + /// An interpreter stays looked-up-able until nothing holds its state. + /// + /// Dropping the handle is not the end of its life: `new_thread()` workers + /// hold their own reference, and a collection in progress holds one for + /// every live interpreter while the world is stopped. So the registry entry + /// goes away eventually rather than at the drop. + fn wait_until_unregistered(id: i64) { + use core::time::Duration; + use std::time::Instant; + + let deadline = Instant::now() + Duration::from_secs(30); + while runtime::lookup_interpreter(id).is_some() { + assert!( + Instant::now() < deadline, + "interpreter {id} still registered long after its last reference" + ); + std::thread::yield_now(); + } + } + + /// A collection snapshots the registry and then reads tracked objects with + /// the interpreters it found parked. An interpreter that registered inside + /// that window would be missing from the snapshot, so nothing would stop it + /// and its bootstrap would run under the scan; registration therefore waits + /// for the stop to end. + #[cfg(feature = "threading")] + #[test] + fn registering_waits_for_an_in_flight_stop() { + use core::time::Duration; + use std::sync::mpsc; + + // Stands in for a collector between its snapshot and its restart. + let admission = runtime::lock_admission_for_stop(); + + let (tx, rx) = mpsc::channel(); + let worker = std::thread::spawn(move || { + let interp = Interpreter::without_stdlib(Default::default()); + tx.send(interp.id()).expect("receiver is alive"); + interp + }); + + assert!( + matches!( + rx.recv_timeout(Duration::from_millis(200)), + Err(mpsc::RecvTimeoutError::Timeout) + ), + "an interpreter registered while a stop-the-world was in flight" + ); + + drop(admission); + let id = rx + .recv_timeout(Duration::from_secs(30)) + .expect("registration proceeds once the world restarts"); + assert!(runtime::lookup_interpreter(id).is_some()); + drop(worker.join().expect("worker did not panic")); + wait_until_unregistered(id); + } + + /// Dropping a subinterpreter releases it; main remains. + #[test] + fn drop_subinterpreter_unregisters() { + let main = Interpreter::without_stdlib(Default::default()); + let sub_id = { + let sub = main.create_subinterpreter(); + let id = sub.id(); + assert!(runtime::lookup_interpreter(id).is_some()); + id + }; + wait_until_unregistered(sub_id); + assert!(runtime::lookup_interpreter(main.id()).is_some()); + } + + /// Each interpreter has its own `sys.modules` / builtins module instance. + #[test] + fn subinterpreters_isolate_modules() { + let main = Interpreter::without_stdlib(Default::default()); + let sub = main.create_subinterpreter(); + + let (main_sys_ptr, main_builtins_ptr, main_ctx_ptr, main_state_ptr) = main.enter(|vm| { + ( + vm.sys_module.as_object() as *const _, + vm.builtins.as_object() as *const _, + PyRc::as_ptr(&vm.ctx), + PyRc::as_ptr(&vm.state), + ) + }); + let (sub_sys_ptr, sub_builtins_ptr, sub_ctx_ptr, sub_state_ptr) = sub.enter(|vm| { + ( + vm.sys_module.as_object() as *const _, + vm.builtins.as_object() as *const _, + PyRc::as_ptr(&vm.ctx), + PyRc::as_ptr(&vm.state), + ) + }); + + assert_ne!(main_sys_ptr, sub_sys_ptr); + assert_ne!(main_builtins_ptr, sub_builtins_ptr); + // Distinct per-interpreter state. + assert_ne!(main_state_ptr, sub_state_ptr); + // Shared process-wide type context (immortal / builtin types). + assert_eq!(main_ctx_ptr, sub_ctx_ptr); + } + + /// Mutations to interpreter-owned modules must not leak between interpreters. + #[test] + fn subinterpreters_behaviorally_isolate_builtins_and_sys_modules() { + const PROBE: &str = "__rustpython_subinterpreter_isolation_probe__"; + + let main = Interpreter::without_stdlib(Default::default()); + let sub = main.create_subinterpreter(); + + main.enter(|vm| { + vm.builtins + .set_attr(PROBE, vm.ctx.new_int(11_i32), vm) + .unwrap(); + vm.sys_module + .get_attr("modules", vm) + .unwrap() + .set_item(PROBE, vm.ctx.new_int(12_i32).into(), vm) + .unwrap(); + }); + + sub.enter(|vm| { + assert!(vm.builtins.get_attr(PROBE, vm).is_err()); + let modules = vm.sys_module.get_attr("modules", vm).unwrap(); + assert!(modules.get_item(PROBE, vm).is_err()); + + vm.builtins + .set_attr(PROBE, vm.ctx.new_int(21_i32), vm) + .unwrap(); + modules + .set_item(PROBE, vm.ctx.new_int(22_i32).into(), vm) + .unwrap(); + }); + + main.enter(|vm| { + let builtin_probe = vm.builtins.get_attr(PROBE, vm).unwrap(); + assert_eq!(*int::get_value(&builtin_probe), 11_i32.to_bigint().unwrap()); + + let module_probe = vm + .sys_module + .get_attr("modules", vm) + .unwrap() + .get_item(PROBE, vm) + .unwrap(); + assert_eq!(*int::get_value(&module_probe), 12_i32.to_bigint().unwrap()); + }); + } + + /// Creating a subinterpreter while the parent is entered must not corrupt + /// the parent's current-VM / thread-slot state. + #[test] + fn create_subinterpreter_while_parent_entered() { + let main = Interpreter::without_stdlib(Default::default()); + main.enter(|vm| { + let before = vm.state.interpreter_id; + let sub = main.create_subinterpreter(); + assert_ne!(sub.id(), before); + // Still the parent after create returns. + assert_eq!(vm.state.interpreter_id, before); + // Can still use the parent VM. + let n: PyObjectRef = vm.ctx.new_int(7_i32).into(); + assert_eq!(int::get_value(&n), &7_i32.to_bigint().unwrap()); + // And the sub is independently usable after parent section. + drop(sub); + }); + } + + /// Sequential enter of main then sub on the same OS thread is safe. + #[test] + fn sequential_enter_main_and_sub() { + let main = Interpreter::without_stdlib(Default::default()); + let sub = main.create_subinterpreter(); + + main.enter(|vm| { + assert!(vm.state.is_main_interpreter()); + let a: PyObjectRef = vm.ctx.new_int(1_i32).into(); + let b: PyObjectRef = vm.ctx.new_int(2_i32).into(); + let res = vm._add(&a, &b).unwrap(); + assert_eq!(*int::get_value(&res), 3_i32.to_bigint().unwrap()); + }); + sub.enter(|vm| { + assert!(!vm.state.is_main_interpreter()); + let a: PyObjectRef = vm.ctx.new_int(10_i32).into(); + let b: PyObjectRef = vm.ctx.new_int(5_i32).into(); + let res = vm._mul(&a, &b).unwrap(); + assert_eq!(*int::get_value(&res), 50_i32.to_bigint().unwrap()); + }); + // Re-enter main after sub. + main.enter(|vm| { + assert!(vm.state.is_main_interpreter()); + }); + } + + /// Concurrent use of main + subinterpreter on different OS threads. + #[cfg(feature = "threading")] + #[test] + fn concurrent_main_and_subinterpreter_threads() { + use alloc::sync::Arc; + use core::sync::atomic::{AtomicUsize, Ordering}; + + let main = Interpreter::without_stdlib(Default::default()); + let sub = main.create_subinterpreter(); + let counter = Arc::new(AtomicUsize::new(0)); + + let c1 = Arc::clone(&counter); + let h_main = main.enter(|vm| { + let thread_vm = vm.new_thread(); + let c = Arc::clone(&c1); + std::thread::spawn(move || { + thread_vm.run(|vm| { + for _ in 0..100 { + let a: PyObjectRef = vm.ctx.new_int(1_i32).into(); + let b: PyObjectRef = vm.ctx.new_int(1_i32).into(); + let _ = vm._add(&a, &b).unwrap(); + c.fetch_add(1, Ordering::Relaxed); + } + assert!(vm.state.is_main_interpreter()); + }); + }) + }); + + let c2 = Arc::clone(&counter); + let h_sub = sub.enter(|vm| { + let thread_vm = vm.new_thread(); + let c = Arc::clone(&c2); + std::thread::spawn(move || { + thread_vm.run(|vm| { + for _ in 0..100 { + let a: PyObjectRef = vm.ctx.new_int(2_i32).into(); + let b: PyObjectRef = vm.ctx.new_int(3_i32).into(); + let _ = vm._mul(&a, &b).unwrap(); + c.fetch_add(1, Ordering::Relaxed); + } + assert!(!vm.state.is_main_interpreter()); + }); + }) + }); + + h_main.join().expect("main worker panicked"); + h_sub.join().expect("sub worker panicked"); + assert_eq!(counter.load(Ordering::Relaxed), 200); + } + + /// Entering one interpreter must not serialize entry into another interpreter. + #[cfg(feature = "threading")] + #[test] + fn main_and_subinterpreter_run_sections_overlap() { + use alloc::sync::Arc; + use core::time::Duration; + use std::{ + sync::{Condvar, Mutex}, + time::Instant, + }; + + #[derive(Default)] + struct OverlapState { + entered: usize, + release: bool, + } + + let main = Interpreter::without_stdlib(Default::default()); + let sub = main.create_subinterpreter(); + let state = Arc::new((Mutex::new(OverlapState::default()), Condvar::new())); + + let spawn_worker = |interpreter: &Interpreter| { + let state = Arc::clone(&state); + interpreter.enter(|vm| { + let thread_vm = vm.new_thread(); + std::thread::spawn(move || { + thread_vm.run(|vm| { + let a: PyObjectRef = vm.ctx.new_int(20_i32).into(); + let b: PyObjectRef = vm.ctx.new_int(22_i32).into(); + assert_eq!( + *int::get_value(&vm._add(&a, &b).unwrap()), + 42_i32.to_bigint().unwrap() + ); + + let (lock, ready) = &*state; + let mut state = lock.lock().unwrap(); + state.entered += 1; + ready.notify_all(); + while !state.release { + state = ready.wait(state).unwrap(); + } + }); + }) + }) + }; + + let main_worker = spawn_worker(&main); + let sub_worker = spawn_worker(&sub); + + let (lock, ready) = &*state; + let deadline = Instant::now() + Duration::from_secs(30); + let mut state_guard = lock.lock().unwrap(); + while state_guard.entered < 2 { + let now = Instant::now(); + if now >= deadline { + break; + } + let (next, _) = ready.wait_timeout(state_guard, deadline - now).unwrap(); + state_guard = next; + } + let overlapped = state_guard.entered == 2; + state_guard.release = true; + ready.notify_all(); + drop(state_guard); + + main_worker.join().expect("main worker panicked"); + sub_worker.join().expect("subinterpreter worker panicked"); + assert!( + overlapped, + "main and subinterpreter run sections were serialized" + ); + } + + /// A busy interpreter must not prevent another interpreter from making progress. + #[cfg(feature = "threading")] + #[test] + fn busy_main_interpreter_does_not_block_subinterpreter() { + use alloc::sync::Arc; + use core::{ + sync::atomic::{AtomicBool, Ordering}, + time::Duration, + }; + use std::time::Instant; + + let main = Interpreter::without_stdlib(Default::default()); + let sub = main.create_subinterpreter(); + let main_started = Arc::new(AtomicBool::new(false)); + let sub_finished = Arc::new(AtomicBool::new(false)); + + let main_started_worker = Arc::clone(&main_started); + let sub_finished_worker = Arc::clone(&sub_finished); + let main_worker = main.enter(|vm| { + let thread_vm = vm.new_thread(); + std::thread::spawn(move || { + thread_vm.run(|vm| { + main_started_worker.store(true, Ordering::Release); + let deadline = Instant::now() + Duration::from_secs(30); + let mut operations = 0; + while !sub_finished_worker.load(Ordering::Acquire) && Instant::now() < deadline + { + let a: PyObjectRef = vm.ctx.new_int(20_i32).into(); + let b: PyObjectRef = vm.ctx.new_int(22_i32).into(); + let result = vm._add(&a, &b).unwrap(); + assert_eq!(*int::get_value(&result), 42_i32.to_bigint().unwrap()); + operations += 1; + std::thread::yield_now(); + } + (sub_finished_worker.load(Ordering::Acquire), operations) + }) + }) + }); + + let main_started_worker = Arc::clone(&main_started); + let sub_finished_worker = Arc::clone(&sub_finished); + let sub_worker = sub.enter(|vm| { + let thread_vm = vm.new_thread(); + std::thread::spawn(move || { + while !main_started_worker.load(Ordering::Acquire) { + std::thread::yield_now(); + } + thread_vm.run(|vm| { + let a: PyObjectRef = vm.ctx.new_int(6_i32).into(); + let b: PyObjectRef = vm.ctx.new_int(7_i32).into(); + let result = vm._mul(&a, &b).unwrap(); + assert_eq!(*int::get_value(&result), 42_i32.to_bigint().unwrap()); + sub_finished_worker.store(true, Ordering::Release); + }); + }) + }); + + let (sub_progressed_while_main_was_busy, main_operations) = + main_worker.join().expect("main worker panicked"); + sub_worker.join().expect("subinterpreter worker panicked"); + + assert!(main_operations > 0); + assert!( + sub_progressed_while_main_was_busy, + "subinterpreter made no progress until the busy main interpreter exited" + ); + } + + /// `new_thread` on a subinterpreter shares that subinterpreter's state, not main's. + #[cfg(feature = "threading")] + #[test] + fn subinterpreter_new_thread_shares_sub_state() { + let main = Interpreter::without_stdlib(Default::default()); + let sub = main.create_subinterpreter(); + let sub_id = sub.id(); + + let handle = sub.enter(|vm| { + let thread_vm = vm.new_thread(); + std::thread::spawn(move || { + thread_vm.run(|vm| { + assert_eq!(vm.state.interpreter_id, sub_id); + assert!(!vm.state.is_main_interpreter()); + }); + }) + }); + handle.join().expect("thread panicked"); + } + + /// Multiple subinterpreters can each run bytecode via compile+exec. + #[cfg(feature = "rustpython-compiler")] + #[test] + fn subinterpreter_runs_python_code() { + use crate::compiler::Mode; + + let main = Interpreter::without_stdlib(Default::default()); + let sub = main.create_subinterpreter(); + + sub.enter(|vm| { + let scope = vm.new_scope_with_builtins(); + let source = "x = 40 + 2\n"; + let code = vm + .compile(source, Mode::Exec, "") + .map_err(|err| err.into_pyexception(vm, Some(source))) + .unwrap(); + vm.run_code_obj(code, scope.clone()).unwrap(); + let x = scope.globals.get_item("x", vm).unwrap(); + assert_eq!(*int::get_value(&x), 42_i32.to_bigint().unwrap()); + }); + } + + /// Subclassing a shared type records the subclass on an object every + /// interpreter reaches, but only the interpreter that created it lists it. + fn run(vm: &VirtualMachine, scope: &crate::scope::Scope, source: &str) { + let code = vm + .compile(source, crate::compiler::Mode::Exec, "") + .map_err(|err| err.into_pyexception(vm, Some(source))) + .unwrap(); + vm.run_code_obj(code, scope.clone()).unwrap(); + } + + #[test] + fn subinterpreter_subclasses_are_scoped_to_their_interpreter() { + use crate::scope::Scope; + + fn lists_subclass(vm: &VirtualMachine, scope: &Scope, name: &str) -> bool { + run( + vm, + scope, + &format!("found = any(c.__name__ == {name:?} for c in int.__subclasses__())\n"), + ); + let found = scope.globals.get_item("found", vm).unwrap(); + found.try_to_bool(vm).unwrap() + } + + let main = Interpreter::without_stdlib(Default::default()); + let sub = main.create_subinterpreter(); + + // The scopes are what keep the classes alive; a subclass list holds + // only weak references, so both must outlive every assertion below. + let main_scope = main.enter(|vm| { + let scope = vm.new_scope_with_builtins(); + run(vm, &scope, "class MainOnly(int): pass\n"); + scope + }); + let sub_scope = sub.enter(|vm| { + let scope = vm.new_scope_with_builtins(); + run(vm, &scope, "class SubOnly(int): pass\n"); + scope + }); + + main.enter(|vm| { + assert!(lists_subclass(vm, &main_scope, "MainOnly")); + assert!(!lists_subclass(vm, &main_scope, "SubOnly")); + // A subclass built before either interpreter existed belongs to the + // shared context, so it stays visible to both. + assert!(lists_subclass(vm, &main_scope, "bool")); + }); + sub.enter(|vm| { + assert!(lists_subclass(vm, &sub_scope, "SubOnly")); + assert!(!lists_subclass(vm, &sub_scope, "MainOnly")); + assert!(lists_subclass(vm, &sub_scope, "bool")); + }); + + main.enter(|_| drop(main_scope)); + sub.enter(|_| drop(sub_scope)); + } + + /// A cycle allocated in one interpreter is not the parent's to collect. + #[test] + fn collections_only_reach_the_collecting_interpreter() { + use core::time::Duration; + use std::time::Instant; + + const CYCLE: &str = "class Node:\n pass\n\ + a = Node()\n\ + b = Node()\n\ + a.other = b\n\ + b.other = a\n\ + del a\n\ + del b\n"; + + fn live_nodes(vm: &VirtualMachine) -> usize { + vm.state + .gc + .get_objects(None) + .iter() + .filter(|obj| &*obj.class().name() == "Node") + .count() + } + + let main = Interpreter::without_stdlib(Default::default()); + let sub = main.create_subinterpreter(); + + let sub_scope = sub.enter(|vm| { + let scope = vm.new_scope_with_builtins(); + run(vm, &scope, CYCLE); + assert_eq!(live_nodes(vm), 2); + scope + }); + + // A collection in the parent walks its own tracked objects and leaves + // the sub's cycle where it is. Collections are serialized process-wide + // by a `try_lock`, so one running elsewhere in the suite makes + // `collect_force` a no-op; retry until this one gets to run. Each retry + // waits outside `enter`, since a thread that is entered but not running + // bytecode never reaches a safepoint, and the collection this is + // waiting for cannot stop it. + let deadline = Instant::now() + Duration::from_secs(30); + while !main.enter(|vm| vm.state.gc.collect_force(2).candidates > 0) { + assert!( + Instant::now() < deadline, + "no collection ran in the parent interpreter" + ); + std::thread::sleep(Duration::from_millis(5)); + } + sub.enter(|vm| assert_eq!(live_nodes(vm), 2)); + + sub.enter(|_| drop(sub_scope)); + } + + /// And it is not the parent's to enumerate either. + #[test] + fn get_objects_only_reports_the_calling_interpreter() { + fn tracks_class(vm: &VirtualMachine, name: &str) -> bool { + vm.state + .gc + .get_objects(None) + .iter() + .any(|obj| &*obj.class().name() == name) + } + + let main = Interpreter::without_stdlib(Default::default()); + let sub = main.create_subinterpreter(); + + let main_scope = main.enter(|vm| { + let scope = vm.new_scope_with_builtins(); + run(vm, &scope, "class MainNode:\n pass\nkeep = MainNode()\n"); + scope + }); + let sub_scope = sub.enter(|vm| { + let scope = vm.new_scope_with_builtins(); + run(vm, &scope, "class SubNode:\n pass\nkeep = SubNode()\n"); + scope + }); + + main.enter(|vm| { + assert!(tracks_class(vm, "MainNode")); + assert!(!tracks_class(vm, "SubNode")); + }); + sub.enter(|vm| { + assert!(tracks_class(vm, "SubNode")); + assert!(!tracks_class(vm, "MainNode")); + }); + + main.enter(|_| drop(main_scope)); + sub.enter(|_| drop(sub_scope)); + } + + /// The runtime can own a subinterpreter by id and hand it back on destroy. + #[cfg(feature = "threading")] + #[test] + fn runtime_owned_interpreter_lifecycle() { + let main = Interpreter::without_stdlib(Default::default()); + let sub = main.create_subinterpreter(); + let id = sub.id(); + + assert_eq!(runtime::store_owned_interpreter(sub), id); + assert!(runtime::is_owned_interpreter(id)); + assert!(runtime::lookup_interpreter(id).is_some()); + // The owned table is process-global and other tests store into it in + // parallel, so only this entry's own membership is deterministic. + assert!(runtime::owned_interpreter_count() >= 1); + + // Reclaiming removes ownership but keeps the interpreter alive while the + // returned handle is held. + let reclaimed = runtime::take_owned_interpreter(id).expect("owned by runtime"); + assert_eq!(reclaimed.id(), id); + assert!(!runtime::is_owned_interpreter(id)); + assert!(runtime::lookup_interpreter(id).is_some()); + assert!(runtime::take_owned_interpreter(id).is_none()); + + // Dropping the reclaimed handle releases it. + drop(reclaimed); + wait_until_unregistered(id); + } + + /// `create_owned_subinterpreter` stores the sub and returns only its id. + #[cfg(feature = "threading")] + #[test] + fn create_owned_subinterpreter_returns_id() { + let main = Interpreter::without_stdlib(Default::default()); + let id = main.create_owned_subinterpreter(); + assert!(runtime::is_owned_interpreter(id)); + assert_ne!(id, main.id()); + + let sub = runtime::take_owned_interpreter(id).expect("owned by runtime"); + assert_eq!(sub.id(), id); + assert!(!sub.is_main()); + } + + /// A collection must stop every interpreter, not just the collecting one: + /// the generation lists are process-global, so the reachability walk reads + /// objects owned by other interpreters while their threads would otherwise + /// still be mutating them. + #[cfg(all(feature = "threading", feature = "rustpython-compiler"))] + #[test] + fn gc_collect_is_safe_while_another_interpreter_runs() { + use crate::compiler::Mode; + use alloc::sync::Arc; + use core::{ + sync::atomic::{AtomicBool, Ordering}, + time::Duration, + }; + use std::time::Instant; + + // Each interpreter churns reference cycles so both contribute tracked + // objects to the shared generation lists. + const CHURN: &str = "\ +for _ in range(40): + a = {} + b = {'peer': a} + a['peer'] = b +"; + + let main = Interpreter::without_stdlib(Default::default()); + let sub = main.create_subinterpreter(); + let stop = Arc::new(AtomicBool::new(false)); + + let run_source = |vm: &VirtualMachine, source: &str| { + let scope = vm.new_scope_with_builtins(); + let code = vm + .compile(source, Mode::Exec, "") + .map_err(|err| err.into_pyexception(vm, Some(source))) + .unwrap(); + vm.run_code_obj(code, scope).unwrap(); + }; + + // Subinterpreter thread: allocate cycles continuously. + let stop_worker = Arc::clone(&stop); + let churner = sub.enter(|vm| { + let thread_vm = vm.new_thread(); + std::thread::spawn(move || { + thread_vm.run(|vm| { + while !stop_worker.load(Ordering::Acquire) { + run_source(vm, CHURN); + } + }); + }) + }); + + // Main interpreter: force collections while the sub keeps mutating. + main.enter(|vm| { + run_source(vm, CHURN); + let deadline = Instant::now() + Duration::from_secs(2); + let mut collections = 0; + while Instant::now() < deadline && collections < 20 { + vm.state.gc.collect_force(2); + collections += 1; + } + assert!(collections > 0); + }); + + stop.store(true, Ordering::Release); + churner.join().expect("churn worker panicked"); + } + + /// A thread entered in one interpreter can park another interpreter's + /// threads. This is what makes a collection safe: the generation lists are + /// process-global, so the collector must be able to stop every interpreter, + /// not only its own. + #[cfg(all(feature = "threading", feature = "rustpython-compiler"))] + #[test] + fn stop_the_world_parks_threads_of_another_interpreter() { + use crate::compiler::Mode; + use alloc::sync::Arc; + use core::{ + sync::atomic::{AtomicBool, AtomicU64, Ordering}, + time::Duration, + }; + + let main = Interpreter::without_stdlib(Default::default()); + let sub = main.create_subinterpreter(); + let sub_state = sub.enter(|vm| vm.state.clone()); + + let progress = Arc::new(AtomicU64::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + + // Sub-interpreter worker: runs bytecode (so it reaches safepoints) and + // reports progress every iteration. + let progress_worker = Arc::clone(&progress); + let stop_worker = Arc::clone(&stop); + let worker = sub.enter(|vm| { + let thread_vm = vm.new_thread(); + std::thread::spawn(move || { + thread_vm.run(|vm| { + let source = "x = 1 + 1\n"; + let code = vm + .compile(source, Mode::Exec, "") + .map_err(|err| err.into_pyexception(vm, Some(source))) + .unwrap(); + while !stop_worker.load(Ordering::Acquire) { + let scope = vm.new_scope_with_builtins(); + vm.run_code_obj(code.clone(), scope).unwrap(); + progress_worker.fetch_add(1, Ordering::Release); + } + }); + }) + }); + + // Wait until the worker is actually running. + while progress.load(Ordering::Acquire) == 0 { + std::thread::yield_now(); + } + + main.enter(|_vm| { + // Stop the *subinterpreter* from a thread whose current interpreter + // is main — the cross-interpreter stop a collection performs. + sub_state.stop_the_world.stop_the_world(&sub_state); + + let parked_at = progress.load(Ordering::Acquire); + std::thread::sleep(Duration::from_millis(50)); + assert_eq!( + progress.load(Ordering::Acquire), + parked_at, + "subinterpreter thread kept running while its world was stopped" + ); + + sub_state.stop_the_world.start_the_world(&sub_state); + }); + + // After restart the worker makes progress again. + let resumed_from = progress.load(Ordering::Acquire); + while progress.load(Ordering::Acquire) == resumed_from { + std::thread::yield_now(); + } + + stop.store(true, Ordering::Release); + worker.join().expect("worker panicked"); + } + + /// Entering a subinterpreter from inside the parent's `enter` must attach + /// the subinterpreter's thread slot (and detach the parent's). Otherwise the + /// thread runs the sub's bytecode with a DETACHED slot, and a collector + /// stopping that interpreter force-parks the slot and wrongly concludes the + /// world is stopped while this thread keeps mutating objects. + #[cfg(all(feature = "threading", feature = "rustpython-compiler"))] + #[test] + fn nested_enter_of_subinterpreter_is_stoppable() { + use crate::compiler::Mode; + use alloc::sync::Arc; + use core::{ + sync::atomic::{AtomicBool, AtomicU64, Ordering}, + time::Duration, + }; + + let main = Interpreter::without_stdlib(Default::default()); + let sub = main.create_subinterpreter(); + let sub_state = sub.enter(|vm| vm.state.clone()); + + let progress = Arc::new(AtomicU64::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + + // Worker runs the SUB nested inside an active MAIN section. + let progress_worker = Arc::clone(&progress); + let stop_worker = Arc::clone(&stop); + let main_vm = main.enter(|vm| vm.new_thread()); + let sub_vm = sub.enter(|vm| vm.new_thread()); + let worker = std::thread::spawn(move || { + main_vm.run(|_main| { + sub_vm.run(|vm| { + let source = "x = 1 + 1\n"; + let code = vm + .compile(source, Mode::Exec, "") + .map_err(|err| err.into_pyexception(vm, Some(source))) + .unwrap(); + while !stop_worker.load(Ordering::Acquire) { + let scope = vm.new_scope_with_builtins(); + vm.run_code_obj(code.clone(), scope).unwrap(); + progress_worker.fetch_add(1, Ordering::Release); + } + }); + }); + }); + + while progress.load(Ordering::Acquire) == 0 { + std::thread::yield_now(); + } + + sub_state.stop_the_world.stop_the_world(&sub_state); + let parked_at = progress.load(Ordering::Acquire); + std::thread::sleep(Duration::from_millis(50)); + assert_eq!( + progress.load(Ordering::Acquire), + parked_at, + "nested subinterpreter thread kept running while the sub's world was stopped" + ); + sub_state.stop_the_world.start_the_world(&sub_state); + + let resumed_from = progress.load(Ordering::Acquire); + while progress.load(Ordering::Acquire) == resumed_from { + std::thread::yield_now(); + } + + stop.store(true, Ordering::Release); + worker.join().expect("nested worker panicked"); + } + + /// The process main id is recorded once and is stable across later creates. + #[test] + fn process_main_id_recorded_and_stable() { + // At least one main exists by now (this one, if not an earlier test), so + // `get_main()` is populated. + let main = Interpreter::without_stdlib(Default::default()); + let recorded = runtime::main_interpreter_id().expect("a process main exists"); + + // Recording is once-only: further interpreters do not displace it. + let _sub = main.create_subinterpreter(); + let _main2 = Interpreter::without_stdlib(Default::default()); + assert_eq!(runtime::main_interpreter_id(), Some(recorded)); + } } diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 6009f421e12..54d3e813eec 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -13,6 +13,7 @@ mod interpreter; mod method; #[cfg(feature = "rustpython-compiler")] mod python_run; +pub mod runtime; mod setting; pub mod thread; mod vm_new; @@ -61,16 +62,22 @@ use std::{ pub use context::Context; pub use interpreter::{Interpreter, InterpreterBuilder}; pub(crate) use method::PyMethod; +pub use runtime::{InterpreterInfo, InterpreterWhence, MAIN_INTERPRETER_ID}; pub use setting::{CheckHashPycsMode, Paths, PyConfig, Settings}; pub const MAX_MEMORY_SIZE: usize = isize::MAX as usize; // Objects are live when they are on stack, or referenced by a name (for now) -/// Top level container of a python virtual machine. In theory you could -/// create more instances of this struct and have them operate fully isolated. +/// Per-thread execution context for a single interpreter (≈ CPython `PyThreadState`). /// -/// To construct this, please refer to the [`Interpreter`] +/// A `VirtualMachine` holds thread-local eval state (exceptions, recursion, frames, +/// datastack) plus shared references to interpreter-owned data (`state`, +/// `builtins`, `sys_module`, `ctx`). Multiple VMs may share the same +/// [`PyGlobalState`] via `VirtualMachine::new_thread`; distinct interpreters +/// each have their own `PyGlobalState` (see [`Interpreter::create_subinterpreter`]). +/// +/// To construct the main VM of an interpreter, use [`Interpreter`]. pub struct VirtualMachine { pub builtins: PyRef, pub sys_module: PyRef, @@ -93,6 +100,11 @@ pub struct VirtualMachine { pub state: PyRc, pub initialized: bool, recursion_depth: Cell, + /// Depth of native recursion that pushes no Python frame, counted only + /// where the stack pointer cannot be read. Everywhere else the native + /// stack itself answers, and nothing needs counting. + #[cfg(any(miri, target_env = "musl"))] + native_recursion_depth: Cell, /// C stack soft limit for detecting stack overflow (like c_stack_soft_limit) #[cfg_attr(any(miri, target_env = "musl"), allow(dead_code))] c_stack_soft_limit: Cell, @@ -110,11 +122,11 @@ pub struct VirtualMachine { /// pointer here before returning `ExecutionResult::TailCall`. /// Access only via `set_pending_tailcall` / `take_pending_tailcall`. pending_tailcall_frame: Cell>, - /// Owned references that keep callee raw pointers valid during TailCall. - /// Set by `tailcall_prepare_frame`, drained by the trampoline into - /// its local `owned_refs` Vec. Uses UnsafeCell because the VM is - /// per-thread and this field is only accessed on the owning thread. - pub(crate) pending_tailcall_refs: core::cell::UnsafeCell>, + /// Owned reference that keeps callee raw pointers valid during TailCall. + /// Set by the exact-call handlers and moved into the trampoline's + /// `SuspendedFrame`. Uses UnsafeCell because the VM is per-thread and this + /// field is only accessed on the owning thread. + pending_tailcall_owner: core::cell::UnsafeCell>, } /// Non-owning frame pointer for the non-unix threading frames stack. @@ -257,9 +269,9 @@ impl StopTheWorldState { } #[inline] - fn init_thread_countdown(&self, vm: &VirtualMachine) -> i64 { + fn init_thread_countdown(&self, state: &PyGlobalState) -> i64 { let requester = self.requester.load(Ordering::Relaxed); - let registry = vm.state.thread_frames.lock(); + let registry = state.thread_frames.lock(); // Keep requested/count initialization serialized with thread-slot // registration (which also takes this lock), matching the // HEAD_LOCK-guarded stop-the-world bookkeeping. @@ -288,10 +300,10 @@ impl StopTheWorldState { /// Try to CAS detached threads directly to SUSPENDED and check whether /// stop countdown reached zero after parking detached threads. - fn park_detached_threads(&self, vm: &VirtualMachine) -> bool { + fn park_detached_threads(&self, state: &PyGlobalState) -> bool { use thread::{THREAD_ATTACHED, THREAD_DETACHED, THREAD_SUSPENDED}; let requester = self.requester.load(Ordering::Relaxed); - let registry = vm.state.thread_frames.lock(); + let registry = state.thread_frames.lock(); let mut attached_seen = 0u64; let mut forced_parks = 0u64; @@ -377,7 +389,7 @@ impl StopTheWorldState { /// is only ever `try_lock`'d. The active requester therefore force-parks /// this thread, finishes its whole stop→start span, releases the exclusion, /// and only then does this thread resume and acquire it. - fn acquire_exclusion(&self) { + fn acquire_exclusion(&self, state: &PyGlobalState) { if self .exclusion .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) @@ -386,7 +398,7 @@ impl StopTheWorldState { return; } loop { - crate::vm::thread::suspend_if_needed(self); + crate::vm::thread::suspend_if_needed(state); std::thread::yield_now(); if self .exclusion @@ -413,23 +425,23 @@ impl StopTheWorldState { /// Takes the shared exclusion first so at most one requester (fork or GC) /// drives the stop→start span at a time; it is released by /// `start_the_world`/`reset_after_fork`. - pub fn stop_the_world(&self, vm: &VirtualMachine) { - self.acquire_exclusion(); + pub fn stop_the_world(&self, state: &PyGlobalState) { + self.acquire_exclusion(state); let start = std::time::Instant::now(); let requester_ident = crate::stdlib::_thread::get_ident(); self.requester.store(requester_ident, Ordering::Relaxed); self.stats_stop_calls.fetch_add(1, Ordering::Relaxed); - let initial_countdown = self.init_thread_countdown(vm); + let initial_countdown = self.init_thread_countdown(state); stw_trace(format_args!("stop begin requester={requester_ident}")); // Park detached threads and set stop bits, then confirm every other // thread is SUSPENDED. The completion condition is level-triggered // (`all_non_requester_suspended`) so an already-suspended thread that // was counted but will not notify again cannot stall the stop. - self.park_detached_threads(vm); - if initial_countdown == 0 || self.all_non_requester_suspended(vm) { + self.park_detached_threads(state); + if initial_countdown == 0 || self.all_non_requester_suspended(state) { self.world_stopped.store(true, Ordering::Release); #[cfg(debug_assertions)] - self.debug_assert_all_non_requester_suspended(vm); + self.debug_assert_all_non_requester_suspended(state); stw_trace(format_args!( "stop end requester={requester_ident} wait_ns=0 polls=0" )); @@ -438,8 +450,8 @@ impl StopTheWorldState { let mut polls = 0u64; loop { - self.park_detached_threads(vm); - if self.all_non_requester_suspended(vm) { + self.park_detached_threads(state); + if self.all_non_requester_suspended(state) { break; } polls = polls.saturating_add(1); @@ -447,7 +459,7 @@ impl StopTheWorldState { // Re-check under the wait mutex first to avoid a lost-wake race: // a thread may have suspended and notified right before we enter wait. let guard = self.notify_mutex.lock().unwrap(); - if self.all_non_requester_suspended(vm) { + if self.all_non_requester_suspended(state) { drop(guard); break; } @@ -476,18 +488,18 @@ impl StopTheWorldState { } self.world_stopped.store(true, Ordering::Release); #[cfg(debug_assertions)] - self.debug_assert_all_non_requester_suspended(vm); + self.debug_assert_all_non_requester_suspended(state); stw_trace(format_args!( "stop end requester={requester_ident} wait_ns={wait_ns} polls={polls}" )); } /// Resume all suspended threads (`start_the_world`). - pub fn start_the_world(&self, vm: &VirtualMachine) { + pub fn start_the_world(&self, state: &PyGlobalState) { use thread::{THREAD_DETACHED, THREAD_SUSPENDED}; let requester = self.requester.load(Ordering::Relaxed); stw_trace(format_args!("start begin requester={requester}")); - let registry = vm.state.thread_frames.lock(); + let registry = state.thread_frames.lock(); // Clear the request flag BEFORE waking threads. Otherwise a thread // returning from allow_threads → attach_thread could observe // `requested == true`, re-suspend itself, and stay parked forever. @@ -521,7 +533,7 @@ impl StopTheWorldState { self.thread_countdown.store(0, Ordering::Release); self.requester.store(0, Ordering::Relaxed); #[cfg(debug_assertions)] - self.debug_assert_all_non_requester_detached(vm); + self.debug_assert_all_non_requester_detached(state); // Release the exclusion last, ending the stop→start span so the next // requester (fork or GC) can proceed. self.release_exclusion(); @@ -604,10 +616,10 @@ impl StopTheWorldState { /// lost-decrement race under rapid back-to-back stops: a thread that is /// already SUSPENDED when a new stop counts it neither notifies nor is /// force-parked again, so an edge-based countdown could never reach zero. - fn all_non_requester_suspended(&self, vm: &VirtualMachine) -> bool { + fn all_non_requester_suspended(&self, state: &PyGlobalState) -> bool { use thread::THREAD_SUSPENDED; let requester = self.requester.load(Ordering::Relaxed); - let registry = vm.state.thread_frames.lock(); + let registry = state.thread_frames.lock(); #[expect( clippy::iter_over_hash_type, @@ -625,10 +637,10 @@ impl StopTheWorldState { } #[cfg(debug_assertions)] - fn debug_assert_all_non_requester_suspended(&self, vm: &VirtualMachine) { + fn debug_assert_all_non_requester_suspended(&self, state: &PyGlobalState) { use thread::THREAD_SUSPENDED; let requester = self.requester.load(Ordering::Relaxed); - let registry = vm.state.thread_frames.lock(); + let registry = state.thread_frames.lock(); #[expect( clippy::iter_over_hash_type, @@ -648,10 +660,10 @@ impl StopTheWorldState { } #[cfg(debug_assertions)] - fn debug_assert_all_non_requester_detached(&self, vm: &VirtualMachine) { + fn debug_assert_all_non_requester_detached(&self, state: &PyGlobalState) { use thread::THREAD_SUSPENDED; let requester = self.requester.load(Ordering::Relaxed); - let registry = vm.state.thread_frames.lock(); + let registry = state.thread_frames.lock(); #[expect( clippy::iter_over_hash_type, @@ -732,14 +744,30 @@ pub(crate) struct CallableCache { pub builtin_any: Option, } +/// Per-interpreter shared state (≈ CPython `PyInterpreterState`). +/// +/// Not process-global: each [`Interpreter`] (main or subinterpreter) owns its own +/// `PyGlobalState`. Process-wide pieces live elsewhere (`Context::genesis`, +/// GC, the interpreter registry in [`runtime`]). pub struct PyGlobalState { + /// Unique process-global interpreter id (main is [`MAIN_INTERPRETER_ID`]). + pub interpreter_id: i64, + /// How this interpreter was created. + pub whence: runtime::InterpreterWhence, + /// True for every top-level (non-sub) interpreter, each of which keeps its + /// own signal and main-thread bookkeeping. Only the first one registered + /// becomes *the* process main — see [`runtime::main_interpreter_id`]. + pub is_main: bool, pub config: PyConfig, pub module_defs: BTreeMap<&'static str, &'static builtins::PyModuleDef>, pub frozen: HashMap<&'static str, FrozenModule, rapidhash::quality::RandomState>, pub stacksize: AtomicCell, pub thread_count: AtomicCell, pub hash_secret: HashSecret, - pub atexit_funcs: PyMutex>>, + /// Registered `atexit` callbacks, newest first. Shared ownership so + /// `atexit.unregister` can keep the entry it is comparing alive while the + /// list is unlocked, and still recognize it afterwards by identity. + pub atexit_funcs: PyMutex>>, pub codec_registry: CodecsRegistry, pub finalizing: AtomicBool, pub warnings: WarningsState, @@ -777,6 +805,16 @@ pub struct PyGlobalState { /// Stop-the-world state for pre-fork thread suspension #[cfg(feature = "threading")] pub stop_the_world: StopTheWorldState, + /// This interpreter's garbage collector policy and results. + pub gc: crate::gc_state::GcInterpreterState, +} + +impl PyGlobalState { + #[inline] + #[must_use] + pub fn is_main_interpreter(&self) -> bool { + self.is_main + } } pub fn process_hash_secret_seed() -> u32 { @@ -828,11 +866,12 @@ pub(crate) struct IframeEntryState { struct SuspendedFrame { iframe: *mut crate::frame::InterpreterFrame, entry_state: IframeEntryState, - /// Owned references that keep callee's raw pointers (code, globals, - /// builtins borrowed from PyFunction) valid. Drained from - /// `vm.pending_tailcall_refs` when the callee's TailCall is consumed. - /// Dropped when this SuspendedFrame is popped (after callee returns/errors). - owned_refs: Vec, + /// Function that owns the callee's raw pointers (code, globals, builtins, + /// closure, and func_obj). Moved from `vm.pending_tailcall_owner` when the + /// callee's TailCall is consumed. + /// Dropped as soon as this SuspendedFrame is popped — the callee has + /// returned or raised and its frame is already released by then. + callee_owner: PyObjectRef, /// True for the initial frame passed into the trampoline by the caller. /// The caller owns the datastack allocation for the entry frame, so the /// trampoline must NOT release it — only callee-allocated frames are @@ -865,6 +904,13 @@ impl VirtualMachine { unsafe { (*self.datastack.get()).push(size) } } + /// Bump-allocate a full frame, returning whether the same cleared LIFO + /// block and size were reused. + #[inline(always)] + pub(crate) fn datastack_push_frame(&self, size: usize) -> (*mut u8, bool) { + unsafe { (*self.datastack.get()).push_frame(size) } + } + /// Check whether the thread data stack currently has room for `size` bytes. #[inline(always)] pub(crate) fn datastack_has_space(&self, size: usize) -> bool { @@ -881,6 +927,12 @@ impl VirtualMachine { unsafe { (*self.datastack.get()).pop(base) } } + /// Pop a full frame after its localsplus slots have been cleared. + #[inline(always)] + pub(crate) unsafe fn datastack_pop_frame(&self, base: *mut u8, size: usize) { + unsafe { (*self.datastack.get()).pop_frame(base, size) } + } + /// Temporarily detach the current thread (ATTACHED → DETACHED) while /// running `f`, then re-attach afterwards. Allows `stop_the_world` to /// park this thread during blocking syscalls. @@ -947,6 +999,8 @@ impl VirtualMachine { state, initialized: false, recursion_depth: Cell::new(0), + #[cfg(any(miri, target_env = "musl"))] + native_recursion_depth: Cell::new(0), c_stack_soft_limit: Cell::new(Self::calculate_c_stack_soft_limit()), async_gen_firstiter: RefCell::new(None), async_gen_finalizer: RefCell::new(None), @@ -955,7 +1009,7 @@ impl VirtualMachine { callable_cache: CallableCache::default(), audit_hooks: RefCell::new(vec![]), pending_tailcall_frame: Cell::new(None), - pending_tailcall_refs: core::cell::UnsafeCell::new(Vec::with_capacity(2)), + pending_tailcall_owner: core::cell::UnsafeCell::new(None), }; if vm.state.hash_secret.hash_str("") @@ -1083,9 +1137,12 @@ impl VirtualMachine { assert!(!self.initialized, "Double Initialize Error"); - // Initialize main thread ident before any threading operations + // Process main-thread identity is owned by the main interpreter only + // (used for signal handling / `_thread._is_main_interpreter` helpers). #[cfg(feature = "threading")] - stdlib::_thread::init_main_thread_ident(self); + if self.state.is_main_interpreter() { + stdlib::_thread::init_main_thread_ident(self); + } stdlib::builtins::init_module(self, &self.builtins); let callable_cache_init = self.init_callable_cache(); @@ -1406,6 +1463,22 @@ impl VirtualMachine { .set(Some(PendingFrame(core::ptr::NonNull::from(iframe)))); } + /// Store the function that owns the fields borrowed by the pending callee. + #[inline(always)] + pub(crate) fn set_pending_tailcall_owner(&self, owner: PyObjectRef) { + let slot = unsafe { &mut *self.pending_tailcall_owner.get() }; + debug_assert!(slot.is_none(), "pending TailCall owner was not consumed"); + *slot = Some(owner); + } + + /// Take the pending callee owner, resetting the side channel. + #[inline(always)] + fn take_pending_tailcall_owner(&self) -> PyObjectRef { + unsafe { &mut *self.pending_tailcall_owner.get() } + .take() + .expect("TailCall without pending owner") + } + /// Take the pending tailcall frame pointer, resetting the side channel. #[inline(always)] fn take_pending_tailcall(&self) -> *mut crate::frame::InterpreterFrame { @@ -1466,14 +1539,11 @@ impl VirtualMachine { } let initial_ptr = self.take_pending_tailcall(); - // Drain the refs that keep the initial callee's raw pointers alive. - let initial_refs = unsafe { &mut *self.pending_tailcall_refs.get() } - .drain(..) - .collect(); + let initial_owner = self.take_pending_tailcall_owner(); frame_stack.push(SuspendedFrame { iframe: iframe as *mut crate::frame::InterpreterFrame, entry_state, - owned_refs: initial_refs, + callee_owner: initial_owner, is_entry: true, }); let mut action = Action::EnterCallee(initial_ptr); @@ -1486,8 +1556,8 @@ impl VirtualMachine { Ok(state) => state, Err(exc) => { unsafe { - if let Some(base) = callee.release_datastack_frame() { - self.datastack_pop(base); + if let Some((base, size)) = callee.release_datastack_frame() { + self.datastack_pop_frame(base, size); } } action = Action::Unwind(exc); @@ -1498,13 +1568,11 @@ impl VirtualMachine { let result = crate::frame::run_iframe(callee, self); match result { Ok(ExecutionResult::TailCall) => { - let refs = unsafe { &mut *self.pending_tailcall_refs.get() } - .drain(..) - .collect(); + let callee_owner = self.take_pending_tailcall_owner(); frame_stack.push(SuspendedFrame { iframe: callee_ptr, entry_state: callee_entry, - owned_refs: refs, + callee_owner, is_entry: false, }); action = Action::EnterCallee(self.take_pending_tailcall()); @@ -1512,8 +1580,8 @@ impl VirtualMachine { Ok(ExecutionResult::Return(value)) => { self.exit_iframe(callee_entry); unsafe { - if let Some(base) = callee.release_datastack_frame() { - self.datastack_pop(base); + if let Some((base, size)) = callee.release_datastack_frame() { + self.datastack_pop_frame(base, size); } } action = Action::ReturnValue(value); @@ -1522,8 +1590,8 @@ impl VirtualMachine { Err(exc) => { self.exit_iframe(callee_entry); unsafe { - if let Some(base) = callee.release_datastack_frame() { - self.datastack_pop(base); + if let Some((base, size)) = callee.release_datastack_frame() { + self.datastack_pop_frame(base, size); } } action = Action::Unwind(exc); @@ -1539,34 +1607,38 @@ impl VirtualMachine { let SuspendedFrame { iframe: caller_iframe_ptr, entry_state: caller_entry, - owned_refs: _caller_refs, + callee_owner, is_entry: caller_is_entry, } = caller; + // The callee's frame was released before this action was + // formed, and a materialized frame object holds its own + // references, so nothing borrows the callee's function any + // more. Release it here, at the callee's return, rather than + // holding it across the caller's next stretch of bytecode. + drop(callee_owner); let caller_iframe = unsafe { &mut *caller_iframe_ptr }; caller_iframe.localsplus.push_stack(value); let result = crate::frame::run_iframe(caller_iframe, self); match result { Ok(ExecutionResult::TailCall) => { - let refs = unsafe { &mut *self.pending_tailcall_refs.get() } - .drain(..) - .collect(); - drop(_caller_refs); + let next_callee_owner = self.take_pending_tailcall_owner(); frame_stack.push(SuspendedFrame { iframe: caller_iframe_ptr, entry_state: caller_entry, - owned_refs: refs, + callee_owner: next_callee_owner, is_entry: caller_is_entry, }); action = Action::EnterCallee(self.take_pending_tailcall()); } Ok(ExecutionResult::Return(value)) => { - drop(_caller_refs); self.exit_iframe(caller_entry); if !caller_is_entry { unsafe { - if let Some(base) = caller_iframe.release_datastack_frame() { - self.datastack_pop(base); + if let Some((base, size)) = + caller_iframe.release_datastack_frame() + { + self.datastack_pop_frame(base, size); } } } @@ -1574,12 +1646,13 @@ impl VirtualMachine { } Ok(ExecutionResult::Yield(_)) => panic!("Yield in non-generator frame"), Err(exc) => { - drop(_caller_refs); self.exit_iframe(caller_entry); if !caller_is_entry { unsafe { - if let Some(base) = caller_iframe.release_datastack_frame() { - self.datastack_pop(base); + if let Some((base, size)) = + caller_iframe.release_datastack_frame() + { + self.datastack_pop_frame(base, size); } } } @@ -1595,9 +1668,13 @@ impl VirtualMachine { let SuspendedFrame { iframe: caller_iframe_ptr, entry_state: caller_entry, - owned_refs: _caller_refs, + callee_owner, is_entry: caller_is_entry, } = caller; + // Released at the callee's return, for the same reason as + // in `ReturnValue`: the exception carries owned references + // through its traceback, not borrows into the callee frame. + drop(callee_owner); let caller_iframe = unsafe { &mut *caller_iframe_ptr }; let handled = @@ -1609,27 +1686,23 @@ impl VirtualMachine { let result = crate::frame::run_iframe(caller_iframe, self); match result { Ok(ExecutionResult::TailCall) => { - let refs = unsafe { &mut *self.pending_tailcall_refs.get() } - .drain(..) - .collect(); - drop(_caller_refs); + let next_callee_owner = self.take_pending_tailcall_owner(); frame_stack.push(SuspendedFrame { iframe: caller_iframe_ptr, entry_state: caller_entry, - owned_refs: refs, + callee_owner: next_callee_owner, is_entry: caller_is_entry, }); action = Action::EnterCallee(self.take_pending_tailcall()); } Ok(ExecutionResult::Return(value)) => { - drop(_caller_refs); self.exit_iframe(caller_entry); if !caller_is_entry { unsafe { - if let Some(base) = + if let Some((base, size)) = caller_iframe.release_datastack_frame() { - self.datastack_pop(base); + self.datastack_pop_frame(base, size); } } } @@ -1639,14 +1712,13 @@ impl VirtualMachine { panic!("Yield in non-generator frame") } Err(new_exc) => { - drop(_caller_refs); self.exit_iframe(caller_entry); if !caller_is_entry { unsafe { - if let Some(base) = + if let Some((base, size)) = caller_iframe.release_datastack_frame() { - self.datastack_pop(base); + self.datastack_pop_frame(base, size); } } } @@ -1655,12 +1727,13 @@ impl VirtualMachine { } } Ok(Some(ExecutionResult::Return(value))) => { - drop(_caller_refs); self.exit_iframe(caller_entry); if !caller_is_entry { unsafe { - if let Some(base) = caller_iframe.release_datastack_frame() { - self.datastack_pop(base); + if let Some((base, size)) = + caller_iframe.release_datastack_frame() + { + self.datastack_pop_frame(base, size); } } } @@ -1670,12 +1743,13 @@ impl VirtualMachine { panic!("Unexpected execution result in trampoline unwind") } Err(new_exc) => { - drop(_caller_refs); self.exit_iframe(caller_entry); if !caller_is_entry { unsafe { - if let Some(base) = caller_iframe.release_datastack_frame() { - self.datastack_pop(base); + if let Some((base, size)) = + caller_iframe.release_datastack_frame() + { + self.datastack_pop_frame(base, size); } } } @@ -1767,14 +1841,14 @@ impl VirtualMachine { // Phase 4: GC collect — modules removed from sys.modules are freed, // exposing cycles (e.g., dict ↔ function.__globals__). GC collects // these and calls __del__ while module dicts are still intact. - crate::gc_state::gc_state().collect_force(2); + self.state.gc.collect_force(2); // Phase 5: Clear module dicts in reverse import order using 2-pass algorithm. // Skip builtins and sys — those are cleared last. self.finalize_clear_module_dicts(&module_weakrefs); // Phase 6: GC collect — pick up anything freed by dict clearing. - crate::gc_state::gc_state().collect_force(2); + self.state.gc.collect_force(2); // Phase 7: Clear sys and builtins dicts last self.finalize_clear_sys_builtins_dict(); @@ -1940,6 +2014,14 @@ impl VirtualMachine { const STACK_MARGIN_BYTES: usize = (if cfg!(debug_assertions) { 16384 } else { 4096 }) * core::mem::size_of::(); + /// How deep native recursion may go where the stack cannot be measured + /// (`Py_C_RECURSION_LIMIT`). A native step costs far more stack than a + /// Python one and debug builds cost more again, so this sits well under + /// what a default stack holds rather than at what it would just fit. + #[cfg(any(miri, target_env = "musl"))] + const NATIVE_RECURSION_LIMIT_UNMEASURED: usize = + if cfg!(debug_assertions) { 500 } else { 1500 }; + /// Get the stack boundaries using platform-specific APIs. /// Returns (base, top) where base is the lowest address and top is the highest. #[cfg(all(not(miri), not(target_env = "musl"), windows))] @@ -2041,16 +2123,34 @@ impl VirtualMachine { /// Used to run the body of a (possibly) recursive function. It will raise a /// RecursionError if recursive functions are nested far too many times, /// preventing a stack overflow. + /// `Py_EnterRecursiveCall`: bounds native recursion that pushes no Python + /// frame, against the native stack. That is a separate budget from the + /// frame limit `sys.setrecursionlimit()` sets, so nesting counted here does + /// not come out of what Python code has left to call with. pub fn with_recursion PyResult>(&self, _where: &str, f: F) -> PyResult { - self.check_recursive_call(_where)?; - - // Native stack guard: check C stack like _Py_MakeRecCheck - if self.check_c_stack_overflow() { - return Err(self.new_recursion_error(_where.to_string())); + // `check_c_stack_overflow()` answers no unconditionally where the stack + // pointer cannot be read, which would leave this guard with nothing to + // stop. A count of the nesting stands in for the measurement there. + #[cfg(any(miri, target_env = "musl"))] + let counted_too_deep = + self.native_recursion_depth.get() >= Self::NATIVE_RECURSION_LIMIT_UNMEASURED; + #[cfg(not(any(miri, target_env = "musl")))] + let counted_too_deep = false; + + if counted_too_deep || self.check_c_stack_overflow() { + return Err( + self.new_recursion_error(format!("maximum recursion depth exceeded {_where}")) + ); } - self.recursion_depth.update(|d| d + 1); - scopeguard::defer! { self.recursion_depth.update(|d| d - 1) } + #[cfg(any(miri, target_env = "musl"))] + let _native_depth_guard = { + self.native_recursion_depth.update(|d| d + 1); + scopeguard::guard((), |()| { + self.native_recursion_depth.update(|d| d.saturating_sub(1)) + }) + }; + f() } @@ -2132,7 +2232,7 @@ impl VirtualMachine { self.restore_exception(saved_exc); } // Clear previous before popping — it may point to a stack-allocated - // iframe that will be freed when the caller's with_iframe exits. + // iframe that will be freed when the caller releases its frame. { #[allow(unused_imports)] use rustpython_common::atomic::Radium; @@ -2245,6 +2345,9 @@ impl VirtualMachine { core::sync::atomic::Ordering::Relaxed, ); } + // The slots above are the last write this thread makes into + // the frame object, so it is now readable from anywhere. + fo.iframe().detach(); if !old_chain.is_null() { let prev_iframe = unsafe { &*old_chain }; let back_fo = prev_iframe.materialize_chain(self); @@ -2261,7 +2364,7 @@ impl VirtualMachine { self.restore_exception(saved_exc); } // Clear previous before popping — it may point to a stack-allocated - // iframe that will be freed when the caller's with_iframe exits. + // iframe that will be freed when the caller releases its frame. { #[allow(unused_imports)] use rustpython_common::atomic::Radium; @@ -2284,8 +2387,10 @@ impl VirtualMachine { if mat_ptr != 0 { let fo = unsafe { &*(mat_ptr as *const crate::Py) }; unsafe { - crate::gc_state::gc_state() - .track_object(core::ptr::NonNull::from(fo.as_object())); + crate::gc_state::gc_state().track_object( + core::ptr::NonNull::from(fo.as_object()), + crate::gc_state::current_owner(), + ); let live_iframe = &*iframe_ptr; live_iframe.cold().temporary_refs.lock().clear(); } @@ -2293,20 +2398,6 @@ impl VirtualMachine { } } - pub fn with_iframe( - &self, - iframe: &mut crate::frame::InterpreterFrame, - f: impl FnOnce(&mut crate::frame::InterpreterFrame) -> PyResult, - ) -> PyResult { - let state = self.enter_iframe(iframe)?; - // Ensure exit_iframe runs even if f(iframe) panics. - let guard = scopeguard::guard(state, |s| self.exit_iframe(s)); - let result = f(iframe); - let state = scopeguard::ScopeGuard::into_inner(guard); - self.exit_iframe(state); - result - } - /// FrameObject execution for generator/coroutine resume. /// Pushes a new exc_info slot (gi_exc_state) onto the chain, /// linking the generator's saved handled-exception. @@ -2348,7 +2439,7 @@ impl VirtualMachine { frame.iframe().owner.store(old_owner, core::sync::atomic::Ordering::Release); self.pop_exception(); // Clear previous before popping — it may point to a stack-allocated - // iframe that will be freed when the caller's with_iframe exits. + // iframe that will be freed when the caller releases its frame. { #[allow(unused_imports)] use rustpython_common::atomic::Radium; @@ -2548,12 +2639,28 @@ impl VirtualMachine { // Objects/listobject.c. Each branch takes an atomic snapshot to avoid // race conditions from concurrent mutation (no GIL). let cls = value.class(); - let list_borrow; let slice = if cls.is(self.ctx.types.tuple_type) { value.downcast_ref::().unwrap().as_slice() } else if cls.is(self.ctx.types.list_type) { - list_borrow = value.downcast_ref::().unwrap().borrow_vec(); - &list_borrow + // The list is re-read on every step, the way map_iterable_object() + // does it: func() runs Python, which can mutate or even clear the + // same list, and a borrow held across that call deadlocks it. + let list = value.downcast_ref::().unwrap(); + let mut results = Vec::new(); + let mut i = 0; + loop { + let elem = { + let elements = list.borrow_vec(); + let Some(elem) = elements.get(i) else { + break; + }; + elem.clone() + // free the lock + }; + results.push(func(elem)?); + i += 1; + } + return Ok(results); } else if cls.is(self.ctx.types.dict_type) { let keys = value.downcast_ref::().unwrap().keys_vec(); return keys.into_iter().map(func).collect(); @@ -2742,7 +2849,7 @@ impl VirtualMachine { // Suspend this thread if stop-the-world is in progress #[cfg(feature = "threading")] - thread::suspend_if_needed(&self.state.stop_the_world); + thread::suspend_if_needed(&self.state); // Pass a QSBR checkpoint if requested (deferred memory reclamation). #[cfg(feature = "threading")] @@ -2764,7 +2871,7 @@ impl VirtualMachine { #[cfg(feature = "threading")] pub(crate) fn run_scheduled_gc(&self) { if crate::signal::take_gc_scheduled() { - crate::gc_state::gc_state().collect(0); + self.state.gc.collect(0); } } diff --git a/crates/vm/src/vm/runtime.rs b/crates/vm/src/vm/runtime.rs new file mode 100644 index 00000000000..7c5168c45b1 --- /dev/null +++ b/crates/vm/src/vm/runtime.rs @@ -0,0 +1,326 @@ +//! Process-global runtime support for multiple interpreters (PEP 734 preparation). +//! +//! CPython maps roughly as: +//! - this module ≈ `_PyRuntimeState.interpreters` + ID allocation +//! - [`crate::vm::PyGlobalState`] ≈ `PyInterpreterState` +//! - [`crate::VirtualMachine`] ≈ `PyThreadState` (plus shared refs to interpreter state) +//! +//! Multiple [`crate::Interpreter`] instances can coexist in one process. Each owns +//! an isolated `PyGlobalState` (modules, codecs, thread registry, stop-the-world, …) +//! while sharing the process-wide [`crate::Context`] (builtin types / immortals). + +use crate::common::rc::PyRc; +use crate::vm::PyGlobalState; +use core::sync::atomic::{AtomicI64, Ordering}; +use parking_lot::Mutex; +use std::collections::HashMap; + +/// Where an interpreter state came from (mirrors CPython `_PyInterpreterState_GetWhence`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[repr(i32)] +pub enum InterpreterWhence { + /// Unknown / not recorded. + Unknown = 0, + /// Created as the process main interpreter at runtime init. + Runtime = 1, + /// Legacy C-API creation path (reserved for C-API parity). + LegacyCapi = 2, + /// Modern C-API creation path (reserved for C-API parity). + Capi = 3, + /// Cross-interpreter C-API (reserved). + Xi = 4, + /// Created via the stdlib / Rust subinterpreter API (PEP 734). + Stdlib = 5, +} + +impl InterpreterWhence { + #[must_use] + pub const fn as_i32(self) -> i32 { + self as i32 + } +} + +/// Snapshot of a registered interpreter for enumeration APIs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct InterpreterInfo { + pub id: i64, + pub whence: InterpreterWhence, +} + +struct RegistryEntry { + whence: InterpreterWhence, + /// Weak handle so the registry does not keep interpreters alive. + /// Type matches `PyRc` (Arc when threading, Rc otherwise). + #[cfg(feature = "threading")] + state: alloc::sync::Weak, + #[cfg(not(feature = "threading"))] + state: alloc::rc::Weak, +} + +/// `main_id` value before any main interpreter has been registered. +const NO_MAIN_INTERPRETER: i64 = -1; + +struct InterpreterRegistry { + next_id: AtomicI64, + /// Id of the first registered `is_main` interpreter (PEP 734 `get_main()`), + /// or [`NO_MAIN_INTERPRETER`]. + main_id: AtomicI64, + /// id → entry. Main interpreter is always id 0 when created first. + entries: Mutex>, +} + +impl InterpreterRegistry { + fn new() -> Self { + Self { + // Monotonic ids starting at 0. Concurrent Interpreter construction + // (e.g. cargo test threads) must never share an id. + next_id: AtomicI64::new(0), + main_id: AtomicI64::new(NO_MAIN_INTERPRETER), + entries: Mutex::new(HashMap::new()), + } + } +} + +/// The interpreter registry. +/// +/// With `threading` this is one process-global table. Without it, `PyRc` is +/// `Rc` and each OS thread owns an independent `Context::genesis()` and +/// `GcState`, so the registry is thread-local for the same reason `gc_state()` +/// is: an `Rc` handle must never be reachable from another thread. +/// `static_cell!` provides exactly that split. +fn registry() -> &'static InterpreterRegistry { + rustpython_common::static_cell! { + static REGISTRY: InterpreterRegistry; + } + REGISTRY.get_or_init(InterpreterRegistry::new) +} + +/// Conventional id of the first process main interpreter when allocation is +/// sequential (CPython parity). Concurrent construction may assign other ids; +/// use [`PyGlobalState::is_main`] / [`crate::Interpreter::is_main`] to identify +/// a main interpreter, not this constant alone. +pub const MAIN_INTERPRETER_ID: i64 = 0; + +/// Backs `sys.implementation.supports_isolated_interpreters`. +/// +/// The Rust substrate already isolates interpreters (`PyGlobalState` per +/// interpreter, per-interpreter thread slots / stop-the-world). This stays +/// `false` until the Python-facing `_interpreters` module is wired up; flip it +/// in the commit that lands `_interpreters`. +pub const SUPPORTS_ISOLATED_INTERPRETERS: bool = false; + +/// Id of the main interpreter (PEP 734 `get_main()`), or `None` before any +/// interpreter has been created. +/// +/// This is distinct from [`PyGlobalState::is_main`]: every top-level (non-sub) +/// interpreter carries `is_main` for its own signal / main-thread bookkeeping, +/// but only the first one registered becomes *the* main. +#[must_use] +pub fn main_interpreter_id() -> Option { + match registry().main_id.load(Ordering::Acquire) { + NO_MAIN_INTERPRETER => None, + id => Some(id), + } +} + +/// Allocate a unique interpreter id. +/// +/// Ids are strictly monotonic and never reused for the lifetime of the +/// registry, so concurrent `Interpreter` construction (parallel unit tests, +/// multi-threaded embedding) never shares an id. Without `threading` the +/// registry — like `Context::genesis()` and the GC state — is per OS thread, so +/// ids are unique within a thread rather than across the process. +pub(crate) fn alloc_interpreter_id() -> i64 { + registry().next_id.fetch_add(1, Ordering::Relaxed) +} + +/// Gate between registering an interpreter and a collection's stop-the-world. +/// +/// A collection snapshots the registry, stops every interpreter in the +/// snapshot, and then reads tracked objects with those threads parked. An +/// interpreter that registered after the snapshot was taken would not be in it, +/// so nothing would stop it, and its bootstrap — which runs Python and mutates +/// the shared generation lists — would run underneath that scan. Registration +/// therefore waits for an in-flight stop to end; the next collection's snapshot +/// then contains the new interpreter. +fn admission() -> &'static Mutex<()> { + static ADMISSION: std::sync::OnceLock> = std::sync::OnceLock::new(); + ADMISSION.get_or_init(|| Mutex::new(())) +} + +/// Take the admission gate for the duration of a stop-the-world. +#[cfg(feature = "threading")] +pub(crate) fn lock_admission_for_stop() -> parking_lot::MutexGuard<'static, ()> { + admission().lock() +} + +/// Add the registry entry, behind the admission gate. +/// +/// Only ever called with this thread detached, because the gate is held across +/// a stop-the-world: an attached thread waiting here, or re-attaching while +/// holding the gate, would leave that stop no safepoint to complete at. Nothing +/// under the gate blocks or allocates a tracked object, so this cannot re-enter +/// the collection it waits for. +fn insert_registry_entry(state: &PyRc) { + let _admission = admission().lock(); + let mut entries = registry().entries.lock(); + // Entries are weak and an interpreter's lifetime is decided by its last + // `PyRc` — which outlives the `Interpreter` handle whenever + // `new_thread()` workers are still running — so nothing removes them at a + // fixed point. Reap the dead ones here to bound the table instead. + entries.retain(|_, entry| entry.state.strong_count() > 0); + entries.insert( + state.interpreter_id, + RegistryEntry { + whence: state.whence, + state: PyRc::downgrade(state), + }, + ); +} + +/// Register an interpreter state in the registry. +pub(crate) fn register_interpreter(state: &PyRc) { + let id = state.interpreter_id; + if state.is_main { + // First `is_main` interpreter defines the main for `get_main()`. + // Additional top-level Interpreters (embedding) keep their own `is_main` + // flag but do not displace the recorded main. + let _ = registry().main_id.compare_exchange( + NO_MAIN_INTERPRETER, + id, + Ordering::AcqRel, + Ordering::Relaxed, + ); + } + // A subinterpreter is registered by a thread that is running its parent, so + // detach for the whole insert rather than only for the wait. + let detached = crate::vm::thread::try_with_current_vm(|vm| { + vm.allow_threads(|| insert_registry_entry(state)) + }); + if detached.is_none() { + insert_registry_entry(state); + } +} + +/// Look up a live interpreter state by id. +#[must_use] +pub fn lookup_interpreter(id: i64) -> Option> { + let entries = registry().entries.lock(); + entries.get(&id).and_then(|e| e.state.upgrade()) +} + +/// List all currently registered (still-alive) interpreters. +#[must_use] +pub fn list_interpreters() -> Vec { + let entries = registry().entries.lock(); + let mut out: Vec = entries + .iter() + .filter_map(|(&id, entry)| { + // Drop dead weak refs from the listing. + if entry.state.strong_count() == 0 { + return None; + } + Some(InterpreterInfo { + id, + whence: entry.whence, + }) + }) + .collect(); + out.sort_by_key(|info| info.id); + out +} + +/// Number of registered interpreters that are still alive. +#[must_use] +pub fn interpreter_count() -> usize { + list_interpreters().len() +} + +/// Reset the registry's locks after `fork()`. +/// +/// The tables are reachable from every thread, so a thread that died in the +/// fork may have left one locked; the child would then deadlock the first time +/// it enumerates interpreters (which the collector now does on every stop). +/// +/// # Safety +/// Must only be called after `fork()` in the child process, when no other +/// threads exist and the calling thread holds none of these locks. +#[cfg(all(unix, feature = "threading"))] +pub unsafe fn reinit_after_fork() { + unsafe { + crate::common::lock::reinit_mutex_after_fork(®istry().entries); + crate::common::lock::reinit_mutex_after_fork(owned_interpreters()); + crate::common::lock::reinit_mutex_after_fork(admission()); + } +} + +/// All live interpreter states, ordered by id. +/// +/// Used by the cyclic collector, which must stop every interpreter's threads +/// (not just the collecting one) because GC-tracked objects from all +/// interpreters share one object graph. Ordering is deterministic so that +/// multiple stop-the-world requesters always take exclusions in the same order. +#[must_use] +pub fn live_interpreter_states() -> Vec> { + let entries = registry().entries.lock(); + let mut states: Vec<(i64, PyRc)> = entries + .iter() + .filter_map(|(&id, entry)| entry.state.upgrade().map(|state| (id, state))) + .collect(); + drop(entries); + states.sort_by_key(|(id, _)| *id); + states.into_iter().map(|(_, state)| state).collect() +} + +/// Runtime-owned interpreters (the ownership anchor for the Python +/// `_interpreters` API). +/// +/// A Rust [`crate::Interpreter`] handle is normally owned by its Rust caller. +/// For PEP 734, `_interpreters.create()` returns only an id and the runtime +/// must keep the interpreter alive until `_interpreters.destroy(id)`. These +/// functions hold that ownership, keyed by interpreter id, while the weak +/// [`registry`] above still drives enumeration and lookup. +/// +/// Only available with the `threading` feature: a runtime-owned interpreter is +/// reachable from other OS threads, which requires `Interpreter: Send` (true +/// only when `PyObjectRef` is `Arc`-backed). +#[cfg(feature = "threading")] +fn owned_interpreters() -> &'static Mutex> { + use std::sync::OnceLock; + static OWNED: OnceLock>> = OnceLock::new(); + OWNED.get_or_init(|| Mutex::new(HashMap::new())) +} + +/// Transfer ownership of `interp` to the runtime, returning its id. +#[cfg(feature = "threading")] +pub fn store_owned_interpreter(interp: crate::Interpreter) -> i64 { + let id = interp.id(); + // Ids are strictly monotonic, so this never displaces (and drops) an + // existing entry under the lock. + owned_interpreters().lock().insert(id, interp); + id +} + +/// Reclaim a runtime-owned interpreter, removing it from the owner table. +/// +/// The returned handle is dropped by the caller *outside* the owner lock; its +/// `Drop` unregisters the interpreter from the weak [`registry`]. +#[cfg(feature = "threading")] +#[must_use] +pub fn take_owned_interpreter(id: i64) -> Option { + owned_interpreters().lock().remove(&id) +} + +/// Whether `id` refers to a runtime-owned interpreter. +#[cfg(feature = "threading")] +#[must_use] +pub fn is_owned_interpreter(id: i64) -> bool { + owned_interpreters().lock().contains_key(&id) +} + +/// Number of runtime-owned interpreters currently alive. +#[cfg(feature = "threading")] +#[must_use] +pub fn owned_interpreter_count() -> usize { + owned_interpreters().lock().len() +} diff --git a/crates/vm/src/vm/setting.rs b/crates/vm/src/vm/setting.rs index 7298c95ab08..3c42ca0b6fc 100644 --- a/crates/vm/src/vm/setting.rs +++ b/crates/vm/src/vm/setting.rs @@ -25,6 +25,7 @@ pub struct Paths { /// Combined configuration: user settings + computed paths /// CPython directly exposes every fields under both of them. /// We separate them to maintain better ownership discipline. +#[derive(Clone)] pub struct PyConfig { pub settings: Settings, pub paths: Paths, @@ -39,6 +40,7 @@ impl PyConfig { /// User-configurable settings for the python vm. #[non_exhaustive] +#[derive(Clone)] pub struct Settings { /// -I pub isolated: bool, diff --git a/crates/vm/src/vm/thread.rs b/crates/vm/src/vm/thread.rs index 1bab539a0a6..3f83d88fe70 100644 --- a/crates/vm/src/vm/thread.rs +++ b/crates/vm/src/vm/thread.rs @@ -5,16 +5,13 @@ use crate::builtins::PyBaseExceptionRef; #[cfg(feature = "threading")] use alloc::sync::Arc; -#[cfg(all(unix, feature = "threading"))] -use crate::frame::FrameObject; use crate::frame::InterpreterFrame; -#[cfg(all(unix, feature = "threading"))] -use crate::{AsObject, Py, PyObject, VirtualMachine}; -#[cfg(all(not(unix), feature = "threading"))] -use crate::{AsObject, PyObject, VirtualMachine}; -#[cfg(not(feature = "threading"))] +#[cfg(feature = "threading")] +use crate::vm::PyGlobalState; use crate::{AsObject, PyObject, VirtualMachine}; #[cfg(all(unix, feature = "threading"))] +use crate::{Py, frame::FrameObject}; +#[cfg(all(unix, feature = "threading"))] use core::sync::atomic::AtomicPtr; use core::{ cell::{Cell, RefCell}, @@ -22,6 +19,8 @@ use core::{ sync::atomic::{AtomicUsize, Ordering}, }; use itertools::Itertools; +#[cfg(feature = "threading")] +use std::collections::HashMap; use std::thread_local; // Thread states for stop-the-world support. @@ -47,7 +46,7 @@ pub struct ThreadSlot { /// thread at a safepoint and supplies the happens-before edge, so the /// pointer and the frames it reaches are quiescent and alive at read time. #[cfg(unix)] - pub top_frame: AtomicPtr, + pub top_frame: AtomicPtr>, /// Raw InterpreterFrame pointer, published alongside top_frame so /// cross-thread readers (sys._current_frames) can materialize /// stack-allocated frames that have no FrameObject. @@ -90,7 +89,17 @@ thread_local! { pub(crate) static COROUTINE_ORIGIN_TRACKING_DEPTH: Cell = const { Cell::new(0) }; - /// Current thread's slot for sys._current_frames() and sys._current_exceptions() + /// Per-interpreter thread slots for this OS thread (PEP 734 multi-interpreter). + /// + /// CPython keeps a `PyThreadState` per (thread, interpreter) pair. RustPython + /// mirrors that: each interpreter's `PyGlobalState.thread_frames` gets its own + /// [`ThreadSlot`] for this OS thread. `CURRENT_THREAD_SLOT` always points at + /// the slot for the currently entered interpreter. + #[cfg(feature = "threading")] + static INTERP_THREAD_SLOTS: RefCell> = + RefCell::new(HashMap::new()); + + /// Current thread's slot for the currently entered interpreter. #[cfg(feature = "threading")] static CURRENT_THREAD_SLOT: RefCell> = const { RefCell::new(None) }; @@ -107,7 +116,22 @@ thread_local! { /// initialized; the `Arc` in `CURRENT_THREAD_SLOT` keeps the /// pointee alive until `cleanup_current_thread_frames` clears this. #[cfg(all(unix, feature = "threading"))] - static CURRENT_TOP_FRAME_SLOT: Cell<*const AtomicPtr> = + static CURRENT_TOP_FRAME_SLOT: Cell<*const AtomicPtr>> = + const { Cell::new(core::ptr::null()) }; + + /// Cached pointer to this thread's `ThreadSlot::top_iframe` for the hot + /// light-frame push/pop path. The slot's Arc keeps the pointee alive. + #[cfg(feature = "threading")] + static CURRENT_TOP_IFRAME_SLOT: Cell<*const AtomicUsize> = + const { Cell::new(core::ptr::null()) }; + + /// Cached pointer to this thread's `ThreadSlot::stop_requested`, for the + /// safepoint the dispatch loop takes once per instruction. Reading it + /// through `CURRENT_THREAD_SLOT` costs a `RefCell` borrow — two stores to + /// thread-local memory — where this costs one relaxed load. The slot's Arc + /// keeps the pointee alive, as with the frame pointers above. + #[cfg(feature = "threading")] + static CURRENT_STOP_REQUESTED: Cell<*const core::sync::atomic::AtomicBool> = const { Cell::new(core::ptr::null()) }; } @@ -131,15 +155,42 @@ pub fn with_current_vm(f: impl FnOnce(&VirtualMachine) -> R) -> R { } fn set_current_vm(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { + // Attach to this VM's interpreter, detaching the enclosing one if this is a + // switch between interpreters on the same OS thread. + #[cfg(feature = "threading")] + let switched = begin_interpreter_section(vm); + VM_STACK.with(|vms| { vms.borrow_mut().push(vm.into()); scopeguard::defer! { vms.borrow_mut().pop(); + #[cfg(feature = "threading")] + end_interpreter_section(switched); } f() }) } +/// Pointer to the GC state of the interpreter running on this thread. +/// +/// The pointee belongs to the `PyGlobalState` of the VM on top of `VM_STACK`, +/// which is borrowed for the whole `set_current_vm` scope — so the pointer stays +/// valid as long as the caller remains inside that scope. +pub(crate) fn current_gc_state() -> Option> { + // Reached from every tracked allocation, including ones a thread-local + // destructor makes while the VM stack is being torn down, so neither a + // destroyed key nor an outstanding borrow may panic here. + VM_STACK + .try_with(|vms| { + let vm = vms.try_borrow().ok()?.last().copied()?; + // SAFETY: entries in VM_STACK either borrow a VM for the dynamic + // scope of a set_current_vm()/enter_vm() call or point at GILSTATE_VM. + Some(NonNull::from(&unsafe { vm.as_ref() }.state.gc)) + }) + .ok() + .flatten() +} + pub fn try_with_current_vm(f: impl FnOnce(&VirtualMachine) -> R) -> Option { VM_STACK.with(|vms| { let vm = vms.borrow().last().copied()?; @@ -150,27 +201,8 @@ pub fn try_with_current_vm(f: impl FnOnce(&VirtualMachine) -> R) -> Option } pub fn enter_vm(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { - // Outermost enter_vm: transition DETACHED → ATTACHED - #[cfg(feature = "threading")] - let was_outermost = !current_vm_is_set(); - - // Initialize thread slot for this thread if not already done - #[cfg(feature = "threading")] - init_thread_slot_if_needed(vm); - - #[cfg(feature = "threading")] - if was_outermost { - attach_thread(vm); - } - - scopeguard::defer! { - // Outermost exit: transition ATTACHED → DETACHED - #[cfg(feature = "threading")] - if was_outermost { - detach_thread(); - } - } - + // Attach/detach is handled by `set_current_vm`, which pairs it with the + // VM_STACK push so that switching interpreters mid-stack stays consistent. set_current_vm(vm, f) } @@ -188,29 +220,19 @@ pub fn enter_vm(vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { #[must_use] pub(crate) struct VmBootstrapGuard { #[cfg(feature = "threading")] - was_outermost: bool, + switched: bool, } impl VmBootstrapGuard { pub(crate) fn new(vm: &VirtualMachine) -> Self { - // Outermost: transition DETACHED → ATTACHED #[cfg(feature = "threading")] - let was_outermost = !current_vm_is_set(); - - // Initialize thread slot for this thread if not already done - #[cfg(feature = "threading")] - init_thread_slot_if_needed(vm); - - #[cfg(feature = "threading")] - if was_outermost { - attach_thread(vm); - } + let switched = begin_interpreter_section(vm); VM_STACK.with(|vms| vms.borrow_mut().push(vm.into())); Self { #[cfg(feature = "threading")] - was_outermost, + switched, } } } @@ -221,11 +243,8 @@ impl Drop for VmBootstrapGuard { vms.borrow_mut().pop(); }); - // Outermost exit: transition ATTACHED → DETACHED #[cfg(feature = "threading")] - if self.was_outermost { - detach_thread(); - } + end_interpreter_section(self.switched); } } @@ -287,7 +306,13 @@ pub fn restore_current_thread(state: SavedThreadState) { // SAFETY: borrowed VMs remain alive for the dynamic save/restore scope, // while an owned GILState VM was restored above before this dereference. - attach_thread(unsafe { vm.as_ref() }); + let vm = unsafe { vm.as_ref() }; + // Point CURRENT_THREAD_SLOT at the restored interpreter before attach. + // After subinterpreter bootstrap, CURRENT may still refer to the temporary + // subinterpreter slot (DETACHED); attaching that would leave the parent + // slot detached and later confuse outermost detach. + init_thread_slot_if_needed(vm); + attach_thread(vm); VM_STACK.with(|vms| *vms.borrow_mut() = vm_stack); } @@ -340,43 +365,130 @@ pub fn release_current_thread(state: CurrentVmAttachState) { detach_thread(); } -/// Initialize thread slot for current thread if not already initialized. -/// Called automatically by enter_vm(). +/// Ensure this OS thread has a [`ThreadSlot`] registered with `vm`'s interpreter +/// and make it the current slot. +/// +/// Called automatically by `enter_vm()` / `VmBootstrapGuard` whenever a VM +/// becomes current. Switching between interpreters on the same OS thread swaps +/// `CURRENT_THREAD_SLOT` to that interpreter's slot (creating one if needed). #[cfg(feature = "threading")] fn init_thread_slot_if_needed(vm: &VirtualMachine) { - CURRENT_THREAD_SLOT.with(|slot| { - if slot.borrow().is_none() { - let thread_id = crate::stdlib::_thread::get_ident(); - let mut registry = vm.state.thread_frames.lock(); - let new_slot = Arc::new(ThreadSlot { - #[cfg(unix)] - top_frame: AtomicPtr::new(core::ptr::null_mut()), - top_iframe: AtomicUsize::new(0), - #[cfg(not(unix))] - frames: parking_lot::Mutex::new(Vec::new()), - exception: crate::PyAtomicRef::from(None::), - state: core::sync::atomic::AtomicI32::new( - if vm.state.stop_the_world.requested.load(Ordering::Acquire) { - // Match init_threadstate(): new thread-state starts - // suspended while stop-the-world is active. - THREAD_SUSPENDED - } else { - THREAD_DETACHED - }, - ), - stop_requested: core::sync::atomic::AtomicBool::new(false), - thread: std::thread::current(), - qsbr: crate::object::qsbr::QSBR.register(), - }); - registry.insert(thread_id, new_slot.clone()); - drop(registry); - #[cfg(all(unix, feature = "threading"))] - CURRENT_TOP_FRAME_SLOT.with(|c| c.set(&new_slot.top_frame)); - *slot.borrow_mut() = Some(new_slot); + let slot = ensure_thread_slot(vm); + set_current_thread_slot(slot); +} + +/// Look up (creating if needed) this thread's [`ThreadSlot`] for `vm`'s +/// interpreter, without making it the current slot. +#[cfg(feature = "threading")] +fn ensure_thread_slot(vm: &VirtualMachine) -> CurrentFrameSlot { + let interp_id = vm.state.interpreter_id; + INTERP_THREAD_SLOTS.with(|slots| { + let mut slots = slots.borrow_mut(); + if let Some(existing) = slots.get(&interp_id) { + return existing.clone(); } + + let thread_id = crate::stdlib::_thread::get_ident(); + let mut registry = vm.state.thread_frames.lock(); + let new_slot = Arc::new(ThreadSlot { + #[cfg(unix)] + top_frame: AtomicPtr::new(core::ptr::null_mut()), + top_iframe: AtomicUsize::new(0), + #[cfg(not(unix))] + frames: parking_lot::Mutex::new(Vec::new()), + exception: crate::PyAtomicRef::from(None::), + state: core::sync::atomic::AtomicI32::new( + if vm.state.stop_the_world.requested.load(Ordering::Acquire) { + // Match init_threadstate(): new thread-state starts + // suspended while stop-the-world is active. + THREAD_SUSPENDED + } else { + THREAD_DETACHED + }, + ), + stop_requested: core::sync::atomic::AtomicBool::new(false), + thread: std::thread::current(), + qsbr: crate::object::qsbr::QSBR.register(), + }); + registry.insert(thread_id, new_slot.clone()); + drop(registry); + slots.insert(interp_id, new_slot.clone()); + new_slot + }) +} + +/// Make `slot` the current thread slot (and the cached top-frame pointer). +#[cfg(feature = "threading")] +fn set_current_thread_slot(slot: CurrentFrameSlot) { + #[cfg(unix)] + CURRENT_TOP_FRAME_SLOT.with(|c| c.set(&slot.top_frame)); + CURRENT_TOP_IFRAME_SLOT.with(|c| c.set(&slot.top_iframe)); + CURRENT_STOP_REQUESTED.with(|c| c.set(&slot.stop_requested)); + CURRENT_THREAD_SLOT.with(|current| { + *current.borrow_mut() = Some(slot); }); } +/// Whether the current thread slot is ATTACHED. +#[cfg(feature = "threading")] +fn current_slot_is_attached() -> bool { + CURRENT_THREAD_SLOT.with(|slot| { + slot.borrow() + .as_ref() + .is_some_and(|s| s.state.load(Ordering::Acquire) == THREAD_ATTACHED) + }) +} + +/// Attach this thread to `vm`'s interpreter for the duration of a section, +/// detaching whichever interpreter it was attached to (≈ `_PyThreadState_Swap`). +/// +/// A thread must never be ATTACHED to two interpreters at once: stop-the-world +/// treats an ATTACHED slot as "running this interpreter's bytecode" and a +/// DETACHED slot as parkable without cooperation, so running interpreter B's +/// code while B's slot is DETACHED would let a collector conclude B is stopped +/// while this thread keeps mutating the (process-global) object graph. +/// +/// Returns whether the attachment changed, i.e. whether the matching +/// [`end_interpreter_section`] must undo it. +#[cfg(feature = "threading")] +fn begin_interpreter_section(vm: &VirtualMachine) -> bool { + let target = ensure_thread_slot(vm); + let already_current = CURRENT_THREAD_SLOT.with(|slot| { + slot.borrow() + .as_ref() + .is_some_and(|s| Arc::ptr_eq(s, &target)) + }); + if already_current && current_slot_is_attached() { + // Nested section in the same interpreter: already attached. + return false; + } + if !already_current && current_slot_is_attached() { + detach_thread(); + } + set_current_thread_slot(target); + attach_thread(vm); + true +} + +/// Undo [`begin_interpreter_section`]: detach this interpreter and re-attach the +/// enclosing one, if any. Call after the VM has been popped from `VM_STACK`. +#[cfg(feature = "threading")] +fn end_interpreter_section(switched: bool) { + if !switched { + return; + } + if current_slot_is_attached() { + detach_thread(); + } + // The enclosing section, if any, is the VM now on top of the stack. + if let Some(vm_ptr) = VM_STACK.with(|vms| vms.borrow().last().copied()) { + // SAFETY: entries on VM_STACK are valid for their enter/set_current_vm scope. + let vm = unsafe { vm_ptr.as_ref() }; + set_current_thread_slot(ensure_thread_slot(vm)); + attach_thread(vm); + } +} + /// Transition DETACHED → ATTACHED. Blocks if the thread was SUSPENDED by /// a stop-the-world request (like `_PyThreadState_Attach` + `tstate_wait_attach`). #[cfg(feature = "threading")] @@ -426,9 +538,10 @@ fn attach_thread(vm: &VirtualMachine) { // a thread doing rapid allow_threads calls from re-attaching and running // past the requester forever, which would stall stop-the-world. Done // outside the CURRENT_THREAD_SLOT borrow above because suspend re-borrows - // it. Safe against a concurrent start_the_world: suspend_if_needed only - // parks while the request is still live and self-recovers otherwise. - suspend_if_needed(&vm.state.stop_the_world); + // it. Safe against a concurrent start_the_world: suspend_if_needed decides + // whether to park under the registry lock, so it never parks after the + // request has been withdrawn. + suspend_if_needed(&vm.state); } /// Transition ATTACHED → DETACHED (like `_PyThreadState_Detach`). @@ -495,102 +608,111 @@ pub fn allow_threads(_vm: &VirtualMachine, f: impl FnOnce() -> R) -> R { /// Transitions ATTACHED → SUSPENDED and waits until released /// (like `_PyThreadState_Suspend` + `_PyThreadState_Attach`). #[cfg(feature = "threading")] -pub fn suspend_if_needed(stw: &super::StopTheWorldState) { +pub fn suspend_if_needed(state: &PyGlobalState) { let should_suspend = CURRENT_THREAD_SLOT.with(|slot| { slot.borrow() .as_ref() .is_some_and(|s| s.stop_requested.load(Ordering::Relaxed)) }); - if !should_suspend { - return; - } - - if !stw.requested.load(Ordering::Acquire) { - CURRENT_THREAD_SLOT.with(|slot| { - if let Some(s) = slot.borrow().as_ref() { - s.stop_requested.store(false, Ordering::Release); - } - }); - return; + if should_suspend { + do_suspend(state); } - - do_suspend(stw); } #[cfg(feature = "threading")] #[cold] -fn do_suspend(stw: &super::StopTheWorldState) { +fn do_suspend(state: &PyGlobalState) { + let stw = &state.stop_the_world; CURRENT_THREAD_SLOT.with(|slot| { - if let Some(s) = slot.borrow().as_ref() { - // ATTACHED → SUSPENDED - match s.state.compare_exchange( - THREAD_ATTACHED, - THREAD_SUSPENDED, - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => { - // Consumed this thread's stop request bit. - s.stop_requested.store(false, Ordering::Release); - } - Err(THREAD_DETACHED) => { - // Leaving VM; caller will re-check on next entry. - super::stw_trace(format_args!("suspend skip DETACHED")); - return; - } - Err(THREAD_SUSPENDED) => { - // Already parked by another path. - s.stop_requested.store(false, Ordering::Release); - super::stw_trace(format_args!("suspend skip already-suspended")); - return; - } - Err(state) => { - debug_assert!(false, "unexpected thread state in suspend: {state}"); - return; - } + let borrowed = slot.borrow(); + let Some(s) = borrowed.as_ref() else { + return; + }; + + // Decide whether to park while holding the thread registry. Both edges + // of `requested` are written under that lock: `init_thread_countdown` + // sets it, and `start_the_world` clears it and then releases every + // SUSPENDED thread without letting go. Publishing SUSPENDED here is + // therefore either seen by that release pass or never reached, which + // leaves the requester the only writer that takes a thread out of + // SUSPENDED. A completion check that observed this thread parked cannot + // then be invalidated by the thread resuming on its own. + let park = { + let _registry = state.thread_frames.lock(); + if stw.requested.load(Ordering::Acquire) { + Some(s.state.compare_exchange( + THREAD_ATTACHED, + THREAD_SUSPENDED, + Ordering::AcqRel, + Ordering::Acquire, + )) + } else { + // The stop already ended; this thread's request bit is stale. + s.stop_requested.store(false, Ordering::Release); + None } - super::stw_trace(format_args!("suspend ATTACHED->SUSPENDED")); + }; - // Re-check: if start_the_world already ran (cleared `requested`), - // no one will set us back to DETACHED — we must self-recover. - if !stw.requested.load(Ordering::Acquire) { - s.state.store(THREAD_ATTACHED, Ordering::Release); + match park { + None => { + super::stw_trace(format_args!("suspend skip not-requested")); + return; + } + Some(Ok(_)) => { + // Consumed this thread's stop request bit. + s.stop_requested.store(false, Ordering::Release); + } + Some(Err(THREAD_DETACHED)) => { + // Leaving VM; caller will re-check on next entry. + super::stw_trace(format_args!("suspend skip DETACHED")); + return; + } + Some(Err(THREAD_SUSPENDED)) => { + // Already parked by another path. s.stop_requested.store(false, Ordering::Release); - super::stw_trace(format_args!("suspend abort requested-cleared")); + super::stw_trace(format_args!("suspend skip already-suspended")); return; } + Some(Err(state)) => { + debug_assert!(false, "unexpected thread state in suspend: {state}"); + return; + } + } + super::stw_trace(format_args!("suspend ATTACHED->SUSPENDED")); - // Notify the stop-the-world requester that we've parked - stw.notify_suspended(); - super::stw_trace(format_args!("suspend notified-requester")); + // Notify the stop-the-world requester that we've parked. The registry + // is released first: the requester's wait loop takes the notify mutex + // and then the registry, so taking them the other way round here would + // invert the order. + stw.notify_suspended(); + super::stw_trace(format_args!("suspend notified-requester")); - // Wait until start_the_world sets us back to DETACHED - let wait_yields = wait_while_suspended(s); - stw.add_suspend_wait_yields(wait_yields); + // Wait until start_the_world sets us back to DETACHED + let wait_yields = wait_while_suspended(s); + stw.add_suspend_wait_yields(wait_yields); - // Re-attach (DETACHED → ATTACHED), tstate_wait_attach CAS loop. - loop { - match s.state.compare_exchange( - THREAD_DETACHED, - THREAD_ATTACHED, - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => break, - Err(THREAD_SUSPENDED) => { - let extra_wait = wait_while_suspended(s); - stw.add_suspend_wait_yields(extra_wait); - } - Err(THREAD_ATTACHED) => break, - Err(state) => { - debug_assert!(false, "unexpected post-suspend state: {state}"); - break; - } + // Re-attach (DETACHED → ATTACHED), tstate_wait_attach CAS loop. + loop { + match s.state.compare_exchange( + THREAD_DETACHED, + THREAD_ATTACHED, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => break, + Err(THREAD_SUSPENDED) => { + let extra_wait = wait_while_suspended(s); + stw.add_suspend_wait_yields(extra_wait); + } + Err(THREAD_ATTACHED) => break, + Err(state) => { + debug_assert!(false, "unexpected post-suspend state: {state}"); + break; } } - s.stop_requested.store(false, Ordering::Release); - super::stw_trace(format_args!("suspend resume -> ATTACHED")); } + s.stop_requested.store(false, Ordering::Release); + super::stw_trace(format_args!("suspend resume -> ATTACHED")); }); } @@ -598,10 +720,12 @@ fn do_suspend(stw: &super::StopTheWorldState) { #[inline] #[must_use] pub fn stop_requested_for_current_thread() -> bool { - CURRENT_THREAD_SLOT.with(|slot| { - slot.borrow() - .as_ref() - .is_some_and(|s| s.stop_requested.load(Ordering::Relaxed)) + CURRENT_STOP_REQUESTED.with(|cached| { + let flag = cached.get(); + // SAFETY: the pointer is non-null only while `CURRENT_THREAD_SLOT` + // holds the `Arc` that owns the flag; both are cleared + // together in `cleanup_current_thread_frames`. + !flag.is_null() && unsafe { &*flag }.load(Ordering::Relaxed) }) } @@ -692,27 +816,25 @@ pub fn set_current_frame(frame: *const InterpreterFrame) -> *const InterpreterFr // sys._current_frames). #[cfg(feature = "threading")] { - CURRENT_THREAD_SLOT.with(|slot| { - if let Some(s) = slot.borrow().as_ref() { - if !frame.is_null() { - #[cfg(unix)] - { - let frame_obj = unsafe { (*frame).frame_obj() }; - let fo_ptr = match frame_obj { - Some(py) => { - py as *const Py as *const FrameObject - as *mut FrameObject - } - None => core::ptr::null_mut(), - }; - s.top_frame.store(fo_ptr, Ordering::Relaxed); - } - s.top_iframe.store(frame as usize, Ordering::Relaxed); + CURRENT_TOP_IFRAME_SLOT.with(|slot| { + let slot = slot.get(); + if !slot.is_null() { + unsafe { &*slot }.store(frame as usize, Ordering::Relaxed); + } + }); + #[cfg(unix)] + CURRENT_TOP_FRAME_SLOT.with(|slot| { + let slot = slot.get(); + if !slot.is_null() { + let fo_ptr = if frame.is_null() { + core::ptr::null_mut() } else { - #[cfg(unix)] - s.top_frame.store(core::ptr::null_mut(), Ordering::Relaxed); - s.top_iframe.store(0, Ordering::Relaxed); - } + let frame_obj = unsafe { (*frame).frame_obj() }; + frame_obj.map_or(core::ptr::null_mut(), |py| { + py as *const Py as *mut Py + }) + }; + unsafe { &*slot }.store(fo_ptr, Ordering::Relaxed); } }); } @@ -759,16 +881,21 @@ pub fn get_all_current_exceptions(vm: &VirtualMachine) -> Vec<(u64, Option registry.remove(&thread_id), @@ -789,7 +916,6 @@ pub fn cleanup_current_thread_frames(vm: &VirtualMachine) { None }; - #[cfg(feature = "threading")] if let Some(slot) = &_removed && vm.state.stop_the_world.requested.load(Ordering::Acquire) && thread_id != vm.state.stop_the_world.requester_ident() @@ -799,12 +925,23 @@ pub fn cleanup_current_thread_frames(vm: &VirtualMachine) { // Unblock requester countdown progress. vm.state.stop_the_world.notify_thread_gone(); } - // Clear the cached top-frame pointer before dropping the slot Arc so no - // later `set_current_frame` dereferences freed slot memory. - #[cfg(all(unix, feature = "threading"))] - CURRENT_TOP_FRAME_SLOT.with(|c| c.set(core::ptr::null())); + + // If CURRENT pointed at the cleaned slot, clear it (and top-frame cache). CURRENT_THREAD_SLOT.with(|s| { - *s.borrow_mut() = None; + let clear = match (s.borrow().as_ref(), slot_to_clean.as_ref()) { + (Some(cur), Some(cleaned)) => Arc::ptr_eq(cur, cleaned), + (Some(_), None) => false, + (None, _) => false, + }; + if clear { + *s.borrow_mut() = None; + #[cfg(all(unix, feature = "threading"))] + CURRENT_TOP_FRAME_SLOT.with(|c| c.set(core::ptr::null())); + #[cfg(feature = "threading")] + CURRENT_TOP_IFRAME_SLOT.with(|c| c.set(core::ptr::null())); + #[cfg(feature = "threading")] + CURRENT_STOP_REQUESTED.with(|c| c.set(core::ptr::null())); + } }); } @@ -843,7 +980,7 @@ pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) { core::ptr::null_mut() } else { match unsafe { (*top_iframe).frame_obj() } { - Some(fo) => fo as *const Py as *const FrameObject as *mut FrameObject, + Some(fo) => fo as *const Py as *mut Py, None => core::ptr::null_mut(), } } @@ -865,6 +1002,10 @@ pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) { }); #[cfg(all(unix, feature = "threading"))] CURRENT_TOP_FRAME_SLOT.with(|c| c.set(&new_slot.top_frame)); + #[cfg(feature = "threading")] + CURRENT_TOP_IFRAME_SLOT.with(|c| c.set(&new_slot.top_iframe)); + #[cfg(feature = "threading")] + CURRENT_STOP_REQUESTED.with(|c| c.set(&new_slot.stop_requested)); // Lock is safe: reinit_locks_after_fork() already reset it to unlocked. let mut registry = vm.state.thread_frames.lock(); @@ -873,7 +1014,23 @@ pub fn reinit_frame_slot_after_fork(vm: &VirtualMachine) { drop(registry); CURRENT_THREAD_SLOT.with(|s| { - *s.borrow_mut() = Some(new_slot); + *s.borrow_mut() = Some(new_slot.clone()); + }); + INTERP_THREAD_SLOTS.with(|slots| { + slots.borrow_mut().insert(vm.state.interpreter_id, new_slot); + }); +} + +/// Drop this thread's cached slots for every interpreter except `keep_id`. +/// +/// After `fork()` only the calling thread survives, and the other +/// interpreters' registries are cleared; a cached slot would otherwise stay +/// current for an interpreter that no longer lists it, hiding the thread from +/// that interpreter's stop-the-world. The next enter builds a fresh slot. +#[cfg(feature = "threading")] +pub fn purge_other_interpreter_slots_after_fork(keep_id: i64) { + INTERP_THREAD_SLOTS.with(|slots| { + slots.borrow_mut().retain(|&id, _| id == keep_id); }); } @@ -1015,6 +1172,8 @@ impl VirtualMachine { state: self.state.clone(), initialized: self.initialized, recursion_depth: Cell::new(0), + #[cfg(any(miri, target_env = "musl"))] + native_recursion_depth: Cell::new(0), c_stack_soft_limit: Cell::new(Self::calculate_c_stack_soft_limit()), async_gen_firstiter: RefCell::new(None), async_gen_finalizer: RefCell::new(None), @@ -1023,7 +1182,7 @@ impl VirtualMachine { callable_cache: self.callable_cache.clone(), audit_hooks: RefCell::new(vec![]), pending_tailcall_frame: Cell::new(None), - pending_tailcall_refs: core::cell::UnsafeCell::new(Vec::with_capacity(2)), + pending_tailcall_owner: core::cell::UnsafeCell::new(None), }; ThreadedVirtualMachine { vm } } diff --git a/crates/vm/src/vm/vm_new.rs b/crates/vm/src/vm/vm_new.rs index 6110a3d5b1a..82f382ca6d7 100644 --- a/crates/vm/src/vm/vm_new.rs +++ b/crates/vm/src/vm/vm_new.rs @@ -517,7 +517,7 @@ impl VirtualMachine { self.new_os_subtype_error(exc_type.to_owned(), Some(errno), msg) } - pub fn new_unicode_decode_error_real( + pub fn new_unicode_decode_error( &self, encoding: PyStrRef, object: PyBytesRef, @@ -996,12 +996,6 @@ impl VirtualMachine { define_exception_fn!(fn new_type_error, type_error, TypeError); define_exception_fn!(fn new_system_error, system_error, SystemError); - // TODO: remove & replace with new_unicode_decode_error_real - define_exception_fn!(fn new_unicode_decode_error, unicode_decode_error, UnicodeDecodeError); - - // TODO: remove & replace with new_unicode_encode_error_real - define_exception_fn!(fn new_unicode_encode_error, unicode_encode_error, UnicodeEncodeError); - define_exception_fn!(fn new_value_error, value_error, ValueError); define_exception_fn!(fn new_buffer_error, buffer_error, BufferError); diff --git a/crates/vm/src/vm/vm_ops.rs b/crates/vm/src/vm/vm_ops.rs index 692444fc7de..dc31e508218 100644 --- a/crates/vm/src/vm/vm_ops.rs +++ b/crates/vm/src/vm/vm_ops.rs @@ -168,6 +168,27 @@ impl VirtualMachine { } } + /// `vec![0; len]` for a length that came from Python, where a request too + /// large to satisfy is a `MemoryError` rather than an aborted process. + /// + /// The bytes are left for the allocator to zero, so a large request costs + /// no more than the pages that are actually written to. + pub fn new_zeroed_bytes(&self, len: usize) -> PyResult> { + if len == 0 { + return Ok(Vec::new()); + } + let layout = + core::alloc::Layout::array::(len).map_err(|_| self.new_memory_error(""))?; + // SAFETY: `len` is not zero, so neither is the layout's size. + let ptr = unsafe { alloc::alloc::alloc_zeroed(layout) }; + if ptr.is_null() { + return Err(self.new_memory_error("")); + } + // SAFETY: `ptr` was just allocated by the global allocator for exactly + // this many bytes, and every one of them is initialized to zero. + Ok(unsafe { Vec::from_raw_parts(ptr, len, len) }) + } + /// Calling scheme used for binary operations: /// /// Order operations are tried until either a valid result or error: diff --git a/crates/wasm/src/convert.rs b/crates/wasm/src/convert.rs index 3e07b27d4a0..9349c727942 100644 --- a/crates/wasm/src/convert.rs +++ b/crates/wasm/src/convert.rs @@ -2,10 +2,13 @@ use crate::js_module; use crate::vm_class::{WASMVirtualMachine, stored_vm_from_wasm}; -use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, SyntaxError, Uint8Array}; +use js_sys::{ + Array, ArrayBuffer, JsString, Map, Object, Promise, Reflect, SyntaxError, Uint8Array, +}; +use rustpython_common::wtf8::{Wtf8, Wtf8Buf}; use rustpython_vm::{ AsObject, Py, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine, - builtins::{PyBaseException, PyBaseExceptionRef}, + builtins::{PyBaseException, PyBaseExceptionRef, PyDict, PyList, PyStr, PyTuple}, compiler::{CompileError, ParseError, parser::LexicalErrorType, parser::ParseErrorType}, exceptions, function::{ArgBytesLike, FuncArgs}, @@ -13,6 +16,26 @@ use rustpython_vm::{ }; use wasm_bindgen::{JsCast, closure::Closure, prelude::*}; +pub(crate) fn js_string_to_wtf8(value: &JsString) -> Wtf8Buf { + Wtf8Buf::from_wide(&value.iter().collect::>()) +} + +fn wtf8_to_js_string(value: &Wtf8) -> JsString { + const CHUNK_SIZE: usize = 8192; + + if let Ok(value) = value.as_str() { + return value.into(); + } + + value + .encode_wide() + .collect::>() + .chunks(CHUNK_SIZE) + .map(JsString::from_char_code) + .collect::() + .join("") +} + #[wasm_bindgen(inline_js = r" export class PyError extends Error { constructor(info) { @@ -119,12 +142,9 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { if let Some(ref kwargs) = kwargs { for pair in object_entries(kwargs) { let (key, val) = pair?; - py_func_args.kwargs.insert( - // JS strings coming in are UTF-16; go through Rust `String` - // (kwargs keys are now WTF-8, so convert String -> Wtf8Buf). - String::from(js_sys::JsString::from(key)).into(), - js_to_py(vm, val), - ); + py_func_args + .kwargs + .insert(js_string_to_wtf8(&key.into()), js_to_py(vm, val)); } } let result = py_obj.call(py_func_args, vm); @@ -151,17 +171,44 @@ pub fn py_to_js(vm: &VirtualMachine, py_obj: PyObjectRef) -> JsValue { } if let Ok(bytes) = ArgBytesLike::try_from_borrowed_object(vm, &py_obj) { - bytes.with_ref(|bytes| unsafe { + return bytes.with_ref(|bytes| unsafe { // `Uint8Array::view` is an `unsafe fn` because it provides // a direct view into the WASM linear memory; if you were to allocate // something with Rust that view would probably become invalid. It's safe // because we then copy the array using `Uint8Array::slice`. let view = Uint8Array::view(bytes); view.slice(0, bytes.len() as u32).into() - }) + }); + } + py_serde_to_js(vm, &py_obj).unwrap_or(JsValue::UNDEFINED) +} + +fn py_serde_to_js( + vm: &VirtualMachine, + py_obj: &PyObjectRef, +) -> Result { + if let Some(value) = py_obj.downcast_ref::() { + Ok(wtf8_to_js_string(value.as_wtf8()).into()) + } else if let Some(value) = py_obj.downcast_ref::() { + let array = Array::new(); + for item in value.borrow_vec().iter() { + array.push(&py_serde_to_js(vm, item)?); + } + Ok(array.into()) + } else if let Some(value) = py_obj.downcast_ref::() { + let array = Array::new(); + for item in value { + array.push(&py_serde_to_js(vm, item)?); + } + Ok(array.into()) + } else if let Some(value) = py_obj.downcast_ref::() { + let map = Map::new(); + for (key, value) in value { + map.set(&py_serde_to_js(vm, &key)?, &py_serde_to_js(vm, &value)?); + } + Ok(map.into()) } else { - py_serde::serialize(vm, &py_obj, &serde_wasm_bindgen::Serializer::new()) - .unwrap_or(JsValue::UNDEFINED) + py_serde::serialize(vm, py_obj, &serde_wasm_bindgen::Serializer::new()) } } @@ -199,6 +246,15 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { .map(|val| js_to_py(vm, val.expect("Iteration over array failed"))) .collect(); vm.ctx.new_list(elems).into() + } else if let Some(map) = js_val.dyn_ref::() { + let dict = vm.ctx.new_dict(); + for entry in map.entries() { + let entry = Array::from(&entry.expect("Iteration over map failed")); + let key = js_to_py(vm, entry.get(0)); + dict.set_item(&*key, js_to_py(vm, entry.get(1)), vm) + .unwrap(); + } + dict.into() } else if ArrayBuffer::is_view(&js_val) || js_val.is_instance_of::() { // unchecked_ref because if it's not an ArrayBuffer it could either be a TypedArray // or a DataView, but they all have a `buffer` property @@ -216,12 +272,8 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { for pair in object_entries(&Object::from(js_val)) { let (key, val) = pair.expect("iteration over object to not fail"); let py_val = js_to_py(vm, val); - dict.set_item( - String::from(js_sys::JsString::from(key)).as_str(), - py_val, - vm, - ) - .unwrap(); + dict.set_item(&*js_string_to_wtf8(&key.into()), py_val, vm) + .unwrap(); } dict.into() } @@ -232,9 +284,7 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { move |args: FuncArgs, vm: &VirtualMachine| -> PyResult { let this = Object::new(); for (k, v) in args.kwargs { - // WTF-8 -> JS string: lone surrogates in the key become U+FFFD - // (wasm-bindgen only accepts Rust `String`); acceptable at this boundary. - Reflect::set(&this, &k.to_string().into(), &py_to_js(vm, v)) + Reflect::set(&this, &wtf8_to_js_string(&k).into(), &py_to_js(vm, v)) .expect("property to be settable"); } let js_args = args @@ -253,6 +303,8 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { } else if js_val.is_undefined() { // Because `JSON.stringify(undefined)` returns undefined vm.ctx.none() + } else if js_val.is_string() { + vm.ctx.new_str(js_string_to_wtf8(&js_val.into())).into() } else { py_serde::deserialize(vm, serde_wasm_bindgen::Deserializer::from(js_val)) .unwrap_or_else(|_| vm.ctx.none()) diff --git a/crates/wasm/src/vm_class.rs b/crates/wasm/src/vm_class.rs index 5e09af0ee95..80f3ece1358 100644 --- a/crates/wasm/src/vm_class.rs +++ b/crates/wasm/src/vm_class.rs @@ -325,9 +325,12 @@ impl WASMVirtualMachine { if let Some(imports) = imports { for entry in convert::object_entries(&imports) { let (key, value) = entry?; - let key: String = Object::from(key).to_string().into(); attrs - .set_item(key.as_str(), convert::js_to_py(vm, value), vm) + .set_item( + &*convert::js_string_to_wtf8(&key.into()), + convert::js_to_py(vm, value), + vm, + ) .into_js(vm)?; } } @@ -356,10 +359,10 @@ impl WASMVirtualMachine { let py_module = vm.new_module(&name, vm.ctx.new_dict(), None); for entry in convert::object_entries(&module) { let (key, value) = entry?; - let key = Object::from(key).to_string(); - extend_module!(vm, &py_module, { - String::from(key) => convert::js_to_py(vm, value), - }); + let key = vm.ctx.new_str(convert::js_string_to_wtf8(&key.into())); + py_module + .set_attr(&key, convert::js_to_py(vm, value), vm) + .into_js(vm)?; } let sys_modules = vm.sys_module.get_attr("modules", vm).into_js(vm)?; diff --git a/extra_tests/snippets/builtin_bytes.py b/extra_tests/snippets/builtin_bytes.py index 4f861364488..3cbed79c069 100644 --- a/extra_tests/snippets/builtin_bytes.py +++ b/extra_tests/snippets/builtin_bytes.py @@ -747,3 +747,22 @@ def __new__(cls, value): assert "123A".istitle(), f"{s}" assert not "123a".istitle(), f"{s}" assert not "123A\ta".istitle(), f"{s}" + + +def test_huge_size(): + # sizes that cannot be allocated are MemoryError, not an aborted process + for factory in (bytes, bytearray): + assert_raises(MemoryError, lambda factory=factory: factory(2**62)) + for meth in ("center", "ljust", "rjust", "zfill"): + assert_raises( + MemoryError, + lambda factory=factory, meth=meth: getattr(factory(b"a"), meth)( + 1 << 62 + ), + ) + assert_raises( + OverflowError, lambda factory=factory: factory(b"\ta").expandtabs(2**31) + ) + + +test_huge_size() diff --git a/extra_tests/snippets/builtin_compile.py b/extra_tests/snippets/builtin_compile.py index 49295bf26d2..73247e50df1 100644 --- a/extra_tests/snippets/builtin_compile.py +++ b/extra_tests/snippets/builtin_compile.py @@ -145,3 +145,8 @@ def _check_flags_error(flags): assert exc.args[0] == "incomplete input", repr(exc) else: raise AssertionError("expected _IncompleteInputError") + +# The source is encoded before it is parsed, so a lone surrogate has to be +# reported rather than assumed away. +with assert_raises(UnicodeEncodeError): + compile(chr(0xD800), "", "eval") diff --git a/extra_tests/snippets/builtin_eval.py b/extra_tests/snippets/builtin_eval.py index 2f2405c8d9e..1648a1a271d 100644 --- a/extra_tests/snippets/builtin_eval.py +++ b/extra_tests/snippets/builtin_eval.py @@ -1,3 +1,5 @@ +from testutils import assert_raises + assert 3 == eval("1+2") code = compile("5+3", "x.py", "eval") @@ -75,3 +77,8 @@ def make_closure(): assert False, "eval with code containing free variables should fail" except NameError as e: pass + +# The source is encoded before it is parsed, so a lone surrogate has to be +# reported rather than assumed away. +with assert_raises(UnicodeEncodeError): + eval(chr(0xD800)) diff --git a/extra_tests/snippets/builtin_exceptions.py b/extra_tests/snippets/builtin_exceptions.py index 8879e130bc2..080294a3c8a 100644 --- a/extra_tests/snippets/builtin_exceptions.py +++ b/extra_tests/snippets/builtin_exceptions.py @@ -1,4 +1,5 @@ import builtins +import itertools import pickle import platform import sys @@ -393,3 +394,19 @@ class SubError(MyError): assert err.exceptions[0].args == ("x",) else: assert False, "except* handler did not run" + +# The exceptions argument is a sequence, so an arbitrary iterable must be +# rejected rather than drained. +try: + ExceptionGroup("m", itertools.count()) +except TypeError: + pass +else: + assert False, "ExceptionGroup accepted an unbounded iterable" + +# ImportError.__reduce__ has to cope with the exception carrying no args. +assert pickle.loads(pickle.dumps(ImportError())).args == () +restored = pickle.loads(pickle.dumps(ImportError("m", name="n", path="p"))) +assert restored.args == ("m",) +assert restored.name == "n" +assert restored.path == "p" diff --git a/extra_tests/snippets/builtin_exec.py b/extra_tests/snippets/builtin_exec.py index 2eae90e91c5..cfb88c15dc1 100644 --- a/extra_tests/snippets/builtin_exec.py +++ b/extra_tests/snippets/builtin_exec.py @@ -1,3 +1,5 @@ +from testutils import assert_raises + exec("def square(x):\n return x * x\n") assert 16 == square(4) # noqa: F821 @@ -71,3 +73,8 @@ def f(): f() + +# The source is encoded before it is parsed, so a lone surrogate has to be +# reported rather than assumed away. +with assert_raises(UnicodeEncodeError): + exec(chr(0xD800)) diff --git a/extra_tests/snippets/builtin_float.py b/extra_tests/snippets/builtin_float.py index 1417c5ae174..c459c2d0da6 100644 --- a/extra_tests/snippets/builtin_float.py +++ b/extra_tests/snippets/builtin_float.py @@ -561,3 +561,12 @@ def _check_msg(call, exc_type, expected_msg): assert repr(1.5) == "1.5" assert repr(0.1) == "0.1" assert repr(100.0) == "100.0" + + +# float() takes at most one positional argument; the exact-float fast path +# must not let extra ones through. +assert_raises(TypeError, float, 1.5, True) +assert_raises(TypeError, float, 1.5, 2, 3) +assert_raises(TypeError, float, "1.5", 2) +assert float(1.5) == 1.5 +assert float() == 0.0 diff --git a/extra_tests/snippets/builtin_hash.py b/extra_tests/snippets/builtin_hash.py index 9b2c8388790..818ee523f30 100644 --- a/extra_tests/snippets/builtin_hash.py +++ b/extra_tests/snippets/builtin_hash.py @@ -1,3 +1,5 @@ +import sys + from testutils import assert_raises @@ -28,3 +30,20 @@ def __hash__(self): with assert_raises(TypeError): hash([]) + +# Hashing a deeply nested tuple must not run off the native stack: the hash +# slot dispatch is what recurses, so that is where the depth is checked. + +if sys.implementation.name == "rustpython": + # Deep enough to reach the native stack guard; CPython, which also runs + # this snippet, dies on the same value. + deep_tuple = () + for _ in range(100_000): + deep_tuple = (deep_tuple,) + with assert_raises(RecursionError): + hash(deep_tuple) + # a dict key and a set member are hashed on insertion, same dispatch + with assert_raises(RecursionError): + {deep_tuple: 1} + with assert_raises(RecursionError): + {deep_tuple} diff --git a/extra_tests/snippets/builtin_list.py b/extra_tests/snippets/builtin_list.py index d62cae03b50..44492092bad 100644 --- a/extra_tests/snippets/builtin_list.py +++ b/extra_tests/snippets/builtin_list.py @@ -1,3 +1,5 @@ +import sys + from testutils import assert_raises x = [1, 2, 3] @@ -923,3 +925,8 @@ def __eq__(self, other): list1 = rewrite_list_eq([poc()]) list1.remove(list1) assert list1 == [] + +# The repeat count is multiplied by the element size; a count that overflows +# that product must raise instead of wrapping into a short allocation. +with assert_raises(MemoryError): + [1] * sys.maxsize diff --git a/extra_tests/snippets/builtin_memoryview.py b/extra_tests/snippets/builtin_memoryview.py index f206056ebfd..34928041cd2 100644 --- a/extra_tests/snippets/builtin_memoryview.py +++ b/extra_tests/snippets/builtin_memoryview.py @@ -90,3 +90,588 @@ def test_delitem(): test_delitem() + + +def test_empty_view_offset(): + # An empty view keeps the offset slicing left it, which can sit outside the + # exporter, and reaches no byte through it. + ba = bytearray(range(17)) + assert bytes(memoryview(ba)[::-9][-30::-9]) == b"" + assert bytes(memoryview(ba)[-30::-1]) == b"" + v = memoryview(ba)[::-9][-30::-9] + assert v.shape == (0,) + assert v.strides == (81,) + assert v.suboffsets == () + b24 = bytearray(range(24)) + assert bytes(memoryview(b24).cast("B", [4, 6])[-30::-1]) == b"" + + +test_empty_view_offset() + + +def test_exported_suboffsets(): + mv = memoryview(bytearray(b"abcdef"))[::-1] + exported = mv.__buffer__(284) + assert exported.suboffsets == () + assert bytes(exported) == b"fedcba" + assert ( + bytes(memoryview(memoryview(bytearray(b"abcdefg"))[::2].__buffer__(284))) + == b"aceg" + ) + + +test_exported_suboffsets() + + +def test_setitem_slice_strided_source(): + src = bytearray(b"abcdef") + dst = bytearray(b"......") + memoryview(dst)[:] = memoryview(src)[::-1] + assert bytes(dst) == b"fedcba" + dst = bytearray(b"...") + memoryview(dst)[:] = memoryview(src)[::2] + assert bytes(dst) == b"ace" + + +test_setitem_slice_strided_source() + + +def test_zero_dim_position(): + z = memoryview(bytearray(range(8)))[4:5].cast("B", []) + assert z[()] == 4 + assert z.tolist() == 4 + w = bytearray(range(8)) + memoryview(w)[4:5].cast("B", [])[()] = 99 + assert w[4] == 99 + assert w[0] == 0 + + +test_zero_dim_position() + + +def test_cast_zero_dim_size(): + assert_raises(TypeError, lambda: memoryview(bytearray(range(8))).cast("B", [])) + assert memoryview(bytearray(b"a")).cast("B", []).nbytes == 1 + + +test_cast_zero_dim_size() + + +def test_hash_format(): + assert_raises(ValueError, lambda: hash(memoryview(b"abcd").cast("I"))) + hash(memoryview(b"abcd").cast("b")) + hash(memoryview(b"abcdef")[::2]) + hash(memoryview(b"a").cast("B", [])) + + +test_hash_format() + + +def test_cast_keeps_exports(): + ba = bytearray(b"abc") + mv = memoryview(ba) + cast = mv.cast("B") + mv.release() + assert_raises(BufferError, lambda: ba.clear()) + cast.release() + ba.clear() + assert bytes(ba) == b"" + + +test_cast_keeps_exports() + + +def test_setitem_converts_before_writing(): + ba = bytearray(b"abc") + mv = memoryview(ba) + + class Idx: + def __index__(self): + return len(bytes(ba)) + + mv[0] = Idx() + assert bytes(ba) == b"\x03bc" + + +test_setitem_converts_before_writing() + + +def test_pep688_exporter_aliasing(): + def exporter(view_factory): + class C: + def __buffer__(self, flags): + return view_factory() + + def __release_buffer__(self, view): + pass + + return C() + + ba = bytearray(b"abc") + memoryview(ba)[:] = exporter(lambda: memoryview(ba)) + assert bytes(ba) == b"abc" + + ba = bytearray(b"abcdef") + memoryview(ba)[0:3] = exporter(lambda: memoryview(ba)[3:6]) + assert bytes(ba) == b"defdef" + + ba = bytearray(b"abcdef") + memoryview(ba)[3:6] = exporter(lambda: memoryview(ba)[0:3]) + assert bytes(ba) == b"abcabc" + + ba = bytearray(b"abcdef") + memoryview(ba)[:] = exporter(lambda: memoryview(ba)[::-1]) + assert bytes(ba) == b"fedcba" + + ba = bytearray(b"abcdef") + memoryview(ba)[::2] = exporter(lambda: memoryview(ba)[0:3]) + assert bytes(ba) == b"abbdcf" + + ba = bytearray(b"abcdef") + mv = memoryview(exporter(lambda: memoryview(ba))) + mv[:] = exporter(lambda: memoryview(ba)) + assert bytes(ba) == b"abcdef" + mv[:] = ba + assert bytes(ba) == b"abcdef" + + +test_pep688_exporter_aliasing() + + +def test_release_buffer_waits_for_last_view(): + class C(bytearray): + calls = 0 + + def __release_buffer__(self, view): + type(self).calls += 1 + super().__release_buffer__(view) + + c = C(b"abcdef") + a = memoryview(c) + b = memoryview(a) + a.release() + assert C.calls == 0 + assert b.tobytes() == b"abcdef" + b.release() + assert C.calls == 1 + + class D: + n = 0 + + def __init__(self): + self.b = bytearray(b"abcdef") + + def __buffer__(self, flags): + return memoryview(self.b) + + def __release_buffer__(self, view): + type(self).n += 1 + + d = D() + m = memoryview(d) + m2 = memoryview(m) + m3 = m.cast("B") + m.release() + m2.release() + assert D.n == 0 + m3.release() + assert D.n == 1 + + # Two acquisitions are two exports, each released on its own. + D.n = 0 + d = D() + a1 = memoryview(d) + a2 = memoryview(d) + a1.release() + assert D.n == 1 + a2.release() + assert D.n == 2 + + +test_release_buffer_waits_for_last_view() + + +def test_failed_request_does_not_release(): + import inspect + import mmap + + class M(mmap.mmap): + calls = 0 + + def __release_buffer__(self, view): + type(self).calls += 1 + super().__release_buffer__(view) + + m = M(-1, 10, access=mmap.ACCESS_READ) + assert_raises(BufferError, lambda: m.__buffer__(inspect.BufferFlags.WRITABLE)) + assert M.calls == 0 + + +test_failed_request_does_not_release() + + +def test_request_shapes_exported_descriptor(): + import array + + a = array.array("I", [1, 2, 3]) + assert a.__buffer__(0).format == "B" + assert a.__buffer__(28).format == "I" + + m = memoryview(a) + b = m.__buffer__(0) + assert (b.format, b.itemsize, b.ndim, b.shape, b.strides) == ("B", 4, 1, (3,), (4,)) + assert m.__buffer__(28).format == "I" + + b = a.__buffer__(0) + assert b[0] == 1 + assert b.tolist() == [1, 2, 3] + assert len(b.tobytes()) == 12 + b[0] = 9 + assert a[0] == 9 + + n = memoryview(bytearray(b"abcdef" * 4)).cast("I", (2, 3)) + assert n.__buffer__(0).ndim == 1 + assert n.__buffer__(0).shape == (6,) + assert n.__buffer__(8).ndim == 2 + assert n.__buffer__(8).format == "B" + + +test_request_shapes_exported_descriptor() + + +def test_release_during_index_conversion(): + # CHECK_RELEASED_AGAIN: the conversion that produces the value, and the one + # that produced the index, both run Python that can release the view. + ba = bytearray(b"abcdefgh") + mv = memoryview(ba) + + class Writer: + def __index__(self): + mv.release() + ba.clear() + return 7 + + try: + mv[7] = Writer() + raise AssertionError("write into a released view") + except ValueError as e: + assert "released memoryview" in str(e), e + + ba = bytearray(b"abcdefgh") + mv = memoryview(ba) + + class Reader: + def __index__(self): + mv.release() + ba.clear() + return 7 + + try: + mv[Reader()] + raise AssertionError("read from a released view") + except ValueError as e: + assert "released memoryview" in str(e), e + + # A release that does not resize still forbids the write. + ba = bytearray(b"abcd") + mv = memoryview(ba) + + class Quiet: + def __index__(self): + mv.release() + return 65 + + try: + mv[0] = Quiet() + raise AssertionError("write into a released view") + except ValueError as e: + assert "released memoryview" in str(e), e + assert bytes(ba) == b"abcd" + + +test_release_during_index_conversion() + + +def test_cast_rejects_non_native_format(): + # get_native_fmtchar + for fmt in ["", "ii", " 0; a 0 used to divide by zero while + # checking the product against SSIZE_MAX + for shape in ([0], [0, 4], [4, 0], [-1, 4], [0, 0]): + assert_raises( + ValueError, lambda shape=shape: memoryview(b"abcd").cast("B", shape) + ) + + class Index: + def __index__(self): + return 4 + + for shape in ([2.0, 2], [Index()], ["4"]): + assert_raises( + TypeError, lambda shape=shape: memoryview(b"abcd").cast("B", shape) + ) + + assert memoryview(b"abcd").cast("B", [True, 4]).tolist() == [[97, 98, 99, 100]] + + +test_cast_arguments() + + +def test_negative_stride(): + # A reversed view starts at its last byte, so walking it from there runs + # off the front of the exported slice. + assert memoryview(b"dcba") == memoryview(b"abcd")[::-1] + assert memoryview(b"abcd")[::-1] == memoryview(b"dcba") + assert not memoryview(b"abcd") == memoryview(b"abcd")[::-1] + + b = bytearray(b"____") + memoryview(b)[0:4] = memoryview(b"abcd")[::-1] + assert b == bytearray(b"dcba"), b + + a = array.array("i", [1, 2, 3]) + assert memoryview(array.array("i", [3, 2, 1])) == memoryview(a)[::-1] + assert memoryview(a)[::-1].tolist() == [3, 2, 1] + + +test_negative_stride() + + +def test_write_through_same_object(): + # Reading the source and writing the destination lock the same object + # when they overlap, and converting a value runs Python that can reach it. + b = bytearray(b"abcd") + memoryview(b)[0:4] = b + assert b == bytearray(b"abcd"), b + + b = bytearray(b"abcd") + memoryview(b)[0:4] = memoryview(b)[::-1] + assert b == bytearray(b"dcba"), b + + b = bytearray(b"abcd") + memoryview(b)[0:2] = memoryview(b)[2:4] + assert b == bytearray(b"cdcd"), b + + b = bytearray(b"abcd") + view = memoryview(b) + + class Index: + def __index__(self): + view[1] = 66 + return 65 + + view[0] = Index() + assert b == bytearray(b"ABcd"), b + + +test_write_through_same_object() + + +def test_cast_between_non_byte_formats(): + # A cast re-divides bytes into items; going from one item type straight to + # another would reinterpret what is already there. + view = memoryview(b"abcd").cast("i") + for fmt in ("h", "i", "f"): + try: + view.cast(fmt) + except TypeError as e: + assert "cannot cast between two non-byte formats" in str(e), e + else: + raise AssertionError(f"expected TypeError for cast to {fmt!r}") + + # Either side being bytes is allowed. + assert view.cast("B").tolist() == [97, 98, 99, 100] + assert view.cast("b").format == "b" + assert view.cast("c").tolist() == [b"a", b"b", b"c", b"d"] + assert memoryview(b"abcd").cast("c").cast("i").format == "i" + + +def test_cast_to_zero_dim(): + # A zero-dimensional view holds exactly one item, so the buffer has to be + # that one item and no more. + assert memoryview(b"abcd").cast("I", shape=()).tobytes() == b"abcd" + assert memoryview(b"a").cast("B", shape=()).tobytes() == b"a" + + for source, fmt in ((b"abcd", "B"), (b"abcdefgh", "I"), (b"ab", "b")): + try: + memoryview(source).cast(fmt, shape=()) + except TypeError as e: + assert "product(shape) * itemsize != buffer size" in str(e), e + else: + raise AssertionError(f"expected TypeError for {source!r} as {fmt!r}") + + +def test_hash_restricted_to_byte_formats(): + # The hash is over the bytes, so it agrees with the hash of those bytes + # only where an item is a byte. + data = b"abcdefgh" + assert hash(memoryview(data)) == hash(data) + assert hash(memoryview(data).cast("c")) == hash(data) + assert hash(memoryview(data).cast("b")) == hash(data) + + for fmt in ("I", "i", "h", "d"): + try: + hash(memoryview(data).cast(fmt)) + except ValueError as e: + assert "hashing is restricted to formats" in str(e), e + else: + raise AssertionError(f"expected ValueError for format {fmt!r}") + + +def test_tobytes_order(): + view = memoryview(b"abcdefgh") + for order in (None, "C", "F", "A"): + assert view.tobytes(order=order) == b"abcdefgh", order + + # A multidimensional view is laid out C-contiguously, so a Fortran-ordered + # copy walks it down the columns instead. + grid = memoryview(b"abcdefgh").cast("B", shape=(2, 4)) + assert grid.tolist() == [[97, 98, 99, 100], [101, 102, 103, 104]] + assert grid.tobytes() == b"abcdefgh" + assert grid.tobytes(order="C") == b"abcdefgh" + assert grid.tobytes(order="A") == b"abcdefgh" + assert grid.tobytes(order="F") == b"aebfcgdh" + + cube = memoryview(b"abcdefgh").cast("B", shape=(2, 2, 2)) + assert cube.tobytes(order="F") == b"aecgbfdh" + + for order in ("Z", "c", "f", ""): + try: + view.tobytes(order=order) + except ValueError as e: + assert str(e) == "order must be 'C', 'F' or 'A'", e + else: + raise AssertionError(f"expected ValueError for order {order!r}") + + +test_cast_between_non_byte_formats() +test_cast_to_zero_dim() +test_hash_restricted_to_byte_formats() +test_tobytes_order() diff --git a/extra_tests/snippets/builtin_property.py b/extra_tests/snippets/builtin_property.py index de64e526228..397d41fb075 100644 --- a/extra_tests/snippets/builtin_property.py +++ b/extra_tests/snippets/builtin_property.py @@ -85,3 +85,10 @@ def foo(self): p2 = property("a", doc="pdoc") # assert p2.__doc__ == 'pdoc' + + +# property() takes at most four arguments, and `name` is not one of them: +# the name slot is filled by __set_name__ and the __name__ setter instead. +assert_raises(TypeError, property, None, None, None, None, None) +assert_raises(TypeError, property, "a", "b", "c", "d", "e") +assert_raises(TypeError, property, name="x") diff --git a/extra_tests/snippets/builtin_round.py b/extra_tests/snippets/builtin_round.py index e94d9754204..83725420a40 100644 --- a/extra_tests/snippets/builtin_round.py +++ b/extra_tests/snippets/builtin_round.py @@ -93,3 +93,21 @@ def __round__(self, ndigits=None): assert round(1.0, 1000) == 1.0 assert round(1.0, -1000) == 0.0 assert round(1.7976931348623157e308, 0) == 1.7976931348623157e308 + + +# round() normalizes an int subclass to an exact int, like CPython's long_long(). +assert round(True) == 1 +assert type(round(True)) is int +assert type(round(True, 0)) is int +assert type(round(False)) is int + + +class MyInt(int): + pass + + +assert round(MyInt(5)) == 5 +assert type(round(MyInt(5))) is int +assert type(round(MyInt(5), 2)) is int +# A negative ndigits already produced a fresh exact int. +assert type(round(MyInt(15), -1)) is int diff --git a/extra_tests/snippets/builtin_str.py b/extra_tests/snippets/builtin_str.py index fde9deb8e0b..684bd66a1ff 100644 --- a/extra_tests/snippets/builtin_str.py +++ b/extra_tests/snippets/builtin_str.py @@ -170,6 +170,15 @@ assert "aaa".count("a", 2, 2) == 0 assert "aaa".count("a", 2, 1) == 0 +# An empty needle is counted in characters, not in encoded positions. +assert "".count("") == 1 +assert "abc".count("") == 4 +assert "가나다".count("") == 4 +assert "가나다".count("", 1) == 3 +assert "가나다".count("", 1, 2) == 2 +assert "가나다".count("", 4, 4) == 0 +assert "a\U0001f600b".count("") == 4 + assert "___a__".find("a") == 3 assert "___a__".find("a", -10) == 3 assert "___a__".find("a", -3) == 3 @@ -891,3 +900,18 @@ class MyString(str): assert id(b) != id(b * 1) assert id(b) != id(1 * b) assert id(b) != id(b * 2) + + +def test_huge_width(): + # A width that cannot be allocated is a MemoryError, not an aborted + # process, and a tabsize wider than a C int does not fit at all. + for meth in ("center", "ljust", "rjust", "zfill"): + assert_raises(MemoryError, lambda meth=meth: getattr("a", meth)(1 << 62)) + assert_raises(OverflowError, lambda: "\ta".expandtabs(1 << 62)) + assert_raises(OverflowError, lambda: "\ta".expandtabs(2**31)) + # The widest tabsize that still fits is accepted. With no tab to expand + # there is nothing to lay out, so the width is never allocated. + assert "a".expandtabs(2**31 - 1) == "a" + + +test_huge_width() diff --git a/extra_tests/snippets/builtin_str_unicode_slice.py b/extra_tests/snippets/builtin_str_unicode_slice.py index 252f84b1c72..1d35c6c483c 100644 --- a/extra_tests/snippets/builtin_str_unicode_slice.py +++ b/extra_tests/snippets/builtin_str_unicode_slice.py @@ -59,3 +59,32 @@ def expect_index_error(s, index): assert len(hebrew_text[30:10:-3]) == 7 assert hebrew_text[30:10:-1] == "א ,םיִהֹלֱא אָרָּב ," assert len(hebrew_text[30:10:-1]) == 20 + + +# A stepped slice whose span is an exact multiple of the step ends on the last +# character it collects rather than one past it, so the character count is the +# span divided by the step and not one more. The subject goes through a +# variable because a constant subscript is folded at compile time and would +# never reach the runtime slice at all. +def stepped(s, step): + return s[::step] + + +for subject, step, expected in [ + ("a\u00e9c", 3, "a"), + ("가나다라", 2, "가다"), + ("가나다라마바", 3, "가라"), + ("가나다라", -2, "라나"), + ("가나다라마바", -3, "바다"), + ("\U0001f600\U0001f601\U0001f602\U0001f603", 2, "\U0001f600\U0001f602"), +]: + sliced = stepped(subject, step) + assert sliced == expected, (subject, step, sliced) + assert len(sliced) == len(expected), (subject, step, len(sliced)) + # An overstated count makes the string claim characters its buffer does not + # hold, which reversed() then reads past. + assert list(reversed(sliced)) == list(expected)[::-1] + +assert len(stepped(hebrew_text, 2)) == 30 +assert len(stepped(hebrew_text, 4)) == 15 +assert len(stepped(hebrew_text, -2)) == 30 diff --git a/extra_tests/snippets/builtin_tuple.py b/extra_tests/snippets/builtin_tuple.py index fc2f8d5bb75..a679d2a99a8 100644 --- a/extra_tests/snippets/builtin_tuple.py +++ b/extra_tests/snippets/builtin_tuple.py @@ -1,3 +1,5 @@ +import sys + from testutils import assert_raises assert (1, 2) == (1, 2) @@ -93,3 +95,8 @@ def __eq__(self, x): assert (float("inf"), float("inf")) >= (float("inf"), float("inf")) assert not (float("inf"), float("inf")) < (float("inf"), float("inf")) assert not (float("inf"), float("inf")) > (float("inf"), float("inf")) + +# The repeat count is multiplied by the element size; a count that overflows +# that product must raise instead of wrapping into a short allocation. +with assert_raises(MemoryError): + (1,) * sys.maxsize diff --git a/extra_tests/snippets/builtin_type.py b/extra_tests/snippets/builtin_type.py index 8cb0a09a215..15a330aea19 100644 --- a/extra_tests/snippets/builtin_type.py +++ b/extra_tests/snippets/builtin_type.py @@ -687,3 +687,33 @@ def foo(): code = compile(stmts, "", "exec") assert code.co_names == ("blah", "foo") + + +# A slot descriptor carries the layout it was defined for. Reached from another +# class, it has to report that rather than read the slot at its own offset, +# whether the access is fresh or has been seen often enough to be specialized. + + +class WideSlots: + __slots__ = ("s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7") + + +class NarrowSlots: + __slots__ = ("only",) + + +class NoSlots: + __slots__ = () + + +NarrowSlots.borrowed = WideSlots.__dict__["s7"] +NoSlots.borrowed = WideSlots.__dict__["s7"] + +for owner in (NarrowSlots(), NoSlots()): + for _ in range(1000): + with assert_raises(TypeError): + owner.borrowed + with assert_raises(TypeError): + owner.borrowed = 1 + with assert_raises(TypeError): + del owner.borrowed diff --git a/extra_tests/snippets/forbidden_instantiation.py b/extra_tests/snippets/forbidden_instantiation.py index 50b6f58f07f..50a0e2cf635 100644 --- a/extra_tests/snippets/forbidden_instantiation.py +++ b/extra_tests/snippets/forbidden_instantiation.py @@ -1,3 +1,4 @@ +import re from types import ( AsyncGeneratorType, BuiltinFunctionType, @@ -62,3 +63,9 @@ def check_forbidden_instantiation(typ, reverse=False): for typ in internal_types: with assert_raises(TypeError): typ() + +# a match object carries state that only the matcher can fill in +with assert_raises(TypeError): + re.Match() +with assert_raises(TypeError): + re.Match.__new__(re.Match) diff --git a/extra_tests/snippets/operator_comparison.py b/extra_tests/snippets/operator_comparison.py index 71231f033dc..35a2083e94d 100644 --- a/extra_tests/snippets/operator_comparison.py +++ b/extra_tests/snippets/operator_comparison.py @@ -87,3 +87,50 @@ def test_type_error(x, y): assert not math.nan < 123 assert not math.nan >= 123 assert not math.nan <= 123 + + +# str and bytes comparisons, through a function so that the operands are not +# constants the compiler can fold, and in a loop so the specialized comparison +# is reached. +def cmp_all(a, b): + return (a == b, a != b, a < b, a <= b, a > b, a >= b) + + +def check(a, b, expected): + for _ in range(200): + assert cmp_all(a, b) == expected, (a, b, cmp_all(a, b), expected) + + +EQ = (True, False, False, True, False, True) +LT = (False, True, True, True, False, False) +GT = (False, True, False, False, True, True) + +same = "abc" * 3 +check(same, same, EQ) # the very same object +check(same, "abcabcabc", EQ) # equal, distinct objects +check("abc", "abd", LT) # same length, differing content +check("abc", "abcd", LT) # a prefix is less than what extends it +check("abcd", "abc", GT) +check("", "a", LT) +check("", "", EQ) +check("\ud800", "\ud800", EQ) # lone surrogates are compared as themselves +check("\ud800", "\udfff", LT) +check("a\U0001f600", "a\U0001f600", EQ) +check("가나다", "가나다", EQ) +check("가나", "가나다", LT) + +# Comparing with a non-string is never an error for == and !=. +assert not "abc" == 3 +assert "abc" != 3 + +bsame = b"abc" * 3 +check(bsame, bsame, EQ) +check(bsame, b"abcabcabc", EQ) +check(b"abc", b"abd", LT) +check(b"abc", b"abcd", LT) +check(b"abcd", b"abc", GT) +check(bytearray(b"abc"), bytearray(b"abcd"), LT) +check(bytearray(b"abc"), b"abc", EQ) # bytearray and bytes compare by content +check(b"abc", bytearray(b"abd"), LT) +assert not b"abc" == "abc" +assert b"abc" != "abc" diff --git a/extra_tests/snippets/recursion.py b/extra_tests/snippets/recursion.py index 2d3b2205d68..4b61a74b438 100644 --- a/extra_tests/snippets/recursion.py +++ b/extra_tests/snippets/recursion.py @@ -11,3 +11,36 @@ class Foo(object): # Since the default __str__ implementation calls __repr__ and __repr__ is # actually __str__, str(foo) should raise a RecursionError. assert_raises(RecursionError, str, foo) + + +# A __call__ that is the object being called dispatches through the call slot +# again, and none of that pushes a Python frame. + + +class Caller: + pass + + +caller = Caller() +Caller.__call__ = caller +assert_raises(RecursionError, caller) + + +# The same shape through the descriptor protocol: resolving the attribute +# fetches __get__, which is the descriptor itself. + + +class Descr: + pass + + +descr = Descr() +Descr.__get__ = descr +Descr.x = descr +try: + descr.x +except (RecursionError, TypeError): + # RecursionError here, TypeError from the call of a non-callable elsewhere + pass +else: + raise AssertionError("descr.x should not resolve") diff --git a/extra_tests/snippets/stdlib_array.py b/extra_tests/snippets/stdlib_array.py index ed2a8f22369..c2de6ac1ec8 100644 --- a/extra_tests/snippets/stdlib_array.py +++ b/extra_tests/snippets/stdlib_array.py @@ -143,3 +143,36 @@ def write(self, chunk): arr = array("b", range(128)) arr.tofile(_ReenteringWriter(arr)) assert len(arr) == 129 + + +def test_setitem_reentrant(): + # Converting the value runs Python, which can reach the array, so the + # array is not locked while it happens. + a = array("i", [1, 2, 3]) + + class Index: + def __index__(self): + a[1] = 9 + return 7 + + a[0] = Index() + assert a == array("i", [7, 9, 3]), a + + +test_setitem_reentrant() + + +def test_frombytes_of_itself(): + # Resizing is refused while a buffer is exported, before any lock is taken. + # The typecode is "b" so the view's items are bytes and the resize is what + # the call is refused for. + a = array("b", [1, 2, 3]) + m = memoryview(a) + with assert_raises(BufferError): + a.frombytes(m) + del m + + # A view of wider items is not a source of bytes at all. + wide = array("i", [1, 2, 3]) + with assert_raises(TypeError): + wide.frombytes(memoryview(wide)) diff --git a/extra_tests/snippets/stdlib_asyncio.py b/extra_tests/snippets/stdlib_asyncio.py new file mode 100644 index 00000000000..a6a55509036 --- /dev/null +++ b/extra_tests/snippets/stdlib_asyncio.py @@ -0,0 +1,102 @@ +"""The private _asyncio accessors, reached directly instead of through a loop. + +CPython's _asyncio rejects every call below with "loop ... is not the running +loop" before it gets anywhere, and does not expose _current_tasks at all, so +these only run where they are reachable. +""" + +import sys + +from testutils import assert_raises + +if sys.implementation.name != "rustpython": + sys.exit(0) + +import _asyncio + + +def _task(): + pass + + +# The "already entered" message formats both tasks; a plain function used to be +# formatted as the wrong type there. +_asyncio._enter_task(0, _task) +with assert_raises(RuntimeError) as cm: + _asyncio._enter_task(0, _task) +assert "Cannot enter into task" in str(cm.exception), cm.exception +assert " str: # print(get_win_folder_via_ctypes("CSIDL_DOWNLOADS")) +# A value wider than the C type is masked down to it instead of failing an +# unchecked conversion. +assert ctypes.c_char_p(2**64).value is None +assert ctypes.c_int(2**64 + 7).value == 7 +buf = (ctypes.c_int * 1)() +int_ptr = ctypes.cast(buf, ctypes.POINTER(ctypes.c_int)) +int_ptr[0] = 2**64 + 5 +assert int_ptr[0] == 5 + +# A slice assignment is length-checked against the slice, so the right-hand +# side must not be drained first. +array3 = (ctypes.c_int * 3)() +try: + array3[0:3] = itertools.count() +except ValueError: + pass +else: + assert False, "slice assignment accepted an unbounded iterable" +array3[0:3] = [7, 8, 9] +assert list(array3) == [7, 8, 9] + print("done") diff --git a/extra_tests/snippets/stdlib_ctypes_calls.py b/extra_tests/snippets/stdlib_ctypes_calls.py index 1de29931429..cc4e8020511 100644 --- a/extra_tests/snippets/stdlib_ctypes_calls.py +++ b/extra_tests/snippets/stdlib_ctypes_calls.py @@ -1,6 +1,7 @@ # Exercises the migrated _ctypes foreign-call path (routed through the unified # host_env `call` entry point): scalar int/double arguments and returns, -# pointer (c_char_p / c_void_p) returns, and a use_errno round-trip. +# pointer (c_char_p / c_void_p) returns, a use_errno round-trip, and the +# argument conversion an untyped call performs. # # Prints "OK" and exits 0; any failed assertion aborts. Output is identical # under CPython and RustPython on the same platform. @@ -61,4 +62,17 @@ libc.strtol(b"9" * 40, None, 10) assert get_errno() == errno.ERANGE, (get_errno(), errno.ERANGE) +# 7. A float has no implicit conversion to an integer argument: converting it +# would pass a truncated value where the callee expects an int or a pointer. +libc.abs.argtypes = None +for bad in (1.5, 0.0, 1e300): + try: + libc.abs(bad) + except (TypeError, ctypes.ArgumentError): + pass + else: + assert False, f"{bad!r} was accepted as an integer argument" +assert libc.abs(-3) == 3 +assert libc.abs(True) == 1 + print("OK") diff --git a/extra_tests/snippets/stdlib_gc.py b/extra_tests/snippets/stdlib_gc.py new file mode 100644 index 00000000000..6c3169beedb --- /dev/null +++ b/extra_tests/snippets/stdlib_gc.py @@ -0,0 +1,68 @@ +"""The cycle collector has to walk the internal fields of containers and +iterators. + +Every type below is built into the cycle + + node -> node.__dict__ -> wrapper -> container -> node + +so the only path back to `node` runs through a field of the wrapper. A type +that reports nothing while being traversed, or reports the objects it iterates +instead of the iterator it holds, leaves its own reference unaccounted for: the +cycle is then classified as reachable and `node` is never freed. +""" + +import gc +import itertools +import weakref +from collections import defaultdict, deque + + +class Node: + pass + + +def collects(wrap): + """Report whether the collector breaks the cycle built around wrap().""" + + def build(): + container = [] + node = Node() + container.append(node) + node.held = wrap(container) + return weakref.ref(node) + + gc.collect() + ref = build() + gc.collect() + return ref() is None + + +# containers keeping their items in a field of their own +assert collects(deque) +assert collects(lambda c: defaultdict(int, {"k": c})) +assert collects(lambda c: classmethod(lambda cls: c)) + +# iterators: the wrapper holds an iterator, and that iterator holds the +# container +assert collects(iter) +assert collects(lambda c: map(str, c)) +assert collects(lambda c: filter(None, c)) +assert collects(lambda c: zip(c)) +assert collects(enumerate) +assert collects(reversed) +assert collects(itertools.chain) +assert collects(itertools.cycle) +assert collects(lambda c: itertools.islice(c, 5)) +assert collects(itertools.groupby) +assert collects(itertools.accumulate) +assert collects(lambda c: itertools.starmap(str, c)) +assert collects(lambda c: itertools.takewhile(bool, c)) +assert collects(lambda c: itertools.dropwhile(bool, c)) +assert collects(lambda c: itertools.filterfalse(None, c)) +assert collects(lambda c: itertools.compress(c, [1])) +assert collects(lambda c: itertools.product(c)) +assert collects(lambda c: itertools.combinations(c, 1)) +# tee holds its buffer through a second object, which has to be walked too +assert collects(lambda c: itertools.tee(c)[0]) + +print("ok") diff --git a/extra_tests/snippets/stdlib_hashlib.py b/extra_tests/snippets/stdlib_hashlib.py index c5feb709e17..f3400aed57d 100644 --- a/extra_tests/snippets/stdlib_hashlib.py +++ b/extra_tests/snippets/stdlib_hashlib.py @@ -1,5 +1,9 @@ +import _md5 +import _sha1 import hashlib +from testutils import assert_raises + # print(hashlib.md5) h = hashlib.md5() h.update(b"a") @@ -48,3 +52,15 @@ assert ( h.hexdigest() == "25738bfe4cc104131e1b45bece4dfd4e7e1d6f0dffda1211e996e9d5d3b66e81" ) + +# The single-algorithm modules set up their own types rather than relying on +# hashlib having done it. + +assert _md5.md5(b"").hexdigest() == "d41d8cd98f00b204e9800998ecf8427e" +assert _sha1.sha1(b"").hexdigest() == "da39a3ee5e6b4b0d3255bfef95601890afd80709" + +# a derived key wider than a C int does not fit, and never gets allocated. +# Which OverflowError comes out depends on the width of a C long: where it is +# narrower than the length asked for, converting the argument fails first. +with assert_raises(OverflowError): + hashlib.pbkdf2_hmac("sha256", b"password", b"salt", 1, 2**62) diff --git a/extra_tests/snippets/stdlib_imp.py b/extra_tests/snippets/stdlib_imp.py index 835b50d6171..9fd5f8a36fa 100644 --- a/extra_tests/snippets/stdlib_imp.py +++ b/extra_tests/snippets/stdlib_imp.py @@ -1,6 +1,8 @@ import _imp import time as import_time +from testutils import assert_raises + assert _imp.is_builtin("time") == True assert _imp.is_builtin("os") == False assert _imp.is_builtin("not existing module") == False @@ -29,3 +31,14 @@ def __init__(self, name): hello = _imp.init_frozen("__hello__") assert hello.initialized == True + +# withdata is keyword-only +with assert_raises(TypeError): + _imp.find_frozen("x", True) +assert _imp.find_frozen("_this_module_does_not_exist_") is None + +# and it hands back the marshalled code that get_frozen_object() takes +data, ispkg, origname = _imp.find_frozen("__hello__", withdata=True) +assert ispkg is False +assert origname == "__hello__" +assert _imp.get_frozen_object("__hello__", data).co_name == "" diff --git a/extra_tests/snippets/stdlib_io.py b/extra_tests/snippets/stdlib_io.py index f17eae5b172..8346ddbb62d 100644 --- a/extra_tests/snippets/stdlib_io.py +++ b/extra_tests/snippets/stdlib_io.py @@ -197,3 +197,48 @@ def __index__(self): f"cannot fit '{truncated_non_ascii_type_name}' into an index-sized integer", lambda: setattr(textio, "_CHUNK_SIZE", NonAsciiNamedChunkSize()), ) + + +# A buffer size or read size that cannot be allocated is a MemoryError, not an +# aborted process. +assert_raises(MemoryError, lambda: BufferedReader(BytesIO(b"a"), buffer_size=2**62)) +assert_raises(MemoryError, lambda: BufferedReader(BytesIO(b"a")).read(2**62)) +assert_raises(MemoryError, lambda: BufferedReader(BytesIO(b"a")).read1(2**62)) + + +def _text_cookie( + start_pos=0, + dec_flags=0, + bytes_to_feed=0, + chars_to_skip=0, + need_eof=0, + bytes_to_skip=0, +): + packed = ( + start_pos.to_bytes(8, "little", signed=True) + + dec_flags.to_bytes(4, "little", signed=True) + + bytes_to_feed.to_bytes(4, "little", signed=True) + + chars_to_skip.to_bytes(4, "little", signed=True) + + bytes([need_eof]) + + bytes_to_skip.to_bytes(4, "little", signed=True) + ) + return int.from_bytes(packed, "little") + + +# A cookie names a position both in characters and in bytes, and everything +# read back from it indexes what was decoded, so a position past the end is +# refused rather than stored. +for _bad in ( + _text_cookie(bytes_to_feed=10, chars_to_skip=1000, bytes_to_skip=0), + _text_cookie(bytes_to_feed=10, chars_to_skip=100000, bytes_to_skip=3), + _text_cookie(bytes_to_feed=10, chars_to_skip=1, bytes_to_skip=1000), +): + _textio = TextIOWrapper(BytesIO(b"hello world " * 20), encoding="utf-8") + _textio.read(1) + try: + _textio.seek(_bad) + except (OSError, OverflowError): + pass + else: + assert _textio.read(50) is not None + _textio.tell() diff --git a/extra_tests/snippets/stdlib_io_blocking_buffer.py b/extra_tests/snippets/stdlib_io_blocking_buffer.py new file mode 100644 index 00000000000..2119111dc2e --- /dev/null +++ b/extra_tests/snippets/stdlib_io_blocking_buffer.py @@ -0,0 +1,176 @@ +"""Transfers that wait for a peer must not hold the buffer they were given. + +A pipe or a socket answers when the other end does, which may be never. The +buffer is exported for the whole call, so it cannot be resized meanwhile, but +nothing else about it changes: another thread can still read it, write to it, +and the interpreter can still stop the world. An implementation that holds the +buffer's storage for the duration of the wait takes all of that away, and a +thread parked on that storage never reaches a safepoint, so a collection that +wants every thread stopped ends up waiting for the peer too. +""" + +import gc +import os +import socket +import threading +import time + +# The peer acts after DELAY; the checks below have to finish well inside it. +DELAY = 1.0 +SLACK = DELAY / 2 + + +def measure(buf, expected_len, writable): + """Time each operation on `buf` that does not need the peer, separately, so + a failure names the one that waited rather than the group.""" + elapsed = {} + + def timed(name, operation): + start = time.monotonic() + value = operation() + elapsed[name] = time.monotonic() - start + return value + + assert timed("len", lambda: len(buf)) == expected_len, len(buf) + assert isinstance(timed("bytes", lambda: bytes(buf)), bytes) + if writable: + timed("setitem", lambda: buf.__setitem__(0, buf[0])) + timed("gc.collect", gc.collect) + return elapsed + + +def run(buf, blocking_call, release_peer, writable): + started = threading.Event() + expected_len = len(buf) + result = [] + + def transfer(): + started.set() + result.append(blocking_call(buf)) + + def peer(): + time.sleep(DELAY) + release_peer() + + threads = [threading.Thread(target=transfer), threading.Thread(target=peer)] + for t in threads: + t.start() + started.wait() + time.sleep(0.2) # the transfer is now waiting on its peer + + elapsed = measure(buf, expected_len, writable) + waited = ["%s %.2fs" % item for item in elapsed.items() if item[1] >= SLACK] + assert not waited, "waited on the peer: " + ", ".join(waited) + + # The transfer is still in flight, so its export is still held and the + # buffer cannot be resized. An operating system that took the whole + # transfer without a peer leaves nothing here to observe. + assert not result, "the transfer finished without its peer" + try: + buf.append(0) + except BufferError: + pass + else: + raise AssertionError("append during an export should raise BufferError") + + for t in threads: + t.join() + return result[0] + + +# --- reading: the buffer is written into, so nothing else may touch it at all + + +read_fd, write_fd = os.pipe() +pipe = open(read_fd, "rb", buffering=0) +try: + target = bytearray(16) + n = run(target, pipe.readinto, lambda: os.write(write_fd, b"pipe"), writable=False) + assert n == 4, n + assert bytes(target[:4]) == b"pipe", bytes(target) +finally: + pipe.close() + os.close(write_fd) + +if hasattr(socket, "socketpair"): + left, right = socket.socketpair() + try: + target = bytearray(16) + n = run(target, left.recv_into, lambda: right.send(b"socket"), writable=False) + assert n == 6, n + assert bytes(target[:6]) == b"socket", bytes(target) + finally: + left.close() + right.close() + + +# --- writing: the buffer is only read, so it stays writable meanwhile + + +read_fd, write_fd = os.pipe() +sink = open(write_fd, "wb", buffering=0) +try: + # More than any pipe will hold, so the write cannot finish on its own. + source = bytearray(4 * 1024 * 1024) + drained = [] + + def drain(): + with open(read_fd, "rb", buffering=0) as f: + while True: + chunk = f.read(1 << 16) + if not chunk: + break + drained.append(len(chunk)) + + reader = threading.Thread(target=drain, daemon=True) + # One unbuffered write() reports what it transferred, which a signal can + # cut short, so the reader is measured against that rather than the source. + written = run(source, sink.write, reader.start, writable=True) + sink.close() + reader.join() + assert sum(drained) == written, (sum(drained), written) +finally: + if not sink.closed: + sink.close() + +if hasattr(socket, "socketpair"): + left, right = socket.socketpair() + try: + # How much a connection holds before it makes the sender wait is the + # operating system's to decide, and asking for a small send buffer does + # not settle it -- a socketpair is already connected, and on Windows it + # is a loopback pair whose receiver has a window of its own. So fill it + # until it refuses rather than guess a size that outruns it. + left.setblocking(False) + filled = 0 + while True: + try: + filled += left.send(bytes(1 << 16)) + except (BlockingIOError, InterruptedError): + break + left.setblocking(True) + + source = bytearray(1 << 16) + received = [] + + def receive(): + wanted = filled + len(source) + while sum(received) < wanted: + chunk = right.recv(1 << 16) + if not chunk: + break + received.append(len(chunk)) + + reader = threading.Thread(target=receive, daemon=True) + run(source, left.sendall, reader.start, writable=True) + reader.join() + assert sum(received) == filled + len(source), ( + sum(received), + filled, + len(source), + ) + finally: + left.close() + right.close() + +print("ok") diff --git a/extra_tests/snippets/stdlib_io_bytesio.py b/extra_tests/snippets/stdlib_io_bytesio.py index ba8ae20015e..9344c50d947 100644 --- a/extra_tests/snippets/stdlib_io_bytesio.py +++ b/extra_tests/snippets/stdlib_io_bytesio.py @@ -106,3 +106,11 @@ def test_07(): test_05() test_06() test_07() + + +# Reading into a buffer that views this same object locks it twice unless the +# read finishes first. +_bio = BytesIO(b"x" * 60) +assert _bio.readinto(_bio.getbuffer()) == 60 +_bio = BytesIO(b"x" * 60) +assert _bio.readinto(memoryview(_bio.getbuffer())) == 60 diff --git a/extra_tests/snippets/stdlib_io_stringio.py b/extra_tests/snippets/stdlib_io_stringio.py index 5419eef2bb2..0adf0edac0b 100644 --- a/extra_tests/snippets/stdlib_io_stringio.py +++ b/extra_tests/snippets/stdlib_io_stringio.py @@ -69,9 +69,25 @@ def test_05(): assert f.readline() == "" +def test_06(): + f = StringIO(newline=None) + f.write("\r") + f.__init__("x\n", newline=None) + assert f.newlines == "\n" + + f.close() + try: + f.newlines + except ValueError: + pass + else: + assert False + + if __name__ == "__main__": test_01() test_02() test_03() test_04() test_05() + test_06() diff --git a/extra_tests/snippets/stdlib_itertools.py b/extra_tests/snippets/stdlib_itertools.py index ce7a494713a..029d0d4229a 100644 --- a/extra_tests/snippets/stdlib_itertools.py +++ b/extra_tests/snippets/stdlib_itertools.py @@ -524,3 +524,19 @@ def __iter__(self): assert next(it) == (2, None) with assert_raises(StopIteration): next(it) + +# r is an arbitrary Python int: one too large for an index must raise +# OverflowError, and a representable one that cannot be allocated must raise +# MemoryError. +for factory in ( + itertools.combinations, + itertools.combinations_with_replacement, + itertools.permutations, +): + with assert_raises(OverflowError): + factory(range(5), 2**64) + +with assert_raises(MemoryError): + itertools.combinations(range(5), 2**44) +with assert_raises(MemoryError): + itertools.combinations_with_replacement(range(5), 2**44) diff --git a/extra_tests/snippets/stdlib_lzma.py b/extra_tests/snippets/stdlib_lzma.py new file mode 100644 index 00000000000..5ebce3c7fb1 --- /dev/null +++ b/extra_tests/snippets/stdlib_lzma.py @@ -0,0 +1,22 @@ +import itertools +import lzma + +from testutils import assert_raises + +# A raw-format compressor needs the filter chain's length before it can build +# it, so a filter argument that is not a sequence has to be rejected instead of +# being drained. +with assert_raises(TypeError): + lzma.LZMACompressor( + format=lzma.FORMAT_RAW, + filters=({"id": lzma.FILTER_LZMA2} for _ in itertools.count()), + ) + +compressor = lzma.LZMACompressor( + format=lzma.FORMAT_RAW, filters=[{"id": lzma.FILTER_LZMA2}] +) +compressed = compressor.compress(b"data") + compressor.flush() +decompressor = lzma.LZMADecompressor( + format=lzma.FORMAT_RAW, filters=[{"id": lzma.FILTER_LZMA2}] +) +assert decompressor.decompress(compressed) == b"data" diff --git a/extra_tests/snippets/stdlib_marshal.py b/extra_tests/snippets/stdlib_marshal.py index db843ff65d5..4e224fb313f 100644 --- a/extra_tests/snippets/stdlib_marshal.py +++ b/extra_tests/snippets/stdlib_marshal.py @@ -74,6 +74,85 @@ def test_roundtrip(self): assert eval(loaded) == eval(orig) + def test_roundtrip_non_constant_co_consts(self): + # `code.replace` accepts any marshalable object, including values the + # compiler constant representation cannot describe. + orig = compile("1 + 1", "", "eval").replace( + co_consts=([1, 2], {"a": 3}, {4, 5}, 6) + ) + + loaded = marshal.loads(marshal.dumps(orig)) + + self.assertEqual(loaded.co_consts, ([1, 2], {"a": 3}, {4, 5}, 6)) + + def test_roundtrip_shared_co_const(self): + # A constant shared with the enclosing object is written once and both + # readers resolve the same reference. + shared = ["shared"] + orig = compile("1 + 1", "", "eval").replace(co_consts=(shared,)) + + loaded_code, loaded_shared = marshal.loads(marshal.dumps((orig, shared))) + + self.assertIs(loaded_code.co_consts[0], loaded_shared) + + +class AllowCodeTests(unittest.TestCase): + """allow_code is answered where a code object is written or read, so a + graph that walks back on itself is not a second walk of its own.""" + + def test_recursive_value(self): + recursive = [] + recursive.append(recursive) + loaded = marshal.loads( + marshal.dumps(recursive, allow_code=False), allow_code=False + ) + self.assertIs(loaded[0], loaded) + + def test_too_deeply_nested(self): + nested = [] + for _ in range(100_000): + nested = [nested] + with self.assertRaises(ValueError): + marshal.dumps(nested, allow_code=False) + + def test_code_is_rejected(self): + code = compile("1", "", "exec") + for value in (code, [code], (code,), {0: code}): + with self.assertRaises(ValueError): + marshal.dumps(value, allow_code=False) + data = marshal.dumps(value) + with self.assertRaises(ValueError): + marshal.loads(data, allow_code=False) + + +class BadDataTests(unittest.TestCase): + def test_container_size_out_of_range(self): + import struct + + # a length is signed, so the top bit set is out of range rather than + # four billion items to reserve room for + for marker in b"([<>": + data = bytes([marker | 0x80]) + struct.pack("H", "'H' format requires 0 <= number <= 65535"), + (">i", "'i' format requires -2147483648 <= number <= 2147483647"), + ("N", "'N' format requires 0 <= number <= 18446744073709551615"), + ("P", "int too large to convert"), +): + try: + struct.pack(fmt, 10**30) + except struct.error as e: + assert str(e) == message, (fmt, str(e)) + else: + raise AssertionError(f"expected struct.error for {fmt!r}") + +try: + struct.pack("B", "x") +except struct.error as e: + assert str(e) == "required argument is not an integer", e +else: + raise AssertionError("expected struct.error") + + +# __init__ reads a new format into a Struct that already holds one. +s = struct.Struct(">h") +s.__init__(">hh") +assert s.format == ">hh" +assert s.size == 4 +assert s.pack(1, 2) == b"\x00\x01\x00\x02" +assert s.unpack(b"\x00\x01\x00\x02") == (1, 2) + +# A format that cannot be read leaves the Struct as it was. +for bad in ("\udc00", "$"): + with assert_raises((UnicodeEncodeError, struct.error)): + s.__init__(bad) + assert s.format == ">hh" + assert s.pack(1, 2) == b"\x00\x01\x00\x02" + + +# A subclass may do its own __init__ and pass the format up. +class BigShort(struct.Struct): + def __init__(self): + super().__init__(">h") + + +assert BigShort().pack(12345) == b"\x30\x39" + +# Until __init__ runs there is no format to answer with. +blank = struct.Struct.__new__(struct.Struct) +assert blank.size == -1 +for call in ( + lambda: blank.format, + lambda: blank.pack(1), + lambda: blank.unpack(b"aa"), + lambda: blank.unpack_from(b"aaaa"), + lambda: blank.pack_into(bytearray(4), 0, 1), + lambda: blank.iter_unpack(b"aa"), + lambda: repr(blank), +): + with assert_raises(RuntimeError): + call() diff --git a/extra_tests/snippets/stdlib_sys.py b/extra_tests/snippets/stdlib_sys.py index 155fc905a73..9dba301fb01 100644 --- a/extra_tests/snippets/stdlib_sys.py +++ b/extra_tests/snippets/stdlib_sys.py @@ -1,6 +1,7 @@ import os import subprocess import sys +import warnings from testutils import assert_raises @@ -158,3 +159,18 @@ def test_getframemodulename(): test_getframemodulename.__module__ = "awesome_module" assert test_getframemodulename() == "awesome_module" + +# An unimportable $PYTHONBREAKPOINT warns, and the hook has to survive that +# warning being turned into an exception. +saved_breakpoint_env = os.environ.get("PYTHONBREAKPOINT") +os.environ["PYTHONBREAKPOINT"] = "nonexistent_xyz.foo" +try: + with warnings.catch_warnings(): + warnings.simplefilter("error") + with assert_raises(RuntimeWarning): + sys.breakpointhook() +finally: + if saved_breakpoint_env is None: + del os.environ["PYTHONBREAKPOINT"] + else: + os.environ["PYTHONBREAKPOINT"] = saved_breakpoint_env diff --git a/extra_tests/snippets/stdlib_threading_contextvars.py b/extra_tests/snippets/stdlib_threading_contextvars.py new file mode 100644 index 00000000000..7947ef74fd4 --- /dev/null +++ b/extra_tests/snippets/stdlib_threading_contextvars.py @@ -0,0 +1,72 @@ +"""Stress contextvars from several threads at once. + +A Context holds the variable map, and both the map and the per-variable cache +are shared between every thread that touches the Context. Reading and writing +them has to be done under a lock rather than a cell borrow. + +Dropping a value that a set() or reset() displaced can run a __del__ that comes +straight back into the same Context, so the displaced value has to be released +after the lock is, not while it is held. +""" + +import contextvars +import threading + +ROUNDS = 2000 + +var = contextvars.ContextVar("v", default=0) +shared = contextvars.Context() +errors = [] + + +class Reentrant: + """__del__ runs while the variable that held this value is being replaced.""" + + def __del__(self): + try: + var.get() + except Exception: # a different context, or no value: not what is tested + pass + + +def churn(): + try: + for i in range(ROUNDS): + token = var.set(Reentrant()) + var.get() + var.reset(token) + var.set(i) + var.get() + contextvars.copy_context() + except Exception as exc: # noqa: BLE001 + errors.append(exc) + + +def run_in_shared(): + for i in range(ROUNDS): + try: + shared.run(var.set, i) + except RuntimeError: + # the Context is already entered by another thread + pass + + +threads = [threading.Thread(target=churn) for _ in range(4)] +threads += [threading.Thread(target=run_in_shared) for _ in range(4)] +for t in threads: + t.start() +for t in threads: + t.join() + +assert not errors, errors + +# the map itself still behaves +ctx = contextvars.copy_context() +ctx.run(var.set, 42) +assert ctx[var] == 42 +assert var in ctx +assert list(ctx) == [var] +assert ctx.get(var) == 42 +assert len(ctx) == 1 + +print("ok") diff --git a/extra_tests/snippets/stdlib_threading_current_frames.py b/extra_tests/snippets/stdlib_threading_current_frames.py new file mode 100644 index 00000000000..e93a222148e --- /dev/null +++ b/extra_tests/snippets/stdlib_threading_current_frames.py @@ -0,0 +1,100 @@ +"""Take sys._current_frames() while other threads are running Python. + +The frame each thread is executing is published for cross-thread readers, and +_current_frames() takes a reference to it with the world stopped. A reader that +disagrees with the publisher about what the published pointer addresses reads +and reference-counts the wrong memory, which corrupts a neighbouring object +rather than failing at the read: the damage surfaces later, in the thread that +owns it, as a crash or a wedge. + +Workers therefore run ordinary Python calls (which publish a frame) in a tight +loop while the main thread hammers _current_frames(). +""" + +import sys +import threading +import time + +DURATION = 1.5 + + +def leaf(): + return sum(range(8)) + + +def nest(n): + if n: + return nest(n - 1) + return leaf() + + +def worker(stop): + while not stop.is_set(): + nest(16) + + +def frames_are_sane(frames): + # Every key is a thread id, every value a frame of this process. + for tid, frame in frames.items(): + assert isinstance(tid, int), tid + assert tid > 0, tid + assert type(frame).__name__ == "frame", frame + assert isinstance(frame.f_lineno, int), frame + assert isinstance(frame.f_code.co_name, str), frame + + +# The main thread sees itself where it stands. +me = sys._current_frames()[threading.get_ident()] +assert me is sys._getframe(), me + +stop = threading.Event() +threads = [threading.Thread(target=worker, args=(stop,)) for _ in range(4)] +for t in threads: + t.start() + +deadline = time.time() + DURATION +calls = 0 +while time.time() < deadline: + frames_are_sane(sys._current_frames()) + calls += 1 +stop.set() +for t in threads: + t.join() + +assert calls > 0, calls + + +# A thread parked in a call the main thread can name is reported inside it, +# with its callers reachable through f_back. +entered = threading.Event() +leave = threading.Event() +seen = [] + + +def g456(): + seen.append(threading.get_ident()) + entered.set() + leave.wait() + + +def f123(): + g456() + + +t = threading.Thread(target=f123) +t.start() +entered.wait() +try: + chain = [] + frame = sys._current_frames()[seen[0]] + while frame is not None: + chain.append(frame.f_code.co_name) + frame = frame.f_back + assert "g456" in chain, chain + assert "f123" in chain, chain + assert chain.index("g456") < chain.index("f123"), chain +finally: + leave.set() + t.join() + +print("ok") diff --git a/extra_tests/snippets/stdlib_threading_generator.py b/extra_tests/snippets/stdlib_threading_generator.py new file mode 100644 index 00000000000..e606ef11cec --- /dev/null +++ b/extra_tests/snippets/stdlib_threading_generator.py @@ -0,0 +1,95 @@ +"""Resume one generator from several threads at once. + +A generator is resumed by one thread at a time, and whether the sent value is +pushed onto the frame's value stack depends on whether the generator has +already started. Deciding that before the generator is claimed reads a frame +another thread can advance in the meantime, and resuming it then leaves the +stack short of what the code after the yield expects. + +Every yielded value still has to reach exactly one caller: threads that lose +the race get a ValueError instead of a value. +""" + +import threading + +WORKERS = 4 +ROUNDS = 400 + + +def counter(): + yield 1 + yield 2 + yield 3 + + +gens = [counter() for _ in range(ROUNDS)] +received = [[] for _ in range(ROUNDS)] +start = threading.Barrier(WORKERS) +errors = [] + + +def worker(): + try: + for index, gen in enumerate(gens): + start.wait() + for _ in range(3): + try: + received[index].append(next(gen)) + except StopIteration: + break + except ValueError: + # another thread is running this generator + pass + except Exception as exc: # noqa: BLE001 + errors.append(exc) + # the other workers are waiting at the barrier for this one + start.abort() + + +threads = [threading.Thread(target=worker) for _ in range(WORKERS)] +for t in threads: + t.start() +for t in threads: + t.join() + +assert not errors, errors +for got in received: + # no value handed out twice, and none skipped + assert sorted(got) == list(range(1, len(got) + 1)), got + + +# a generator that is closed while it is being resumed stays consistent +def loop(): + while True: + yield 1 + + +shared = loop() +closed = threading.Barrier(2) + + +def resumer(): + closed.wait() + for _ in range(ROUNDS): + try: + next(shared) + except (StopIteration, ValueError): + pass + + +def closer(): + closed.wait() + try: + shared.close() + except ValueError: + # the generator was running + pass + + +pair = [threading.Thread(target=resumer), threading.Thread(target=closer)] +for t in pair: + t.start() +for t in pair: + t.join() + +print("ok") diff --git a/extra_tests/snippets/stdlib_threading_itertools_cycle.py b/extra_tests/snippets/stdlib_threading_itertools_cycle.py new file mode 100644 index 00000000000..b50a31b2443 --- /dev/null +++ b/extra_tests/snippets/stdlib_threading_itertools_cycle.py @@ -0,0 +1,26 @@ +"""Stress itertools.cycle from several threads at once. + +cycle() advances its index and wraps it back to zero when it reaches the end of +the saved items. Doing that in two separate steps lets another thread observe +the index past the end and read out of bounds, so the update has to be a single +atomic step. +""" + +import itertools +import threading + +shared_cycle = itertools.cycle([1, 2, 3]) + + +def spin(): + for _ in range(20000): + next(shared_cycle) + + +threads = [threading.Thread(target=spin) for _ in range(4)] +for t in threads: + t.start() +for t in threads: + t.join() + +print("ok") diff --git a/extra_tests/snippets/stdlib_threading_itertools_tee.py b/extra_tests/snippets/stdlib_threading_itertools_tee.py new file mode 100644 index 00000000000..e13e20731f8 --- /dev/null +++ b/extra_tests/snippets/stdlib_threading_itertools_tee.py @@ -0,0 +1,57 @@ +"""Advance the iterators of one tee() from several threads at once. + +Every tee iterator reads its position, asks the shared buffer for that item and +then moves the position on. Reading and moving it on has to be one step, and +the buffer has to stay claimed until the value it fetched from the source is +cached: otherwise two callers work on the same index, a fetched value is +dropped, and the buffer is left to be filled out of order. + +A caller that loses the race gets a RuntimeError, never a value another caller +has already been handed. +""" + +import itertools +import threading + +ROUNDS = 200 +WORKERS = 4 + +errors = [] + + +def drain(iterator, out): + for _ in range(ROUNDS): + try: + out.append(next(iterator)) + except StopIteration: + break + except RuntimeError: + # another thread is advancing this tee + pass + except Exception as exc: # noqa: BLE001 + errors.append(exc) + break + + +for _ in range(10): + first, second = itertools.tee(iter(range(ROUNDS * WORKERS))) + taken = [[] for _ in range(WORKERS)] + threads = [ + threading.Thread(target=drain, args=(first if i % 2 else second, taken[i])) + for i in range(WORKERS) + ] + for t in threads: + t.start() + for t in threads: + t.join() + + assert not errors, errors + for got in taken: + # one iterator hands out ascending values, each of them once + assert got == sorted(set(got)), got + for side in (taken[1], taken[3]), (taken[0], taken[2]): + # the two threads sharing an iterator split its values between them + shared = side[0] + side[1] + assert len(shared) == len(set(shared)), shared + +print("ok") diff --git a/extra_tests/snippets/stdlib_threading_set_repr.py b/extra_tests/snippets/stdlib_threading_set_repr.py new file mode 100644 index 00000000000..e2ce2d94357 --- /dev/null +++ b/extra_tests/snippets/stdlib_threading_set_repr.py @@ -0,0 +1,44 @@ +"""Stress set repr against concurrent mutation. + +repr() checks that the set is non-empty and then reads its first element. The +two steps are separate, so another thread can empty the set in between; the +read has to cope with that rather than trusting the earlier check. + +Threads that observe a mutation mid-iteration raise RuntimeError, which is a +legitimate outcome here; a regression shows up as a crash instead. +""" + +import threading + +shared_set = {1, 2, 3, 4, 5} +stop = False + + +def mutate(): + while not stop: + try: + shared_set.clear() + shared_set.update({1, 2, 3}) + except RuntimeError: # changed size during iteration + pass + + +def read(): + for _ in range(20000): + try: + repr(shared_set) + except RuntimeError: # changed size during iteration + pass + + +mutators = [threading.Thread(target=mutate) for _ in range(2)] +readers = [threading.Thread(target=read) for _ in range(2)] +for t in mutators + readers: + t.start() +for t in readers: + t.join() +stop = True +for t in mutators: + t.join() + +print("ok") diff --git a/extra_tests/snippets/stdlib_time.py b/extra_tests/snippets/stdlib_time.py index 68ceab89521..b74d5bbc638 100644 --- a/extra_tests/snippets/stdlib_time.py +++ b/extra_tests/snippets/stdlib_time.py @@ -82,3 +82,16 @@ assert monotonic_elapsed >= 0.01 assert perf_elapsed >= 0.01 + +# The optional second argument fills the fields that are not part of the +# sequence. +fields = (2024, 1, 2, 3, 4, 5, 6, 7, 0) +assert time.struct_time(fields).tm_zone is None +assert time.struct_time(fields, {"tm_zone": "UTC"}).tm_zone == "UTC" +assert time.struct_time(fields, {"tm_gmtoff": 60}).tm_gmtoff == 60 +try: + time.struct_time(fields, ["tm_zone", "UTC"]) +except TypeError: + pass +else: + assert False, "struct_time accepted a non-dict second argument" diff --git a/extra_tests/snippets/stdlib_traceback.py b/extra_tests/snippets/stdlib_traceback.py index c2cc5773dbc..b1b11a75503 100644 --- a/extra_tests/snippets/stdlib_traceback.py +++ b/extra_tests/snippets/stdlib_traceback.py @@ -1,5 +1,9 @@ +import itertools import traceback +import _suggestions +from testutils import assert_raises + try: 1 / 0 except ZeroDivisionError as ex: @@ -25,3 +29,10 @@ except ZeroDivisionError as ex2: tb = traceback.extract_tb(ex2.__traceback__) assert len(tb) == 1 + +# The candidate list backing "Did you mean" suggestions is a list; an arbitrary +# iterable must be rejected rather than drained. + +with assert_raises(TypeError): + _suggestions._generate_suggestions(itertools.count(), "x") +assert _suggestions._generate_suggestions(["value"], "valu") == "value" diff --git a/extra_tests/snippets/stdlib_types.py b/extra_tests/snippets/stdlib_types.py index cdecf12dd2b..4bccd2985bf 100644 --- a/extra_tests/snippets/stdlib_types.py +++ b/extra_tests/snippets/stdlib_types.py @@ -1,5 +1,6 @@ import _ast import platform +import sys import types from testutils import assert_raises @@ -34,3 +35,26 @@ def _run_missing_type_params_regression(): _run_missing_type_params_regression() + +if sys.implementation.name == "rustpython": + # __parameters__ is computed when the alias is built, and the walk descends + # into every list and tuple argument, so a self-referential or deeply + # nested argument must be caught. CPython, which also runs this snippet, + # does not walk into plain lists at all. + self_referential = [] + self_referential.append(self_referential) + with assert_raises(RecursionError): + list[self_referential] + + nested = [0] + for _ in range(100_000): + nested = [nested] + with assert_raises(RecursionError): + list[nested] + + # hashing an alias walks the same shape + deep_alias = int + for _ in range(100_000): + deep_alias = list[deep_alias] + with assert_raises(RecursionError): + hash(deep_alias) diff --git a/extra_tests/snippets/stdlib_typing.py b/extra_tests/snippets/stdlib_typing.py index 07348945842..4082d683f8d 100644 --- a/extra_tests/snippets/stdlib_typing.py +++ b/extra_tests/snippets/stdlib_typing.py @@ -1,6 +1,9 @@ from collections.abc import Awaitable, Callable from typing import TypeVar +import _typing +from testutils import assert_raises + T = TypeVar("T") @@ -35,3 +38,28 @@ def __init__( def method(self, value: Union[int, float]) -> Union[str, bytes]: return str(value) + + +# _idfunc takes exactly one argument, checked before the argument is read. + +assert _typing._idfunc(1) == 1 +with assert_raises(TypeError): + _typing._idfunc() + + +# ParamSpecArgs shows a non-ParamSpec origin by its repr, which is where the +# recursion guard lives; nesting them deeply must not walk the native stack. + +from typing import ParamSpec, ParamSpecArgs + +spec = ParamSpec("spec") +assert repr(spec.args) == "spec.args" +assert repr(spec.kwargs) == "spec.kwargs" + +nested = object() +for _ in range(2000): + nested = ParamSpecArgs(nested) +try: + repr(nested) +except RecursionError: + pass diff --git a/extra_tests/snippets/syntax_try.py b/extra_tests/snippets/syntax_try.py index 1f46caae3e7..5610cb23e6a 100644 --- a/extra_tests/snippets/syntax_try.py +++ b/extra_tests/snippets/syntax_try.py @@ -285,3 +285,85 @@ def y(): try: pass """) + + +# leaving the try block early emits an extra copy of the finally body, which +# must not consume the symbol tables of the nested scopes it contains +def return_from_try(): + log = [] + try: + return "returned" + finally: + log.append((lambda x: x * 2)(3)) + log.append({t for t in [1, 2]}) + log.append([t for t in [3]]) + log.append({k: k for k in [4]}) + + def nested(): + return 5 + + class Nested: + value = 6 + + assert log == [6, {1, 2}, [3], {4: 4}], log + assert nested() == 5 + assert Nested.value == 6 + + +assert return_from_try() == "returned" + + +def break_and_continue_from_try(): + seen = [] + for i in range(4): + try: + if i == 1: + continue + if i == 3: + break + seen.append(i) + finally: + seen.append({t for t in [i]}) + return seen + + +assert break_and_continue_from_try() == [0, {0}, {1}, 2, {2}, {3}] + + +def return_from_try_runs_finally_once(): + log = [] + + def inner(): + try: + return "value" + finally: + log.append(sorted({t for t in "ab"})) + + assert inner() == "value" + return log + + +assert return_from_try_runs_finally_once() == [["a", "b"]] + + +def generator_return_from_try(): + log = [] + + def gen(): + try: + return (yield "yielded") + finally: + log.append([t for t in "z"]) + + g = gen() + assert g.send(None) == "yielded" + try: + g.send("sent") + except StopIteration as stop: + assert stop.value == "sent", stop.value + else: + assert False, "generator did not stop" + return log + + +assert generator_return_from_try() == [["z"]]