Allows to introspect Python modules from cdylib: first step

.github/workflows/ci.yml

@@ -757,6 +757,52 @@ jobs:
         with:
           path: ~/.cache/cargo-xwin
           key: cargo-xwin-cache
+
+  test-introspection:
+    needs: [fmt]
+    strategy:
+      matrix:
+        platform: [
+          {
+            os: "macos-latest",
+            python-architecture: "arm64",
+            rust-target: "aarch64-apple-darwin",
+          },
+          {
+            os: "ubuntu-latest",
+            python-architecture: "x64",
+            rust-target: "x86_64-unknown-linux-gnu",
+          },
+          {
+            os: "windows-latest",
+            python-architecture: "x64",
+            rust-target: "x86_64-pc-windows-msvc",
+          },
+          {
+            os: "windows-latest",
+            python-architecture: "x86",
+            rust-target: "i686-pc-windows-msvc",
+          },
+        ]
+    runs-on: ${{ matrix.platform.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          targets: ${{ matrix.platform.rust-target }}
+          components: rust-src
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          architecture: ${{ matrix.platform.python-architecture }}
+      - uses: Swatinem/rust-cache@v2
+        with:
+          save-if: ${{ github.event_name != 'merge_group' }}
+      - run: python -m pip install --upgrade pip && pip install nox
+      - run: nox -s test-introspection
+    env:
+      CARGO_BUILD_TARGET: ${{ matrix.platform.rust-target }}
+
   conclusion:
     needs:
       - fmt
@@ -775,6 +821,7 @@ jobs:
       - check-feature-powerset
       - test-cross-compilation
       - test-cross-compilation-windows
+      - test-introspection
     if: always()
     runs-on: ubuntu-latest
     steps:

Cargo.toml

@@ -78,7 +78,7 @@ experimental-async = ["macros", "pyo3-macros/experimental-async"]
 
 # Enables pyo3::inspect module and additional type information on FromPyObject
 # and IntoPy traits
-experimental-inspect = []
+experimental-inspect = ["pyo3-macros/experimental-inspect"]
 
 # Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc.
 macros = ["pyo3-macros", "indoc", "unindent"]
@@ -145,6 +145,7 @@ members = [
     "pyo3-build-config",
     "pyo3-macros",
     "pyo3-macros-backend",
+    "pyo3-introspection",
     "pytests",
     "examples",
 ]

guide/src/features.md

@@ -59,7 +59,9 @@ The feature has some unfinished refinements and performance improvements. To hel
 
 ### `experimental-inspect`
 
-This feature adds the `pyo3::inspect` module, as well as `IntoPy::type_output` and `FromPyObject::type_input` APIs to produce Python type "annotations" for Rust types.
+This feature adds to the built binaries introspection data that can be then retrieved using the `pyo3-introspection` crate to generate [type stubs](https://typing.readthedocs.io/en/latest/source/stubs.html).
+
+Also, this feature adds the `pyo3::inspect` module, as well as `IntoPy::type_output` and `FromPyObject::type_input` APIs to produce Python type "annotations" for Rust types.
 
 This is a first step towards adding first-class support for generating type annotations automatically in PyO3, however work is needed to finish this off. All feedback and offers of help welcome on [issue #2454](https://github.com/PyO3/pyo3/issues/2454).
 

newsfragments/3977.added.md

@@ -0,0 +1 @@
+Basic introspection and stub generation based on metadata embedded in produced cdylib.

noxfile.py

@@ -816,6 +816,26 @@ def update_ui_tests(session: nox.Session):
     _run_cargo(session, *command, "--features=abi3,full,jiff-02", env=env)
 
 
+@nox.session(name="test-introspection")
+def test_introspection(session: nox.Session):
+    session.install("maturin")
+    target = os.environ.get("CARGO_BUILD_TARGET")
+    for options in ([], ["--release"]):
+        if target is not None:
+            options += ("--target", target)
+        session.run_always("maturin", "develop", "-m", "./pytests/Cargo.toml", *options)
+        # We look for the built library
+        lib_file = None
+        for file in Path(session.virtualenv.location).rglob("pyo3_pytests.*"):
+            if file.is_file():
+                lib_file = str(file.resolve())
+        _run_cargo_test(
+            session,
+            package="pyo3-introspection",
+            env={"PYO3_PYTEST_LIB_PATH": lib_file},
+        )
+
+
 def _build_docs_for_ffi_check(session: nox.Session) -> None:
     # pyo3-ffi-check needs to scrape docs of pyo3-ffi
     env = os.environ.copy()
@@ -932,6 +952,7 @@ def _run_cargo_test(
     *,
     package: Optional[str] = None,
     features: Optional[str] = None,
+    env: Optional[Dict[str, str]] = None,
 ) -> None:
     command = ["cargo"]
     if "careful" in session.posargs:
@@ -946,7 +967,7 @@ def _run_cargo_test(
     if features:
         command.append(f"--features={features}")
 
-    _run(session, *command, external=True)
+    _run(session, *command, external=True, env=env or {})
 
 
 def _run_cargo_publish(session: nox.Session, *, package: str) -> None:

pyo3-introspection/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "pyo3-introspection"
+version = "0.25.0-dev"
+description = "Introspect dynamic libraries built with PyO3 to get metadata about the exported Python types"
+authors = ["PyO3 Project and Contributors <https://github.com/PyO3>"]
+homepage = "https://github.com/pyo3/pyo3"
+repository = "https://github.com/pyo3/pyo3"
+license = "MIT OR Apache-2.0"
+edition = "2021"
+
+[dependencies]
+anyhow = "1"
+goblin = "0.9.0"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+
+[lints]
+workspace = true

pyo3-introspection/LICENSE-APACHE

@@ -0,0 +1 @@
+../LICENSE-APACHE

pyo3-introspection/LICENSE-MIT

@@ -0,0 +1 @@
+../LICENSE-MIT