aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorStephen Rothwell <sfr@canb.auug.org.au>2021-06-17 17:22:25 +1000
committerStephen Rothwell <sfr@canb.auug.org.au>2021-06-17 17:22:25 +1000
commit7cf6b8b2124fb6e3ad5c89f55170a6ce78fabd7c (patch)
treed1574e35601f2d3cc1a8bb82822e40eccf9c566b
parent7696cf52e1d3e25826e68cb475fab5a1a56dc578 (diff)
parentfef0214731ccdff3f674940e579903ccc9638f93 (diff)
downloadlinux-next-7cf6b8b2124fb6e3ad5c89f55170a6ce78fabd7c.tar.gz
Merge remote-tracking branch 'rust/rust-next'
Notice: this object is not reachable from any branch.
# Conflicts: # Makefile # include/uapi/linux/android/binder.h # kernel/printk/printk.c
Notice: this object is not reachable from any branch.
-rw-r--r--.gitignore5
-rw-r--r--.rustfmt.toml12
-rw-r--r--Documentation/doc-guide/kernel-doc.rst3
-rw-r--r--Documentation/index.rst1
-rw-r--r--Documentation/kbuild/kbuild.rst4
-rw-r--r--Documentation/process/changes.rst9
-rw-r--r--Documentation/rust/arch-support.rst32
-rw-r--r--Documentation/rust/coding.rst92
-rw-r--r--Documentation/rust/docs.rst109
-rw-r--r--Documentation/rust/index.rst20
-rw-r--r--Documentation/rust/quick-start.rst220
-rw-r--r--Documentation/rust/rust-project.json35
-rw-r--r--MAINTAINERS14
-rw-r--r--Makefile147
-rw-r--r--arch/arm/rust/target.json28
-rw-r--r--arch/arm64/rust/target.json40
-rw-r--r--arch/powerpc/rust/target.json30
-rw-r--r--arch/x86/rust/target.json42
-rw-r--r--drivers/android/Kconfig7
-rw-r--r--drivers/android/Makefile2
-rw-r--r--drivers/android/allocation.rs252
-rw-r--r--drivers/android/context.rs80
-rw-r--r--drivers/android/defs.rs92
-rw-r--r--drivers/android/node.rs479
-rw-r--r--drivers/android/process.rs950
-rw-r--r--drivers/android/range_alloc.rs191
-rw-r--r--drivers/android/rust_binder.rs128
-rw-r--r--drivers/android/thread.rs821
-rw-r--r--drivers/android/transaction.rs206
-rw-r--r--include/linux/kallsyms.h2
-rw-r--r--include/linux/spinlock.h17
-rw-r--r--include/uapi/linux/android/binder.h28
-rw-r--r--init/Kconfig27
-rw-r--r--kernel/kallsyms.c7
-rw-r--r--kernel/livepatch/core.c4
-rw-r--r--kernel/printk/printk.c1
-rw-r--r--lib/Kconfig.debug106
-rw-r--r--rust/.gitignore5
-rw-r--r--rust/Makefile152
-rw-r--r--rust/compiler_builtins.rs156
-rw-r--r--rust/exports.c16
-rw-r--r--rust/helpers.c94
-rw-r--r--rust/kernel/allocator.rs68
-rw-r--r--rust/kernel/bindings.rs22
-rw-r--r--rust/kernel/bindings_helper.h18
-rw-r--r--rust/kernel/buffer.rs39
-rw-r--r--rust/kernel/c_types.rs133
-rw-r--r--rust/kernel/chrdev.rs162
-rw-r--r--rust/kernel/error.rs106
-rw-r--r--rust/kernel/file_operations.rs674
-rw-r--r--rust/kernel/lib.rs204
-rw-r--r--rust/kernel/linked_list.rs245
-rw-r--r--rust/kernel/miscdev.rs109
-rw-r--r--rust/kernel/module_param.rs497
-rw-r--r--rust/kernel/pages.rs173
-rw-r--r--rust/kernel/prelude.rs22
-rw-r--r--rust/kernel/print.rs463
-rw-r--r--rust/kernel/random.rs50
-rw-r--r--rust/kernel/raw_list.rs361
-rw-r--r--rust/kernel/static_assert.rs38
-rw-r--r--rust/kernel/sync/arc.rs184
-rw-r--r--rust/kernel/sync/condvar.rs136
-rw-r--r--rust/kernel/sync/guard.rs82
-rw-r--r--rust/kernel/sync/locked_by.rs112
-rw-r--r--rust/kernel/sync/mod.rs85
-rw-r--r--rust/kernel/sync/mutex.rs101
-rw-r--r--rust/kernel/sync/spinlock.rs108
-rw-r--r--rust/kernel/sysctl.rs185
-rw-r--r--rust/kernel/types.rs73
-rw-r--r--rust/kernel/user_ptr.rs282
-rw-r--r--rust/module.rs764
-rw-r--r--samples/Kconfig2
-rw-r--r--samples/Makefile1
-rw-r--r--samples/rust/Kconfig113
-rw-r--r--samples/rust/Makefile12
-rw-r--r--samples/rust/rust_chrdev.rs55
-rw-r--r--samples/rust/rust_minimal.rs40
-rw-r--r--samples/rust/rust_miscdev.rs145
-rw-r--r--samples/rust/rust_module_parameters.rs72
-rw-r--r--samples/rust/rust_print.rs58
-rw-r--r--samples/rust/rust_random.rs60
-rw-r--r--samples/rust/rust_semaphore.rs176
-rw-r--r--samples/rust/rust_semaphore_c.c212
-rw-r--r--samples/rust/rust_stack_probing.rs42
-rw-r--r--samples/rust/rust_sync.rs84
-rw-r--r--scripts/Makefile.build19
-rw-r--r--scripts/Makefile.lib12
l---------scripts/dummy-tools/elfedit1
-rw-r--r--scripts/kallsyms.c33
-rw-r--r--scripts/kconfig/confdata.c67
-rwxr-xr-xscripts/rust-version.sh31
-rw-r--r--tools/include/linux/kallsyms.h2
-rw-r--r--tools/include/linux/lockdep.h2
-rw-r--r--tools/lib/perf/include/perf/event.h2
-rw-r--r--tools/lib/symbol/kallsyms.h2
95 files changed, 11357 insertions, 48 deletions
diff --git a/.gitignore b/.gitignore
index 7afd412dadd2c..48c68948f476d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,6 +37,7 @@
*.o
*.o.*
*.patch
+*.rmeta
*.s
*.so
*.so.dbg
@@ -96,6 +97,7 @@ modules.order
!.gitattributes
!.gitignore
!.mailmap
+!.rustfmt.toml
#
# Generated include files
@@ -161,3 +163,6 @@ x509.genkey
# Documentation toolchain
sphinx_*/
+
+# Rust analyzer configuration
+/rust-project.json
diff --git a/.rustfmt.toml b/.rustfmt.toml
new file mode 100644
index 0000000000000..5893c0e3cbde1
--- /dev/null
+++ b/.rustfmt.toml
@@ -0,0 +1,12 @@
+edition = "2018"
+format_code_in_doc_comments = true
+newline_style = "Unix"
+
+# Unstable options that help catching some mistakes in formatting and that we may want to enable
+# when they become stable.
+#
+# They are kept here since they are useful to run from time to time.
+#reorder_impl_items = true
+#comment_width = 100
+#wrap_comments = true
+#normalize_comments = true
diff --git a/Documentation/doc-guide/kernel-doc.rst b/Documentation/doc-guide/kernel-doc.rst
index 79aaa55d6bcf2..724e2ffddff12 100644
--- a/Documentation/doc-guide/kernel-doc.rst
+++ b/Documentation/doc-guide/kernel-doc.rst
@@ -11,6 +11,9 @@ when it is embedded in source files.
reasons. The kernel source contains tens of thousands of kernel-doc
comments. Please stick to the style described here.
+.. note:: kernel-doc does not cover Rust code: please see
+ Documentation/rust/docs.rst instead.
+
The kernel-doc structure is extracted from the comments, and proper
`Sphinx C Domain`_ function and type descriptions with anchors are
generated from them. The descriptions are filtered for special kernel-doc
diff --git a/Documentation/index.rst b/Documentation/index.rst
index 54ce34fd6fbda..1b13c2445e87b 100644
--- a/Documentation/index.rst
+++ b/Documentation/index.rst
@@ -82,6 +82,7 @@ merged much easier.
maintainer/index
fault-injection/index
livepatch/index
+ rust/index
Kernel API documentation
diff --git a/Documentation/kbuild/kbuild.rst b/Documentation/kbuild/kbuild.rst
index 2d1fc03d346ea..1109d18d9377d 100644
--- a/Documentation/kbuild/kbuild.rst
+++ b/Documentation/kbuild/kbuild.rst
@@ -57,6 +57,10 @@ CFLAGS_MODULE
-------------
Additional module specific options to use for $(CC).
+KRUSTCFLAGS
+-----------
+Additional options to the Rust compiler (for built-in and modules).
+
LDFLAGS_MODULE
--------------
Additional options used for $(LD) when linking modules.
diff --git a/Documentation/process/changes.rst b/Documentation/process/changes.rst
index d3a8557b66a1a..7a1d2837d2c3d 100644
--- a/Documentation/process/changes.rst
+++ b/Documentation/process/changes.rst
@@ -31,6 +31,8 @@ you probably needn't concern yourself with pcmciautils.
====================== =============== ========================================
GNU C 4.9 gcc --version
Clang/LLVM (optional) 10.0.1 clang --version
+rustc (optional) nightly rustc --version
+bindgen (optional) 0.56.0 bindgen --version
GNU make 3.81 make --version
binutils 2.23 ld -v
flex 2.5.35 flex --version
@@ -55,6 +57,7 @@ iptables 1.4.2 iptables -V
openssl & libcrypto 1.0.0 openssl version
bc 1.06.95 bc --version
Sphinx\ [#f1]_ 1.3 sphinx-build --version
+rustdoc (optional) nightly rustdoc --version
====================== =============== ========================================
.. [#f1] Sphinx is needed only to build the Kernel documentation
@@ -329,6 +332,12 @@ Sphinx
Please see :ref:`sphinx_install` in :ref:`Documentation/doc-guide/sphinx.rst <sphinxdoc>`
for details about Sphinx requirements.
+rustdoc
+-------
+
+``rustdoc`` is used to generate Rust documentation. Please see
+:ref:`Documentation/rust/docs.rst <rust_docs>` for more information.
+
Getting updated software
========================
diff --git a/Documentation/rust/arch-support.rst b/Documentation/rust/arch-support.rst
new file mode 100644
index 0000000000000..e93a8ceb3f82d
--- /dev/null
+++ b/Documentation/rust/arch-support.rst
@@ -0,0 +1,32 @@
+.. _rust_arch_support:
+
+Arch Support
+============
+
+Currently, the Rust compiler (``rustc``) uses LLVM for code generation,
+which limits the supported architectures we can target. In addition, support
+for building the kernel with LLVM/Clang varies (see :ref:`kbuild_llvm`),
+which ``bindgen`` relies on through ``libclang``.
+
+Below is a general summary of architectures that currently work. Level of
+support corresponds to ``S`` values in the ``MAINTAINERS`` file.
+
+.. list-table::
+ :widths: 10 10 10
+ :header-rows: 1
+
+ * - Architecture
+ - Level of support
+ - Constraints
+ * - ``arm``
+ - Maintained
+ - ``armv6`` and compatible only, ``RUST_OPT_LEVEL >= 2``
+ * - ``arm64``
+ - Maintained
+ - None
+ * - ``powerpc``
+ - Maintained
+ - ``ppc64le`` only, ``RUST_OPT_LEVEL < 2`` requires ``CONFIG_THREAD_SHIFT=15``
+ * - ``x86``
+ - Maintained
+ - ``x86_64`` only
diff --git a/Documentation/rust/coding.rst b/Documentation/rust/coding.rst
new file mode 100644
index 0000000000000..46da5fb0974cb
--- /dev/null
+++ b/Documentation/rust/coding.rst
@@ -0,0 +1,92 @@
+.. _rust_coding:
+
+Coding
+======
+
+This document describes how to write Rust code in the kernel.
+
+
+Coding style
+------------
+
+The code is automatically formatted using the ``rustfmt`` tool. This is very
+good news!
+
+- If you contribute from time to time to the kernel, you do not need to learn
+ and remember one more style guide. You will also need less patch roundtrips
+ to land a change.
+
+- If you are a reviewer or a maintainer, you will not need to spend time on
+ pointing out style issues anymore.
+
+.. note:: Conventions on comments and documentation are not checked by
+ ``rustfmt``. Thus we still need to take care of those: please see
+ :ref:`Documentation/rust/docs.rst <rust_docs>`.
+
+We use the tool's default settings. This means we are following the idiomatic
+Rust style. For instance, we use 4 spaces for indentation rather than tabs.
+
+Typically, you will want to instruct your editor/IDE to format while you type,
+when you save or at commit time. However, if for some reason you want
+to reformat the entire kernel Rust sources at some point, you may run::
+
+ make rustfmt
+
+To check if everything is formatted (printing a diff otherwise), e.g. if you
+have configured a CI for a tree as a maintainer, you may run::
+
+ make rustfmtcheck
+
+Like ``clang-format`` for the rest of the kernel, ``rustfmt`` works on
+individual files, and does not require a kernel configuration. Sometimes it may
+even work with broken code.
+
+
+Extra lints
+-----------
+
+While ``rustc`` is a very helpful compiler, some extra lints and analysis are
+available via ``clippy``, a Rust linter. To enable it, pass ``CLIPPY=1`` to
+the same invocation you use for compilation, e.g.::
+
+ make ARCH=... CROSS_COMPILE=... CC=... -j... CLIPPY=1
+
+At the moment, we do not enforce a "clippy-free" compilation, so you can treat
+the output the same way as the extra warning levels for C, e.g. like ``W=2``.
+Still, we use the default configuration, which is relatively conservative, so
+it is a good idea to read any output it may produce from time to time and fix
+the pointed out issues. The list of enabled lists will be likely tweaked over
+time, and extra levels may end up being introduced, e.g. ``CLIPPY=2``.
+
+
+Abstractions vs. bindings
+-------------------------
+
+We don't have abstractions for all the kernel internal APIs and concepts,
+but we would like to expand coverage as time goes on. Unless there is
+a good reason not to, always use the abstractions instead of calling
+the C bindings directly.
+
+If you are writing some code that requires a call to an internal kernel API
+or concept that isn't abstracted yet, consider providing an (ideally safe)
+abstraction for everyone to use.
+
+
+Conditional compilation
+-----------------------
+
+Rust code has access to conditional compilation based on the kernel
+configuration:
+
+.. code-block:: rust
+
+ #[cfg(CONFIG_X)] // `CONFIG_X` is enabled (`y` or `m`)
+ #[cfg(CONFIG_X="y")] // `CONFIG_X` is enabled as a built-in (`y`)
+ #[cfg(CONFIG_X="m")] // `CONFIG_X` is enabled as a module (`m`)
+ #[cfg(not(CONFIG_X))] // `CONFIG_X` is disabled
+
+
+Documentation
+-------------
+
+Please see :ref:`Documentation/rust/docs.rst <rust_docs>`.
diff --git a/Documentation/rust/docs.rst b/Documentation/rust/docs.rst
new file mode 100644
index 0000000000000..58c5f98ccb353
--- /dev/null
+++ b/Documentation/rust/docs.rst
@@ -0,0 +1,109 @@
+.. _rust_docs:
+
+Docs
+====
+
+Rust kernel code is not documented like C kernel code (i.e. via kernel-doc).
+Instead, we use the usual system for documenting Rust code: the ``rustdoc``
+tool, which uses Markdown (a *very* lightweight markup language).
+
+This document describes how to make the most out of the kernel documentation
+for Rust.
+
+
+Reading the docs
+----------------
+
+An advantage of using Markdown is that it attempts to make text look almost as
+you would have written it in plain text. This makes the documentation quite
+pleasant to read even in its source form.
+
+However, the generated HTML docs produced by ``rustdoc`` provide a *very* nice
+experience, including integrated instant search, clickable items (types,
+functions, constants, etc. -- including to all the standard Rust library ones
+that we use in the kernel, e.g. ``core``), categorization, links to the source
+code, etc.
+
+Like for the rest of the kernel documentation, pregenerated HTML docs for
+the libraries (crates) inside ``rust/`` that are used by the rest of the kernel
+are available at `kernel.org`_.
+
+// TODO: link when ready
+
+.. _kernel.org: http://kernel.org/
+
+Otherwise, you can generate them locally. This is quite fast (same order as
+compiling the code itself) and you do not need any special tools or environment.
+This has the added advantage that they will be tailored to your particular
+kernel configuration. To generate them, simply use the ``rustdoc`` target with
+the same invocation you use for compilation, e.g.::
+
+ make ARCH=... CROSS_COMPILE=... CC=... -j... rustdoc
+
+
+Writing the docs
+----------------
+
+If you already know Markdown, learning how to write Rust documentation will be
+a breeze. If not, understanding the basics is a matter of minutes reading other
+code. There are also many guides available out there, a particularly nice one
+is at `GitHub`_.
+
+.. _GitHub: https://guides.github.com/features/mastering-markdown/#syntax
+
+This is how a well-documented Rust function may look like (derived from the Rust
+standard library)::
+
+ /// Returns the contained [`Some`] value, consuming the `self` value,
+ /// without checking that the value is not [`None`].
+ ///
+ /// # Safety
+ ///
+ /// Calling this method on [`None`] is *[undefined behavior]*.
+ ///
+ /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html
+ ///
+ /// # Examples
+ ///
+ /// ```
+ /// let x = Some("air");
+ /// assert_eq!(unsafe { x.unwrap_unchecked() }, "air");
+ /// ```
+ pub unsafe fn unwrap_unchecked(self) -> T {
+ match self {
+ Some(val) => val,
+
+ // SAFETY: the safety contract must be upheld by the caller.
+ None => unsafe { hint::unreachable_unchecked() },
+ }
+ }
+
+This example showcases a few ``rustdoc`` features and some common conventions
+(that we also follow in the kernel):
+
+* The first paragraph must be a single sentence briefly describing what
+ the documented item does. Further explanations must go in extra paragraphs.
+
+* ``unsafe`` functions must document the preconditions needed for a call to be
+ safe under a ``Safety`` section.
+
+* While not shown here, if a function may panic, the conditions under which
+ that happens must be described under a ``Panics`` section.
+
+* If providing examples of usage would help readers, they must be written in
+ a section called ``Examples``.
+
+* Rust items (functions, types, constants...) will be automatically linked
+ (``rustdoc`` will find out the URL for you).
+
+* Following the Rust standard library conventions, any ``unsafe`` block must be
+ preceded by a ``SAFETY`` comment describing why the code inside is sound.
+
+ While sometimes the reason might look trivial and therefore unneeded, writing
+ these comments is not just a good way of documenting what has been taken into
+ account, but also that there are no *extra* implicit constraints.
+
+To learn more about how to write documentation for Rust and extra features,
+please take a look at the ``rustdoc`` `book`_.
+
+.. _book: https://doc.rust-lang.org/rustdoc/how-to-write-documentation.html
diff --git a/Documentation/rust/index.rst b/Documentation/rust/index.rst
new file mode 100644
index 0000000000000..257cf2b200b8a
--- /dev/null
+++ b/Documentation/rust/index.rst
@@ -0,0 +1,20 @@
+Rust
+====
+
+Documentation related to Rust within the kernel. If you are starting out,
+read the :ref:`Documentation/rust/quick-start.rst <rust_quick_start>` guide.
+
+.. toctree::
+ :maxdepth: 1
+
+ quick-start
+ coding
+ docs
+ arch-support
+
+.. only:: subproject and html
+
+ Indices
+ =======
+
+ * :ref:`genindex`
diff --git a/Documentation/rust/quick-start.rst b/Documentation/rust/quick-start.rst
new file mode 100644
index 0000000000000..e78125d9289bd
--- /dev/null
+++ b/Documentation/rust/quick-start.rst
@@ -0,0 +1,220 @@
+.. _rust_quick_start:
+
+Quick Start
+===========
+
+This document describes how to get started with kernel development in Rust.
+If you have worked previously with Rust, this will only take a moment.
+
+Please note that, at the moment, a very restricted subset of architectures
+is supported, see :doc:`/rust/arch-support`.
+
+
+Requirements: Building
+----------------------
+
+This section explains how to fetch the tools needed for building.
+
+Some of these requirements might be available from your Linux distribution
+under names like ``rustc``, ``rust-src``, ``rust-bindgen``, etc. However,
+at the time of writing, they are likely to not be recent enough.
+
+
+rustc
+*****
+
+A recent *nightly* Rust toolchain (with, at least, ``rustc``) is required,
+e.g. ``nightly-2021-02-20``. Our goal is to use a stable toolchain as soon
+as possible, but for the moment we depend on a handful of nightly features.
+
+If you are using ``rustup``, run::
+
+ rustup default nightly-2021-02-20
+
+Please avoid the very latest nightlies (>= nightly-2021-03-05) until
+https://github.com/Rust-for-Linux/linux/issues/135 is resolved.
+
+Otherwise, fetch a standalone installer or install ``rustup`` from:
+
+ https://www.rust-lang.org
+
+
+Rust standard library source
+****************************
+
+The Rust standard library source is required because the build system will
+cross-compile ``core`` and ``alloc``.
+
+If you are using ``rustup``, run::
+
+ rustup component add rust-src
+
+Otherwise, if you used a standalone installer, you can clone the Rust
+repository into the installation folder of your nightly toolchain::
+
+ git clone --recurse-submodules https://github.com/rust-lang/rust $(rustc --print sysroot)/lib/rustlib/src/rust
+
+
+libclang
+********
+
+``libclang`` (part of LLVM) is used by ``bindgen`` to understand the C code
+in the kernel, which means you will need a recent LLVM installed; like when
+you compile the kernel with ``CC=clang`` or ``LLVM=1``.
+
+Your Linux distribution is likely to have a suitable one available, so it is
+best if you check that first.
+
+There are also some binaries for several systems and architectures uploaded at:
+
+ https://releases.llvm.org/download.html
+
+For Debian-based distributions, you can also fetch them from:
+
+ https://apt.llvm.org
+
+Otherwise, building LLVM takes quite a while, but it is not a complex process:
+
+ https://llvm.org/docs/GettingStarted.html#getting-the-source-code-and-building-llvm
+
+
+bindgen
+*******
+
+The bindings to the C side of the kernel are generated at build time using
+the ``bindgen`` tool. A recent version should work, e.g. ``0.56.0``.
+
+Install it via (this will build the tool from source)::
+
+ cargo install --locked --version 0.56.0 bindgen
+
+
+Requirements: Developing
+------------------------
+
+This section explains how to fetch the tools needed for developing. That is,
+if you only want to build the kernel, you do not need them.
+
+
+rustfmt
+*******
+
+The ``rustfmt`` tool is used to automatically format all the Rust kernel code,
+including the generated C bindings (for details, please see
+:ref:`Documentation/rust/coding.rst <rust_coding>`).
+
+If you are using ``rustup``, its ``default`` profile already installs the tool,
+so you should be good to go. If you are using another profile, you can install
+the component manually::
+
+ rustup component add rustfmt
+
+The standalone installers also come with ``rustfmt``.
+
+
+clippy
+******
+
+``clippy`` is a Rust linter. Installing it allows you to get extra warnings
+for Rust code passing ``CLIPPY=1`` to ``make`` (for details, please see
+:ref:`Documentation/rust/coding.rst <rust_coding>`).
+
+If you are using ``rustup``, its ``default`` profile already installs the tool,
+so you should be good to go. If you are using another profile, you can install
+the component manually::
+
+ rustup component add clippy
+
+The standalone installers also come with ``clippy``.
+
+
+rustdoc
+*******
+
+If you install the ``rustdoc`` tool, then you will be able to generate pretty
+HTML documentation for Rust code, including for the libraries (crates) inside
+``rust/`` that are used by the rest of the kernel (for details, please see
+:ref:`Documentation/rust/docs.rst <rust_docs>`).
+
+If you are using ``rustup``, its ``default`` profile already installs the tool,
+so you should be good to go. If you are using another profile, you can install
+the component manually::
+
+ rustup component add rustdoc
+
+The standalone installers also come with ``rustdoc``.
+
+
+rust-analyzer
+*************
+
+The `rust-analyzer <https://rust-analyzer.github.io/>`_ language server can
+be used with many editors to enable syntax highlighting, completion, go to
+definition, and other features.
+
+``rust-analyzer`` will need to be
+`configured <https://rust-analyzer.github.io/manual.html#non-cargo-based-projects>`_
+to work with the kernel by adding a ``rust-project.json`` file in the root folder.
+The example ``Documentation/rust/rust-project.json`` can
+be used after updating ``sysroot_src`` and including the relevant modules.
+The path to ``sysroot_src`` is given by::
+
+ $(rustc --print sysroot)/lib/rustlib/src/rust/library
+
+
+Configuration
+-------------
+
+``Rust support`` (``CONFIG_RUST``) needs to be enabled in the ``General setup``
+menu. The option is only shown if the build system can locate ``rustc``.
+In turn, this will make visible the rest of options that depend on Rust.
+
+Afterwards, go to::
+
+ Kernel hacking
+ -> Sample kernel code
+ -> Rust samples
+
+And enable some sample modules either as built-in or as loadable.
+
+
+Building
+--------
+
+Building a kernel with Clang or a complete LLVM toolchain is the best supported
+setup at the moment. That is::
+
+ make ARCH=... CROSS_COMPILE=... CC=clang -j...
+
+or::
+
+ make ARCH=... CROSS_COMPILE=... LLVM=1 -j...
+
+Using GCC also works for some configurations, but it is *very* experimental at
+the moment.
+
+
+Hacking
+-------
+
+If you want to dive deeper, take a look at the source code of the samples
+at ``samples/rust/``, the Rust support code under ``rust/`` and
+the ``Rust hacking`` menu under ``Kernel hacking``.
+
+If you use GDB/Binutils and Rust symbols aren't getting demangled, the reason
+is your toolchain doesn't support Rust's new v0 mangling scheme yet. There are
+a few ways out:
+
+ - If you don't mind building your own tools, we provide the following fork
+ with the support cherry-picked from GCC on top of very recent releases:
+
+ https://github.com/Rust-for-Linux/binutils-gdb/releases/tag/gdb-10.1-release-rust
+ https://github.com/Rust-for-Linux/binutils-gdb/releases/tag/binutils-2_35_1-rust
+
+ - If you only need GDB and can enable ``CONFIG_DEBUG_INFO``, do so:
+ some versions of GDB (e.g. vanilla GDB 10.1) are able to use
+ the pre-demangled names embedded in the debug info.
+
+ - If you don't need loadable module support, you may compile without
+ the ``-Zsymbol-mangling-version=v0`` flag. However, we don't maintain
+ support for that, so avoid it unless you are in a hurry.
diff --git a/Documentation/rust/rust-project.json b/Documentation/rust/rust-project.json
new file mode 100644
index 0000000000000..e0f64b8dec1e3
--- /dev/null
+++ b/Documentation/rust/rust-project.json
@@ -0,0 +1,35 @@
+{
+ "sysroot_src": <path-to-sysroot-source>,
+ "crates": [
+ {
+ "root_module": "rust/module.rs",
+ "edition": "2018",
+ "deps": []
+ },
+ {
+ "root_module": "rust/kernel/lib.rs",
+ "edition": "2018",
+ "deps": [
+ {
+ "crate": 0,
+ "name": "module"
+ }
+ ]
+ },
+ {
+ "root_module": <path-to-module>,
+ "edition": "2018",
+ "deps": [
+ {
+ "crate": 0,
+ "name": "module"
+ },
+ {
+ "crate": 1,
+ "name": "kernel"
+ }
+ ]
+ },
+ <...entries for other modules...>
+ ]
+} \ No newline at end of file
diff --git a/MAINTAINERS b/MAINTAINERS
index 0c396f360c164..285e7c02e9e5a 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -16061,6 +16061,20 @@ L: linux-rdma@vger.kernel.org
S: Maintained
F: drivers/infiniband/ulp/rtrs/
+RUST
+M: Miguel Ojeda <ojeda@kernel.org>
+M: Alex Gaynor <alex.gaynor@gmail.com>
+M: Wedson Almeida Filho <wedsonaf@google.com>
+L: rust-for-linux@vger.kernel.org
+S: Maintained
+W: https://github.com/Rust-for-Linux/linux
+B: https://github.com/Rust-for-Linux/linux/issues
+T: git https://github.com/Rust-for-Linux/linux.git rust-next
+F: rust/
+F: samples/rust/
+F: Documentation/rust/
+K: \b(?i:rust)\b
+
RXRPC SOCKETS (AF_RXRPC)
M: David Howells <dhowells@redhat.com>
M: Marc Dionne <marc.dionne@auristor.com>
diff --git a/Makefile b/Makefile
index 2d3725f6a0393..cc416528c21d2 100644
--- a/Makefile
+++ b/Makefile
@@ -120,6 +120,13 @@ endif
export KBUILD_CHECKSRC
+# Enable "clippy" (a linter) as part of the Rust compilation.
+#
+# Use 'make CLIPPY=1' to enable it.
+ifeq ("$(origin CLIPPY)", "command line")
+ KBUILD_CLIPPY := $(CLIPPY)
+endif
+
# Use make M=dir or set the environment variable KBUILD_EXTMOD to specify the
# directory of external module to build. Setting M= takes precedence.
ifeq ("$(origin M)", "command line")
@@ -263,7 +270,7 @@ no-dot-config-targets := $(clean-targets) \
cscope gtags TAGS tags help% %docs check% coccicheck \
$(version_h) headers headers_% archheaders archscripts \
%asm-generic kernelversion %src-pkg dt_binding_check \
- outputmakefile
+ outputmakefile rustfmt rustfmtcheck
# Installation targets should not require compiler. Unfortunately, vdso_install
# is an exception where build artifacts may be updated. This must be fixed.
no-compiler-targets := $(no-dot-config-targets) install dtbs_install \
@@ -452,6 +459,10 @@ OBJDUMP = $(CROSS_COMPILE)objdump
READELF = $(CROSS_COMPILE)readelf
STRIP = $(CROSS_COMPILE)strip
endif
+RUSTC = rustc
+RUSTFMT = rustfmt
+CLIPPY_DRIVER = clippy-driver
+BINDGEN = bindgen
PAHOLE = pahole
RESOLVE_BTFIDS = $(objtree)/tools/bpf/resolve_btfids/resolve_btfids
LEX = flex
@@ -475,9 +486,11 @@ CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
-Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)
NOSTDINC_FLAGS :=
CFLAGS_MODULE =
+RUSTCFLAGS_MODULE =
AFLAGS_MODULE =
LDFLAGS_MODULE =
CFLAGS_KERNEL =
+RUSTCFLAGS_KERNEL =
AFLAGS_KERNEL =
LDFLAGS_vmlinux =
@@ -506,15 +519,30 @@ KBUILD_CFLAGS := -Wall -Wundef -Werror=strict-prototypes -Wno-trigraphs \
-Werror=return-type -Wno-format-security \
-std=gnu89
KBUILD_CPPFLAGS := -D__KERNEL__
+KBUILD_RUSTCFLAGS := --emit=dep-info,obj,metadata --edition=2018 \
+ -Cpanic=abort -Cembed-bitcode=n -Clto=n -Crpath=n \
+ -Cforce-unwind-tables=n -Ccodegen-units=1 \
+ -Zbinary_dep_depinfo=y -Zsymbol-mangling-version=v0
KBUILD_AFLAGS_KERNEL :=
KBUILD_CFLAGS_KERNEL :=
+KBUILD_RUSTCFLAGS_KERNEL :=
KBUILD_AFLAGS_MODULE := -DMODULE
KBUILD_CFLAGS_MODULE := -DMODULE
+KBUILD_RUSTCFLAGS_MODULE := --cfg MODULE
KBUILD_LDFLAGS_MODULE :=
KBUILD_LDFLAGS :=
CLANG_FLAGS :=
-export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC
+ifeq ($(KBUILD_CLIPPY),1)
+ RUSTC_OR_CLIPPY_QUIET := CLIPPY
+ RUSTC_OR_CLIPPY = $(CLIPPY_DRIVER)
+else
+ RUSTC_OR_CLIPPY_QUIET := RUSTC
+ RUSTC_OR_CLIPPY = $(RUSTC)
+endif
+export RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY
+
+export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC RUSTC BINDGEN
export CPP AR NM STRIP OBJCOPY OBJDUMP READELF PAHOLE RESOLVE_BTFIDS LEX YACC AWK INSTALLKERNEL
export PERL PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ ZSTD
@@ -522,9 +550,10 @@ export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE
export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS
export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE
+export KBUILD_RUSTCFLAGS RUSTCFLAGS_KERNEL RUSTCFLAGS_MODULE
export KBUILD_AFLAGS AFLAGS_KERNEL AFLAGS_MODULE
-export KBUILD_AFLAGS_MODULE KBUILD_CFLAGS_MODULE KBUILD_LDFLAGS_MODULE
-export KBUILD_AFLAGS_KERNEL KBUILD_CFLAGS_KERNEL
+export KBUILD_AFLAGS_MODULE KBUILD_CFLAGS_MODULE KBUILD_RUSTCFLAGS_MODULE KBUILD_LDFLAGS_MODULE
+export KBUILD_AFLAGS_KERNEL KBUILD_CFLAGS_KERNEL KBUILD_RUSTCFLAGS_KERNEL
# Files to ignore in find ... statements
@@ -574,24 +603,29 @@ outputmakefile:
{ echo "# this is build directory, ignore it"; echo "*"; } > .gitignore
endif
-# The expansion should be delayed until arch/$(SRCARCH)/Makefile is included.
-# Some architectures define CROSS_COMPILE in arch/$(SRCARCH)/Makefile.
-# CC_VERSION_TEXT is referenced from Kconfig (so it needs export),
-# and from include/config/auto.conf.cmd to detect the compiler upgrade.
-CC_VERSION_TEXT = $(subst $(pound),,$(shell $(CC) --version 2>/dev/null | head -n 1))
+TENTATIVE_CLANG_FLAGS := -Werror=unknown-warning-option
-ifneq ($(findstring clang,$(CC_VERSION_TEXT)),)
ifneq ($(CROSS_COMPILE),)
-CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%))
+TENTATIVE_CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%))
endif
ifeq ($(LLVM_IAS),1)
-CLANG_FLAGS += -integrated-as
+TENTATIVE_CLANG_FLAGS += -integrated-as
else
-CLANG_FLAGS += -no-integrated-as
+TENTATIVE_CLANG_FLAGS += -no-integrated-as
GCC_TOOLCHAIN_DIR := $(dir $(shell which $(CROSS_COMPILE)elfedit))
-CLANG_FLAGS += --prefix=$(GCC_TOOLCHAIN_DIR)$(notdir $(CROSS_COMPILE))
+TENTATIVE_CLANG_FLAGS += --prefix=$(GCC_TOOLCHAIN_DIR)$(notdir $(CROSS_COMPILE))
endif
-CLANG_FLAGS += -Werror=unknown-warning-option
+
+export TENTATIVE_CLANG_FLAGS
+
+# The expansion should be delayed until arch/$(SRCARCH)/Makefile is included.
+# Some architectures define CROSS_COMPILE in arch/$(SRCARCH)/Makefile.
+# CC_VERSION_TEXT is referenced from Kconfig (so it needs export),
+# and from include/config/auto.conf.cmd to detect the compiler upgrade.
+CC_VERSION_TEXT = $(shell $(CC) --version 2>/dev/null | head -n 1 | sed 's/\#//g')
+
+ifneq ($(findstring clang,$(CC_VERSION_TEXT)),)
+CLANG_FLAGS += $(TENTATIVE_CLANG_FLAGS)
KBUILD_CFLAGS += $(CLANG_FLAGS)
KBUILD_AFLAGS += $(CLANG_FLAGS)
export CLANG_FLAGS
@@ -726,7 +760,7 @@ $(KCONFIG_CONFIG):
quiet_cmd_syncconfig = SYNC $@
cmd_syncconfig = $(MAKE) -f $(srctree)/Makefile syncconfig
-%/config/auto.conf %/config/auto.conf.cmd %/generated/autoconf.h: $(KCONFIG_CONFIG)
+%/config/auto.conf %/config/auto.conf.cmd %/generated/autoconf.h %/generated/rustc_cfg: $(KCONFIG_CONFIG)
+$(call cmd,syncconfig)
else # !may-sync-config
# External modules and some install targets need include/generated/autoconf.h
@@ -752,12 +786,43 @@ KBUILD_CFLAGS += $(call cc-disable-warning, format-truncation)
KBUILD_CFLAGS += $(call cc-disable-warning, format-overflow)
KBUILD_CFLAGS += $(call cc-disable-warning, address-of-packed-member)
+ifdef CONFIG_RUST_DEBUG_ASSERTIONS
+KBUILD_RUSTCFLAGS += -Cdebug-assertions=y
+else
+KBUILD_RUSTCFLAGS += -Cdebug-assertions=n
+endif
+
+ifdef CONFIG_RUST_OVERFLOW_CHECKS
+KBUILD_RUSTCFLAGS += -Coverflow-checks=y
+else
+KBUILD_RUSTCFLAGS += -Coverflow-checks=n
+endif
+
ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE
KBUILD_CFLAGS += -O2
+KBUILD_RUSTCFLAGS_OPT_LEVEL_MAP := 2
else ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3
KBUILD_CFLAGS += -O3
+KBUILD_RUSTCFLAGS_OPT_LEVEL_MAP := 3
else ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE
KBUILD_CFLAGS += -Os
+KBUILD_RUSTCFLAGS_OPT_LEVEL_MAP := z
+endif
+
+ifdef CONFIG_RUST_OPT_LEVEL_SIMILAR_AS_CHOSEN_FOR_C
+KBUILD_RUSTCFLAGS += -Copt-level=$(KBUILD_RUSTCFLAGS_OPT_LEVEL_MAP)
+else ifdef CONFIG_RUST_OPT_LEVEL_0
+KBUILD_RUSTCFLAGS += -Copt-level=0
+else ifdef CONFIG_RUST_OPT_LEVEL_1
+KBUILD_RUSTCFLAGS += -Copt-level=1
+else ifdef CONFIG_RUST_OPT_LEVEL_2
+KBUILD_RUSTCFLAGS += -Copt-level=2
+else ifdef CONFIG_RUST_OPT_LEVEL_3
+KBUILD_RUSTCFLAGS += -Copt-level=3
+else ifdef CONFIG_RUST_OPT_LEVEL_S
+KBUILD_RUSTCFLAGS += -Copt-level=s
+else ifdef CONFIG_RUST_OPT_LEVEL_Z
+KBUILD_RUSTCFLAGS += -Copt-level=z
endif
# Tell gcc to never replace conditional load with a non-conditional one
@@ -807,6 +872,7 @@ KBUILD_CFLAGS += $(call cc-disable-warning, unused-but-set-variable)
KBUILD_CFLAGS += $(call cc-disable-warning, unused-const-variable)
ifdef CONFIG_FRAME_POINTER
KBUILD_CFLAGS += -fno-omit-frame-pointer -fno-optimize-sibling-calls
+KBUILD_RUSTCFLAGS += -Cforce-frame-pointers=y
else
# Some targets (ARM with Thumb2, for example), can't be built with frame
# pointers. For those, we don't have FUNCTION_TRACER automatically
@@ -844,6 +910,8 @@ ifdef CONFIG_CC_IS_GCC
DEBUG_CFLAGS += $(call cc-ifversion, -lt, 0500, $(call cc-option, -fno-var-tracking-assignments))
endif
+DEBUG_RUSTCFLAGS :=
+
ifdef CONFIG_DEBUG_INFO
ifdef CONFIG_DEBUG_INFO_SPLIT
@@ -854,6 +922,11 @@ endif
ifneq ($(LLVM_IAS),1)
KBUILD_AFLAGS += -Wa,-gdwarf-2
+ifdef CONFIG_DEBUG_INFO_REDUCED
+DEBUG_RUSTCFLAGS += -Cdebuginfo=1
+else
+DEBUG_RUSTCFLAGS += -Cdebuginfo=2
+endif
endif
ifndef CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT
@@ -878,6 +951,9 @@ endif # CONFIG_DEBUG_INFO
KBUILD_CFLAGS += $(DEBUG_CFLAGS)
export DEBUG_CFLAGS
+KBUILD_RUSTCFLAGS += $(DEBUG_RUSTCFLAGS)
+export DEBUG_RUSTCFLAGS
+
ifdef CONFIG_FUNCTION_TRACER
ifdef CONFIG_FTRACE_MCOUNT_USE_CC
CC_FLAGS_FTRACE += -mrecord-mcount
@@ -1033,10 +1109,11 @@ include $(addprefix $(srctree)/, $(include-y))
# Do not add $(call cc-option,...) below this line. When you build the kernel
# from the clean source tree, the GCC plugins do not exist at this point.
-# Add user supplied CPPFLAGS, AFLAGS and CFLAGS as the last assignments
+# Add user supplied CPPFLAGS, AFLAGS, CFLAGS and RUSTCFLAGS as the last assignments
KBUILD_CPPFLAGS += $(KCPPFLAGS)
KBUILD_AFLAGS += $(KAFLAGS)
KBUILD_CFLAGS += $(KCFLAGS)
+KBUILD_RUSTCFLAGS += $(KRUSTCFLAGS)
KBUILD_LDFLAGS_MODULE += --build-id=sha1
LDFLAGS_vmlinux += --build-id=sha1
@@ -1105,6 +1182,10 @@ export MODULES_NSDEPS := $(extmod_prefix)modules.nsdeps
ifeq ($(KBUILD_EXTMOD),)
core-y += kernel/ certs/ mm/ fs/ ipc/ security/ crypto/ block/
+ifdef CONFIG_RUST
+core-y += rust/
+endif
+
vmlinux-dirs := $(patsubst %/,%,$(filter %/, \
$(core-y) $(core-m) $(drivers-y) $(drivers-m) \
$(libs-y) $(libs-m)))
@@ -1205,6 +1286,9 @@ archprepare: outputmakefile archheaders archscripts scripts include/config/kerne
prepare0: archprepare
$(Q)$(MAKE) $(build)=scripts/mod
$(Q)$(MAKE) $(build)=.
+ifdef CONFIG_RUST
+ $(Q)$(MAKE) $(build)=rust
+endif
# All the preparing..
prepare: prepare0
@@ -1618,6 +1702,13 @@ help:
@echo ' kselftest-merge - Merge all the config dependencies of'
@echo ' kselftest to existing .config.'
@echo ''
+ @echo 'Rust targets:'
+ @echo ' rustfmt - Reformat all the Rust code in the kernel'
+ @echo ' rustfmtcheck - Checks if all the Rust code in the kernel'
+ @echo ' is formatted, printing a diff otherwise.'
+ @echo ' rustdoc - Generate Rust documentation'
+ @echo ' (requires kernel .config)'
+ @echo ''
@$(if $(dtstree), \
echo 'Devicetree:'; \
echo '* dtbs - Build device tree blobs for enabled boards'; \
@@ -1689,6 +1780,27 @@ PHONY += $(DOC_TARGETS)
$(DOC_TARGETS):
$(Q)$(MAKE) $(build)=Documentation $@
+
+# Rust targets
+# ---------------------------------------------------------------------------
+
+# Documentation target
+#
+# Using the singular to avoid running afoul of `no-dot-config-targets`.
+PHONY += rustdoc
+rustdoc: prepare0
+ $(Q)$(MAKE) $(build)=rust $@
+
+# Formatting targets
+PHONY += rustfmt rustfmtcheck
+
+rustfmt:
+ find -name '*.rs' | xargs $(RUSTFMT)
+
+rustfmtcheck:
+ find -name '*.rs' | xargs $(RUSTFMT) --check
+
+
# Misc
# ---------------------------------------------------------------------------
@@ -1846,6 +1958,7 @@ clean: $(clean-dirs)
$(call cmd,rmfiles)
@find $(if $(KBUILD_EXTMOD), $(KBUILD_EXTMOD), .) $(RCS_FIND_IGNORE) \
\( -name '*.[aios]' -o -name '*.ko' -o -name '.*.cmd' \
+ -o -name '*.rmeta' \
-o -name '*.ko.*' \
-o -name '*.dtb' -o -name '*.dtbo' -o -name '*.dtb.S' -o -name '*.dt.yaml' \
-o -name '*.dwo' -o -name '*.lst' \
diff --git a/arch/arm/rust/target.json b/arch/arm/rust/target.json
new file mode 100644
index 0000000000000..37710eb727b1c
--- /dev/null
+++ b/arch/arm/rust/target.json
@@ -0,0 +1,28 @@
+{
+ "arch": "arm",
+ "crt-static-respected": true,
+ "data-layout": "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64",
+ "dynamic-linking": true,
+ "env": "gnu",
+ "executables": true,
+ "features": "+strict-align,+v6",
+ "function-sections": false,
+ "has-elf-tls": true,
+ "has-rpath": true,
+ "is-builtin": true,
+ "linker-is-gnu": true,
+ "llvm-target": "arm-unknown-linux-gnueabi",
+ "max-atomic-width": 64,
+ "os": "linux",
+ "position-independent-executables": true,
+ "pre-link-args": {
+ "gcc": [
+ "-Wl,--as-needed",
+ "-Wl,-z,noexecstack"
+ ]
+ },
+ "relocation-model": "static",
+ "target-family": "unix",
+ "target-mcount": "\u0001__gnu_mcount_nc",
+ "target-pointer-width": "32"
+}
diff --git a/arch/arm64/rust/target.json b/arch/arm64/rust/target.json
new file mode 100644
index 0000000000000..44953e2725c4a
--- /dev/null
+++ b/arch/arm64/rust/target.json
@@ -0,0 +1,40 @@
+{
+ "arch": "aarch64",
+ "data-layout": "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128",
+ "disable-redzone": true,
+ "eliminate-frame-pointer": false,
+ "emit-debug-gdb-scripts": false,
+ "env": "gnu",
+ "features": "+strict-align,+neon,+fp-armv8",
+ "function-sections": false,
+ "is-builtin": true,
+ "linker-flavor": "gcc",
+ "linker-is-gnu": true,
+ "llvm-target": "aarch64-unknown-none",
+ "max-atomic-width": 128,
+ "needs-plt": true,
+ "os": "none",
+ "panic-strategy": "abort",
+ "position-independent-executables": true,
+ "pre-link-args": {
+ "gcc": [
+ "-Wl,--as-needed",
+ "-Wl,-z,noexecstack",
+ "-m64"
+ ]
+ },
+ "relocation-model": "static",
+ "relro-level": "full",
+ "stack-probes": {
+ "kind": "inline-or-call",
+ "min-llvm-version-for-inline": [
+ 11,
+ 0,
+ 1
+ ]
+ },
+ "target-c-int-width": "32",
+ "target-endian": "little",
+ "target-pointer-width": "64",
+ "vendor": ""
+}
diff --git a/arch/powerpc/rust/target.json b/arch/powerpc/rust/target.json
new file mode 100644
index 0000000000000..1e53f83080927
--- /dev/null
+++ b/arch/powerpc/rust/target.json
@@ -0,0 +1,30 @@
+{
+ "arch": "powerpc64",
+ "code-mode": "kernel",
+ "cpu": "ppc64le",
+ "data-layout": "e-m:e-i64:64-n32:64",
+ "env": "gnu",
+ "features": "-altivec,-vsx,-hard-float",
+ "function-sections": false,
+ "is-builtin": true,
+ "linker-flavor": "gcc",
+ "linker-is-gnu": true,
+ "llvm-target": "powerpc64le-elf",
+ "max-atomic-width": 64,
+ "os": "none",
+ "panic-strategy": "abort",
+ "position-independent-executables": true,
+ "pre-link-args": {
+ "gcc": [
+ "-Wl,--as-needed",
+ "-Wl,-z,noexecstack",
+ "-m64"
+ ]
+ },
+ "relocation-model": "static",
+ "relro-level": "full",
+ "target-family": "unix",
+ "target-mcount": "_mcount",
+ "target-endian": "little",
+ "target-pointer-width": "64"
+}
diff --git a/arch/x86/rust/target.json b/arch/x86/rust/target.json
new file mode 100644
index 0000000000000..6e1759cd45bf5
--- /dev/null
+++ b/arch/x86/rust/target.json
@@ -0,0 +1,42 @@
+{
+ "arch": "x86_64",
+ "code-model": "kernel",
+ "cpu": "x86-64",
+ "data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128",
+ "disable-redzone": true,
+ "eliminate-frame-pointer": false,
+ "emit-debug-gdb-scripts": false,
+ "env": "gnu",
+ "features": "-mmx,-sse,-sse2,-sse3,-ssse3,-sse4.1,-sse4.2,-3dnow,-3dnowa,-avx,-avx2,+soft-float",
+ "function-sections": false,
+ "is-builtin": true,
+ "linker-flavor": "gcc",
+ "linker-is-gnu": true,
+ "llvm-target": "x86_64-elf",
+ "max-atomic-width": 64,
+ "needs-plt": true,
+ "os": "none",
+ "panic-strategy": "abort",
+ "position-independent-executables": true,
+ "pre-link-args": {
+ "gcc": [
+ "-Wl,--as-needed",
+ "-Wl,-z,noexecstack",
+ "-m64"
+ ]
+ },
+ "relocation-model": "static",
+ "relro-level": "full",
+ "stack-probes": {
+ "kind": "inline-or-call",
+ "min-llvm-version-for-inline": [
+ 11,
+ 0,
+ 1
+ ]
+ },
+ "target-c-int-width": "32",
+ "target-endian": "little",
+ "target-pointer-width": "64",
+ "vendor": "unknown"
+}
diff --git a/drivers/android/Kconfig b/drivers/android/Kconfig
index 53b22e26266c3..7aef7f38f9887 100644
--- a/drivers/android/Kconfig
+++ b/drivers/android/Kconfig
@@ -20,6 +20,13 @@ config ANDROID_BINDER_IPC
Android process, using Binder to identify, invoke and pass arguments
between said processes.
+config ANDROID_BINDER_IPC_RUST
+ bool "Android Binder IPC Driver in Rust"
+ depends on MMU && RUST
+ default n
+ help
+ Implementation of the Binder IPC in Rust.
+
config ANDROID_BINDERFS
bool "Android Binderfs filesystem"
depends on ANDROID_BINDER_IPC
diff --git a/drivers/android/Makefile b/drivers/android/Makefile
index c9d3d0c99c257..c428f2ce2f05e 100644
--- a/drivers/android/Makefile
+++ b/drivers/android/Makefile
@@ -4,3 +4,5 @@ ccflags-y += -I$(src) # needed for trace events
obj-$(CONFIG_ANDROID_BINDERFS) += binderfs.o
obj-$(CONFIG_ANDROID_BINDER_IPC) += binder.o binder_alloc.o
obj-$(CONFIG_ANDROID_BINDER_IPC_SELFTEST) += binder_alloc_selftest.o
+
+obj-$(CONFIG_ANDROID_BINDER_IPC_RUST) += rust_binder.o
diff --git a/drivers/android/allocation.rs b/drivers/android/allocation.rs
new file mode 100644
index 0000000000000..80b96628c778c
--- /dev/null
+++ b/drivers/android/allocation.rs
@@ -0,0 +1,252 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use alloc::sync::Arc;
+use core::mem::{size_of, MaybeUninit};
+use kernel::{bindings, pages::Pages, prelude::*, user_ptr::UserSlicePtrReader, Error};
+
+use crate::{
+ node::NodeRef,
+ process::{AllocationInfo, Process},
+ thread::{BinderError, BinderResult},
+};
+
+pub(crate) struct Allocation<'a> {
+ pub(crate) offset: usize,
+ size: usize,
+ pub(crate) ptr: usize,
+ pages: Arc<[Pages<0>]>,
+ pub(crate) process: &'a Process,
+ allocation_info: Option<AllocationInfo>,
+ free_on_drop: bool,
+}
+
+impl<'a> Allocation<'a> {
+ pub(crate) fn new(
+ process: &'a Process,
+ offset: usize,
+ size: usize,
+ ptr: usize,
+ pages: Arc<[Pages<0>]>,
+ ) -> Self {
+ Self {
+ process,
+ offset,
+ size,
+ ptr,
+ pages,
+ allocation_info: None,
+ free_on_drop: true,
+ }
+ }
+
+ fn iterate<T>(&self, mut offset: usize, mut size: usize, mut cb: T) -> KernelResult
+ where
+ T: FnMut(&Pages<0>, usize, usize) -> KernelResult,
+ {
+ // Check that the request is within the buffer.
+ if offset.checked_add(size).ok_or(Error::EINVAL)? > self.size {
+ return Err(Error::EINVAL);
+ }
+ offset += self.offset;
+ let mut page_index = offset >> bindings::PAGE_SHIFT;
+ offset &= (1 << bindings::PAGE_SHIFT) - 1;
+ while size > 0 {
+ let available = core::cmp::min(size, (1 << bindings::PAGE_SHIFT) as usize - offset);
+ cb(&self.pages[page_index], offset, available)?;
+ size -= available;
+ page_index += 1;
+ offset = 0;
+ }
+ Ok(())
+ }
+
+ pub(crate) fn copy_into(
+ &self,
+ reader: &mut UserSlicePtrReader,
+ offset: usize,
+ size: usize,
+ ) -> KernelResult {
+ self.iterate(offset, size, |page, offset, to_copy| {
+ page.copy_into_page(reader, offset, to_copy)
+ })
+ }
+
+ pub(crate) fn read<T>(&self, offset: usize) -> KernelResult<T> {
+ let mut out = MaybeUninit::<T>::uninit();
+ let mut out_offset = 0;
+ self.iterate(offset, size_of::<T>(), |page, offset, to_copy| {
+ // SAFETY: Data buffer is allocated on the stack.
+ unsafe {
+ page.read(
+ (out.as_mut_ptr() as *mut u8).add(out_offset),
+ offset,
+ to_copy,
+ )
+ }?;
+ out_offset += to_copy;
+ Ok(())
+ })?;
+ // SAFETY: We just initialised the data.
+ Ok(unsafe { out.assume_init() })
+ }
+
+ pub(crate) fn write<T>(&self, offset: usize, obj: &T) -> KernelResult {
+ let mut obj_offset = 0;
+ self.iterate(offset, size_of::<T>(), |page, offset, to_copy| {
+ // SAFETY: The sum of `offset` and `to_copy` is bounded by the size of T.
+ let obj_ptr = unsafe { (obj as *const T as *const u8).add(obj_offset) };
+ // SAFETY: We have a reference to the object, so the pointer is valid.
+ unsafe { page.write(obj_ptr, offset, to_copy) }?;
+ obj_offset += to_copy;
+ Ok(())
+ })
+ }
+
+ pub(crate) fn keep_alive(mut self) {
+ self.process
+ .buffer_make_freeable(self.offset, self.allocation_info.take());
+ self.free_on_drop = false;
+ }
+
+ pub(crate) fn set_info(&mut self, info: AllocationInfo) {
+ self.allocation_info = Some(info);
+ }
+
+ fn cleanup_object(&self, index_offset: usize, view: &AllocationView) -> KernelResult {
+ let offset = self.read(index_offset)?;
+ let header = view.read::<bindings::binder_object_header>(offset)?;
+ // TODO: Handle other types.
+ match header.type_ {
+ bindings::BINDER_TYPE_WEAK_BINDER | bindings::BINDER_TYPE_BINDER => {
+ let obj = view.read::<bindings::flat_binder_object>(offset)?;
+ let strong = header.type_ == bindings::BINDER_TYPE_BINDER;
+ // SAFETY: The type is `BINDER_TYPE_{WEAK_}BINDER`, so the `binder` field is
+ // populated.
+ let ptr = unsafe { obj.__bindgen_anon_1.binder } as usize;
+ let cookie = obj.cookie as usize;
+ self.process.update_node(ptr, cookie, strong, false);
+ Ok(())
+ }
+ bindings::BINDER_TYPE_WEAK_HANDLE | bindings::BINDER_TYPE_HANDLE => {
+ let obj = view.read::<bindings::flat_binder_object>(offset)?;
+ let strong = header.type_ == bindings::BINDER_TYPE_HANDLE;
+ // SAFETY: The type is `BINDER_TYPE_{WEAK_}HANDLE`, so the `handle` field is
+ // populated.
+ let handle = unsafe { obj.__bindgen_anon_1.handle } as _;
+ self.process.update_ref(handle, false, strong)
+ }
+ _ => Ok(()),
+ }
+ }
+}
+
+impl Drop for Allocation<'_> {
+ fn drop(&mut self) {
+ if !self.free_on_drop {
+ return;
+ }
+
+ if let Some(info) = &self.allocation_info {
+ let view = AllocationView::new(self, info.offsets.start);
+ for i in info.offsets.clone().step_by(size_of::<usize>()) {
+ if self.cleanup_object(i, &view).is_err() {
+ pr_warn!("Error cleaning up object at offset {}\n", i)
+ }
+ }
+ }
+
+ self.process.buffer_raw_free(self.ptr);
+ }
+}
+
+pub(crate) struct AllocationView<'a> {
+ alloc: &'a Allocation<'a>,
+ limit: usize,
+}
+
+impl<'a> AllocationView<'a> {
+ pub(crate) fn new(alloc: &'a Allocation, limit: usize) -> Self {
+ AllocationView { alloc, limit }
+ }
+
+ pub fn read<T>(&self, offset: usize) -> KernelResult<T> {
+ if offset.checked_add(size_of::<T>()).ok_or(Error::EINVAL)? > self.limit {
+ return Err(Error::EINVAL);
+ }
+ self.alloc.read(offset)
+ }
+
+ pub fn write<T>(&self, offset: usize, obj: &T) -> KernelResult {
+ if offset.checked_add(size_of::<T>()).ok_or(Error::EINVAL)? > self.limit {
+ return Err(Error::EINVAL);
+ }
+ self.alloc.write(offset, obj)
+ }
+
+ pub(crate) fn transfer_binder_object<T>(
+ &self,
+ offset: usize,
+ strong: bool,
+ get_node: T,
+ ) -> BinderResult
+ where
+ T: FnOnce(&bindings::flat_binder_object) -> BinderResult<NodeRef>,
+ {
+ // TODO: Do we want this function to take a &mut self?
+ let obj = self.read::<bindings::flat_binder_object>(offset)?;
+ let node_ref = get_node(&obj)?;
+
+ if core::ptr::eq(&*node_ref.node.owner, self.alloc.process) {
+ // The receiving process is the owner of the node, so send it a binder object (instead
+ // of a handle).
+ let (ptr, cookie) = node_ref.node.get_id();
+ let newobj = bindings::flat_binder_object {
+ hdr: bindings::binder_object_header {
+ type_: if strong {
+ bindings::BINDER_TYPE_BINDER
+ } else {
+ bindings::BINDER_TYPE_WEAK_BINDER
+ },
+ },
+ flags: obj.flags,
+ __bindgen_anon_1: bindings::flat_binder_object__bindgen_ty_1 { binder: ptr as _ },
+ cookie: cookie as _,
+ };
+ self.write(offset, &newobj)?;
+
+ // Increment the user ref count on the node. It will be decremented as part of the
+ // destruction of the buffer, when we see a binder or weak-binder object.
+ node_ref.node.update_refcount(true, strong);
+ } else {
+ // The receiving process is different from the owner, so we need to insert a handle to
+ // the binder object.
+ let handle = self
+ .alloc
+ .process
+ .insert_or_update_handle(node_ref, false)?;
+
+ let newobj = bindings::flat_binder_object {
+ hdr: bindings::binder_object_header {
+ type_: if strong {
+ bindings::BINDER_TYPE_HANDLE
+ } else {
+ bindings::BINDER_TYPE_WEAK_HANDLE
+ },
+ },
+ flags: obj.flags,
+ // TODO: To avoid padding, we write to `binder` instead of `handle` here. We need a
+ // better solution though.
+ __bindgen_anon_1: bindings::flat_binder_object__bindgen_ty_1 {
+ binder: handle as _,
+ },
+ ..bindings::flat_binder_object::default()
+ };
+ if self.write(offset, &newobj).is_err() {
+ // Decrement ref count on the handle we just created.
+ let _ = self.alloc.process.update_ref(handle, false, strong);
+ return Err(BinderError::new_failed());
+ }
+ }
+ Ok(())
+ }
+}
diff --git a/drivers/android/context.rs b/drivers/android/context.rs
new file mode 100644
index 0000000000000..1ab2b26c8fd3a
--- /dev/null
+++ b/drivers/android/context.rs
@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: GPL-2.0
+
+extern crate alloc;
+
+use alloc::sync::Arc;
+use core::pin::Pin;
+use kernel::{bindings, prelude::*, sync::Mutex, Error};
+
+use crate::{
+ node::NodeRef,
+ thread::{BinderError, BinderResult},
+};
+
+struct Manager {
+ node: Option<NodeRef>,
+ uid: Option<bindings::kuid_t>,
+}
+
+pub(crate) struct Context {
+ manager: Mutex<Manager>,
+}
+
+unsafe impl Send for Context {}
+unsafe impl Sync for Context {}
+
+impl Context {
+ pub(crate) fn new() -> KernelResult<Pin<Arc<Self>>> {
+ let mut ctx_ref = Arc::try_new(Self {
+ // SAFETY: Init is called below.
+ manager: unsafe {
+ Mutex::new(Manager {
+ node: None,
+ uid: None,
+ })
+ },
+ })?;
+ let ctx = Arc::get_mut(&mut ctx_ref).unwrap();
+
+ // SAFETY: `manager` is also pinned when `ctx` is.
+ let manager = unsafe { Pin::new_unchecked(&ctx.manager) };
+ kernel::mutex_init!(manager, "Context::manager");
+
+ // SAFETY: `ctx_ref` is pinned behind the `Arc` reference.
+ Ok(unsafe { Pin::new_unchecked(ctx_ref) })
+ }
+
+ pub(crate) fn set_manager_node(&self, node_ref: NodeRef) -> KernelResult {
+ let mut manager = self.manager.lock();
+ if manager.node.is_some() {
+ return Err(Error::EBUSY);
+ }
+ // TODO: Call security_binder_set_context_mgr.
+
+ // TODO: Get the actual caller id.
+ let caller_uid = bindings::kuid_t::default();
+ if let Some(ref uid) = manager.uid {
+ if uid.val != caller_uid.val {
+ return Err(Error::EPERM);
+ }
+ }
+
+ manager.node = Some(node_ref);
+ manager.uid = Some(caller_uid);
+ Ok(())
+ }
+
+ pub(crate) fn unset_manager_node(&self) {
+ let node_ref = self.manager.lock().node.take();
+ drop(node_ref);
+ }
+
+ pub(crate) fn get_manager_node(&self, strong: bool) -> BinderResult<NodeRef> {
+ self.manager
+ .lock()
+ .node
+ .as_ref()
+ .ok_or_else(BinderError::new_dead)?
+ .clone(strong)
+ }
+}
diff --git a/drivers/android/defs.rs b/drivers/android/defs.rs
new file mode 100644
index 0000000000000..619bb1a234200
--- /dev/null
+++ b/drivers/android/defs.rs
@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use core::ops::{Deref, DerefMut};
+use kernel::{
+ bindings,
+ bindings::*,
+ user_ptr::{ReadableFromBytes, WritableToBytes},
+};
+
+macro_rules! pub_no_prefix {
+ ($prefix:ident, $($newname:ident),+) => {
+ $(pub const $newname: u32 = concat_idents!($prefix, $newname);)+
+ };
+}
+
+pub_no_prefix!(
+ binder_driver_return_protocol_,
+ BR_OK,
+ BR_ERROR,
+ BR_TRANSACTION,
+ BR_REPLY,
+ BR_DEAD_REPLY,
+ BR_TRANSACTION_COMPLETE,
+ BR_INCREFS,
+ BR_ACQUIRE,
+ BR_RELEASE,
+ BR_DECREFS,
+ BR_NOOP,
+ BR_SPAWN_LOOPER,
+ BR_DEAD_BINDER,
+ BR_CLEAR_DEATH_NOTIFICATION_DONE,
+ BR_FAILED_REPLY
+);
+
+pub_no_prefix!(
+ binder_driver_command_protocol_,
+ BC_TRANSACTION,
+ BC_REPLY,
+ BC_FREE_BUFFER,
+ BC_INCREFS,
+ BC_ACQUIRE,
+ BC_RELEASE,
+ BC_DECREFS,
+ BC_INCREFS_DONE,
+ BC_ACQUIRE_DONE,
+ BC_REGISTER_LOOPER,
+ BC_ENTER_LOOPER,
+ BC_EXIT_LOOPER,
+ BC_REQUEST_DEATH_NOTIFICATION,
+ BC_CLEAR_DEATH_NOTIFICATION,
+ BC_DEAD_BINDER_DONE
+);
+
+macro_rules! decl_wrapper {
+ ($newname:ident, $wrapped:ty) => {
+ #[derive(Copy, Clone, Default)]
+ pub(crate) struct $newname($wrapped);
+
+ // TODO: This must be justified by inspecting the type, so should live outside the macro or
+ // the macro should be somehow marked unsafe.
+ unsafe impl ReadableFromBytes for $newname {}
+ unsafe impl WritableToBytes for $newname {}
+
+ impl Deref for $newname {
+ type Target = $wrapped;
+ fn deref(&self) -> &Self::Target {
+ &self.0
+ }
+ }
+
+ impl DerefMut for $newname {
+ fn deref_mut(&mut self) -> &mut Self::Target {
+ &mut self.0
+ }
+ }
+ };
+}
+
+decl_wrapper!(BinderNodeDebugInfo, bindings::binder_node_debug_info);
+decl_wrapper!(BinderNodeInfoForRef, bindings::binder_node_info_for_ref);
+decl_wrapper!(FlatBinderObject, bindings::flat_binder_object);
+decl_wrapper!(BinderTransactionData, bindings::binder_transaction_data);
+decl_wrapper!(BinderWriteRead, bindings::binder_write_read);
+decl_wrapper!(BinderVersion, bindings::binder_version);
+
+impl BinderVersion {
+ pub(crate) fn current() -> Self {
+ Self(bindings::binder_version {
+ protocol_version: bindings::BINDER_CURRENT_PROTOCOL_VERSION as _,
+ })
+ }
+}
diff --git a/drivers/android/node.rs b/drivers/android/node.rs
new file mode 100644
index 0000000000000..94c8adddbb17a
--- /dev/null
+++ b/drivers/android/node.rs
@@ -0,0 +1,479 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use alloc::sync::Arc;
+use core::{
+ pin::Pin,
+ sync::atomic::{AtomicU64, Ordering},
+};
+use kernel::{
+ linked_list::{GetLinks, Links, List},
+ prelude::*,
+ sync::{Guard, LockedBy, Mutex, Ref, SpinLock},
+ user_ptr::UserSlicePtrWriter,
+};
+
+use crate::{
+ defs::*,
+ process::{Process, ProcessInner},
+ thread::{BinderError, BinderResult, Thread},
+ DeliverToRead,
+};
+
+struct CountState {
+ count: usize,
+ has_count: bool,
+ is_biased: bool,
+}
+
+impl CountState {
+ fn new() -> Self {
+ Self {
+ count: 0,
+ has_count: false,
+ is_biased: false,
+ }
+ }
+
+ fn add_bias(&mut self) {
+ self.count += 1;
+ self.is_biased = true;
+ }
+}
+
+struct NodeInner {
+ strong: CountState,
+ weak: CountState,
+ death_list: List<Arc<NodeDeath>>,
+}
+
+struct NodeDeathInner {
+ dead: bool,
+ cleared: bool,
+ notification_done: bool,
+
+ /// Indicates whether the normal flow was interrupted by removing the handle. In this case, we
+ /// need behave as if the death notification didn't exist (i.e., we don't deliver anything to
+ /// the user.
+ aborted: bool,
+}
+
+pub(crate) struct NodeDeath {
+ node: Arc<Node>,
+ process: Ref<Process>,
+ // TODO: Make this private.
+ pub(crate) cookie: usize,
+ work_links: Links<dyn DeliverToRead>,
+ // TODO: Add the moment we're using this for two lists, which isn't safe because we want to
+ // remove from the list without knowing the list it's in. We need to separate this out.
+ death_links: Links<NodeDeath>,
+ inner: SpinLock<NodeDeathInner>,
+}
+
+impl NodeDeath {
+ /// Constructs a new node death notification object.
+ ///
+ /// # Safety
+ ///
+ /// The caller must call `NodeDeath::init` before using the notification object.
+ pub(crate) unsafe fn new(node: Arc<Node>, process: Ref<Process>, cookie: usize) -> Self {
+ Self {
+ node,
+ process,
+ cookie,
+ work_links: Links::new(),
+ death_links: Links::new(),
+ inner: SpinLock::new(NodeDeathInner {
+ dead: false,
+ cleared: false,
+ notification_done: false,
+ aborted: false,
+ }),
+ }
+ }
+
+ pub(crate) fn init(self: Pin<&Self>) {
+ // SAFETY: `inner` is pinned when `self` is.
+ let inner = unsafe { self.map_unchecked(|s| &s.inner) };
+ kernel::spinlock_init!(inner, "NodeDeath::inner");
+ }
+
+ /// Sets the cleared flag to `true`.
+ ///
+ /// It removes `self` from the node's death notification list if needed. It must only be called
+ /// once.
+ ///
+ /// Returns whether it needs to be queued.
+ pub(crate) fn set_cleared(self: &Arc<Self>, abort: bool) -> bool {
+ let (needs_removal, needs_queueing) = {
+ // Update state and determine if we need to queue a work item. We only need to do it
+ // when the node is not dead or if the user already completed the death notification.
+ let mut inner = self.inner.lock();
+ inner.cleared = true;
+ if abort {
+ inner.aborted = true;
+ }
+ (!inner.dead, !inner.dead || inner.notification_done)
+ };
+
+ // Remove death notification from node.
+ if needs_removal {
+ let mut owner_inner = self.node.owner.inner.lock();
+ let node_inner = self.node.inner.access_mut(&mut owner_inner);
+ unsafe { node_inner.death_list.remove(self) };
+ }
+
+ needs_queueing
+ }
+
+ /// Sets the 'notification done' flag to `true`.
+ ///
+ /// Returns whether it needs to be queued.
+ pub(crate) fn set_notification_done(self: Arc<Self>, thread: &Thread) {
+ let needs_queueing = {
+ let mut inner = self.inner.lock();
+ inner.notification_done = true;
+ inner.cleared
+ };
+
+ if needs_queueing {
+ let _ = thread.push_work_if_looper(self);
+ }
+ }
+
+ /// Sets the 'dead' flag to `true` and queues work item if needed.
+ pub(crate) fn set_dead(self: Arc<Self>) {
+ let needs_queueing = {
+ let mut inner = self.inner.lock();
+ if inner.cleared {
+ false
+ } else {
+ inner.dead = true;
+ true
+ }
+ };
+
+ if needs_queueing {
+ // Push the death notification to the target process. There is nothing else to do if
+ // it's already dead.
+ let process = self.process.clone();
+ let _ = process.push_work(self);
+ }
+ }
+}
+
+impl GetLinks for NodeDeath {
+ type EntryType = NodeDeath;
+ fn get_links(data: &NodeDeath) -> &Links<NodeDeath> {
+ &data.death_links
+ }
+}
+
+impl DeliverToRead for NodeDeath {
+ fn do_work(
+ self: Arc<Self>,
+ _thread: &Thread,
+ writer: &mut UserSlicePtrWriter,
+ ) -> KernelResult<bool> {
+ let done = {
+ let inner = self.inner.lock();
+ if inner.aborted {
+ return Ok(true);
+ }
+ inner.cleared && (!inner.dead || inner.notification_done)
+ };
+
+ let cookie = self.cookie;
+ let cmd = if done {
+ BR_CLEAR_DEATH_NOTIFICATION_DONE
+ } else {
+ let process = self.process.clone();
+ let mut process_inner = process.inner.lock();
+ let inner = self.inner.lock();
+ if inner.aborted {
+ return Ok(true);
+ }
+ // We're still holding the inner lock, so it cannot be aborted while we insert it into
+ // the delivered list.
+ process_inner.death_delivered(self.clone());
+ BR_DEAD_BINDER
+ };
+
+ writer.write(&cmd)?;
+ writer.write(&cookie)?;
+
+ // Mimic the original code: we stop processing work items when we get to a death
+ // notification.
+ Ok(cmd != BR_DEAD_BINDER)
+ }
+
+ fn get_links(&self) -> &Links<dyn DeliverToRead> {
+ &self.work_links
+ }
+}
+
+pub(crate) struct Node {
+ pub(crate) global_id: u64,
+ ptr: usize,
+ cookie: usize,
+ pub(crate) owner: Ref<Process>,
+ inner: LockedBy<NodeInner, Mutex<ProcessInner>>,
+ links: Links<dyn DeliverToRead>,
+}
+
+impl Node {
+ pub(crate) fn new(ptr: usize, cookie: usize, owner: Ref<Process>) -> Self {
+ static NEXT_ID: AtomicU64 = AtomicU64::new(1);
+ let inner = LockedBy::new(
+ &owner.inner,
+ NodeInner {
+ strong: CountState::new(),
+ weak: CountState::new(),
+ death_list: List::new(),
+ },
+ );
+ Self {
+ global_id: NEXT_ID.fetch_add(1, Ordering::Relaxed),
+ ptr,
+ cookie,
+ owner,
+ inner,
+ links: Links::new(),
+ }
+ }
+
+ pub(crate) fn get_id(&self) -> (usize, usize) {
+ (self.ptr, self.cookie)
+ }
+
+ pub(crate) fn next_death(
+ &self,
+ guard: &mut Guard<Mutex<ProcessInner>>,
+ ) -> Option<Arc<NodeDeath>> {
+ self.inner.access_mut(guard).death_list.pop_front()
+ }
+
+ pub(crate) fn add_death(&self, death: Arc<NodeDeath>, guard: &mut Guard<Mutex<ProcessInner>>) {
+ self.inner.access_mut(guard).death_list.push_back(death);
+ }
+
+ pub(crate) fn update_refcount_locked(
+ &self,
+ inc: bool,
+ strong: bool,
+ biased: bool,
+ owner_inner: &mut ProcessInner,
+ ) -> bool {
+ let inner = self.inner.access_from_mut(owner_inner);
+
+ // Get a reference to the state we'll update.
+ let state = if strong {
+ &mut inner.strong
+ } else {
+ &mut inner.weak
+ };
+
+ // Update biased state: if the count is not biased, there is nothing to do; otherwise,
+ // we're removing the bias, so mark the state as such.
+ if biased {
+ if !state.is_biased {
+ return false;
+ }
+
+ state.is_biased = false;
+ }
+
+ // Update the count and determine whether we need to push work.
+ // TODO: Here we may want to check the weak count being zero but the strong count being 1,
+ // because in such cases, we won't deliver anything to userspace, so we shouldn't queue
+ // either.
+ if inc {
+ state.count += 1;
+ !state.has_count
+ } else {
+ state.count -= 1;
+ state.count == 0 && state.has_count
+ }
+ }
+
+ pub(crate) fn update_refcount(self: &Arc<Self>, inc: bool, strong: bool) {
+ self.owner
+ .inner
+ .lock()
+ .update_node_refcount(self, inc, strong, false, None);
+ }
+
+ pub(crate) fn populate_counts(
+ &self,
+ out: &mut BinderNodeInfoForRef,
+ guard: &Guard<Mutex<ProcessInner>>,
+ ) {
+ let inner = self.inner.access(guard);
+ out.strong_count = inner.strong.count as _;
+ out.weak_count = inner.weak.count as _;
+ }
+
+ pub(crate) fn populate_debug_info(
+ &self,
+ out: &mut BinderNodeDebugInfo,
+ guard: &Guard<Mutex<ProcessInner>>,
+ ) {
+ out.ptr = self.ptr as _;
+ out.cookie = self.cookie as _;
+ let inner = self.inner.access(&guard);
+ if inner.strong.has_count {
+ out.has_strong_ref = 1;
+ }
+ if inner.weak.has_count {
+ out.has_weak_ref = 1;
+ }
+ }
+
+ pub(crate) fn force_has_count(&self, guard: &mut Guard<Mutex<ProcessInner>>) {
+ let inner = self.inner.access_mut(guard);
+ inner.strong.has_count = true;
+ inner.weak.has_count = true;
+ }
+
+ fn write(&self, writer: &mut UserSlicePtrWriter, code: u32) -> KernelResult {
+ writer.write(&code)?;
+ writer.write(&self.ptr)?;
+ writer.write(&self.cookie)?;
+ Ok(())
+ }
+}
+
+impl DeliverToRead for Node {
+ fn do_work(
+ self: Arc<Self>,
+ _thread: &Thread,
+ writer: &mut UserSlicePtrWriter,
+ ) -> KernelResult<bool> {
+ let mut owner_inner = self.owner.inner.lock();
+ let inner = self.inner.access_mut(&mut owner_inner);
+ let strong = inner.strong.count > 0;
+ let has_strong = inner.strong.has_count;
+ let weak = strong || inner.weak.count > 0;
+ let has_weak = inner.weak.has_count;
+ inner.weak.has_count = weak;
+ inner.strong.has_count = strong;
+
+ if !weak {
+ // Remove the node if there are no references to it.
+ owner_inner.remove_node(self.ptr);
+ } else {
+ if !has_weak {
+ inner.weak.add_bias();
+ }
+
+ if !has_strong && strong {
+ inner.strong.add_bias();
+ }
+ }
+
+ drop(owner_inner);
+
+ // This could be done more compactly but we write out all the posibilities for
+ // compatibility with the original implementation wrt the order of events.
+ if weak && !has_weak {
+ self.write(writer, BR_INCREFS)?;
+ }
+
+ if strong && !has_strong {
+ self.write(writer, BR_ACQUIRE)?;
+ }
+
+ if !strong && has_strong {
+ self.write(writer, BR_RELEASE)?;
+ }
+
+ if !weak && has_weak {
+ self.write(writer, BR_DECREFS)?;
+ }
+
+ Ok(true)
+ }
+
+ fn get_links(&self) -> &Links<dyn DeliverToRead> {
+ &self.links
+ }
+}
+
+pub struct NodeRef {
+ pub(crate) node: Arc<Node>,
+ strong_count: usize,
+ weak_count: usize,
+}
+
+impl NodeRef {
+ pub(crate) fn new(node: Arc<Node>, strong_count: usize, weak_count: usize) -> Self {
+ Self {
+ node,
+ strong_count,
+ weak_count,
+ }
+ }
+
+ pub(crate) fn absorb(&mut self, mut other: Self) {
+ self.strong_count += other.strong_count;
+ self.weak_count += other.weak_count;
+ other.strong_count = 0;
+ other.weak_count = 0;
+ }
+
+ pub(crate) fn clone(&self, strong: bool) -> BinderResult<NodeRef> {
+ if strong && self.strong_count == 0 {
+ return Err(BinderError::new_failed());
+ }
+
+ Ok(self
+ .node
+ .owner
+ .inner
+ .lock()
+ .new_node_ref(self.node.clone(), strong, None))
+ }
+
+ /// Updates (increments or decrements) the number of references held against the node. If the
+ /// count being updated transitions from 0 to 1 or from 1 to 0, the node is notified by having
+ /// its `update_refcount` function called.
+ ///
+ /// Returns whether `self` should be removed (when both counts are zero).
+ pub(crate) fn update(&mut self, inc: bool, strong: bool) -> bool {
+ if strong && self.strong_count == 0 {
+ return false;
+ }
+
+ let (count, other_count) = if strong {
+ (&mut self.strong_count, self.weak_count)
+ } else {
+ (&mut self.weak_count, self.strong_count)
+ };
+
+ if inc {
+ if *count == 0 {
+ self.node.update_refcount(true, strong);
+ }
+ *count += 1;
+ } else {
+ *count -= 1;
+ if *count == 0 {
+ self.node.update_refcount(false, strong);
+ return other_count == 0;
+ }
+ }
+
+ false
+ }
+}
+
+impl Drop for NodeRef {
+ fn drop(&mut self) {
+ if self.strong_count > 0 {
+ self.node.update_refcount(false, true);
+ }
+
+ if self.weak_count > 0 {
+ self.node.update_refcount(false, false);
+ }
+ }
+}
diff --git a/drivers/android/process.rs b/drivers/android/process.rs
new file mode 100644
index 0000000000000..9d4d85646432c
--- /dev/null
+++ b/drivers/android/process.rs
@@ -0,0 +1,950 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use alloc::{collections::btree_map::BTreeMap, sync::Arc, vec::Vec};
+use core::{
+ mem::{swap, MaybeUninit},
+ ops::Range,
+ pin::Pin,
+};
+use kernel::{
+ bindings, c_types,
+ file_operations::{File, FileOpener, FileOperations, IoctlCommand, IoctlHandler, PollTable},
+ linked_list::List,
+ pages::Pages,
+ prelude::*,
+ sync::{Guard, Mutex, Ref, RefCount, RefCounted},
+ user_ptr::{UserSlicePtr, UserSlicePtrReader},
+ Error,
+};
+
+use crate::{
+ allocation::Allocation,
+ context::Context,
+ defs::*,
+ node::{Node, NodeDeath, NodeRef},
+ range_alloc::RangeAllocator,
+ thread::{BinderError, BinderResult, Thread},
+ DeliverToRead, DeliverToReadListAdapter, Either,
+};
+
+// TODO: Review this:
+// Lock order: Process::node_refs -> Process::inner -> Thread::inner
+
+extern "C" {
+ fn rust_helper_current_pid() -> c_types::c_int;
+}
+
+pub(crate) struct AllocationInfo {
+ /// Range within the allocation where we can find the offsets to the object descriptors.
+ pub(crate) offsets: Range<usize>,
+}
+
+struct Mapping {
+ address: usize,
+ alloc: RangeAllocator<AllocationInfo>,
+ pages: Arc<[Pages<0>]>,
+}
+
+impl Mapping {
+ fn new(address: usize, size: usize, pages: Arc<[Pages<0>]>) -> KernelResult<Self> {
+ let alloc = RangeAllocator::new(size)?;
+ Ok(Self {
+ address,
+ alloc,
+ pages,
+ })
+ }
+}
+
+// TODO: Make this private.
+pub(crate) struct ProcessInner {
+ is_manager: bool,
+ is_dead: bool,
+ threads: BTreeMap<i32, Arc<Thread>>,
+ ready_threads: List<Arc<Thread>>,
+ work: List<DeliverToReadListAdapter>,
+ mapping: Option<Mapping>,
+ nodes: BTreeMap<usize, Arc<Node>>,
+
+ delivered_deaths: List<Arc<NodeDeath>>,
+
+ /// The number of requested threads that haven't registered yet.
+ requested_thread_count: u32,
+
+ /// The maximum number of threads used by the process thread pool.
+ max_threads: u32,
+
+ /// The number of threads the started and registered with the thread pool.
+ started_thread_count: u32,
+}
+
+impl ProcessInner {
+ fn new() -> Self {
+ Self {
+ is_manager: false,
+ is_dead: false,
+ threads: BTreeMap::new(),
+ ready_threads: List::new(),
+ work: List::new(),
+ mapping: None,
+ nodes: BTreeMap::new(),
+ requested_thread_count: 0,
+ max_threads: 0,
+ started_thread_count: 0,
+ delivered_deaths: List::new(),
+ }
+ }
+
+ fn push_work(&mut self, work: Arc<dyn DeliverToRead>) -> BinderResult {
+ // Try to find a ready thread to which to push the work.
+ if let Some(thread) = self.ready_threads.pop_front() {
+ // Push to thread while holding state lock. This prevents the thread from giving up
+ // (for example, because of a signal) when we're about to deliver work.
+ thread.push_work(work)
+ } else if self.is_dead {
+ Err(BinderError::new_dead())
+ } else {
+ // There are no ready threads. Push work to process queue.
+ self.work.push_back(work);
+
+ // Wake up polling threads, if any.
+ for thread in self.threads.values() {
+ thread.notify_if_poll_ready();
+ }
+ Ok(())
+ }
+ }
+
+ // TODO: Should this be private?
+ pub(crate) fn remove_node(&mut self, ptr: usize) {
+ self.nodes.remove(&ptr);
+ }
+
+ /// Updates the reference count on the given node.
+ // TODO: Decide if this should be private.
+ pub(crate) fn update_node_refcount(
+ &mut self,
+ node: &Arc<Node>,
+ inc: bool,
+ strong: bool,
+ biased: bool,
+ othread: Option<&Thread>,
+ ) {
+ let push = node.update_refcount_locked(inc, strong, biased, self);
+
+ // If we decided that we need to push work, push either to the process or to a thread if
+ // one is specified.
+ if push {
+ if let Some(thread) = othread {
+ thread.push_work_deferred(node.clone());
+ } else {
+ let _ = self.push_work(node.clone());
+ // Nothing to do: `push_work` may fail if the process is dead, but that's ok as in
+ // that case, it doesn't care about the notification.
+ }
+ }
+ }
+
+ // TODO: Make this private.
+ pub(crate) fn new_node_ref(
+ &mut self,
+ node: Arc<Node>,
+ strong: bool,
+ thread: Option<&Thread>,
+ ) -> NodeRef {
+ self.update_node_refcount(&node, true, strong, false, thread);
+ let strong_count = if strong { 1 } else { 0 };
+ NodeRef::new(node, strong_count, 1 - strong_count)
+ }
+
+ /// Returns an existing node with the given pointer and cookie, if one exists.
+ ///
+ /// Returns an error if a node with the given pointer but a different cookie exists.
+ fn get_existing_node(&self, ptr: usize, cookie: usize) -> KernelResult<Option<Arc<Node>>> {
+ match self.nodes.get(&ptr) {
+ None => Ok(None),
+ Some(node) => {
+ let (_, node_cookie) = node.get_id();
+ if node_cookie == cookie {
+ Ok(Some(node.clone()))
+ } else {
+ Err(Error::EINVAL)
+ }
+ }
+ }
+ }
+
+ /// Returns a reference to an existing node with the given pointer and cookie. It requires a
+ /// mutable reference because it needs to increment the ref count on the node, which may
+ /// require pushing work to the work queue (to notify userspace of 0 to 1 transitions).
+ fn get_existing_node_ref(
+ &mut self,
+ ptr: usize,
+ cookie: usize,
+ strong: bool,
+ thread: Option<&Thread>,
+ ) -> KernelResult<Option<NodeRef>> {
+ Ok(match self.get_existing_node(ptr, cookie)? {
+ None => None,
+ Some(node) => Some(self.new_node_ref(node, strong, thread)),
+ })
+ }
+
+ fn register_thread(&mut self) -> bool {
+ if self.requested_thread_count == 0 {
+ return false;
+ }
+
+ self.requested_thread_count -= 1;
+ self.started_thread_count += 1;
+ true
+ }
+
+ /// Finds a delivered death notification with the given cookie, removes it from the thread's
+ /// delivered list, and returns it.
+ fn pull_delivered_death(&mut self, cookie: usize) -> Option<Arc<NodeDeath>> {
+ let mut cursor = self.delivered_deaths.cursor_front_mut();
+ while let Some(death) = cursor.current() {
+ if death.cookie == cookie {
+ return cursor.remove_current();
+ }
+ cursor.move_next();
+ }
+ None
+ }
+
+ pub(crate) fn death_delivered(&mut self, death: Arc<NodeDeath>) {
+ self.delivered_deaths.push_back(death);
+ }
+}
+
+struct ArcReservation<T> {
+ mem: Arc<MaybeUninit<T>>,
+}
+
+impl<T> ArcReservation<T> {
+ fn new() -> KernelResult<Self> {
+ Ok(Self {
+ mem: Arc::try_new(MaybeUninit::<T>::uninit())?,
+ })
+ }
+
+ fn commit(mut self, data: T) -> Arc<T> {
+ // SAFETY: Memory was allocated and properly aligned by using `MaybeUninit`.
+ unsafe {
+ Arc::get_mut(&mut self.mem)
+ .unwrap()
+ .as_mut_ptr()
+ .write(data);
+ }
+
+ // SAFETY: We have just initialised the memory block, and we know it's compatible with `T`
+ // because we used `MaybeUninit`.
+ unsafe { Arc::from_raw(Arc::into_raw(self.mem) as _) }
+ }
+}
+
+struct NodeRefInfo {
+ node_ref: NodeRef,
+ death: Option<Arc<NodeDeath>>,
+}
+
+impl NodeRefInfo {
+ fn new(node_ref: NodeRef) -> Self {
+ Self {
+ node_ref,
+ death: None,
+ }
+ }
+}
+
+struct ProcessNodeRefs {
+ by_handle: BTreeMap<u32, NodeRefInfo>,
+ by_global_id: BTreeMap<u64, u32>,
+}
+
+impl ProcessNodeRefs {
+ fn new() -> Self {
+ Self {
+ by_handle: BTreeMap::new(),
+ by_global_id: BTreeMap::new(),
+ }
+ }
+}
+
+pub(crate) struct Process {
+ ctx: Arc<Context>,
+ ref_count: RefCount,
+
+ // TODO: For now this a mutex because we have allocations in BTreeMap and RangeAllocator while
+ // holding the lock. We may want to split up the process state at some point to use a spin lock
+ // for the other fields; we can also get rid of allocations in BTreeMap once we replace it.
+ // TODO: Make this private again.
+ pub(crate) inner: Mutex<ProcessInner>,
+
+ // References are in a different mutex to avoid recursive acquisition when
+ // incrementing/decrementing a node in another process.
+ node_refs: Mutex<ProcessNodeRefs>,
+}
+
+unsafe impl Send for Process {}
+unsafe impl Sync for Process {}
+
+impl Process {
+ fn new(ctx: Arc<Context>) -> KernelResult<Ref<Self>> {
+ let mut proc_ref = Ref::try_new(Self {
+ ref_count: RefCount::new(),
+ ctx,
+ // SAFETY: `inner` is initialised in the call to `mutex_init` below.
+ inner: unsafe { Mutex::new(ProcessInner::new()) },
+ // SAFETY: `node_refs` is initialised in the call to `mutex_init` below.
+ node_refs: unsafe { Mutex::new(ProcessNodeRefs::new()) },
+ })?;
+ let process = Ref::get_mut(&mut proc_ref).ok_or(Error::EINVAL)?;
+ // SAFETY: `inner` is pinned behind the `Arc` reference.
+ let pinned = unsafe { Pin::new_unchecked(&process.inner) };
+ kernel::mutex_init!(pinned, "Process::inner");
+ // SAFETY: `node_refs` is pinned behind the `Arc` reference.
+ let pinned = unsafe { Pin::new_unchecked(&process.node_refs) };
+ kernel::mutex_init!(pinned, "Process::node_refs");
+ Ok(proc_ref)
+ }
+
+ /// Attemps to fetch a work item from the process queue.
+ pub(crate) fn get_work(&self) -> Option<Arc<dyn DeliverToRead>> {
+ self.inner.lock().work.pop_front()
+ }
+
+ /// Attemps to fetch a work item from the process queue. If none is available, it registers the
+ /// given thread as ready to receive work directly.
+ ///
+ /// This must only be called when the thread is not participating in a transaction chain; when
+ /// it is, work will always be delivered directly to the thread (and not through the process
+ /// queue).
+ pub(crate) fn get_work_or_register<'a>(
+ &'a self,
+ thread: &'a Arc<Thread>,
+ ) -> Either<Arc<dyn DeliverToRead>, Registration<'a>> {
+ let mut inner = self.inner.lock();
+
+ // Try to get work from the process queue.
+ if let Some(work) = inner.work.pop_front() {
+ return Either::Left(work);
+ }
+
+ // Register the thread as ready.
+ Either::Right(Registration::new(self, thread, &mut inner))
+ }
+
+ fn get_thread(&self, id: i32) -> KernelResult<Arc<Thread>> {
+ // TODO: Consider using read/write locks here instead.
+ {
+ let inner = self.inner.lock();
+ if let Some(thread) = inner.threads.get(&id) {
+ return Ok(thread.clone());
+ }
+ }
+
+ // Allocate a new `Thread` without holding any locks.
+ let ta = Thread::new(id, Ref::new_from(self))?;
+
+ let mut inner = self.inner.lock();
+
+ // Recheck. It's possible the thread was create while we were not holding the lock.
+ if let Some(thread) = inner.threads.get(&id) {
+ return Ok(thread.clone());
+ }
+
+ // TODO: We need a better solution here. Since this allocates, we cannot do it while
+ // holding a spinlock because it could block. It could panic too.
+ inner.threads.insert(id, ta.clone());
+ Ok(ta)
+ }
+
+ pub(crate) fn push_work(&self, work: Arc<dyn DeliverToRead>) -> BinderResult {
+ self.inner.lock().push_work(work)
+ }
+
+ fn set_as_manager(&self, info: Option<FlatBinderObject>, thread: &Thread) -> KernelResult {
+ let (ptr, cookie) = if let Some(obj) = info {
+ (unsafe { obj.__bindgen_anon_1.binder }, obj.cookie)
+ } else {
+ (0, 0)
+ };
+ let node_ref = self.get_node(ptr as _, cookie as _, true, Some(thread))?;
+ let node = node_ref.node.clone();
+ self.ctx.set_manager_node(node_ref)?;
+ self.inner.lock().is_manager = true;
+
+ // Force the state of the node to prevent the delivery of acquire/increfs.
+ let mut owner_inner = node.owner.inner.lock();
+ node.force_has_count(&mut owner_inner);
+ Ok(())
+ }
+
+ pub(crate) fn get_node(
+ &self,
+ ptr: usize,
+ cookie: usize,
+ strong: bool,
+ thread: Option<&Thread>,
+ ) -> KernelResult<NodeRef> {
+ // Try to find an existing node.
+ {
+ let mut inner = self.inner.lock();
+ if let Some(node) = inner.get_existing_node_ref(ptr, cookie, strong, thread)? {
+ return Ok(node);
+ }
+ }
+
+ // Allocate the node before reacquiring the lock.
+ let node = Arc::try_new(Node::new(ptr, cookie, Ref::new_from(self)))?;
+
+ let mut inner = self.inner.lock();
+ if let Some(node) = inner.get_existing_node_ref(ptr, cookie, strong, thread)? {
+ return Ok(node);
+ }
+
+ // TODO: Avoid allocation while holding lock.
+ inner.nodes.insert(ptr, node.clone());
+ Ok(inner.new_node_ref(node, strong, thread))
+ }
+
+ pub(crate) fn insert_or_update_handle(
+ &self,
+ node_ref: NodeRef,
+ is_mananger: bool,
+ ) -> KernelResult<u32> {
+ let mut refs = self.node_refs.lock();
+
+ // Do a lookup before inserting.
+ if let Some(handle_ref) = refs.by_global_id.get(&node_ref.node.global_id) {
+ let handle = *handle_ref;
+ let info = refs.by_handle.get_mut(&handle).unwrap();
+ info.node_ref.absorb(node_ref);
+ return Ok(handle);
+ }
+
+ // Find id.
+ let mut target = if is_mananger { 0 } else { 1 };
+ for handle in refs.by_handle.keys() {
+ if *handle > target {
+ break;
+ }
+ if *handle == target {
+ target = target.checked_add(1).ok_or(Error::ENOMEM)?;
+ }
+ }
+
+ // Ensure the process is still alive while we insert a new reference.
+ let inner = self.inner.lock();
+ if inner.is_dead {
+ return Err(Error::ESRCH);
+ }
+ // TODO: Two allocations below.
+ refs.by_global_id.insert(node_ref.node.global_id, target);
+ refs.by_handle.insert(target, NodeRefInfo::new(node_ref));
+ Ok(target)
+ }
+
+ pub(crate) fn get_transaction_node(&self, handle: u32) -> BinderResult<NodeRef> {
+ // When handle is zero, try to get the context manager.
+ if handle == 0 {
+ self.ctx.get_manager_node(true)
+ } else {
+ self.get_node_from_handle(handle, true)
+ }
+ }
+
+ pub(crate) fn get_node_from_handle(&self, handle: u32, strong: bool) -> BinderResult<NodeRef> {
+ self.node_refs
+ .lock()
+ .by_handle
+ .get(&handle)
+ .ok_or(Error::ENOENT)?
+ .node_ref
+ .clone(strong)
+ }
+
+ pub(crate) fn remove_from_delivered_deaths(&self, death: &Arc<NodeDeath>) {
+ let mut inner = self.inner.lock();
+ let removed = unsafe { inner.delivered_deaths.remove(death) };
+ drop(inner);
+ drop(removed);
+ }
+
+ pub(crate) fn update_ref(&self, handle: u32, inc: bool, strong: bool) -> KernelResult {
+ if inc && handle == 0 {
+ if let Ok(node_ref) = self.ctx.get_manager_node(strong) {
+ if core::ptr::eq(self, &*node_ref.node.owner) {
+ return Err(Error::EINVAL);
+ }
+ let _ = self.insert_or_update_handle(node_ref, true);
+ return Ok(());
+ }
+ }
+
+ // To preserve original binder behaviour, we only fail requests where the manager tries to
+ // increment references on itself.
+ let mut refs = self.node_refs.lock();
+ if let Some(info) = refs.by_handle.get_mut(&handle) {
+ if info.node_ref.update(inc, strong) {
+ // Clean up death if there is one attached to this node reference.
+ if let Some(death) = info.death.take() {
+ death.set_cleared(true);
+ self.remove_from_delivered_deaths(&death);
+ }
+
+ // Remove reference from process tables.
+ let id = info.node_ref.node.global_id;
+ refs.by_handle.remove(&handle);
+ refs.by_global_id.remove(&id);
+ }
+ }
+ Ok(())
+ }
+
+ /// Decrements the refcount of the given node, if one exists.
+ pub(crate) fn update_node(&self, ptr: usize, cookie: usize, strong: bool, biased: bool) {
+ let mut inner = self.inner.lock();
+ if let Ok(Some(node)) = inner.get_existing_node(ptr, cookie) {
+ inner.update_node_refcount(&node, false, strong, biased, None);
+ }
+ }
+
+ pub(crate) fn inc_ref_done(
+ &self,
+ reader: &mut UserSlicePtrReader,
+ strong: bool,
+ ) -> KernelResult {
+ let ptr = reader.read::<usize>()?;
+ let cookie = reader.read::<usize>()?;
+ self.update_node(ptr, cookie, strong, true);
+ Ok(())
+ }
+
+ pub(crate) fn buffer_alloc(&self, size: usize) -> BinderResult<Allocation> {
+ let mut inner = self.inner.lock();
+ let mapping = inner.mapping.as_mut().ok_or_else(BinderError::new_dead)?;
+
+ let offset = mapping.alloc.reserve_new(size)?;
+ Ok(Allocation::new(
+ self,
+ offset,
+ size,
+ mapping.address + offset,
+ mapping.pages.clone(),
+ ))
+ }
+
+ // TODO: Review if we want an Option or a KernelResult.
+ pub(crate) fn buffer_get(&self, ptr: usize) -> Option<Allocation> {
+ let mut inner = self.inner.lock();
+ let mapping = inner.mapping.as_mut()?;
+ let offset = ptr - mapping.address;
+ let (size, odata) = mapping.alloc.reserve_existing(offset).ok()?;
+ let mut alloc = Allocation::new(self, offset, size, ptr, mapping.pages.clone());
+ if let Some(data) = odata {
+ alloc.set_info(data);
+ }
+ Some(alloc)
+ }
+
+ pub(crate) fn buffer_raw_free(&self, ptr: usize) {
+ let mut inner = self.inner.lock();
+ if let Some(ref mut mapping) = &mut inner.mapping {
+ if mapping
+ .alloc
+ .reservation_abort(ptr - mapping.address)
+ .is_err()
+ {
+ pr_warn!("Offset {} failed to free\n", ptr - mapping.address);
+ }
+ }
+ }
+
+ pub(crate) fn buffer_make_freeable(&self, offset: usize, data: Option<AllocationInfo>) {
+ let mut inner = self.inner.lock();
+ if let Some(ref mut mapping) = &mut inner.mapping {
+ if mapping.alloc.reservation_commit(offset, data).is_err() {
+ pr_warn!("Offset {} failed to be marked freeable\n", offset);
+ }
+ }
+ }
+
+ fn create_mapping(&self, vma: &mut bindings::vm_area_struct) -> KernelResult {
+ let size = core::cmp::min(
+ (vma.vm_end - vma.vm_start) as usize,
+ bindings::SZ_4M as usize,
+ );
+ let page_count = size >> bindings::PAGE_SHIFT;
+
+ // Allocate and map all pages.
+ //
+ // N.B. If we fail halfway through mapping these pages, the kernel will unmap them.
+ let mut pages = Vec::new();
+ pages.try_reserve_exact(page_count)?;
+ let mut address = vma.vm_start as usize;
+ for _ in 0..page_count {
+ let page = Pages::<0>::new()?;
+ page.insert_page(vma, address)?;
+ pages.push(page);
+ address += 1 << bindings::PAGE_SHIFT;
+ }
+
+ // TODO: This allocates memory.
+ let arc = Arc::from(pages);
+
+ // Save pages for later.
+ let mut inner = self.inner.lock();
+ match &inner.mapping {
+ None => inner.mapping = Some(Mapping::new(vma.vm_start as _, size, arc)?),
+ Some(_) => return Err(Error::EBUSY),
+ }
+ Ok(())
+ }
+
+ fn version(&self, data: UserSlicePtr) -> KernelResult {
+ data.writer().write(&BinderVersion::current())
+ }
+
+ pub(crate) fn register_thread(&self) -> bool {
+ self.inner.lock().register_thread()
+ }
+
+ fn remove_thread(&self, thread: Arc<Thread>) {
+ self.inner.lock().threads.remove(&thread.id);
+ thread.release();
+ }
+
+ fn set_max_threads(&self, max: u32) {
+ self.inner.lock().max_threads = max;
+ }
+
+ fn get_node_debug_info(&self, data: UserSlicePtr) -> KernelResult {
+ let (mut reader, mut writer) = data.reader_writer();
+
+ // Read the starting point.
+ let ptr = reader.read::<BinderNodeDebugInfo>()?.ptr as usize;
+ let mut out = BinderNodeDebugInfo::default();
+
+ {
+ let inner = self.inner.lock();
+ for (node_ptr, node) in &inner.nodes {
+ if *node_ptr > ptr {
+ node.populate_debug_info(&mut out, &inner);
+ break;
+ }
+ }
+ }
+
+ writer.write(&out)
+ }
+
+ fn get_node_info_from_ref(&self, data: UserSlicePtr) -> KernelResult {
+ let (mut reader, mut writer) = data.reader_writer();
+ let mut out = reader.read::<BinderNodeInfoForRef>()?;
+
+ if out.strong_count != 0
+ || out.weak_count != 0
+ || out.reserved1 != 0
+ || out.reserved2 != 0
+ || out.reserved3 != 0
+ {
+ return Err(Error::EINVAL);
+ }
+
+ // Only the context manager is allowed to use this ioctl.
+ if !self.inner.lock().is_manager {
+ return Err(Error::EPERM);
+ }
+
+ let node_ref = self
+ .get_node_from_handle(out.handle, true)
+ .or(Err(Error::EINVAL))?;
+
+ // Get the counts from the node.
+ {
+ let owner_inner = node_ref.node.owner.inner.lock();
+ node_ref.node.populate_counts(&mut out, &owner_inner);
+ }
+
+ // Write the result back.
+ writer.write(&out)
+ }
+
+ pub(crate) fn needs_thread(&self) -> bool {
+ let mut inner = self.inner.lock();
+ let ret = inner.requested_thread_count == 0
+ && inner.ready_threads.is_empty()
+ && inner.started_thread_count < inner.max_threads;
+ if ret {
+ inner.requested_thread_count += 1
+ };
+ ret
+ }
+
+ pub(crate) fn request_death(
+ &self,
+ reader: &mut UserSlicePtrReader,
+ thread: &Thread,
+ ) -> KernelResult {
+ let handle: u32 = reader.read()?;
+ let cookie: usize = reader.read()?;
+
+ // TODO: First two should result in error, but not the others.
+
+ // TODO: Do we care about the context manager dying?
+
+ // Queue BR_ERROR if we can't allocate memory for the death notification.
+ let death = ArcReservation::new().map_err(|err| {
+ thread.push_return_work(BR_ERROR);
+ err
+ })?;
+
+ let mut refs = self.node_refs.lock();
+ let info = refs.by_handle.get_mut(&handle).ok_or(Error::EINVAL)?;
+
+ // Nothing to do if there is already a death notification request for this handle.
+ if info.death.is_some() {
+ return Ok(());
+ }
+
+ // SAFETY: `init` is called below.
+ let death = death.commit(unsafe {
+ NodeDeath::new(info.node_ref.node.clone(), Ref::new_from(self), cookie)
+ });
+ // SAFETY: `death` is pinned behind the `Arc` reference.
+ unsafe { Pin::new_unchecked(death.as_ref()) }.init();
+ info.death = Some(death.clone());
+
+ // Register the death notification.
+ {
+ let mut owner_inner = info.node_ref.node.owner.inner.lock();
+ if owner_inner.is_dead {
+ drop(owner_inner);
+ let _ = self.push_work(death);
+ } else {
+ info.node_ref.node.add_death(death, &mut owner_inner);
+ }
+ }
+ Ok(())
+ }
+
+ pub(crate) fn clear_death(
+ &self,
+ reader: &mut UserSlicePtrReader,
+ thread: &Thread,
+ ) -> KernelResult {
+ let handle: u32 = reader.read()?;
+ let cookie: usize = reader.read()?;
+
+ let mut refs = self.node_refs.lock();
+ let info = refs.by_handle.get_mut(&handle).ok_or(Error::EINVAL)?;
+
+ let death = info.death.take().ok_or(Error::EINVAL)?;
+ if death.cookie != cookie {
+ info.death = Some(death);
+ return Err(Error::EINVAL);
+ }
+
+ // Update state and determine if we need to queue a work item. We only need to do it when
+ // the node is not dead or if the user already completed the death notification.
+ if death.set_cleared(false) {
+ let _ = thread.push_work_if_looper(death);
+ }
+
+ Ok(())
+ }
+
+ pub(crate) fn dead_binder_done(&self, cookie: usize, thread: &Thread) {
+ if let Some(death) = self.inner.lock().pull_delivered_death(cookie) {
+ death.set_notification_done(thread);
+ }
+ }
+}
+
+impl IoctlHandler for Process {
+ fn write(&self, _file: &File, cmd: u32, reader: &mut UserSlicePtrReader) -> KernelResult<i32> {
+ let thread = self.get_thread(unsafe { rust_helper_current_pid() })?;
+ match cmd {
+ bindings::BINDER_SET_MAX_THREADS => self.set_max_threads(reader.read()?),
+ bindings::BINDER_SET_CONTEXT_MGR => self.set_as_manager(None, &thread)?,
+ bindings::BINDER_THREAD_EXIT => self.remove_thread(thread),
+ bindings::BINDER_SET_CONTEXT_MGR_EXT => {
+ self.set_as_manager(Some(reader.read()?), &thread)?
+ }
+ _ => return Err(Error::EINVAL),
+ }
+ Ok(0)
+ }
+
+ fn read_write(&self, file: &File, cmd: u32, data: UserSlicePtr) -> KernelResult<i32> {
+ let thread = self.get_thread(unsafe { rust_helper_current_pid() })?;
+ match cmd {
+ bindings::BINDER_WRITE_READ => thread.write_read(data, file.is_blocking())?,
+ bindings::BINDER_GET_NODE_DEBUG_INFO => self.get_node_debug_info(data)?,
+ bindings::BINDER_GET_NODE_INFO_FOR_REF => self.get_node_info_from_ref(data)?,
+ bindings::BINDER_VERSION => self.version(data)?,
+ _ => return Err(Error::EINVAL),
+ }
+ Ok(0)
+ }
+}
+
+unsafe impl RefCounted for Process {
+ fn get_count(&self) -> &RefCount {
+ &self.ref_count
+ }
+}
+
+impl FileOpener<Arc<Context>> for Process {
+ fn open(ctx: &Arc<Context>) -> KernelResult<Self::Wrapper> {
+ let process = Self::new(ctx.clone())?;
+ // SAFETY: Pointer is pinned behind `Ref`.
+ Ok(unsafe { Pin::new_unchecked(process) })
+ }
+}
+
+impl FileOperations for Process {
+ type Wrapper = Pin<Ref<Self>>;
+
+ kernel::declare_file_operations!(ioctl, compat_ioctl, mmap, poll);
+
+ fn release(obj: Self::Wrapper, _file: &File) {
+ // Mark this process as dead. We'll do the same for the threads later.
+ obj.inner.lock().is_dead = true;
+
+ // If this process is the manager, unset it.
+ if obj.inner.lock().is_manager {
+ obj.ctx.unset_manager_node();
+ }
+
+ // TODO: Do this in a worker?
+
+ // Cancel all pending work items.
+ while let Some(work) = obj.get_work() {
+ work.cancel();
+ }
+
+ // Free any resources kept alive by allocated buffers.
+ let omapping = obj.inner.lock().mapping.take();
+ if let Some(mut mapping) = omapping {
+ let address = mapping.address;
+ let pages = mapping.pages.clone();
+ mapping.alloc.for_each(|offset, size, odata| {
+ let ptr = offset + address;
+ let mut alloc = Allocation::new(&obj, offset, size, ptr, pages.clone());
+ if let Some(data) = odata {
+ alloc.set_info(data);
+ }
+ drop(alloc)
+ });
+ }
+
+ // Drop all references. We do this dance with `swap` to avoid destroying the references
+ // while holding the lock.
+ let mut refs = obj.node_refs.lock();
+ let mut node_refs = BTreeMap::new();
+ swap(&mut refs.by_handle, &mut node_refs);
+ drop(refs);
+
+ // Remove all death notifications from the nodes (that belong to a different process).
+ for info in node_refs.values_mut() {
+ let death = if let Some(existing) = info.death.take() {
+ existing
+ } else {
+ continue;
+ };
+
+ death.set_cleared(false);
+ }
+
+ // Do similar dance for the state lock.
+ let mut inner = obj.inner.lock();
+ let mut threads = BTreeMap::new();
+ let mut nodes = BTreeMap::new();
+ swap(&mut inner.threads, &mut threads);
+ swap(&mut inner.nodes, &mut nodes);
+ drop(inner);
+
+ // Release all threads.
+ for thread in threads.values() {
+ thread.release();
+ }
+
+ // Deliver death notifications.
+ for node in nodes.values() {
+ loop {
+ let death = {
+ let mut inner = obj.inner.lock();
+ if let Some(death) = node.next_death(&mut inner) {
+ death
+ } else {
+ break;
+ }
+ };
+
+ death.set_dead();
+ }
+ }
+ }
+
+ fn ioctl(&self, file: &File, cmd: &mut IoctlCommand) -> KernelResult<i32> {
+ cmd.dispatch(self, file)
+ }
+
+ fn compat_ioctl(&self, file: &File, cmd: &mut IoctlCommand) -> KernelResult<i32> {
+ cmd.dispatch(self, file)
+ }
+
+ fn mmap(&self, _file: &File, vma: &mut bindings::vm_area_struct) -> KernelResult {
+ // TODO: Only group leader is allowed to create mappings.
+
+ if vma.vm_start == 0 {
+ return Err(Error::EINVAL);
+ }
+
+ if (vma.vm_flags & (bindings::VM_WRITE as c_types::c_ulong)) != 0 {
+ return Err(Error::EPERM);
+ }
+
+ vma.vm_flags |= (bindings::VM_DONTCOPY | bindings::VM_MIXEDMAP) as c_types::c_ulong;
+ vma.vm_flags &= !(bindings::VM_MAYWRITE as c_types::c_ulong);
+
+ // TODO: Set ops. We need to learn when the user unmaps so that we can stop using it.
+ self.create_mapping(vma)
+ }
+
+ fn poll(&self, file: &File, table: &PollTable) -> KernelResult<u32> {
+ let thread = self.get_thread(unsafe { rust_helper_current_pid() })?;
+ let (from_proc, mut mask) = thread.poll(file, table);
+ if mask == 0 && from_proc && !self.inner.lock().work.is_empty() {
+ mask |= bindings::POLLIN;
+ }
+ Ok(mask)
+ }
+}
+
+pub(crate) struct Registration<'a> {
+ process: &'a Process,
+ thread: &'a Arc<Thread>,
+}
+
+impl<'a> Registration<'a> {
+ fn new(
+ process: &'a Process,
+ thread: &'a Arc<Thread>,
+ guard: &mut Guard<Mutex<ProcessInner>>,
+ ) -> Self {
+ guard.ready_threads.push_back(thread.clone());
+ Self { process, thread }
+ }
+}
+
+impl Drop for Registration<'_> {
+ fn drop(&mut self) {
+ let mut inner = self.process.inner.lock();
+ unsafe { inner.ready_threads.remove(self.thread) };
+ }
+}
diff --git a/drivers/android/range_alloc.rs b/drivers/android/range_alloc.rs
new file mode 100644
index 0000000000000..0278041cdf763
--- /dev/null
+++ b/drivers/android/range_alloc.rs
@@ -0,0 +1,191 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use alloc::boxed::Box;
+use core::ptr::NonNull;
+use kernel::{
+ linked_list::{CursorMut, GetLinks, Links, List},
+ prelude::*,
+ Error,
+};
+
+pub(crate) struct RangeAllocator<T> {
+ list: List<Box<Descriptor<T>>>,
+}
+
+#[derive(Debug, PartialEq, Eq)]
+enum DescriptorState {
+ Free,
+ Reserved,
+ Allocated,
+}
+
+impl<T> RangeAllocator<T> {
+ pub(crate) fn new(size: usize) -> KernelResult<Self> {
+ let desc = Box::try_new(Descriptor::new(0, size))?;
+ let mut list = List::new();
+ list.push_back(desc);
+ Ok(Self { list })
+ }
+
+ fn find_best_match(&self, size: usize) -> Option<NonNull<Descriptor<T>>> {
+ // TODO: Use a binary tree instead of list for this lookup.
+ let mut best = None;
+ let mut best_size = usize::MAX;
+ let mut cursor = self.list.cursor_front();
+ while let Some(desc) = cursor.current() {
+ if desc.state == DescriptorState::Free {
+ if size == desc.size {
+ return Some(NonNull::from(desc));
+ }
+
+ if size < desc.size && desc.size < best_size {
+ best = Some(NonNull::from(desc));
+ best_size = desc.size;
+ }
+ }
+
+ cursor.move_next();
+ }
+ best
+ }
+
+ pub(crate) fn reserve_new(&mut self, size: usize) -> KernelResult<usize> {
+ let desc_ptr = match self.find_best_match(size) {
+ None => return Err(Error::ENOMEM),
+ Some(found) => found,
+ };
+
+ // SAFETY: We hold the only mutable reference to list, so it cannot have changed.
+ let desc = unsafe { &mut *desc_ptr.as_ptr() };
+ if desc.size == size {
+ desc.state = DescriptorState::Reserved;
+ return Ok(desc.offset);
+ }
+
+ // We need to break up the descriptor.
+ let new = Box::try_new(Descriptor::new(desc.offset + size, desc.size - size))?;
+ unsafe { self.list.insert_after(desc_ptr, new) };
+ desc.state = DescriptorState::Reserved;
+ desc.size = size;
+ Ok(desc.offset)
+ }
+
+ fn free_with_cursor(cursor: &mut CursorMut<Box<Descriptor<T>>>) -> KernelResult {
+ let mut size = match cursor.current() {
+ None => return Err(Error::EINVAL),
+ Some(ref mut entry) => {
+ match entry.state {
+ DescriptorState::Free => return Err(Error::EINVAL),
+ DescriptorState::Allocated => return Err(Error::EPERM),
+ DescriptorState::Reserved => {}
+ }
+ entry.state = DescriptorState::Free;
+ entry.size
+ }
+ };
+
+ // Try to merge with the next entry.
+ if let Some(next) = cursor.peek_next() {
+ if next.state == DescriptorState::Free {
+ next.offset -= size;
+ next.size += size;
+ size = next.size;
+ cursor.remove_current();
+ }
+ }
+
+ // Try to merge with the previous entry.
+ if let Some(prev) = cursor.peek_prev() {
+ if prev.state == DescriptorState::Free {
+ prev.size += size;
+ cursor.remove_current();
+ }
+ }
+
+ Ok(())
+ }
+
+ fn find_at_offset(&mut self, offset: usize) -> Option<CursorMut<Box<Descriptor<T>>>> {
+ let mut cursor = self.list.cursor_front_mut();
+ while let Some(desc) = cursor.current() {
+ if desc.offset == offset {
+ return Some(cursor);
+ }
+
+ if desc.offset > offset {
+ return None;
+ }
+
+ cursor.move_next();
+ }
+ None
+ }
+
+ pub(crate) fn reservation_abort(&mut self, offset: usize) -> KernelResult {
+ // TODO: The force case is currently O(n), but could be made O(1) with unsafe.
+ let mut cursor = self.find_at_offset(offset).ok_or(Error::EINVAL)?;
+ Self::free_with_cursor(&mut cursor)
+ }
+
+ pub(crate) fn reservation_commit(&mut self, offset: usize, data: Option<T>) -> KernelResult {
+ // TODO: This is currently O(n), make it O(1).
+ let mut cursor = self.find_at_offset(offset).ok_or(Error::ENOENT)?;
+ let desc = cursor.current().unwrap();
+ desc.state = DescriptorState::Allocated;
+ desc.data = data;
+ Ok(())
+ }
+
+ /// Takes an entry at the given offset from [`DescriptorState::Allocated`] to
+ /// [`DescriptorState::Reserved`].
+ ///
+ /// Returns the size of the existing entry and the data associated with it.
+ pub(crate) fn reserve_existing(&mut self, offset: usize) -> KernelResult<(usize, Option<T>)> {
+ // TODO: This is currently O(n), make it O(log n).
+ let mut cursor = self.find_at_offset(offset).ok_or(Error::ENOENT)?;
+ let desc = cursor.current().unwrap();
+ if desc.state != DescriptorState::Allocated {
+ return Err(Error::ENOENT);
+ }
+ desc.state = DescriptorState::Reserved;
+ Ok((desc.size, desc.data.take()))
+ }
+
+ pub(crate) fn for_each<F: Fn(usize, usize, Option<T>)>(&mut self, callback: F) {
+ let mut cursor = self.list.cursor_front_mut();
+ while let Some(desc) = cursor.current() {
+ if desc.state == DescriptorState::Allocated {
+ callback(desc.offset, desc.size, desc.data.take());
+ }
+
+ cursor.move_next();
+ }
+ }
+}
+
+struct Descriptor<T> {
+ state: DescriptorState,
+ size: usize,
+ offset: usize,
+ links: Links<Descriptor<T>>,
+ data: Option<T>,
+}
+
+impl<T> Descriptor<T> {
+ fn new(offset: usize, size: usize) -> Self {
+ Self {
+ size,
+ offset,
+ state: DescriptorState::Free,
+ links: Links::new(),
+ data: None,
+ }
+ }
+}
+
+impl<T> GetLinks for Descriptor<T> {
+ type EntryType = Self;
+ fn get_links(desc: &Self) -> &Links<Self> {
+ &desc.links
+ }
+}
diff --git a/drivers/android/rust_binder.rs b/drivers/android/rust_binder.rs
new file mode 100644
index 0000000000000..aaef4517d48ff
--- /dev/null
+++ b/drivers/android/rust_binder.rs
@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Binder -- the Android IPC mechanism.
+//!
+//! TODO: This module is a work in progress.
+
+#![no_std]
+#![feature(global_asm, try_reserve, allocator_api, concat_idents)]
+
+use alloc::{boxed::Box, sync::Arc};
+use core::pin::Pin;
+use kernel::{
+ cstr,
+ linked_list::{GetLinks, GetLinksWrapped, Links},
+ miscdev::Registration,
+ prelude::*,
+ user_ptr::UserSlicePtrWriter,
+};
+
+mod allocation;
+mod context;
+mod defs;
+mod node;
+mod process;
+mod range_alloc;
+mod thread;
+mod transaction;
+
+use {context::Context, thread::Thread};
+
+module! {
+ type: BinderModule,
+ name: b"rust_binder",
+ author: b"Wedson Almeida Filho",
+ description: b"Android Binder",
+ license: b"GPL v2",
+ params: {},
+}
+
+enum Either<L, R> {
+ Left(L),
+ Right(R),
+}
+
+trait DeliverToRead {
+ /// Performs work. Returns true if remaining work items in the queue should be processed
+ /// immediately, or false if it should return to caller before processing additional work
+ /// items.
+ fn do_work(
+ self: Arc<Self>,
+ thread: &Thread,
+ writer: &mut UserSlicePtrWriter,
+ ) -> KernelResult<bool>;
+
+ /// Cancels the given work item. This is called instead of [`DeliverToRead::do_work`] when work
+ /// won't be delivered.
+ fn cancel(self: Arc<Self>) {}
+
+ /// Returns the linked list links for the work item.
+ fn get_links(&self) -> &Links<dyn DeliverToRead>;
+}
+
+struct DeliverToReadListAdapter {}
+
+impl GetLinks for DeliverToReadListAdapter {
+ type EntryType = dyn DeliverToRead;
+
+ fn get_links(data: &Self::EntryType) -> &Links<Self::EntryType> {
+ data.get_links()
+ }
+}
+
+impl GetLinksWrapped for DeliverToReadListAdapter {
+ type Wrapped = Arc<dyn DeliverToRead>;
+}
+
+struct DeliverCode {
+ code: u32,
+ links: Links<dyn DeliverToRead>,
+}
+
+impl DeliverCode {
+ fn new(code: u32) -> Self {
+ Self {
+ code,
+ links: Links::new(),
+ }
+ }
+}
+
+impl DeliverToRead for DeliverCode {
+ fn do_work(
+ self: Arc<Self>,
+ _thread: &Thread,
+ writer: &mut UserSlicePtrWriter,
+ ) -> KernelResult<bool> {
+ writer.write(&self.code)?;
+ Ok(true)
+ }
+
+ fn get_links(&self) -> &Links<dyn DeliverToRead> {
+ &self.links
+ }
+}
+
+const fn ptr_align(value: usize) -> usize {
+ let size = core::mem::size_of::<usize>() - 1;
+ (value + size) & !size
+}
+
+unsafe impl Sync for BinderModule {}
+
+struct BinderModule {
+ _reg: Pin<Box<Registration<Arc<Context>>>>,
+}
+
+impl KernelModule for BinderModule {
+ fn init() -> KernelResult<Self> {
+ let pinned_ctx = Context::new()?;
+ let ctx = unsafe { Pin::into_inner_unchecked(pinned_ctx) };
+ let reg = Registration::<Arc<Context>>::new_pinned::<process::Process>(
+ cstr!("rust_binder"),
+ None,
+ ctx,
+ )?;
+ Ok(Self { _reg: reg })
+ }
+}
diff --git a/drivers/android/thread.rs b/drivers/android/thread.rs
new file mode 100644
index 0000000000000..f84fe9df988bc
--- /dev/null
+++ b/drivers/android/thread.rs
@@ -0,0 +1,821 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use alloc::sync::Arc;
+use core::{alloc::AllocError, mem::size_of, pin::Pin};
+use kernel::{
+ bindings,
+ file_operations::{File, PollTable},
+ linked_list::{GetLinks, Links, List},
+ prelude::*,
+ sync::{CondVar, Ref, SpinLock},
+ user_ptr::{UserSlicePtr, UserSlicePtrWriter},
+ Error,
+};
+
+use crate::{
+ allocation::{Allocation, AllocationView},
+ defs::*,
+ process::{AllocationInfo, Process},
+ ptr_align,
+ transaction::Transaction,
+ DeliverCode, DeliverToRead, DeliverToReadListAdapter, Either,
+};
+
+pub(crate) type BinderResult<T = ()> = Result<T, BinderError>;
+
+pub(crate) struct BinderError {
+ pub(crate) reply: u32,
+}
+
+impl BinderError {
+ pub(crate) fn new_failed() -> Self {
+ Self {
+ reply: BR_FAILED_REPLY,
+ }
+ }
+
+ pub(crate) fn new_dead() -> Self {
+ Self {
+ reply: BR_DEAD_REPLY,
+ }
+ }
+}
+
+impl From<Error> for BinderError {
+ fn from(_: Error) -> Self {
+ Self::new_failed()
+ }
+}
+
+impl From<AllocError> for BinderError {
+ fn from(_: AllocError) -> Self {
+ Self::new_failed()
+ }
+}
+
+const LOOPER_REGISTERED: u32 = 0x01;
+const LOOPER_ENTERED: u32 = 0x02;
+const LOOPER_EXITED: u32 = 0x04;
+const LOOPER_INVALID: u32 = 0x08;
+const LOOPER_WAITING: u32 = 0x10;
+const LOOPER_POLL: u32 = 0x20;
+
+struct InnerThread {
+ /// Determines the looper state of the thread. It is a bit-wise combination of the constants
+ /// prefixed with `LOOPER_`.
+ looper_flags: u32,
+
+ /// Determines if thread is dead.
+ is_dead: bool,
+
+ /// Work item used to deliver error codes to the thread that started a transaction. When set to
+ /// `Some(x)`, it will hold the only reference to the object so that it can update the error
+ /// code to be delivered before queuing it.
+ reply_work: Option<Arc<ThreadError>>,
+
+ /// Work item used to deliver error codes to the current thread. When set to `Some(x)`, it will
+ /// hold the only reference to the object so that it can update the error code to be delivered
+ /// before queuing.
+ return_work: Option<Arc<ThreadError>>,
+
+ /// Determines whether the work list below should be processed. When set to false, `work_list`
+ /// is treated as if it were empty.
+ process_work_list: bool,
+ work_list: List<DeliverToReadListAdapter>,
+ current_transaction: Option<Arc<Transaction>>,
+}
+
+impl InnerThread {
+ fn new() -> Self {
+ Self {
+ looper_flags: 0,
+ is_dead: false,
+ process_work_list: false,
+ work_list: List::new(),
+ current_transaction: None,
+ return_work: None,
+ reply_work: None,
+ }
+ }
+
+ fn set_reply_work(&mut self, reply_work: Arc<ThreadError>) {
+ self.reply_work = Some(reply_work);
+ }
+
+ fn push_reply_work(&mut self, code: u32) {
+ let work = self.reply_work.take();
+ self.push_existing_work(work, code);
+ }
+
+ fn set_return_work(&mut self, return_work: Arc<ThreadError>) {
+ self.return_work = Some(return_work);
+ }
+
+ fn push_return_work(&mut self, code: u32) {
+ let work = self.return_work.take();
+ self.push_existing_work(work, code);
+ }
+
+ fn push_existing_work(&mut self, owork: Option<Arc<ThreadError>>, code: u32) {
+ // TODO: Write some warning when the following fails. It should not happen, and
+ // if it does, there is likely something wrong.
+ if let Some(mut work) = owork {
+ if let Some(work_mut) = Arc::get_mut(&mut work) {
+ work_mut.error_code = code;
+ self.push_work(work);
+ }
+ }
+ }
+
+ fn pop_work(&mut self) -> Option<Arc<dyn DeliverToRead>> {
+ if !self.process_work_list {
+ return None;
+ }
+
+ let ret = self.work_list.pop_front();
+ // Once the queue is drained, we stop processing it until a non-deferred item is pushed
+ // again onto it.
+ self.process_work_list = !self.work_list.is_empty();
+ ret
+ }
+
+ fn push_work_deferred(&mut self, work: Arc<dyn DeliverToRead>) {
+ self.work_list.push_back(work);
+ }
+
+ fn push_work(&mut self, work: Arc<dyn DeliverToRead>) {
+ self.push_work_deferred(work);
+ self.process_work_list = true;
+ }
+
+ fn has_work(&self) -> bool {
+ self.process_work_list && !self.work_list.is_empty()
+ }
+
+ /// Fetches the transaction the thread can reply to. If the thread has a pending transaction
+ /// (that it could respond to) but it has also issued a transaction, it must first wait for the
+ /// previously-issued transaction to complete.
+ fn pop_transaction_to_reply(&mut self, thread: &Thread) -> KernelResult<Arc<Transaction>> {
+ let transaction = self.current_transaction.take().ok_or(Error::EINVAL)?;
+
+ if core::ptr::eq(thread, transaction.from.as_ref()) {
+ self.current_transaction = Some(transaction);
+ return Err(Error::EINVAL);
+ }
+
+ // Find a new current transaction for this thread.
+ self.current_transaction = transaction.find_from(thread);
+ Ok(transaction)
+ }
+
+ fn pop_transaction_replied(&mut self, transaction: &Arc<Transaction>) -> bool {
+ match self.current_transaction.take() {
+ None => false,
+ Some(old) => {
+ if !Arc::ptr_eq(transaction, &old) {
+ self.current_transaction = Some(old);
+ return false;
+ }
+ self.current_transaction = old.clone_next();
+ true
+ }
+ }
+ }
+
+ fn looper_enter(&mut self) {
+ self.looper_flags |= LOOPER_ENTERED;
+ if self.looper_flags & LOOPER_REGISTERED != 0 {
+ self.looper_flags |= LOOPER_INVALID;
+ }
+ }
+
+ fn looper_register(&mut self, valid: bool) {
+ self.looper_flags |= LOOPER_REGISTERED;
+ if !valid || self.looper_flags & LOOPER_ENTERED != 0 {
+ self.looper_flags |= LOOPER_INVALID;
+ }
+ }
+
+ fn looper_exit(&mut self) {
+ self.looper_flags |= LOOPER_EXITED;
+ }
+
+ /// Determines whether the thread is part of a pool, i.e., if it is a looper.
+ fn is_looper(&self) -> bool {
+ self.looper_flags & (LOOPER_ENTERED | LOOPER_REGISTERED) != 0
+ }
+
+ /// Determines whether the thread should attempt to fetch work items from the process queue
+ /// (when its own queue is empty). This is case when the thread is not part of a transaction
+ /// stack and it is registered as a looper.
+ fn should_use_process_work_queue(&self) -> bool {
+ self.current_transaction.is_none() && self.is_looper()
+ }
+
+ fn poll(&mut self) -> u32 {
+ self.looper_flags |= LOOPER_POLL;
+ if self.has_work() {
+ bindings::POLLIN
+ } else {
+ 0
+ }
+ }
+}
+
+pub(crate) struct Thread {
+ pub(crate) id: i32,
+ pub(crate) process: Ref<Process>,
+ inner: SpinLock<InnerThread>,
+ work_condvar: CondVar,
+ links: Links<Thread>,
+}
+
+impl Thread {
+ pub(crate) fn new(id: i32, process: Ref<Process>) -> KernelResult<Arc<Self>> {
+ let return_work = Arc::try_new(ThreadError::new(InnerThread::set_return_work))?;
+ let reply_work = Arc::try_new(ThreadError::new(InnerThread::set_reply_work))?;
+ let mut arc = Arc::try_new(Self {
+ id,
+ process,
+ // SAFETY: `inner` is initialised in the call to `spinlock_init` below.
+ inner: unsafe { SpinLock::new(InnerThread::new()) },
+ // SAFETY: `work_condvar` is initalised in the call to `condvar_init` below.
+ work_condvar: unsafe { CondVar::new() },
+ links: Links::new(),
+ })?;
+ {
+ let mut inner = arc.inner.lock();
+ inner.set_reply_work(reply_work);
+ inner.set_return_work(return_work);
+ }
+ let thread = Arc::get_mut(&mut arc).unwrap();
+ // SAFETY: `inner` is pinned behind the `Arc` reference.
+ let inner = unsafe { Pin::new_unchecked(&thread.inner) };
+ kernel::spinlock_init!(inner, "Thread::inner");
+ kernel::condvar_init!(thread.pinned_condvar(), "Thread::work_condvar");
+ Ok(arc)
+ }
+
+ fn pinned_condvar(&self) -> Pin<&CondVar> {
+ unsafe { Pin::new_unchecked(&self.work_condvar) }
+ }
+
+ pub(crate) fn set_current_transaction(&self, transaction: Arc<Transaction>) {
+ self.inner.lock().current_transaction = Some(transaction);
+ }
+
+ /// Attempts to fetch a work item from the thread-local queue. The behaviour if the queue is
+ /// empty depends on `wait`: if it is true, the function waits for some work to be queued (or a
+ /// signal); otherwise it returns indicating that none is available.
+ fn get_work_local(self: &Arc<Self>, wait: bool) -> KernelResult<Arc<dyn DeliverToRead>> {
+ // Try once if the caller does not want to wait.
+ if !wait {
+ return self.inner.lock().pop_work().ok_or(Error::EAGAIN);
+ }
+
+ // Loop waiting only on the local queue (i.e., not registering with the process queue).
+ let cv = self.pinned_condvar();
+ let mut inner = self.inner.lock();
+ loop {
+ if let Some(work) = inner.pop_work() {
+ return Ok(work);
+ }
+
+ inner.looper_flags |= LOOPER_WAITING;
+ let signal_pending = cv.wait(&mut inner);
+ inner.looper_flags &= !LOOPER_WAITING;
+
+ if signal_pending {
+ return Err(Error::ERESTARTSYS);
+ }
+ }
+ }
+
+ /// Attempts to fetch a work item from the thread-local queue, falling back to the process-wide
+ /// queue if none is available locally.
+ ///
+ /// This must only be called when the thread is not participating in a transaction chain. If it
+ /// is, the local version (`get_work_local`) should be used instead.
+ fn get_work(self: &Arc<Self>, wait: bool) -> KernelResult<Arc<dyn DeliverToRead>> {
+ // Try to get work from the thread's work queue, using only a local lock.
+ {
+ let mut inner = self.inner.lock();
+ if let Some(work) = inner.pop_work() {
+ return Ok(work);
+ }
+ }
+
+ // If the caller doesn't want to wait, try to grab work from the process queue.
+ //
+ // We know nothing will have been queued directly to the thread queue because it is not in
+ // a transaction and it is not in the process' ready list.
+ if !wait {
+ return self.process.get_work().ok_or(Error::EAGAIN);
+ }
+
+ // Get work from the process queue. If none is available, atomically register as ready.
+ let reg = match self.process.get_work_or_register(self) {
+ Either::Left(work) => return Ok(work),
+ Either::Right(reg) => reg,
+ };
+
+ let cv = self.pinned_condvar();
+ let mut inner = self.inner.lock();
+ loop {
+ if let Some(work) = inner.pop_work() {
+ return Ok(work);
+ }
+
+ inner.looper_flags |= LOOPER_WAITING;
+ let signal_pending = cv.wait(&mut inner);
+ inner.looper_flags &= !LOOPER_WAITING;
+
+ if signal_pending {
+ // A signal is pending. We need to pull the thread off the list, then check the
+ // state again after it's off the list to ensure that something was not queued in
+ // the meantime. If something has been queued, we just return it (instead of the
+ // error).
+ drop(inner);
+ drop(reg);
+ return self.inner.lock().pop_work().ok_or(Error::ERESTARTSYS);
+ }
+ }
+ }
+
+ pub(crate) fn push_work(&self, work: Arc<dyn DeliverToRead>) -> BinderResult {
+ {
+ let mut inner = self.inner.lock();
+ if inner.is_dead {
+ return Err(BinderError::new_dead());
+ }
+ inner.push_work(work);
+ }
+ self.pinned_condvar().notify_one();
+ Ok(())
+ }
+
+ /// Attempts to push to given work item to the thread if it's a looper thread (i.e., if it's
+ /// part of a thread pool) and is alive. Otherwise, push the work item to the process instead.
+ pub(crate) fn push_work_if_looper(&self, work: Arc<dyn DeliverToRead>) -> BinderResult {
+ let mut inner = self.inner.lock();
+ if inner.is_looper() && !inner.is_dead {
+ inner.push_work(work);
+ Ok(())
+ } else {
+ drop(inner);
+ self.process.push_work(work)
+ }
+ }
+
+ pub(crate) fn push_work_deferred(&self, work: Arc<dyn DeliverToRead>) {
+ self.inner.lock().push_work_deferred(work);
+ }
+
+ fn translate_object(
+ &self,
+ index_offset: usize,
+ alloc: &Allocation,
+ view: &AllocationView,
+ ) -> BinderResult {
+ let offset = alloc.read(index_offset)?;
+ let header = view.read::<bindings::binder_object_header>(offset)?;
+ // TODO: Handle other types.
+ match header.type_ {
+ bindings::BINDER_TYPE_WEAK_BINDER | bindings::BINDER_TYPE_BINDER => {
+ let strong = header.type_ == bindings::BINDER_TYPE_BINDER;
+ view.transfer_binder_object(offset, strong, |obj| {
+ // SAFETY: The type is `BINDER_TYPE_{WEAK_}BINDER`, so `binder` is populated.
+ let ptr = unsafe { obj.__bindgen_anon_1.binder } as _;
+ let cookie = obj.cookie as _;
+ Ok(self.process.get_node(ptr, cookie, strong, Some(self))?)
+ })?;
+ }
+ bindings::BINDER_TYPE_WEAK_HANDLE | bindings::BINDER_TYPE_HANDLE => {
+ let strong = header.type_ == bindings::BINDER_TYPE_HANDLE;
+ view.transfer_binder_object(offset, strong, |obj| {
+ // SAFETY: The type is `BINDER_TYPE_{WEAK_}HANDLE`, so `handle` is populated.
+ let handle = unsafe { obj.__bindgen_anon_1.handle } as _;
+ self.process.get_node_from_handle(handle, strong)
+ })?;
+ }
+ _ => pr_warn!("Unsupported binder object type: {:x}\n", header.type_),
+ }
+ Ok(())
+ }
+
+ fn translate_objects(&self, alloc: &mut Allocation, start: usize, end: usize) -> BinderResult {
+ let view = AllocationView::new(&alloc, start);
+ for i in (start..end).step_by(size_of::<usize>()) {
+ if let Err(err) = self.translate_object(i, alloc, &view) {
+ alloc.set_info(AllocationInfo { offsets: start..i });
+ return Err(err);
+ }
+ }
+ alloc.set_info(AllocationInfo {
+ offsets: start..end,
+ });
+ Ok(())
+ }
+
+ pub(crate) fn copy_transaction_data<'a>(
+ &self,
+ to_process: &'a Process,
+ tr: &BinderTransactionData,
+ ) -> BinderResult<Allocation<'a>> {
+ let data_size = tr.data_size as _;
+ let adata_size = ptr_align(data_size);
+ let offsets_size = tr.offsets_size as _;
+ let aoffsets_size = ptr_align(offsets_size);
+
+ // This guarantees that at least `sizeof(usize)` bytes will be allocated.
+ let len = core::cmp::max(
+ adata_size.checked_add(aoffsets_size).ok_or(Error::ENOMEM)?,
+ size_of::<usize>(),
+ );
+ let mut alloc = to_process.buffer_alloc(len)?;
+
+ // Copy raw data.
+ let mut reader = unsafe { UserSlicePtr::new(tr.data.ptr.buffer as _, data_size) }.reader();
+ alloc.copy_into(&mut reader, 0, data_size)?;
+
+ // Copy offsets if there are any.
+ if offsets_size > 0 {
+ let mut reader =
+ unsafe { UserSlicePtr::new(tr.data.ptr.offsets as _, offsets_size) }.reader();
+ alloc.copy_into(&mut reader, adata_size, offsets_size)?;
+
+ // Traverse the objects specified.
+ self.translate_objects(&mut alloc, adata_size, adata_size + aoffsets_size)?;
+ }
+
+ Ok(alloc)
+ }
+
+ fn unwind_transaction_stack(self: &Arc<Self>) {
+ let mut thread = self.clone();
+ while let Ok(transaction) = {
+ let mut inner = thread.inner.lock();
+ inner.pop_transaction_to_reply(thread.as_ref())
+ } {
+ let reply = Either::Right(BR_DEAD_REPLY);
+ if !transaction.from.deliver_single_reply(reply, &transaction) {
+ break;
+ }
+
+ thread = transaction.from.clone();
+ }
+ }
+
+ pub(crate) fn deliver_reply(
+ &self,
+ reply: Either<Arc<Transaction>, u32>,
+ transaction: &Arc<Transaction>,
+ ) {
+ if self.deliver_single_reply(reply, transaction) {
+ transaction.from.unwind_transaction_stack();
+ }
+ }
+
+ /// Delivers a reply to the thread that started a transaction. The reply can either be a
+ /// reply-transaction or an error code to be delivered instead.
+ ///
+ /// Returns whether the thread is dead. If it is, the caller is expected to unwind the
+ /// transaction stack by completing transactions for threads that are dead.
+ fn deliver_single_reply(
+ &self,
+ reply: Either<Arc<Transaction>, u32>,
+ transaction: &Arc<Transaction>,
+ ) -> bool {
+ {
+ let mut inner = self.inner.lock();
+ if !inner.pop_transaction_replied(transaction) {
+ return false;
+ }
+
+ if inner.is_dead {
+ return true;
+ }
+
+ match reply {
+ Either::Left(work) => inner.push_work(work),
+ Either::Right(code) => inner.push_reply_work(code),
+ }
+ }
+
+ // Notify the thread now that we've released the inner lock.
+ self.pinned_condvar().notify_one();
+ false
+ }
+
+ /// Determines if the given transaction is the current transaction for this thread.
+ fn is_current_transaction(&self, transaction: &Arc<Transaction>) -> bool {
+ let inner = self.inner.lock();
+ match &inner.current_transaction {
+ None => false,
+ Some(current) => Arc::ptr_eq(current, transaction),
+ }
+ }
+
+ fn transaction<T>(self: &Arc<Self>, tr: &BinderTransactionData, inner: T)
+ where
+ T: FnOnce(&Arc<Self>, &BinderTransactionData) -> BinderResult,
+ {
+ if let Err(err) = inner(self, tr) {
+ self.inner.lock().push_return_work(err.reply);
+ }
+ }
+
+ fn reply_inner(self: &Arc<Self>, tr: &BinderTransactionData) -> BinderResult {
+ let orig = self.inner.lock().pop_transaction_to_reply(self)?;
+ if !orig.from.is_current_transaction(&orig) {
+ return Err(BinderError::new_failed());
+ }
+
+ // We need to complete the transaction even if we cannot complete building the reply.
+ (|| -> BinderResult<_> {
+ let completion = Arc::try_new(DeliverCode::new(BR_TRANSACTION_COMPLETE))?;
+ let process = orig.from.process.clone();
+ let reply = Arc::try_new(Transaction::new_reply(self, process, tr)?)?;
+ self.inner.lock().push_work(completion);
+ orig.from.deliver_reply(Either::Left(reply), &orig);
+ Ok(())
+ })()
+ .map_err(|mut err| {
+ // At this point we only return `BR_TRANSACTION_COMPLETE` to the caller, and we must let
+ // the sender know that the transaction has completed (with an error in this case).
+ let reply = Either::Right(BR_FAILED_REPLY);
+ orig.from.deliver_reply(reply, &orig);
+ err.reply = BR_TRANSACTION_COMPLETE;
+ err
+ })
+ }
+
+ /// Determines the current top of the transaction stack. It fails if the top is in another
+ /// thread (i.e., this thread belongs to a stack but it has called another thread). The top is
+ /// [`None`] if the thread is not currently participating in a transaction stack.
+ fn top_of_transaction_stack(&self) -> KernelResult<Option<Arc<Transaction>>> {
+ let inner = self.inner.lock();
+ Ok(if let Some(cur) = &inner.current_transaction {
+ if core::ptr::eq(self, cur.from.as_ref()) {
+ return Err(Error::EINVAL);
+ }
+ Some(cur.clone())
+ } else {
+ None
+ })
+ }
+
+ fn oneway_transaction_inner(self: &Arc<Self>, tr: &BinderTransactionData) -> BinderResult {
+ let handle = unsafe { tr.target.handle };
+ let node_ref = self.process.get_transaction_node(handle)?;
+ let completion = Arc::try_new(DeliverCode::new(BR_TRANSACTION_COMPLETE))?;
+ let transaction = Arc::try_new(Transaction::new(node_ref, None, self, tr)?)?;
+ self.inner.lock().push_work(completion);
+ // TODO: Remove the completion on error?
+ transaction.submit()?;
+ Ok(())
+ }
+
+ fn transaction_inner(self: &Arc<Self>, tr: &BinderTransactionData) -> BinderResult {
+ let handle = unsafe { tr.target.handle };
+ let node_ref = self.process.get_transaction_node(handle)?;
+ // TODO: We need to ensure that there isn't a pending transaction in the work queue. How
+ // could this happen?
+ let top = self.top_of_transaction_stack()?;
+ let completion = Arc::try_new(DeliverCode::new(BR_TRANSACTION_COMPLETE))?;
+ let transaction = Arc::try_new(Transaction::new(node_ref, top, self, tr)?)?;
+
+ // Check that the transaction stack hasn't changed while the lock was released, then update
+ // it with the new transaction.
+ {
+ let mut inner = self.inner.lock();
+ if !transaction.is_stacked_on(&inner.current_transaction) {
+ return Err(BinderError::new_failed());
+ }
+ inner.current_transaction = Some(transaction.clone());
+ }
+
+ // We push the completion as a deferred work so that we wait for the reply before returning
+ // to userland.
+ self.push_work_deferred(completion);
+ // TODO: Remove completion if submission fails?
+ transaction.submit()?;
+ Ok(())
+ }
+
+ fn write(self: &Arc<Self>, req: &mut BinderWriteRead) -> KernelResult {
+ let write_start = req.write_buffer.wrapping_add(req.write_consumed);
+ let write_len = req.write_size - req.write_consumed;
+ let mut reader = unsafe { UserSlicePtr::new(write_start as _, write_len as _).reader() };
+
+ while reader.len() >= size_of::<u32>() && self.inner.lock().return_work.is_some() {
+ let before = reader.len();
+ match reader.read::<u32>()? {
+ BC_TRANSACTION => {
+ let tr = reader.read::<BinderTransactionData>()?;
+ if tr.flags & bindings::transaction_flags_TF_ONE_WAY != 0 {
+ self.transaction(&tr, Self::oneway_transaction_inner)
+ } else {
+ self.transaction(&tr, Self::transaction_inner)
+ }
+ }
+ BC_REPLY => self.transaction(&reader.read()?, Self::reply_inner),
+ BC_FREE_BUFFER => drop(self.process.buffer_get(reader.read()?)),
+ BC_INCREFS => self.process.update_ref(reader.read()?, true, false)?,
+ BC_ACQUIRE => self.process.update_ref(reader.read()?, true, true)?,
+ BC_RELEASE => self.process.update_ref(reader.read()?, false, true)?,
+ BC_DECREFS => self.process.update_ref(reader.read()?, false, false)?,
+ BC_INCREFS_DONE => self.process.inc_ref_done(&mut reader, false)?,
+ BC_ACQUIRE_DONE => self.process.inc_ref_done(&mut reader, true)?,
+ BC_REQUEST_DEATH_NOTIFICATION => self.process.request_death(&mut reader, self)?,
+ BC_CLEAR_DEATH_NOTIFICATION => self.process.clear_death(&mut reader, self)?,
+ BC_DEAD_BINDER_DONE => self.process.dead_binder_done(reader.read()?, self),
+ BC_REGISTER_LOOPER => {
+ let valid = self.process.register_thread();
+ self.inner.lock().looper_register(valid);
+ }
+ BC_ENTER_LOOPER => self.inner.lock().looper_enter(),
+ BC_EXIT_LOOPER => self.inner.lock().looper_exit(),
+
+ // TODO: Add support for BC_TRANSACTION_SG and BC_REPLY_SG.
+ // BC_ATTEMPT_ACQUIRE and BC_ACQUIRE_RESULT are no longer supported.
+ _ => return Err(Error::EINVAL),
+ }
+
+ // Update the number of write bytes consumed.
+ req.write_consumed += (before - reader.len()) as u64;
+ }
+ Ok(())
+ }
+
+ fn read(self: &Arc<Self>, req: &mut BinderWriteRead, wait: bool) -> KernelResult {
+ let read_start = req.read_buffer.wrapping_add(req.read_consumed);
+ let read_len = req.read_size - req.read_consumed;
+ let mut writer = unsafe { UserSlicePtr::new(read_start as _, read_len as _) }.writer();
+ let (in_pool, getter) = {
+ let inner = self.inner.lock();
+ (
+ inner.is_looper(),
+ if inner.should_use_process_work_queue() {
+ Self::get_work
+ } else {
+ Self::get_work_local
+ },
+ )
+ };
+
+ // Reserve some room at the beginning of the read buffer so that we can send a
+ // BR_SPAWN_LOOPER if we need to.
+ if req.read_consumed == 0 {
+ writer.write(&BR_NOOP)?;
+ }
+
+ // Loop doing work while there is room in the buffer.
+ let initial_len = writer.len();
+ while writer.len() >= size_of::<u32>() {
+ match getter(self, wait && initial_len == writer.len()) {
+ Ok(work) => {
+ if !work.do_work(self, &mut writer)? {
+ break;
+ }
+ }
+ Err(err) => {
+ // Propagate the error if we haven't written anything else.
+ if initial_len == writer.len() {
+ return Err(err);
+ } else {
+ break;
+ }
+ }
+ }
+ }
+
+ req.read_consumed += read_len - writer.len() as u64;
+
+ // Write BR_SPAWN_LOOPER if the process needs more threads for its pool.
+ if in_pool && self.process.needs_thread() {
+ let mut writer =
+ unsafe { UserSlicePtr::new(req.read_buffer as _, req.read_size as _) }.writer();
+ writer.write(&BR_SPAWN_LOOPER)?;
+ }
+
+ Ok(())
+ }
+
+ pub(crate) fn write_read(self: &Arc<Self>, data: UserSlicePtr, wait: bool) -> KernelResult {
+ let (mut reader, mut writer) = data.reader_writer();
+ let mut req = reader.read::<BinderWriteRead>()?;
+
+ // TODO: `write(&req)` happens in all exit paths from here on. Find a better way to encode
+ // it.
+
+ // Go through the write buffer.
+ if req.write_size > 0 {
+ if let Err(err) = self.write(&mut req) {
+ req.read_consumed = 0;
+ writer.write(&req)?;
+ return Err(err);
+ }
+ }
+
+ // Go through the work queue.
+ let mut ret = Ok(());
+ if req.read_size > 0 {
+ ret = self.read(&mut req, wait);
+ }
+
+ // Write the request back so that the consumed fields are visible to the caller.
+ writer.write(&req)?;
+ ret
+ }
+
+ pub(crate) fn poll(&self, file: &File, table: &PollTable) -> (bool, u32) {
+ // SAFETY: `free_waiters` is called on release.
+ unsafe { table.register_wait(file, &self.work_condvar) };
+ let mut inner = self.inner.lock();
+ (inner.should_use_process_work_queue(), inner.poll())
+ }
+
+ pub(crate) fn notify_if_poll_ready(&self) {
+ // Determine if we need to notify. This requires the lock.
+ let inner = self.inner.lock();
+ let notify = inner.looper_flags & LOOPER_POLL != 0
+ && inner.should_use_process_work_queue()
+ && !inner.has_work();
+ drop(inner);
+
+ // Now that the lock is no longer held, notify the waiters if we have to.
+ if notify {
+ self.pinned_condvar().notify_one();
+ }
+ }
+
+ pub(crate) fn push_return_work(&self, code: u32) {
+ self.inner.lock().push_return_work(code)
+ }
+
+ pub(crate) fn release(self: &Arc<Thread>) {
+ // Mark the thread as dead.
+ self.inner.lock().is_dead = true;
+
+ // Cancel all pending work items.
+ while let Ok(work) = self.get_work_local(false) {
+ work.cancel();
+ }
+
+ // Complete the transaction stack as far as we can.
+ self.unwind_transaction_stack();
+
+ // Remove epoll items if polling was ever used on the thread.
+ let poller = self.inner.lock().looper_flags & LOOPER_POLL != 0;
+ if poller {
+ self.pinned_condvar().free_waiters();
+
+ unsafe { bindings::synchronize_rcu() };
+ }
+ }
+}
+
+impl GetLinks for Thread {
+ type EntryType = Thread;
+ fn get_links(data: &Thread) -> &Links<Thread> {
+ &data.links
+ }
+}
+
+struct ThreadError {
+ error_code: u32,
+ return_fn: fn(&mut InnerThread, Arc<ThreadError>),
+ links: Links<dyn DeliverToRead>,
+}
+
+impl ThreadError {
+ fn new(return_fn: fn(&mut InnerThread, Arc<ThreadError>)) -> Self {
+ Self {
+ error_code: BR_OK,
+ return_fn,
+ links: Links::new(),
+ }
+ }
+}
+
+impl DeliverToRead for ThreadError {
+ fn do_work(
+ self: Arc<Self>,
+ thread: &Thread,
+ writer: &mut UserSlicePtrWriter,
+ ) -> KernelResult<bool> {
+ let code = self.error_code;
+
+ // Return the `ThreadError` to the thread.
+ (self.return_fn)(&mut *thread.inner.lock(), self);
+
+ // Deliver the error code to userspace.
+ writer.write(&code)?;
+ Ok(true)
+ }
+
+ fn get_links(&self) -> &Links<dyn DeliverToRead> {
+ &self.links
+ }
+}
diff --git a/drivers/android/transaction.rs b/drivers/android/transaction.rs
new file mode 100644
index 0000000000000..98c2c86e12f01
--- /dev/null
+++ b/drivers/android/transaction.rs
@@ -0,0 +1,206 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use alloc::sync::Arc;
+use core::sync::atomic::{AtomicBool, Ordering};
+use kernel::{bindings, linked_list::Links, prelude::*, sync::Ref, user_ptr::UserSlicePtrWriter};
+
+use crate::{
+ defs::*,
+ node::NodeRef,
+ process::Process,
+ ptr_align,
+ thread::{BinderResult, Thread},
+ DeliverToRead, Either,
+};
+
+pub(crate) struct Transaction {
+ // TODO: Node should be released when the buffer is released.
+ node_ref: Option<NodeRef>,
+ stack_next: Option<Arc<Transaction>>,
+ pub(crate) from: Arc<Thread>,
+ to: Ref<Process>,
+ free_allocation: AtomicBool,
+ code: u32,
+ flags: u32,
+ data_size: usize,
+ offsets_size: usize,
+ data_address: usize,
+ links: Links<dyn DeliverToRead>,
+}
+
+impl Transaction {
+ pub(crate) fn new(
+ node_ref: NodeRef,
+ stack_next: Option<Arc<Transaction>>,
+ from: &Arc<Thread>,
+ tr: &BinderTransactionData,
+ ) -> BinderResult<Self> {
+ let to = node_ref.node.owner.clone();
+ let alloc = from.copy_transaction_data(&to, tr)?;
+ let data_address = alloc.ptr;
+ alloc.keep_alive();
+ Ok(Self {
+ node_ref: Some(node_ref),
+ stack_next,
+ from: from.clone(),
+ to,
+ code: tr.code,
+ flags: tr.flags,
+ data_size: tr.data_size as _,
+ data_address,
+ offsets_size: tr.offsets_size as _,
+ links: Links::new(),
+ free_allocation: AtomicBool::new(true),
+ })
+ }
+
+ pub(crate) fn new_reply(
+ from: &Arc<Thread>,
+ to: Ref<Process>,
+ tr: &BinderTransactionData,
+ ) -> BinderResult<Self> {
+ let alloc = from.copy_transaction_data(&to, tr)?;
+ let data_address = alloc.ptr;
+ alloc.keep_alive();
+ Ok(Self {
+ node_ref: None,
+ stack_next: None,
+ from: from.clone(),
+ to,
+ code: tr.code,
+ flags: tr.flags,
+ data_size: tr.data_size as _,
+ data_address,
+ offsets_size: tr.offsets_size as _,
+ links: Links::new(),
+ free_allocation: AtomicBool::new(true),
+ })
+ }
+
+ /// Determines if the transaction is stacked on top of the given transaction.
+ pub(crate) fn is_stacked_on(&self, onext: &Option<Arc<Self>>) -> bool {
+ match (&self.stack_next, onext) {
+ (None, None) => true,
+ (Some(stack_next), Some(next)) => Arc::ptr_eq(stack_next, next),
+ _ => false,
+ }
+ }
+
+ /// Returns a pointer to the next transaction on the transaction stack, if there is one.
+ pub(crate) fn clone_next(&self) -> Option<Arc<Self>> {
+ let next = self.stack_next.as_ref()?;
+ Some(next.clone())
+ }
+
+ /// Searches in the transaction stack for a thread that belongs to the target process. This is
+ /// useful when finding a target for a new transaction: if the node belongs to a process that
+ /// is already part of the transaction stack, we reuse the thread.
+ fn find_target_thread(&self) -> Option<Arc<Thread>> {
+ let process = &self.node_ref.as_ref()?.node.owner;
+
+ let mut it = &self.stack_next;
+ while let Some(transaction) = it {
+ if Ref::ptr_eq(&transaction.from.process, process) {
+ return Some(transaction.from.clone());
+ }
+ it = &transaction.stack_next;
+ }
+ None
+ }
+
+ /// Searches in the transaction stack for a transaction originating at the given thread.
+ pub(crate) fn find_from(&self, thread: &Thread) -> Option<Arc<Transaction>> {
+ let mut it = &self.stack_next;
+ while let Some(transaction) = it {
+ if core::ptr::eq(thread, transaction.from.as_ref()) {
+ return Some(transaction.clone());
+ }
+
+ it = &transaction.stack_next;
+ }
+ None
+ }
+
+ /// Submits the transaction to a work queue. Use a thread if there is one in the transaction
+ /// stack, otherwise use the destination process.
+ pub(crate) fn submit(self: Arc<Self>) -> BinderResult {
+ if let Some(thread) = self.find_target_thread() {
+ thread.push_work(self)
+ } else {
+ let process = self.to.clone();
+ process.push_work(self)
+ }
+ }
+}
+
+impl DeliverToRead for Transaction {
+ fn do_work(
+ self: Arc<Self>,
+ thread: &Thread,
+ writer: &mut UserSlicePtrWriter,
+ ) -> KernelResult<bool> {
+ /* TODO: Initialise the following fields from tr:
+ pub sender_pid: pid_t,
+ pub sender_euid: uid_t,
+ */
+ let mut tr = BinderTransactionData::default();
+
+ if let Some(nref) = &self.node_ref {
+ let (ptr, cookie) = nref.node.get_id();
+ tr.target.ptr = ptr as _;
+ tr.cookie = cookie as _;
+ };
+
+ tr.code = self.code;
+ tr.flags = self.flags;
+ tr.data_size = self.data_size as _;
+ tr.data.ptr.buffer = self.data_address as _;
+ tr.offsets_size = self.offsets_size as _;
+ if tr.offsets_size > 0 {
+ tr.data.ptr.offsets = (self.data_address + ptr_align(self.data_size)) as _;
+ }
+
+ // When `drop` is called, we don't want the allocation to be freed because it is now the
+ // user's reponsibility to free it.
+ self.free_allocation.store(false, Ordering::Relaxed);
+
+ let code = if self.node_ref.is_none() {
+ BR_REPLY
+ } else {
+ BR_TRANSACTION
+ };
+
+ // Write the transaction code and data to the user buffer. On failure we complete the
+ // transaction with an error.
+ if let Err(err) = writer.write(&code).and_then(|_| writer.write(&tr)) {
+ let reply = Either::Right(BR_FAILED_REPLY);
+ self.from.deliver_reply(reply, &self);
+ return Err(err);
+ }
+
+ // When this is not a reply and not an async transaction, update `current_transaction`. If
+ // it's a reply, `current_transaction` has already been updated appropriately.
+ if self.node_ref.is_some() && tr.flags & bindings::transaction_flags_TF_ONE_WAY == 0 {
+ thread.set_current_transaction(self);
+ }
+
+ Ok(false)
+ }
+
+ fn cancel(self: Arc<Self>) {
+ let reply = Either::Right(BR_DEAD_REPLY);
+ self.from.deliver_reply(reply, &self);
+ }
+
+ fn get_links(&self) -> &Links<dyn DeliverToRead> {
+ &self.links
+ }
+}
+
+impl Drop for Transaction {
+ fn drop(&mut self) {
+ if self.free_allocation.load(Ordering::Relaxed) {
+ self.to.buffer_get(self.data_address);
+ }
+ }
+}
diff --git a/include/linux/kallsyms.h b/include/linux/kallsyms.h
index 465060acc9816..5cdc6903abca3 100644
--- a/include/linux/kallsyms.h
+++ b/include/linux/kallsyms.h
@@ -14,7 +14,7 @@
#include <asm/sections.h>
-#define KSYM_NAME_LEN 128
+#define KSYM_NAME_LEN 512
#define KSYM_SYMBOL_LEN (sizeof("%s+%#lx/%#lx [%s]") + (KSYM_NAME_LEN - 1) + \
2*(BITS_PER_LONG*3/10) + (MODULE_NAME_LEN - 1) + 1)
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index 79897841a2cc8..a022992725be3 100644
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -331,12 +331,17 @@ static __always_inline raw_spinlock_t *spinlock_check(spinlock_t *lock)
#ifdef CONFIG_DEBUG_SPINLOCK
-# define spin_lock_init(lock) \
-do { \
- static struct lock_class_key __key; \
- \
- __raw_spin_lock_init(spinlock_check(lock), \
- #lock, &__key, LD_WAIT_CONFIG); \
+static inline void __spin_lock_init(spinlock_t *lock, const char *name,
+ struct lock_class_key *key)
+{
+ __raw_spin_lock_init(spinlock_check(lock), name, key, LD_WAIT_CONFIG);
+}
+
+# define spin_lock_init(lock) \
+do { \
+ static struct lock_class_key __key; \
+ \
+ __spin_lock_init(lock, #lock, &__key); \
} while (0)
#else
diff --git a/include/uapi/linux/android/binder.h b/include/uapi/linux/android/binder.h
index 20e435fe657a1..9fca291e01324 100644
--- a/include/uapi/linux/android/binder.h
+++ b/include/uapi/linux/android/binder.h
@@ -229,19 +229,21 @@ struct binder_frozen_status_info {
__u32 async_recv;
};
-#define BINDER_WRITE_READ _IOWR('b', 1, struct binder_write_read)
-#define BINDER_SET_IDLE_TIMEOUT _IOW('b', 3, __s64)
-#define BINDER_SET_MAX_THREADS _IOW('b', 5, __u32)
-#define BINDER_SET_IDLE_PRIORITY _IOW('b', 6, __s32)
-#define BINDER_SET_CONTEXT_MGR _IOW('b', 7, __s32)
-#define BINDER_THREAD_EXIT _IOW('b', 8, __s32)
-#define BINDER_VERSION _IOWR('b', 9, struct binder_version)
-#define BINDER_GET_NODE_DEBUG_INFO _IOWR('b', 11, struct binder_node_debug_info)
-#define BINDER_GET_NODE_INFO_FOR_REF _IOWR('b', 12, struct binder_node_info_for_ref)
-#define BINDER_SET_CONTEXT_MGR_EXT _IOW('b', 13, struct flat_binder_object)
-#define BINDER_FREEZE _IOW('b', 14, struct binder_freeze_info)
-#define BINDER_GET_FROZEN_INFO _IOWR('b', 15, struct binder_frozen_status_info)
-#define BINDER_ENABLE_ONEWAY_SPAM_DETECTION _IOW('b', 16, __u32)
+enum {
+ BINDER_WRITE_READ = _IOWR('b', 1, struct binder_write_read),
+ BINDER_SET_IDLE_TIMEOUT = _IOW('b', 3, __s64),
+ BINDER_SET_MAX_THREADS = _IOW('b', 5, __u32),
+ BINDER_SET_IDLE_PRIORITY = _IOW('b', 6, __s32),
+ BINDER_SET_CONTEXT_MGR = _IOW('b', 7, __s32),
+ BINDER_THREAD_EXIT = _IOW('b', 8, __s32),
+ BINDER_VERSION = _IOWR('b', 9, struct binder_version),
+ BINDER_GET_NODE_DEBUG_INFO = _IOWR('b', 11, struct binder_node_debug_info),
+ BINDER_GET_NODE_INFO_FOR_REF = _IOWR('b', 12, struct binder_node_info_for_ref),
+ BINDER_SET_CONTEXT_MGR_EXT = _IOW('b', 13, struct flat_binder_object),
+ BINDER_FREEZE = _IOW('b', 14, struct binder_freeze_info),
+ BINDER_GET_FROZEN_INFO = _IOWR('b', 15, struct binder_frozen_status_info),
+ BINDER_ENABLE_ONEWAY_SPAM_DETECTION = _IOW('b', 16, __u32),
+};
/*
* NOTE: Two special error codes you should check for when calling
diff --git a/init/Kconfig b/init/Kconfig
index a61c92066c2e4..b075234001a36 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -60,6 +60,15 @@ config LLD_VERSION
default $(ld-version) if LD_IS_LLD
default 0
+config HAS_RUST
+ depends on ARM64 || CPU_32v6 || CPU_32v6K || (PPC64 && CPU_LITTLE_ENDIAN) || X86_64
+ def_bool $(success,$(RUSTC) --version)
+
+config RUSTC_VERSION
+ depends on HAS_RUST
+ int
+ default $(shell,$(srctree)/scripts/rust-version.sh $(RUSTC))
+
config CC_CAN_LINK
bool
default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(m64-flag)) if 64BIT
@@ -2011,6 +2020,24 @@ config PROFILING
Say Y here to enable the extended profiling support mechanisms used
by profilers.
+config RUST
+ bool "Rust support"
+ depends on HAS_RUST
+ depends on !COMPILE_TEST
+ default n
+ help
+ Enables Rust support in the kernel.
+
+ This allows other Rust-related options, like drivers written in Rust,
+ to be selected.
+
+ It is also required to be able to load external kernel modules
+ written in Rust.
+
+ See Documentation/rust/ for more information.
+
+ If unsure, say N.
+
#
# Place an empty function call at each tracepoint site. Can be
# dynamically changed for a probe function.
diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c
index c851ca0ed3576..9d0c23e1993c1 100644
--- a/kernel/kallsyms.c
+++ b/kernel/kallsyms.c
@@ -73,6 +73,13 @@ static unsigned int kallsyms_expand_symbol(unsigned int off,
*/
off += len + 1;
+ /* If zero, it is a "big" symbol, so a two byte length follows. */
+ if (len == 0) {
+ len = (data[0] << 8) | data[1];
+ data += 2;
+ off += len + 2;
+ }
+
/*
* For every byte on the compressed symbol data, copy the table
* entry for that byte.
diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c
index 335d988bd8111..73874e5edfda6 100644
--- a/kernel/livepatch/core.c
+++ b/kernel/livepatch/core.c
@@ -213,7 +213,7 @@ static int klp_resolve_symbols(Elf64_Shdr *sechdrs, const char *strtab,
* we use the smallest/strictest upper bound possible (56, based on
* the current definition of MODULE_NAME_LEN) to prevent overflows.
*/
- BUILD_BUG_ON(MODULE_NAME_LEN < 56 || KSYM_NAME_LEN != 128);
+ BUILD_BUG_ON(MODULE_NAME_LEN < 56 || KSYM_NAME_LEN != 512);
relas = (Elf_Rela *) relasec->sh_addr;
/* For each rela in this klp relocation section */
@@ -227,7 +227,7 @@ static int klp_resolve_symbols(Elf64_Shdr *sechdrs, const char *strtab,
/* Format: .klp.sym.sym_objname.sym_name,sympos */
cnt = sscanf(strtab + sym->st_name,
- ".klp.sym.%55[^.].%127[^,],%lu",
+ ".klp.sym.%55[^.].%511[^,],%lu",
sym_objname, sym_name, &sympos);
if (cnt != 3) {
pr_err("symbol %s has an incorrectly formatted name\n",
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
index 421c35571797e..114e9963f903f 100644
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -398,6 +398,7 @@ static struct latched_seq clear_seq = {
#define CONSOLE_LOG_MAX 1024
/* the maximum size allowed to be reserved for a record */
+/* Keep in sync with rust/kernel/print.rs */
#define LOG_LINE_MAX (CONSOLE_LOG_MAX - PREFIX_MAX)
#define LOG_LEVEL(v) ((v) & 0x07)
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index a6b68ee531605..35540ac7388e9 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -2616,6 +2616,112 @@ config HYPERV_TESTING
endmenu # "Kernel Testing and Coverage"
+menu "Rust hacking"
+
+config RUST_DEBUG_ASSERTIONS
+ bool "Debug assertions"
+ default n
+ depends on RUST
+ help
+ Enables rustc's `-Cdebug-assertions` codegen option.
+
+ This flag lets you turn `cfg(debug_assertions)` conditional
+ compilation on or off. This can be used to enable extra debugging
+ code in development but not in production. For example, it controls
+ the behavior of the standard library's `debug_assert!` macro.
+
+ Note that this will apply to all Rust code, including `core`.
+
+ If unsure, say N.
+
+config RUST_OVERFLOW_CHECKS
+ bool "Overflow checks"
+ default y
+ depends on RUST
+ help
+ Enables rustc's `-Coverflow-checks` codegen option.
+
+ This flag allows you to control the behavior of runtime integer
+ overflow. When overflow-checks are enabled, a panic will occur
+ on overflow.
+
+ Note that this will apply to all Rust code, including `core`.
+
+ If unsure, say Y.
+
+choice
+ prompt "Optimization level"
+ default RUST_OPT_LEVEL_SIMILAR_AS_CHOSEN_FOR_C
+ depends on RUST
+ help
+ Controls rustc's `-Copt-level` codegen option.
+
+ This flag controls the optimization level.
+
+ If unsure, say "Similar as chosen for C".
+
+config RUST_OPT_LEVEL_SIMILAR_AS_CHOSEN_FOR_C
+ bool "Similar as chosen for C"
+ help
+ This choice will pick a similar optimization level as chosen in
+ the "Compiler optimization level" for C:
+
+ -O2 is currently mapped to -Copt-level=2
+ -O3 is currently mapped to -Copt-level=3
+ -Os is currently mapped to -Copt-level=z
+
+ The mapping may change over time to follow the intended semantics
+ of the choice for C as sensibly as possible.
+
+ This is the default.
+
+config RUST_OPT_LEVEL_0
+ bool "No optimizations (-Copt-level=0)"
+ help
+ Not recommended for most purposes. It may come in handy for debugging
+ suspected optimizer bugs, unexpected undefined behavior, etc.
+
+ Note that this level will *not* enable debug assertions nor overflow
+ checks on its own (like it happens when interacting with rustc
+ directly). Use the corresponding configuration options to control
+ that instead, orthogonally.
+
+ Note this level may cause excessive stack usage, which can lead to stack
+ overflow and subsequent crashes.
+
+config RUST_OPT_LEVEL_1
+ bool "Basic optimizations (-Copt-level=1)"
+ help
+ Useful for debugging without getting too lost, but without
+ the overhead and boilerplate of no optimizations at all.
+
+ Note this level may cause excessive stack usage, which can lead to stack
+ overflow and subsequent crashes.
+
+config RUST_OPT_LEVEL_2
+ bool "Some optimizations (-Copt-level=2)"
+ help
+ The sensible choice in most cases.
+
+config RUST_OPT_LEVEL_3
+ bool "All optimizations (-Copt-level=3)"
+ help
+ Yet more performance (hopefully).
+
+config RUST_OPT_LEVEL_S
+ bool "Optimize for size (-Copt-level=s)"
+ help
+ Smaller kernel, ideally without too much performance loss.
+
+config RUST_OPT_LEVEL_Z
+ bool "Optimize for size, no loop vectorization (-Copt-level=z)"
+ help
+ Like the previous level, but also turn off loop vectorization.
+
+endchoice
+
+endmenu # "Rust"
+
source "Documentation/Kconfig"
endmenu # Kernel hacking
diff --git a/rust/.gitignore b/rust/.gitignore
new file mode 100644
index 0000000000000..8875e08ed0b17
--- /dev/null
+++ b/rust/.gitignore
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0
+
+bindings_generated.rs
+exports_*_generated.h
+doc/ \ No newline at end of file
diff --git a/rust/Makefile b/rust/Makefile
new file mode 100644
index 0000000000000..2d1d9d7af77ca
--- /dev/null
+++ b/rust/Makefile
@@ -0,0 +1,152 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_RUST) += core.o compiler_builtins.o helpers.o
+extra-$(CONFIG_RUST) += exports_core_generated.h
+
+extra-$(CONFIG_RUST) += libmodule.so
+
+extra-$(CONFIG_RUST) += bindings_generated.rs
+obj-$(CONFIG_RUST) += alloc.o kernel.o
+extra-$(CONFIG_RUST) += exports_alloc_generated.h exports_kernel_generated.h
+
+obj-$(CONFIG_RUST) += exports.o
+
+RUSTDOC = rustdoc
+
+quiet_cmd_rustdoc = RUSTDOC $<
+ cmd_rustdoc = \
+ RUST_BINDINGS_FILE=$(abspath $(objtree)/rust/bindings_generated.rs) \
+ $(RUSTDOC) $(filter-out --emit=%, $(rustc_flags)) \
+ $(rustdoc_target_flags) -L $(objtree)/rust/ \
+ --output $(objtree)/rust/doc --crate-name $(subst rustdoc-,,$@) \
+ -Fmissing-docs @$(objtree)/include/generated/rustc_cfg $<
+
+rustdoc: rustdoc-module rustdoc-compiler_builtins rustdoc-kernel
+
+rustdoc-module: private rustdoc_target_flags = --crate-type proc-macro \
+ --extern proc_macro
+rustdoc-module: $(srctree)/rust/module.rs FORCE
+ $(call if_changed,rustdoc)
+
+rustdoc-compiler_builtins: $(srctree)/rust/compiler_builtins.rs FORCE
+ $(call if_changed,rustdoc)
+
+rustdoc-kernel: private rustdoc_target_flags = --extern alloc \
+ --extern module=$(objtree)/rust/libmodule.so
+rustdoc-kernel: $(srctree)/rust/kernel/lib.rs rustdoc-module \
+ $(objtree)/rust/libmodule.so $(objtree)/rust/bindings_generated.rs FORCE
+ $(call if_changed,rustdoc)
+
+ifdef CONFIG_CC_IS_CLANG
+bindgen_c_flags = $(c_flags)
+else
+# bindgen relies on libclang to parse C. Ideally, bindgen would support a GCC
+# plugin backend and/or the Clang driver would be perfectly compatible with GCC.
+#
+# For the moment, here we are tweaking the flags on the fly. Some config
+# options may not work (e.g. `GCC_PLUGIN_RANDSTRUCT` if we end up using one
+# of those structs). We might want to redo how Clang flags are kept track of
+# in the general `Makefile` even for GCC builds, similar to what we did with
+# `TENTATIVE_CLANG_FLAGS`.
+bindgen_skip_c_flags := -mno-fp-ret-in-387 -mpreferred-stack-boundary=% \
+ -mskip-rax-setup -mgeneral-regs-only -msign-return-address=% \
+ -mindirect-branch=thunk-extern -mindirect-branch-register -mrecord-mcount \
+ -mabi=lp64 -mstack-protector-guard% -fconserve-stack -falign-jumps=% \
+ -falign-loops=% -fno-ipa-cp-clone -fno-partial-inlining \
+ -fno-reorder-blocks -fno-allow-store-data-races -fasan-shadow-offset=% \
+ -Wno-packed-not-aligned -Wno-format-truncation -Wno-format-overflow \
+ -Wno-stringop-truncation -Wno-unused-but-set-variable \
+ -Wno-stringop-overflow -Wno-restrict -Wno-maybe-uninitialized \
+ -Werror=designated-init -Wno-zero-length-bounds \
+ --param=% --param asan-%
+
+# PowerPC
+bindgen_skip_c_flags += -mtraceback=no -mno-pointers-to-nested-functions \
+ -mno-string -mno-strict-align
+
+bindgen_extra_c_flags = $(TENTATIVE_CLANG_FLAGS) -Wno-address-of-packed-member
+bindgen_c_flags = $(filter-out $(bindgen_skip_c_flags), $(c_flags)) \
+ $(bindgen_extra_c_flags)
+endif
+
+bindgen_opaque_types := xregs_state desc_struct arch_lbr_state
+
+# To avoid several recompilations in PowerPC, which inserts `-D_TASK_CPU`
+bindgen_c_flags_final = $(filter-out -D_TASK_CPU=%, $(bindgen_c_flags))
+
+quiet_cmd_bindgen = BINDGEN $@
+ cmd_bindgen = \
+ $(BINDGEN) $< $(addprefix --opaque-type , $(bindgen_opaque_types)) \
+ --use-core --with-derive-default --ctypes-prefix c_types \
+ --size_t-is-usize -o $@ -- $(bindgen_c_flags_final) -DMODULE
+
+$(objtree)/rust/bindings_generated.rs: $(srctree)/rust/kernel/bindings_helper.h FORCE
+ $(call if_changed_dep,bindgen)
+
+quiet_cmd_exports = EXPORTS $@
+ cmd_exports = \
+ $(NM) -p --defined-only $< \
+ | grep -E ' (T|R|D) ' | cut -d ' ' -f 3 | grep -E '^(__rust_|_R)' \
+ | xargs -Isymbol \
+ echo 'EXPORT_SYMBOL_RUST_GPL(symbol);' > $@
+
+$(objtree)/rust/exports_core_generated.h: $(objtree)/rust/core.o FORCE
+ $(call if_changed,exports)
+
+$(objtree)/rust/exports_alloc_generated.h: $(objtree)/rust/alloc.o FORCE
+ $(call if_changed,exports)
+
+$(objtree)/rust/exports_kernel_generated.h: $(objtree)/rust/kernel.o FORCE
+ $(call if_changed,exports)
+
+# `-Cpanic=unwind -Cforce-unwind-tables=y` overrides `rustc_flags` in order to
+# avoid the https://github.com/rust-lang/rust/issues/82320 rustc crash.
+quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@
+ cmd_rustc_procmacro = \
+ $(RUSTC_OR_CLIPPY) $(rustc_flags) \
+ --emit=dep-info,link --extern proc_macro \
+ -Cpanic=unwind -Cforce-unwind-tables=y \
+ --crate-type proc-macro --out-dir $(objtree)/rust/ \
+ --crate-name $(patsubst lib%.so,%,$(notdir $@)) $<; \
+ mv $(objtree)/rust/$(patsubst lib%.so,%,$(notdir $@)).d $(depfile); \
+ sed -i '/^\#/d' $(depfile)
+
+$(objtree)/rust/libmodule.so: $(srctree)/rust/module.rs FORCE
+ $(call if_changed_dep,rustc_procmacro)
+
+quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L $@
+ cmd_rustc_library = \
+ RUST_BINDINGS_FILE=$(abspath $(objtree)/rust/bindings_generated.rs) \
+ $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \
+ $(rustc_flags) $(rustc_cross_flags) $(rustc_target_flags) \
+ --crate-type rlib --out-dir $(objtree)/rust/ -L $(objtree)/rust/ \
+ --crate-name $(patsubst %.o,%,$(notdir $@)) $<; \
+ mv $(objtree)/rust/$(patsubst %.o,%,$(notdir $@)).d $(depfile); \
+ sed -i '/^\#/d' $(depfile) \
+ $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
+
+# `$(rustc_flags)` is passed in case the user added `--sysroot`.
+rustc_sysroot = $(shell $(RUSTC) $(rustc_flags) --print sysroot)
+rustc_src = $(rustc_sysroot)/lib/rustlib/src/rust
+
+.SECONDEXPANSION:
+$(objtree)/rust/core.o: private skip_clippy = 1
+$(objtree)/rust/core.o: $$(rustc_src)/library/core/src/lib.rs FORCE
+ $(call if_changed_dep,rustc_library)
+
+$(objtree)/rust/compiler_builtins.o: private rustc_objcopy = -w -W '__*'
+$(objtree)/rust/compiler_builtins.o: $(srctree)/rust/compiler_builtins.rs \
+ $(objtree)/rust/core.o FORCE
+ $(call if_changed_dep,rustc_library)
+
+$(objtree)/rust/alloc.o: private skip_clippy = 1
+$(objtree)/rust/alloc.o: $$(rustc_src)/library/alloc/src/lib.rs \
+ $(objtree)/rust/compiler_builtins.o FORCE
+ $(call if_changed_dep,rustc_library)
+
+# ICE on `--extern module`: https://github.com/rust-lang/rust/issues/56935
+$(objtree)/rust/kernel.o: private rustc_target_flags = --extern alloc \
+ --extern module=$(objtree)/rust/libmodule.so
+$(objtree)/rust/kernel.o: $(srctree)/rust/kernel/lib.rs $(objtree)/rust/alloc.o \
+ $(objtree)/rust/libmodule.so $(objtree)/rust/bindings_generated.rs FORCE
+ $(call if_changed_dep,rustc_library)
diff --git a/rust/compiler_builtins.rs b/rust/compiler_builtins.rs
new file mode 100644
index 0000000000000..58d1a9b4d2182
--- /dev/null
+++ b/rust/compiler_builtins.rs
@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Our own `compiler_builtins`.
+//!
+//! Rust provides [`compiler_builtins`] as a port of LLVM's [`compiler-rt`].
+//! Since we do not need the vast majority of them, we avoid the dependency
+//! by providing this file.
+//!
+//! At the moment, some builtins are required that should not be. For instance,
+//! [`core`] has floating-point functionality which we should not be compiling
+//! in. For the moment, we define them to [`panic!`] at runtime for simplicity.
+//! These are actually a superset of the ones we actually need to define,
+//! but it seems simpler to ban entire categories at once. In the future,
+//! we might be able to remove all this by providing our own custom [`core`]
+//! etc., or perhaps [`core`] itself might provide `cfg` options to disable
+//! enough functionality to avoid requiring some of these.
+//!
+//! In any case, all these symbols are weakened to ensure we do not override
+//! those that may be provided by the rest of the kernel.
+//!
+//! [`compiler_builtins`]: https://github.com/rust-lang/compiler-builtins
+//! [`compiler-rt`]: https://compiler-rt.llvm.org/
+
+#![feature(compiler_builtins)]
+#![compiler_builtins]
+#![no_builtins]
+#![no_std]
+#![deny(clippy::complexity)]
+#![deny(clippy::correctness)]
+#![deny(clippy::perf)]
+#![deny(clippy::style)]
+
+macro_rules! define_panicking_intrinsics(
+ ($reason: tt, { $($ident: ident, )* }) => {
+ $(
+ #[doc(hidden)]
+ #[no_mangle]
+ pub extern "C" fn $ident() {
+ panic!($reason);
+ }
+ )*
+ }
+);
+
+define_panicking_intrinsics!("non-inline stack probes should not be used", {
+ __rust_probestack,
+});
+
+define_panicking_intrinsics!("`f32` should not be used", {
+ __addsf3,
+ __addsf3vfp,
+ __aeabi_fcmpeq,
+ __aeabi_ul2f,
+ __divsf3,
+ __divsf3vfp,
+ __eqsf2,
+ __eqsf2vfp,
+ __fixsfdi,
+ __fixsfsi,
+ __fixsfti,
+ __fixunssfdi,
+ __fixunssfsi,
+ __fixunssfti,
+ __floatdisf,
+ __floatsisf,
+ __floattisf,
+ __floatundisf,
+ __floatunsisf,
+ __floatuntisf,
+ __gesf2,
+ __gesf2vfp,
+ __gtsf2,
+ __gtsf2vfp,
+ __lesf2,
+ __lesf2vfp,
+ __ltsf2,
+ __ltsf2vfp,
+ __mulsf3,
+ __mulsf3vfp,
+ __nesf2,
+ __nesf2vfp,
+ __powisf2,
+ __subsf3,
+ __subsf3vfp,
+ __unordsf2,
+});
+
+define_panicking_intrinsics!("`f64` should not be used", {
+ __adddf3,
+ __adddf3vfp,
+ __aeabi_dcmpeq,
+ __aeabi_ul2d,
+ __divdf3,
+ __divdf3vfp,
+ __eqdf2,
+ __eqdf2vfp,
+ __fixdfdi,
+ __fixdfsi,
+ __fixdfti,
+ __fixunsdfdi,
+ __fixunsdfsi,
+ __fixunsdfti,
+ __floatdidf,
+ __floatsidf,
+ __floattidf,
+ __floatundidf,
+ __floatunsidf,
+ __floatuntidf,
+ __gedf2,
+ __gedf2vfp,
+ __gtdf2,
+ __gtdf2vfp,
+ __ledf2,
+ __ledf2vfp,
+ __ltdf2,
+ __ltdf2vfp,
+ __muldf3,
+ __muldf3vfp,
+ __nedf2,
+ __nedf2vfp,
+ __powidf2,
+ __subdf3,
+ __subdf3vfp,
+ __unorddf2,
+});
+
+define_panicking_intrinsics!("`i128` should not be used", {
+ __ashrti3,
+ __muloti4,
+ __multi3,
+});
+
+define_panicking_intrinsics!("`u128` should not be used", {
+ __ashlti3,
+ __lshrti3,
+ __udivmodti4,
+ __udivti3,
+ __umodti3,
+});
+
+#[cfg(target_arch = "arm")]
+define_panicking_intrinsics!("`u64` division/modulo should not be used", {
+ __aeabi_uldivmod,
+ __mulodi4,
+});
+
+extern "C" {
+ fn rust_helper_BUG() -> !;
+}
+
+#[panic_handler]
+fn panic(_info: &core::panic::PanicInfo) -> ! {
+ unsafe {
+ rust_helper_BUG();
+ }
+}
diff --git a/rust/exports.c b/rust/exports.c
new file mode 100644
index 0000000000000..d7dff1b3b9199
--- /dev/null
+++ b/rust/exports.c
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// A hack to export Rust symbols for loadable modules without having to redo
+// the entire `include/linux/export.h` logic in Rust.
+//
+// This requires the Rust's new/future `v0` mangling scheme because the default
+// one ("legacy") uses invalid characters for C identifiers (thus we cannot use
+// the `EXPORT_SYMBOL_*` macros).
+
+#include <linux/module.h>
+
+#define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym);
+
+#include "exports_core_generated.h"
+#include "exports_alloc_generated.h"
+#include "exports_kernel_generated.h"
diff --git a/rust/helpers.c b/rust/helpers.c
new file mode 100644
index 0000000000000..aa1de0eb0cbe3
--- /dev/null
+++ b/rust/helpers.c
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/bug.h>
+#include <linux/build_bug.h>
+#include <linux/uaccess.h>
+#include <linux/sched/signal.h>
+#include <linux/gfp.h>
+#include <linux/highmem.h>
+
+void rust_helper_BUG(void)
+{
+ BUG();
+}
+
+unsigned long rust_helper_copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+ return copy_from_user(to, from, n);
+}
+
+unsigned long rust_helper_copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+ return copy_to_user(to, from, n);
+}
+
+void rust_helper_spin_lock_init(spinlock_t *lock, const char *name,
+ struct lock_class_key *key)
+{
+#ifdef CONFIG_DEBUG_SPINLOCK
+ __spin_lock_init(lock, name, key);
+#else
+ spin_lock_init(lock);
+#endif
+}
+EXPORT_SYMBOL_GPL(rust_helper_spin_lock_init);
+
+void rust_helper_spin_lock(spinlock_t *lock)
+{
+ spin_lock(lock);
+}
+EXPORT_SYMBOL_GPL(rust_helper_spin_lock);
+
+void rust_helper_spin_unlock(spinlock_t *lock)
+{
+ spin_unlock(lock);
+}
+EXPORT_SYMBOL_GPL(rust_helper_spin_unlock);
+
+void rust_helper_init_wait(struct wait_queue_entry *wq_entry)
+{
+ init_wait(wq_entry);
+}
+EXPORT_SYMBOL_GPL(rust_helper_init_wait);
+
+int rust_helper_current_pid(void)
+{
+ return current->pid;
+}
+EXPORT_SYMBOL_GPL(rust_helper_current_pid);
+
+int rust_helper_signal_pending(void)
+{
+ return signal_pending(current);
+}
+EXPORT_SYMBOL_GPL(rust_helper_signal_pending);
+
+struct page *rust_helper_alloc_pages(gfp_t gfp_mask, unsigned int order)
+{
+ return alloc_pages(gfp_mask, order);
+}
+EXPORT_SYMBOL_GPL(rust_helper_alloc_pages);
+
+void *rust_helper_kmap(struct page *page)
+{
+ return kmap(page);
+}
+EXPORT_SYMBOL_GPL(rust_helper_kmap);
+
+void rust_helper_kunmap(struct page *page)
+{
+ return kunmap(page);
+}
+EXPORT_SYMBOL_GPL(rust_helper_kunmap);
+
+int rust_helper_cond_resched(void)
+{
+ return cond_resched();
+}
+EXPORT_SYMBOL_GPL(rust_helper_cond_resched);
+
+#if !defined(CONFIG_ARM)
+// See https://github.com/rust-lang/rust-bindgen/issues/1671
+static_assert(__builtin_types_compatible_p(size_t, uintptr_t),
+ "size_t must match uintptr_t, what architecture is this??");
+#endif
diff --git a/rust/kernel/allocator.rs b/rust/kernel/allocator.rs
new file mode 100644
index 0000000000000..81104f2d8ffae
--- /dev/null
+++ b/rust/kernel/allocator.rs
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Allocator support.
+
+use core::alloc::{GlobalAlloc, Layout};
+use core::ptr;
+
+use crate::bindings;
+use crate::c_types;
+
+pub struct KernelAllocator;
+
+unsafe impl GlobalAlloc for KernelAllocator {
+ unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
+ // `krealloc()` is used instead of `kmalloc()` because the latter is
+ // an inline function and cannot be bound to as a result.
+ bindings::krealloc(ptr::null(), layout.size(), bindings::GFP_KERNEL) as *mut u8
+ }
+
+ unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) {
+ bindings::kfree(ptr as *const c_types::c_void);
+ }
+}
+
+#[alloc_error_handler]
+fn oom(_layout: Layout) -> ! {
+ panic!("Out of memory!");
+}
+
+// `rustc` only generates these for some crate types. Even then, we would need
+// to extract the object file that has them from the archive. For the moment,
+// let's generate them ourselves instead.
+#[no_mangle]
+pub fn __rust_alloc(size: usize, _align: usize) -> *mut u8 {
+ unsafe { bindings::krealloc(core::ptr::null(), size, bindings::GFP_KERNEL) as *mut u8 }
+}
+
+#[no_mangle]
+pub fn __rust_dealloc(ptr: *mut u8, _size: usize, _align: usize) {
+ unsafe { bindings::kfree(ptr as *const c_types::c_void) };
+}
+
+#[no_mangle]
+pub fn __rust_realloc(ptr: *mut u8, _old_size: usize, _align: usize, new_size: usize) -> *mut u8 {
+ unsafe {
+ bindings::krealloc(
+ ptr as *const c_types::c_void,
+ new_size,
+ bindings::GFP_KERNEL,
+ ) as *mut u8
+ }
+}
+
+#[no_mangle]
+pub fn __rust_alloc_zeroed(size: usize, _align: usize) -> *mut u8 {
+ unsafe {
+ bindings::krealloc(
+ core::ptr::null(),
+ size,
+ bindings::GFP_KERNEL | bindings::__GFP_ZERO,
+ ) as *mut u8
+ }
+}
+
+#[no_mangle]
+pub fn __rust_alloc_error_handler(_size: usize, _align: usize) -> ! {
+ panic!("Out of memory!");
+}
diff --git a/rust/kernel/bindings.rs b/rust/kernel/bindings.rs
new file mode 100644
index 0000000000000..6a300f52335cd
--- /dev/null
+++ b/rust/kernel/bindings.rs
@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Bindings
+//!
+//! Imports the generated bindings by `bindgen`.
+
+#[allow(
+ clippy::all,
+ non_camel_case_types,
+ non_upper_case_globals,
+ non_snake_case,
+ improper_ctypes
+)]
+mod bindings_raw {
+ use crate::c_types;
+ include!(env!("RUST_BINDINGS_FILE"));
+}
+pub use bindings_raw::*;
+
+pub const GFP_KERNEL: gfp_t = BINDINGS_GFP_KERNEL;
+pub const __GFP_ZERO: gfp_t = BINDINGS___GFP_ZERO;
+pub const __GFP_HIGHMEM: gfp_t = ___GFP_HIGHMEM;
diff --git a/rust/kernel/bindings_helper.h b/rust/kernel/bindings_helper.h
new file mode 100644
index 0000000000000..6b08f42cfcceb
--- /dev/null
+++ b/rust/kernel/bindings_helper.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/cdev.h>
+#include <linux/fs.h>
+#include <linux/module.h>
+#include <linux/random.h>
+#include <linux/slab.h>
+#include <linux/sysctl.h>
+#include <linux/uaccess.h>
+#include <linux/version.h>
+#include <linux/miscdevice.h>
+#include <linux/poll.h>
+#include <linux/mm.h>
+#include <uapi/linux/android/binder.h>
+
+// `bindgen` gets confused at certain things
+const gfp_t BINDINGS_GFP_KERNEL = GFP_KERNEL;
+const gfp_t BINDINGS___GFP_ZERO = __GFP_ZERO;
diff --git a/rust/kernel/buffer.rs b/rust/kernel/buffer.rs
new file mode 100644
index 0000000000000..b2502fa968fe9
--- /dev/null
+++ b/rust/kernel/buffer.rs
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Struct for writing to a pre-allocated buffer with the [`write!`] macro.
+
+use core::fmt;
+
+/// A pre-allocated buffer that implements [`core::fmt::Write`].
+///
+/// Consecutive writes will append to what has already been written.
+/// Writes that don't fit in the buffer will fail.
+pub struct Buffer<'a> {
+ slice: &'a mut [u8],
+ pos: usize,
+}
+
+impl<'a> Buffer<'a> {
+ /// Create a new buffer from an existing array.
+ pub fn new(slice: &'a mut [u8]) -> Self {
+ Buffer { slice, pos: 0 }
+ }
+
+ /// Number of bytes that have already been written to the buffer.
+ /// This will always be less than the length of the original array.
+ pub fn bytes_written(&self) -> usize {
+ self.pos
+ }
+}
+
+impl<'a> fmt::Write for Buffer<'a> {
+ fn write_str(&mut self, s: &str) -> fmt::Result {
+ if s.len() > self.slice.len() - self.pos {
+ Err(fmt::Error)
+ } else {
+ self.slice[self.pos..self.pos + s.len()].copy_from_slice(s.as_bytes());
+ self.pos += s.len();
+ Ok(())
+ }
+ }
+}
diff --git a/rust/kernel/c_types.rs b/rust/kernel/c_types.rs
new file mode 100644
index 0000000000000..10486b41efa99
--- /dev/null
+++ b/rust/kernel/c_types.rs
@@ -0,0 +1,133 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! C types for the bindings.
+//!
+//! The bindings generated by `bindgen` use these types to map to the C ones.
+//!
+//! C's standard integer types may differ in width depending on
+//! the architecture, thus we need to conditionally compile those.
+
+#![allow(non_camel_case_types)]
+
+#[cfg(any(target_arch = "arm", target_arch = "x86"))]
+mod c {
+ /// C `void` type.
+ pub type c_void = core::ffi::c_void;
+
+ /// C `char` type.
+ pub type c_char = i8;
+
+ /// C `signed char` type.
+ pub type c_schar = i8;
+
+ /// C `unsigned char` type.
+ pub type c_uchar = u8;
+
+ /// C `short` type.
+ pub type c_short = i16;
+
+ /// C `unsigned short` type.
+ pub type c_ushort = u16;
+
+ /// C `int` type.
+ pub type c_int = i32;
+
+ /// C `unsigned int` type.
+ pub type c_uint = u32;
+
+ /// C `long` type.
+ pub type c_long = i32;
+
+ /// C `unsigned long` type.
+ pub type c_ulong = u32;
+
+ /// C `long long` type.
+ pub type c_longlong = i64;
+
+ /// C `unsigned long long` type.
+ pub type c_ulonglong = u64;
+
+ /// C `ssize_t` type (typically defined in `<sys/types.h>` by POSIX).
+ ///
+ /// For some 32-bit architectures like this one, the kernel defines it as
+ /// `int`, i.e. it is an [`i32`].
+ pub type c_ssize_t = isize;
+
+ /// C `size_t` type (typically defined in `<stddef.h>`).
+ ///
+ /// For some 32-bit architectures like this one, the kernel defines it as
+ /// `unsigned int`, i.e. it is an [`u32`].
+ pub type c_size_t = usize;
+}
+
+#[cfg(any(
+ target_arch = "aarch64",
+ target_arch = "x86_64",
+ target_arch = "powerpc64"
+))]
+mod c {
+ /// C `void` type.
+ pub type c_void = core::ffi::c_void;
+
+ /// C `char` type.
+ pub type c_char = i8;
+
+ /// C `signed char` type.
+ pub type c_schar = i8;
+
+ /// C `unsigned char` type.
+ pub type c_uchar = u8;
+
+ /// C `short` type.
+ pub type c_short = i16;
+
+ /// C `unsigned short` type.
+ pub type c_ushort = u16;
+
+ /// C `int` type.
+ pub type c_int = i32;
+
+ /// C `unsigned int` type.
+ pub type c_uint = u32;
+
+ /// C `long` type.
+ pub type c_long = i64;
+
+ /// C `unsigned long` type.
+ pub type c_ulong = u64;
+
+ /// C `long long` type.
+ pub type c_longlong = i64;
+
+ /// C `unsigned long long` type.
+ pub type c_ulonglong = u64;
+
+ /// C `ssize_t` type (typically defined in `<sys/types.h>` by POSIX).
+ ///
+ /// For 64-bit architectures like this one, the kernel defines it as
+ /// `long`, i.e. it is an [`i64`].
+ pub type c_ssize_t = isize;
+
+ /// C `size_t` type (typically defined in `<stddef.h>`).
+ ///
+ /// For 64-bit architectures like this one, the kernel defines it as
+ /// `unsigned long`, i.e. it is an [`u64`].
+ pub type c_size_t = usize;
+}
+
+pub use c::*;
+
+/// Reads string until null byte is reached and returns slice excluding the
+/// terminating null.
+///
+/// # Safety
+///
+/// The data from the pointer until the null terminator must be valid for reads
+/// and not mutated for all of `'a`. The length of the string must also be less
+/// than `isize::MAX`. See the documentation on
+/// [`core::slice::from_raw_parts()`] for further details on safety of
+/// converting a pointer to a slice.
+pub unsafe fn c_string_bytes<'a>(ptr: *const crate::c_types::c_char) -> &'a [u8] {
+ let length = crate::bindings::strlen(ptr) as usize;
+ &core::slice::from_raw_parts(ptr as *const u8, length)
+}
diff --git a/rust/kernel/chrdev.rs b/rust/kernel/chrdev.rs
new file mode 100644
index 0000000000000..6772a3a925ccf
--- /dev/null
+++ b/rust/kernel/chrdev.rs
@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Character devices.
+//!
+//! Also called "char devices", `chrdev`, `cdev`.
+//!
+//! C header: [`include/linux/cdev.h`](../../../../include/linux/cdev.h)
+//!
+//! Reference: <https://www.kernel.org/doc/html/latest/core-api/kernel-api.html#char-devices>
+
+use alloc::boxed::Box;
+use core::convert::TryInto;
+use core::marker::PhantomPinned;
+use core::mem::MaybeUninit;
+use core::pin::Pin;
+
+use crate::bindings;
+use crate::c_types;
+use crate::error::{Error, KernelResult};
+use crate::file_operations;
+use crate::types::CStr;
+
+struct RegistrationInner<const N: usize> {
+ dev: bindings::dev_t,
+ used: usize,
+ cdevs: [MaybeUninit<bindings::cdev>; N],
+ _pin: PhantomPinned,
+}
+
+/// Character device registration.
+///
+/// May contain up to a fixed number (`N`) of devices. Must be pinned.
+pub struct Registration<const N: usize> {
+ name: CStr<'static>,
+ minors_start: u16,
+ this_module: &'static crate::ThisModule,
+ inner: Option<RegistrationInner<N>>,
+}
+
+impl<const N: usize> Registration<{ N }> {
+ /// Creates a [`Registration`] object for a character device.
+ ///
+ /// This does *not* register the device: see [`Self::register()`].
+ ///
+ /// This associated function is intended to be used when you need to avoid
+ /// a memory allocation, e.g. when the [`Registration`] is a member of
+ /// a bigger structure inside your [`crate::KernelModule`] instance. If you
+ /// are going to pin the registration right away, call
+ /// [`Self::new_pinned()`] instead.
+ pub fn new(
+ name: CStr<'static>,
+ minors_start: u16,
+ this_module: &'static crate::ThisModule,
+ ) -> Self {
+ Registration {
+ name,
+ minors_start,
+ this_module,
+ inner: None,
+ }
+ }
+
+ /// Creates a pinned [`Registration`] object for a character device.
+ ///
+ /// This does *not* register the device: see [`Self::register()`].
+ pub fn new_pinned(
+ name: CStr<'static>,
+ minors_start: u16,
+ this_module: &'static crate::ThisModule,
+ ) -> KernelResult<Pin<Box<Self>>> {
+ Ok(Pin::from(Box::try_new(Self::new(
+ name,
+ minors_start,
+ this_module,
+ ))?))
+ }
+
+ /// Registers a character device.
+ ///
+ /// You may call this once per device type, up to `N` times.
+ pub fn register<T: file_operations::FileOpener<()>>(self: Pin<&mut Self>) -> KernelResult {
+ // SAFETY: We must ensure that we never move out of `this`.
+ let this = unsafe { self.get_unchecked_mut() };
+ if this.inner.is_none() {
+ let mut dev: bindings::dev_t = 0;
+ // SAFETY: Calling unsafe function. `this.name` has `'static`
+ // lifetime.
+ let res = unsafe {
+ bindings::alloc_chrdev_region(
+ &mut dev,
+ this.minors_start.into(),
+ N.try_into()?,
+ this.name.as_ptr() as *const c_types::c_char,
+ )
+ };
+ if res != 0 {
+ return Err(Error::from_kernel_errno(res));
+ }
+ this.inner = Some(RegistrationInner {
+ dev,
+ used: 0,
+ cdevs: [MaybeUninit::<bindings::cdev>::uninit(); N],
+ _pin: PhantomPinned,
+ });
+ }
+
+ let mut inner = this.inner.as_mut().unwrap();
+ if inner.used == N {
+ return Err(Error::EINVAL);
+ }
+ let cdev = inner.cdevs[inner.used].as_mut_ptr();
+ // SAFETY: Calling unsafe functions and manipulating `MaybeUninit`
+ // pointer.
+ unsafe {
+ bindings::cdev_init(
+ cdev,
+ // SAFETY: The adapter doesn't retrieve any state yet, so it's compatible with any
+ // registration.
+ file_operations::FileOperationsVtable::<Self, T>::build(),
+ );
+ (*cdev).owner = this.this_module.0;
+ let rc = bindings::cdev_add(cdev, inner.dev + inner.used as bindings::dev_t, 1);
+ if rc != 0 {
+ return Err(Error::from_kernel_errno(rc));
+ }
+ }
+ inner.used += 1;
+ Ok(())
+ }
+}
+
+impl<const N: usize> file_operations::FileOpenAdapter for Registration<{ N }> {
+ type Arg = ();
+
+ unsafe fn convert(
+ _inode: *mut bindings::inode,
+ _file: *mut bindings::file,
+ ) -> *const Self::Arg {
+ // TODO: Update the SAFETY comment on the call to `FileOperationsVTable::build` above once
+ // this is updated to retrieve state.
+ &()
+ }
+}
+
+// SAFETY: `Registration` does not expose any of its state across threads
+// (it is fine for multiple threads to have a shared reference to it).
+unsafe impl<const N: usize> Sync for Registration<{ N }> {}
+
+impl<const N: usize> Drop for Registration<{ N }> {
+ fn drop(&mut self) {
+ if let Some(inner) = self.inner.as_mut() {
+ // SAFETY: Calling unsafe functions, `0..inner.used` of
+ // `inner.cdevs` are initialized in `Registration::register`.
+ unsafe {
+ for i in 0..inner.used {
+ bindings::cdev_del(inner.cdevs[i].as_mut_ptr());
+ }
+ bindings::unregister_chrdev_region(inner.dev, N.try_into().unwrap());
+ }
+ }
+ }
+}
diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs
new file mode 100644
index 0000000000000..432d866232c13
--- /dev/null
+++ b/rust/kernel/error.rs
@@ -0,0 +1,106 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Kernel errors.
+//!
+//! C header: [`include/uapi/asm-generic/errno-base.h`](../../../include/uapi/asm-generic/errno-base.h)
+
+use crate::{bindings, c_types};
+use alloc::{alloc::AllocError, collections::TryReserveError};
+use core::{num::TryFromIntError, str::Utf8Error};
+
+/// Generic integer kernel error.
+///
+/// The kernel defines a set of integer generic error codes based on C and
+/// POSIX ones. These codes may have a more specific meaning in some contexts.
+pub struct Error(c_types::c_int);
+
+impl Error {
+ /// Invalid argument.
+ pub const EINVAL: Self = Error(-(bindings::EINVAL as i32));
+
+ /// Out of memory.
+ pub const ENOMEM: Self = Error(-(bindings::ENOMEM as i32));
+
+ /// Bad address.
+ pub const EFAULT: Self = Error(-(bindings::EFAULT as i32));
+
+ /// Illegal seek.
+ pub const ESPIPE: Self = Error(-(bindings::ESPIPE as i32));
+
+ /// Try again.
+ pub const EAGAIN: Self = Error(-(bindings::EAGAIN as i32));
+
+ /// Device or resource busy.
+ pub const EBUSY: Self = Error(-(bindings::EBUSY as i32));
+
+ /// Restart the system call.
+ pub const ERESTARTSYS: Self = Error(-(bindings::ERESTARTSYS as i32));
+
+ /// Operation not permitted.
+ pub const EPERM: Self = Error(-(bindings::EPERM as i32));
+
+ /// No such process.
+ pub const ESRCH: Self = Error(-(bindings::ESRCH as i32));
+
+ /// No such file or directory.
+ pub const ENOENT: Self = Error(-(bindings::ENOENT as i32));
+
+ /// Interrupted system call.
+ pub const EINTR: Self = Error(-(bindings::EINTR as i32));
+
+ /// Creates an [`Error`] from a kernel error code.
+ pub fn from_kernel_errno(errno: c_types::c_int) -> Error {
+ Error(errno)
+ }
+
+ /// Returns the kernel error code.
+ pub fn to_kernel_errno(&self) -> c_types::c_int {
+ self.0
+ }
+}
+
+impl From<TryFromIntError> for Error {
+ fn from(_: TryFromIntError) -> Error {
+ Error::EINVAL
+ }
+}
+
+impl From<Utf8Error> for Error {
+ fn from(_: Utf8Error) -> Error {
+ Error::EINVAL
+ }
+}
+
+impl From<TryReserveError> for Error {
+ fn from(_: TryReserveError) -> Error {
+ Error::ENOMEM
+ }
+}
+
+/// A [`Result`] with an [`Error`] error type.
+///
+/// To be used as the return type for functions that may fail.
+///
+/// # Error codes in C and Rust
+///
+/// In C, it is common that functions indicate success or failure through
+/// their return value; modifying or returning extra data through non-`const`
+/// pointer parameters. In particular, in the kernel, functions that may fail
+/// typically return an `int` that represents a generic error code. We model
+/// those as [`Error`].
+///
+/// In Rust, it is idiomatic to model functions that may fail as returning
+/// a [`Result`]. Since in the kernel many functions return an error code,
+/// [`KernelResult`] is a type alias for a [`Result`] that uses [`Error`] as
+/// its error type.
+///
+/// Note that even if a function does not return anything when it succeeds,
+/// it should still be modeled as returning a `KernelResult` rather than
+/// just an [`Error`].
+pub type KernelResult<T = ()> = Result<T, Error>;
+
+impl From<AllocError> for Error {
+ fn from(_: AllocError) -> Error {
+ Error::ENOMEM
+ }
+}
diff --git a/rust/kernel/file_operations.rs b/rust/kernel/file_operations.rs
new file mode 100644
index 0000000000000..da13d4b6b0dbf
--- /dev/null
+++ b/rust/kernel/file_operations.rs
@@ -0,0 +1,674 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! File operations.
+//!
+//! C header: [`include/linux/fs.h`](../../../../include/linux/fs.h)
+
+use core::convert::{TryFrom, TryInto};
+use core::{marker, mem, ops::Deref, pin::Pin, ptr};
+
+use alloc::boxed::Box;
+use alloc::sync::Arc;
+
+use crate::bindings;
+use crate::c_types;
+use crate::error::{Error, KernelResult};
+use crate::sync::{CondVar, Ref, RefCounted};
+use crate::user_ptr::{UserSlicePtr, UserSlicePtrReader, UserSlicePtrWriter};
+
+/// Wraps the kernel's `struct file`.
+///
+/// # Invariants
+///
+/// The pointer [`File::ptr`] is non-null and valid.
+pub struct File {
+ ptr: *const bindings::file,
+}
+
+impl File {
+ /// Constructs a new [`struct file`] wrapper.
+ ///
+ /// # Safety
+ ///
+ /// The pointer `ptr` must be non-null and valid for the lifetime of the object.
+ unsafe fn from_ptr(ptr: *const bindings::file) -> File {
+ // INVARIANTS: the safety contract ensures the type invariant will hold.
+ File { ptr }
+ }
+
+ /// Returns the current seek/cursor/pointer position (`struct file::f_pos`).
+ pub fn pos(&self) -> u64 {
+ // SAFETY: `File::ptr` is guaranteed to be valid by the type invariants.
+ unsafe { (*self.ptr).f_pos as u64 }
+ }
+
+ /// Returns whether the file is in blocking mode.
+ pub fn is_blocking(&self) -> bool {
+ // SAFETY: `File::ptr` is guaranteed to be valid by the type invariants.
+ unsafe { (*self.ptr).f_flags & bindings::O_NONBLOCK == 0 }
+ }
+}
+
+/// Wraps the kernel's `struct poll_table_struct`.
+///
+/// # Invariants
+///
+/// The pointer [`PollTable::ptr`] is null or valid.
+pub struct PollTable {
+ ptr: *mut bindings::poll_table_struct,
+}
+
+impl PollTable {
+ /// Constructors a new `struct poll_table_struct` wrapper.
+ ///
+ /// # Safety
+ ///
+ /// The pointer `ptr` must be either null or a valid pointer for the lifetime of the object.
+ unsafe fn from_ptr(ptr: *mut bindings::poll_table_struct) -> Self {
+ Self { ptr }
+ }
+
+ /// Associates the given file and condition variable to this poll table. It means notifying the
+ /// condition variable will notify the poll table as well; additionally, the association
+ /// between the condition variable and the file will automatically be undone by the kernel when
+ /// the file is destructed. To unilaterally remove the association before then, one can call
+ /// [`CondVar::free_waiters`].
+ ///
+ /// # Safety
+ ///
+ /// If the condition variable is destroyed before the file, then [`CondVar::free_waiters`] must
+ /// be called to ensure that all waiters are flushed out.
+ pub unsafe fn register_wait<'a>(&self, file: &'a File, cv: &'a CondVar) {
+ if self.ptr.is_null() {
+ return;
+ }
+
+ // SAFETY: `PollTable::ptr` is guaranteed to be valid by the type invariants and the null
+ // check above.
+ let table = &*self.ptr;
+ if let Some(proc) = table._qproc {
+ // SAFETY: All pointers are known to be valid.
+ proc(file.ptr as _, cv.wait_list.get(), self.ptr)
+ }
+ }
+}
+
+/// Equivalent to [`std::io::SeekFrom`].
+///
+/// [`std::io::SeekFrom`]: https://doc.rust-lang.org/std/io/enum.SeekFrom.html
+pub enum SeekFrom {
+ /// Equivalent to C's `SEEK_SET`.
+ Start(u64),
+
+ /// Equivalent to C's `SEEK_END`.
+ End(i64),
+
+ /// Equivalent to C's `SEEK_CUR`.
+ Current(i64),
+}
+
+fn from_kernel_result<T>(r: KernelResult<T>) -> T
+where
+ T: TryFrom<c_types::c_int>,
+ T::Error: core::fmt::Debug,
+{
+ match r {
+ Ok(v) => v,
+ Err(e) => T::try_from(e.to_kernel_errno()).unwrap(),
+ }
+}
+
+macro_rules! from_kernel_result {
+ ($($tt:tt)*) => {{
+ from_kernel_result((|| {
+ $($tt)*
+ })())
+ }};
+}
+
+unsafe extern "C" fn open_callback<A: FileOpenAdapter, T: FileOpener<A::Arg>>(
+ inode: *mut bindings::inode,
+ file: *mut bindings::file,
+) -> c_types::c_int {
+ from_kernel_result! {
+ let arg = A::convert(inode, file);
+ let ptr = T::open(&*arg)?.into_pointer();
+ (*file).private_data = ptr as *mut c_types::c_void;
+ Ok(0)
+ }
+}
+
+unsafe extern "C" fn read_callback<T: FileOperations>(
+ file: *mut bindings::file,
+ buf: *mut c_types::c_char,
+ len: c_types::c_size_t,
+ offset: *mut bindings::loff_t,
+) -> c_types::c_ssize_t {
+ from_kernel_result! {
+ let mut data = UserSlicePtr::new(buf as *mut c_types::c_void, len).writer();
+ let f = &*((*file).private_data as *const T);
+ // No `FMODE_UNSIGNED_OFFSET` support, so `offset` must be in [0, 2^63).
+ // See discussion in https://github.com/fishinabarrel/linux-kernel-module-rust/pull/113
+ let read = f.read(&File::from_ptr(file), &mut data, (*offset).try_into()?)?;
+ (*offset) += bindings::loff_t::try_from(read).unwrap();
+ Ok(read as _)
+ }
+}
+
+unsafe extern "C" fn write_callback<T: FileOperations>(
+ file: *mut bindings::file,
+ buf: *const c_types::c_char,
+ len: c_types::c_size_t,
+ offset: *mut bindings::loff_t,
+) -> c_types::c_ssize_t {
+ from_kernel_result! {
+ let mut data = UserSlicePtr::new(buf as *mut c_types::c_void, len).reader();
+ let f = &*((*file).private_data as *const T);
+ // No `FMODE_UNSIGNED_OFFSET` support, so `offset` must be in [0, 2^63).
+ // See discussion in https://github.com/fishinabarrel/linux-kernel-module-rust/pull/113
+ let written = f.write(&mut data, (*offset).try_into()?)?;
+ (*offset) += bindings::loff_t::try_from(written).unwrap();
+ Ok(written as _)
+ }
+}
+
+unsafe extern "C" fn release_callback<T: FileOperations>(
+ _inode: *mut bindings::inode,
+ file: *mut bindings::file,
+) -> c_types::c_int {
+ let ptr = mem::replace(&mut (*file).private_data, ptr::null_mut());
+ T::release(T::Wrapper::from_pointer(ptr as _), &File::from_ptr(file));
+ 0
+}
+
+unsafe extern "C" fn llseek_callback<T: FileOperations>(
+ file: *mut bindings::file,
+ offset: bindings::loff_t,
+ whence: c_types::c_int,
+) -> bindings::loff_t {
+ from_kernel_result! {
+ let off = match whence as u32 {
+ bindings::SEEK_SET => SeekFrom::Start(offset.try_into()?),
+ bindings::SEEK_CUR => SeekFrom::Current(offset),
+ bindings::SEEK_END => SeekFrom::End(offset),
+ _ => return Err(Error::EINVAL),
+ };
+ let f = &*((*file).private_data as *const T);
+ let off = f.seek(&File::from_ptr(file), off)?;
+ Ok(off as bindings::loff_t)
+ }
+}
+
+unsafe extern "C" fn unlocked_ioctl_callback<T: FileOperations>(
+ file: *mut bindings::file,
+ cmd: c_types::c_uint,
+ arg: c_types::c_ulong,
+) -> c_types::c_long {
+ from_kernel_result! {
+ let f = &*((*file).private_data as *const T);
+ // SAFETY: This function is called by the kernel, so it must set `fs` appropriately.
+ let mut cmd = IoctlCommand::new(cmd as _, arg as _);
+ let ret = f.ioctl(&File::from_ptr(file), &mut cmd)?;
+ Ok(ret as _)
+ }
+}
+
+unsafe extern "C" fn compat_ioctl_callback<T: FileOperations>(
+ file: *mut bindings::file,
+ cmd: c_types::c_uint,
+ arg: c_types::c_ulong,
+) -> c_types::c_long {
+ from_kernel_result! {
+ let f = &*((*file).private_data as *const T);
+ // SAFETY: This function is called by the kernel, so it must set `fs` appropriately.
+ let mut cmd = IoctlCommand::new(cmd as _, arg as _);
+ let ret = f.compat_ioctl(&File::from_ptr(file), &mut cmd)?;
+ Ok(ret as _)
+ }
+}
+
+unsafe extern "C" fn mmap_callback<T: FileOperations>(
+ file: *mut bindings::file,
+ vma: *mut bindings::vm_area_struct,
+) -> c_types::c_int {
+ from_kernel_result! {
+ let f = &*((*file).private_data as *const T);
+ f.mmap(&File::from_ptr(file), &mut *vma)?;
+ Ok(0)
+ }
+}
+
+unsafe extern "C" fn fsync_callback<T: FileOperations>(
+ file: *mut bindings::file,
+ start: bindings::loff_t,
+ end: bindings::loff_t,
+ datasync: c_types::c_int,
+) -> c_types::c_int {
+ from_kernel_result! {
+ let start = start.try_into()?;
+ let end = end.try_into()?;
+ let datasync = datasync != 0;
+ let f = &*((*file).private_data as *const T);
+ let res = f.fsync(&File::from_ptr(file), start, end, datasync)?;
+ Ok(res.try_into().unwrap())
+ }
+}
+
+unsafe extern "C" fn poll_callback<T: FileOperations>(
+ file: *mut bindings::file,
+ wait: *mut bindings::poll_table_struct,
+) -> bindings::__poll_t {
+ let f = &*((*file).private_data as *const T);
+ match f.poll(&File::from_ptr(file), &PollTable::from_ptr(wait)) {
+ Ok(v) => v,
+ Err(_) => bindings::POLLERR,
+ }
+}
+
+pub(crate) struct FileOperationsVtable<A, T>(marker::PhantomData<A>, marker::PhantomData<T>);
+
+impl<A: FileOpenAdapter, T: FileOpener<A::Arg>> FileOperationsVtable<A, T> {
+ const VTABLE: bindings::file_operations = bindings::file_operations {
+ open: Some(open_callback::<A, T>),
+ release: Some(release_callback::<T>),
+ read: if T::TO_USE.read {
+ Some(read_callback::<T>)
+ } else {
+ None
+ },
+ write: if T::TO_USE.write {
+ Some(write_callback::<T>)
+ } else {
+ None
+ },
+ llseek: if T::TO_USE.seek {
+ Some(llseek_callback::<T>)
+ } else {
+ None
+ },
+
+ check_flags: None,
+ compat_ioctl: if T::TO_USE.compat_ioctl {
+ Some(compat_ioctl_callback::<T>)
+ } else {
+ None
+ },
+ copy_file_range: None,
+ fallocate: None,
+ fadvise: None,
+ fasync: None,
+ flock: None,
+ flush: None,
+ fsync: if T::TO_USE.fsync {
+ Some(fsync_callback::<T>)
+ } else {
+ None
+ },
+ get_unmapped_area: None,
+ iterate: None,
+ iterate_shared: None,
+ iopoll: None,
+ lock: None,
+ mmap: if T::TO_USE.mmap {
+ Some(mmap_callback::<T>)
+ } else {
+ None
+ },
+ mmap_supported_flags: 0,
+ owner: ptr::null_mut(),
+ poll: if T::TO_USE.poll {
+ Some(poll_callback::<T>)
+ } else {
+ None
+ },
+ read_iter: None,
+ remap_file_range: None,
+ sendpage: None,
+ setlease: None,
+ show_fdinfo: None,
+ splice_read: None,
+ splice_write: None,
+ unlocked_ioctl: if T::TO_USE.ioctl {
+ Some(unlocked_ioctl_callback::<T>)
+ } else {
+ None
+ },
+ write_iter: None,
+ };
+
+ /// Builds an instance of [`struct file_operations`].
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that the adapter is compatible with the way the device is registered.
+ pub(crate) const unsafe fn build() -> &'static bindings::file_operations {
+ &Self::VTABLE
+ }
+}
+
+/// Represents which fields of [`struct file_operations`] should be populated with pointers.
+pub struct ToUse {
+ /// The `read` field of [`struct file_operations`].
+ pub read: bool,
+
+ /// The `write` field of [`struct file_operations`].
+ pub write: bool,
+
+ /// The `llseek` field of [`struct file_operations`].
+ pub seek: bool,
+
+ /// The `unlocked_ioctl` field of [`struct file_operations`].
+ pub ioctl: bool,
+
+ /// The `compat_ioctl` field of [`struct file_operations`].
+ pub compat_ioctl: bool,
+
+ /// The `fsync` field of [`struct file_operations`].
+ pub fsync: bool,
+
+ /// The `mmap` field of [`struct file_operations`].
+ pub mmap: bool,
+
+ /// The `poll` field of [`struct file_operations`].
+ pub poll: bool,
+}
+
+/// A constant version where all values are to set to `false`, that is, all supported fields will
+/// be set to null pointers.
+pub const USE_NONE: ToUse = ToUse {
+ read: false,
+ write: false,
+ seek: false,
+ ioctl: false,
+ compat_ioctl: false,
+ fsync: false,
+ mmap: false,
+ poll: false,
+};
+
+/// Defines the [`FileOperations::TO_USE`] field based on a list of fields to be populated.
+#[macro_export]
+macro_rules! declare_file_operations {
+ () => {
+ const TO_USE: $crate::file_operations::ToUse = $crate::file_operations::USE_NONE;
+ };
+ ($($i:ident),+) => {
+ const TO_USE: kernel::file_operations::ToUse =
+ $crate::file_operations::ToUse {
+ $($i: true),+ ,
+ ..$crate::file_operations::USE_NONE
+ };
+ };
+}
+
+/// Allows the handling of ioctls defined with the `_IO`, `_IOR`, `_IOW`, and `_IOWR` macros.
+///
+/// For each macro, there is a handler function that takes the appropriate types as arguments.
+pub trait IoctlHandler: Sync {
+ /// Handles ioctls defined with the `_IO` macro, that is, with no buffer as argument.
+ fn pure(&self, _file: &File, _cmd: u32, _arg: usize) -> KernelResult<i32> {
+ Err(Error::EINVAL)
+ }
+
+ /// Handles ioctls defined with the `_IOR` macro, that is, with an output buffer provided as
+ /// argument.
+ fn read(&self, _file: &File, _cmd: u32, _writer: &mut UserSlicePtrWriter) -> KernelResult<i32> {
+ Err(Error::EINVAL)
+ }
+
+ /// Handles ioctls defined with the `_IOW` macro, that is, with an input buffer provided as
+ /// argument.
+ fn write(
+ &self,
+ _file: &File,
+ _cmd: u32,
+ _reader: &mut UserSlicePtrReader,
+ ) -> KernelResult<i32> {
+ Err(Error::EINVAL)
+ }
+
+ /// Handles ioctls defined with the `_IOWR` macro, that is, with a buffer for both input and
+ /// output provided as argument.
+ fn read_write(&self, _file: &File, _cmd: u32, _data: UserSlicePtr) -> KernelResult<i32> {
+ Err(Error::EINVAL)
+ }
+}
+
+/// Represents an ioctl command.
+///
+/// It can use the components of an ioctl command to dispatch ioctls using
+/// [`IoctlCommand::dispatch`].
+pub struct IoctlCommand {
+ cmd: u32,
+ arg: usize,
+ user_slice: Option<UserSlicePtr>,
+}
+
+impl IoctlCommand {
+ /// Constructs a new [`IoctlCommand`].
+ fn new(cmd: u32, arg: usize) -> Self {
+ let size = (cmd >> bindings::_IOC_SIZESHIFT) & bindings::_IOC_SIZEMASK;
+
+ // SAFETY: We only create one instance of the user slice per ioctl call, so TOCTOU issues
+ // are not possible.
+ let user_slice = Some(unsafe { UserSlicePtr::new(arg as _, size as _) });
+ Self {
+ cmd,
+ arg,
+ user_slice,
+ }
+ }
+
+ /// Dispatches the given ioctl to the appropriate handler based on the value of the command. It
+ /// also creates a [`UserSlicePtr`], [`UserSlicePtrReader`], or [`UserSlicePtrWriter`]
+ /// depending on the direction of the buffer of the command.
+ ///
+ /// It is meant to be used in implementations of [`FileOperations::ioctl`] and
+ /// [`FileOperations::compat_ioctl`].
+ pub fn dispatch<T: IoctlHandler>(&mut self, handler: &T, file: &File) -> KernelResult<i32> {
+ let dir = (self.cmd >> bindings::_IOC_DIRSHIFT) & bindings::_IOC_DIRMASK;
+ if dir == bindings::_IOC_NONE {
+ return handler.pure(file, self.cmd, self.arg);
+ }
+
+ let data = self.user_slice.take().ok_or(Error::EINVAL)?;
+ const READ_WRITE: u32 = bindings::_IOC_READ | bindings::_IOC_WRITE;
+ match dir {
+ bindings::_IOC_WRITE => handler.write(file, self.cmd, &mut data.reader()),
+ bindings::_IOC_READ => handler.read(file, self.cmd, &mut data.writer()),
+ READ_WRITE => handler.read_write(file, self.cmd, data),
+ _ => Err(Error::EINVAL),
+ }
+ }
+
+ /// Returns the raw 32-bit value of the command and the ptr-sized argument.
+ pub fn raw(&self) -> (u32, usize) {
+ (self.cmd, self.arg)
+ }
+}
+
+/// Trait for extracting file open arguments from kernel data structures.
+///
+/// This is meant to be implemented by registration managers.
+pub trait FileOpenAdapter {
+ /// The type of argument this adapter extracts.
+ type Arg;
+
+ /// Converts untyped data stored in [`struct inode`] and [`struct file`] (when [`struct
+ /// file_operations::open`] is called) into the given type. For example, for `miscdev`
+ /// devices, a pointer to the registered [`struct miscdev`] is stored in [`struct
+ /// file::private_data`].
+ ///
+ /// # Safety
+ ///
+ /// This function must be called only when [`struct file_operations::open`] is being called for
+ /// a file that was registered by the implementer.
+ unsafe fn convert(_inode: *mut bindings::inode, _file: *mut bindings::file)
+ -> *const Self::Arg;
+}
+
+/// Trait for implementers of kernel files.
+///
+/// In addition to the methods in [`FileOperations`], implementers must also provide
+/// [`FileOpener::open`] with a customised argument. This allows a single implementation of
+/// [`FileOperations`] to be used for different types of registrations, for example, `miscdev` and
+/// `chrdev`.
+pub trait FileOpener<T: ?Sized>: FileOperations {
+ /// Creates a new instance of this file.
+ ///
+ /// Corresponds to the `open` function pointer in `struct file_operations`.
+ fn open(context: &T) -> KernelResult<Self::Wrapper>;
+}
+
+impl<T: FileOperations<Wrapper = Box<T>> + Default> FileOpener<()> for T {
+ fn open(_: &()) -> KernelResult<Self::Wrapper> {
+ Ok(Box::try_new(T::default())?)
+ }
+}
+
+/// Corresponds to the kernel's `struct file_operations`.
+///
+/// You implement this trait whenever you would create a `struct file_operations`.
+///
+/// File descriptors may be used from multiple threads/processes concurrently, so your type must be
+/// [`Sync`]. It must also be [`Send`] because [`FileOperations::release`] will be called from the
+/// thread that decrements that associated file's refcount to zero.
+pub trait FileOperations: Send + Sync + Sized {
+ /// The methods to use to populate [`struct file_operations`].
+ const TO_USE: ToUse;
+
+ /// The pointer type that will be used to hold ourselves.
+ type Wrapper: PointerWrapper<Self> = Box<Self>;
+
+ /// Cleans up after the last reference to the file goes away.
+ ///
+ /// Note that the object is moved, so it will be freed automatically unless the implementation
+ /// moves it elsewhere.
+ ///
+ /// Corresponds to the `release` function pointer in `struct file_operations`.
+ fn release(_obj: Self::Wrapper, _file: &File) {}
+
+ /// Reads data from this file to userspace.
+ ///
+ /// Corresponds to the `read` function pointer in `struct file_operations`.
+ fn read(
+ &self,
+ _file: &File,
+ _data: &mut UserSlicePtrWriter,
+ _offset: u64,
+ ) -> KernelResult<usize> {
+ Err(Error::EINVAL)
+ }
+
+ /// Writes data from userspace to this file.
+ ///
+ /// Corresponds to the `write` function pointer in `struct file_operations`.
+ fn write(&self, _data: &mut UserSlicePtrReader, _offset: u64) -> KernelResult<usize> {
+ Err(Error::EINVAL)
+ }
+
+ /// Changes the position of the file.
+ ///
+ /// Corresponds to the `llseek` function pointer in `struct file_operations`.
+ fn seek(&self, _file: &File, _offset: SeekFrom) -> KernelResult<u64> {
+ Err(Error::EINVAL)
+ }
+
+ /// Performs IO control operations that are specific to the file.
+ ///
+ /// Corresponds to the `unlocked_ioctl` function pointer in `struct file_operations`.
+ fn ioctl(&self, _file: &File, _cmd: &mut IoctlCommand) -> KernelResult<i32> {
+ Err(Error::EINVAL)
+ }
+
+ /// Performs 32-bit IO control operations on that are specific to the file on 64-bit kernels.
+ ///
+ /// Corresponds to the `compat_ioctl` function pointer in `struct file_operations`.
+ fn compat_ioctl(&self, _file: &File, _cmd: &mut IoctlCommand) -> KernelResult<i32> {
+ Err(Error::EINVAL)
+ }
+
+ /// Syncs pending changes to this file.
+ ///
+ /// Corresponds to the `fsync` function pointer in `struct file_operations`.
+ fn fsync(&self, _file: &File, _start: u64, _end: u64, _datasync: bool) -> KernelResult<u32> {
+ Err(Error::EINVAL)
+ }
+
+ /// Maps areas of the caller's virtual memory with device/file memory.
+ ///
+ /// Corresponds to the `mmap` function pointer in `struct file_operations`.
+ /// TODO: wrap `vm_area_struct` so that we don't have to expose it.
+ fn mmap(&self, _file: &File, _vma: &mut bindings::vm_area_struct) -> KernelResult {
+ Err(Error::EINVAL)
+ }
+
+ /// Checks the state of the file and optionally registers for notification when the state
+ /// changes.
+ ///
+ /// Corresponds to the `poll` function pointer in `struct file_operations`.
+ fn poll(&self, _file: &File, _table: &PollTable) -> KernelResult<u32> {
+ Ok(bindings::POLLIN | bindings::POLLOUT | bindings::POLLRDNORM | bindings::POLLWRNORM)
+ }
+}
+
+/// Used to convert an object into a raw pointer that represents it.
+///
+/// It can eventually be converted back into the object. This is used to store objects as pointers
+/// in kernel data structures, for example, an implementation of [`FileOperations`] in `struct
+/// file::private_data`.
+pub trait PointerWrapper<T> {
+ /// Returns the raw pointer.
+ fn into_pointer(self) -> *const T;
+
+ /// Returns the instance back from the raw pointer.
+ ///
+ /// # Safety
+ ///
+ /// The passed pointer must come from a previous call to [`PointerWrapper::into_pointer()`].
+ unsafe fn from_pointer(ptr: *const T) -> Self;
+}
+
+impl<T> PointerWrapper<T> for Box<T> {
+ fn into_pointer(self) -> *const T {
+ Box::into_raw(self)
+ }
+
+ unsafe fn from_pointer(ptr: *const T) -> Self {
+ Box::from_raw(ptr as _)
+ }
+}
+
+impl<T: RefCounted> PointerWrapper<T> for Ref<T> {
+ fn into_pointer(self) -> *const T {
+ Ref::into_raw(self)
+ }
+
+ unsafe fn from_pointer(ptr: *const T) -> Self {
+ Ref::from_raw(ptr as _)
+ }
+}
+
+impl<T> PointerWrapper<T> for Arc<T> {
+ fn into_pointer(self) -> *const T {
+ Arc::into_raw(self)
+ }
+
+ unsafe fn from_pointer(ptr: *const T) -> Self {
+ Arc::from_raw(ptr)
+ }
+}
+
+impl<T, W: PointerWrapper<T> + Deref> PointerWrapper<T> for Pin<W> {
+ fn into_pointer(self) -> *const T {
+ // SAFETY: We continue to treat the pointer as pinned by returning just a pointer to it to
+ // the caller.
+ let inner = unsafe { Pin::into_inner_unchecked(self) };
+ inner.into_pointer()
+ }
+
+ unsafe fn from_pointer(p: *const T) -> Self {
+ // SAFETY: The object was originally pinned.
+ Pin::new_unchecked(W::from_pointer(p))
+ }
+}
diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs
new file mode 100644
index 0000000000000..2ad22dc1e6dec
--- /dev/null
+++ b/rust/kernel/lib.rs
@@ -0,0 +1,204 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! The `kernel` crate.
+//!
+//! This crate contains the kernel APIs that have been ported or wrapped for
+//! usage by Rust code in the kernel and is shared by all of them.
+//!
+//! In other words, all the rest of the Rust code in the kernel (e.g. kernel
+//! modules written in Rust) depends on [`core`], [`alloc`] and this crate.
+//!
+//! If you need a kernel C API that is not ported or wrapped yet here, then
+//! do so first instead of bypassing this crate.
+
+#![no_std]
+#![feature(
+ allocator_api,
+ alloc_error_handler,
+ associated_type_defaults,
+ const_fn,
+ const_mut_refs,
+ const_panic,
+ try_reserve
+)]
+#![deny(clippy::complexity)]
+#![deny(clippy::correctness)]
+#![deny(clippy::perf)]
+#![deny(clippy::style)]
+
+// Ensure conditional compilation based on the kernel configuration works;
+// otherwise we may silently break things like initcall handling.
+#[cfg(not(CONFIG_RUST))]
+compile_error!("Missing kernel configuration for conditional compilation");
+
+mod allocator;
+
+#[doc(hidden)]
+pub mod bindings;
+
+pub mod buffer;
+pub mod c_types;
+pub mod chrdev;
+mod error;
+pub mod file_operations;
+pub mod miscdev;
+pub mod pages;
+
+pub mod linked_list;
+mod raw_list;
+
+#[doc(hidden)]
+pub mod module_param;
+
+pub mod prelude;
+pub mod print;
+pub mod random;
+mod static_assert;
+pub mod sync;
+
+#[cfg(CONFIG_SYSCTL)]
+pub mod sysctl;
+
+mod types;
+pub mod user_ptr;
+
+pub use crate::error::{Error, KernelResult};
+pub use crate::types::{CStr, Mode};
+
+/// Page size defined in terms of the `PAGE_SHIFT` macro from C.
+///
+/// [`PAGE_SHIFT`]: ../../../include/asm-generic/page.h
+pub const PAGE_SIZE: usize = 1 << bindings::PAGE_SHIFT;
+
+/// Prefix to appear before log messages printed from within the kernel crate.
+const __LOG_PREFIX: &[u8] = b"rust_kernel\0";
+
+/// The top level entrypoint to implementing a kernel module.
+///
+/// For any teardown or cleanup operations, your type may implement [`Drop`].
+pub trait KernelModule: Sized + Sync {
+ /// Called at module initialization time.
+ ///
+ /// Use this method to perform whatever setup or registration your module
+ /// should do.
+ ///
+ /// Equivalent to the `module_init` macro in the C API.
+ fn init() -> KernelResult<Self>;
+}
+
+/// Equivalent to `THIS_MODULE` in the C API.
+///
+/// C header: `include/linux/export.h`
+pub struct ThisModule(*mut bindings::module);
+
+// SAFETY: `THIS_MODULE` may be used from all threads within a module.
+unsafe impl Sync for ThisModule {}
+
+impl ThisModule {
+ /// Creates a [`ThisModule`] given the `THIS_MODULE` pointer.
+ ///
+ /// # Safety
+ ///
+ /// The pointer must be equal to the right `THIS_MODULE`.
+ pub const unsafe fn from_ptr(ptr: *mut bindings::module) -> ThisModule {
+ ThisModule(ptr)
+ }
+
+ /// Locks the module parameters to access them.
+ ///
+ /// Returns a [`KParamGuard`] that will release the lock when dropped.
+ pub fn kernel_param_lock(&self) -> KParamGuard<'_> {
+ // SAFETY: `kernel_param_lock` will check if the pointer is null and
+ // use the built-in mutex in that case.
+ #[cfg(CONFIG_SYSFS)]
+ unsafe {
+ bindings::kernel_param_lock(self.0)
+ }
+
+ KParamGuard { this_module: self }
+ }
+}
+
+/// Scoped lock on the kernel parameters of [`ThisModule`].
+///
+/// Lock will be released when this struct is dropped.
+pub struct KParamGuard<'a> {
+ this_module: &'a ThisModule,
+}
+
+#[cfg(CONFIG_SYSFS)]
+impl<'a> Drop for KParamGuard<'a> {
+ fn drop(&mut self) {
+ // SAFETY: `kernel_param_lock` will check if the pointer is null and
+ // use the built-in mutex in that case. The existance of `self`
+ // guarantees that the lock is held.
+ unsafe { bindings::kernel_param_unlock(self.this_module.0) }
+ }
+}
+
+/// Calculates the offset of a field from the beginning of the struct it belongs to.
+///
+/// # Example
+///
+/// ```
+/// struct Test {
+/// a: u64,
+/// b: u32,
+/// }
+///
+/// fn test() {
+/// // This prints `8`.
+/// pr_info!("{}\n", offset_of!(Test, b));
+/// }
+/// ```
+#[macro_export]
+macro_rules! offset_of {
+ ($type:ty, $($f:tt)*) => {{
+ let tmp = core::mem::MaybeUninit::<$type>::uninit();
+ let outer = tmp.as_ptr();
+ // To avoid warnings when nesting `unsafe` blocks.
+ #[allow(unused_unsafe)]
+ // SAFETY: The pointer is valid and aligned, just not initialised; `addr_of` ensures that
+ // we don't actually read from `outer` (which would be UB) nor create an intermediate
+ // reference.
+ let inner = unsafe { core::ptr::addr_of!((*outer).$($f)*) } as *const u8;
+ // To avoid warnings when nesting `unsafe` blocks.
+ #[allow(unused_unsafe)]
+ // SAFETY: The two pointers are within the same allocation block.
+ unsafe { inner.offset_from(outer as *const u8) }
+ }}
+}
+
+/// Produces a pointer to an object from a pointer to one of its fields.
+///
+/// # Safety
+///
+/// Callers must ensure that the pointer to the field is in fact a pointer to the specified field,
+/// as opposed to a pointer to another object of the same type.
+///
+/// # Example
+///
+/// ```
+/// struct Test {
+/// a: u64,
+/// b: u32,
+/// }
+///
+/// fn test() {
+/// let test = Test { a: 10, b: 20 };
+/// let b_ptr = &test.b;
+/// let test_alias = unsafe { container_of!(b_ptr, Test, b) };
+/// // This prints `true`.
+/// pr_info!("{}\n", core::ptr::eq(&test, test_alias));
+/// }
+/// ```
+#[macro_export]
+macro_rules! container_of {
+ ($ptr:expr, $type:ty, $($f:tt)*) => {{
+ let offset = $crate::offset_of!($type, $($f)*);
+ ($ptr as *const _ as *const u8).offset(-offset) as *const $type
+ }}
+}
+
+#[global_allocator]
+static ALLOCATOR: allocator::KernelAllocator = allocator::KernelAllocator;
diff --git a/rust/kernel/linked_list.rs b/rust/kernel/linked_list.rs
new file mode 100644
index 0000000000000..5b0b811eae220
--- /dev/null
+++ b/rust/kernel/linked_list.rs
@@ -0,0 +1,245 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Linked lists.
+//!
+//! TODO: This module is a work in progress.
+
+use alloc::{boxed::Box, sync::Arc};
+use core::ptr::NonNull;
+
+pub use crate::raw_list::{Cursor, GetLinks, Links};
+use crate::{raw_list, raw_list::RawList};
+
+// TODO: Use the one from `kernel::file_operations::PointerWrapper` instead.
+/// Wraps an object to be inserted in a linked list.
+pub trait Wrapper<T: ?Sized> {
+ /// Converts the wrapped object into a pointer that represents it.
+ fn into_pointer(self) -> NonNull<T>;
+
+ /// Converts the object back from the pointer representation.
+ ///
+ /// # Safety
+ ///
+ /// The passed pointer must come from a previous call to [`Wrapper::into_pointer()`].
+ unsafe fn from_pointer(ptr: NonNull<T>) -> Self;
+
+ /// Returns a reference to the wrapped object.
+ fn as_ref(&self) -> &T;
+}
+
+impl<T: ?Sized> Wrapper<T> for Box<T> {
+ fn into_pointer(self) -> NonNull<T> {
+ NonNull::new(Box::into_raw(self)).unwrap()
+ }
+
+ unsafe fn from_pointer(ptr: NonNull<T>) -> Self {
+ Box::from_raw(ptr.as_ptr())
+ }
+
+ fn as_ref(&self) -> &T {
+ AsRef::as_ref(self)
+ }
+}
+
+impl<T: ?Sized> Wrapper<T> for Arc<T> {
+ fn into_pointer(self) -> NonNull<T> {
+ NonNull::new(Arc::into_raw(self) as _).unwrap()
+ }
+
+ unsafe fn from_pointer(ptr: NonNull<T>) -> Self {
+ Arc::from_raw(ptr.as_ptr())
+ }
+
+ fn as_ref(&self) -> &T {
+ AsRef::as_ref(self)
+ }
+}
+
+impl<T: ?Sized> Wrapper<T> for &T {
+ fn into_pointer(self) -> NonNull<T> {
+ NonNull::from(self)
+ }
+
+ unsafe fn from_pointer(ptr: NonNull<T>) -> Self {
+ &*ptr.as_ptr()
+ }
+
+ fn as_ref(&self) -> &T {
+ self
+ }
+}
+
+/// A descriptor of wrapped list elements.
+pub trait GetLinksWrapped: GetLinks {
+ /// Specifies which wrapper (e.g., `Box` and `Arc`) wraps the list entries.
+ type Wrapped: Wrapper<Self::EntryType>;
+}
+
+impl<T: ?Sized> GetLinksWrapped for Box<T>
+where
+ Box<T>: GetLinks,
+{
+ type Wrapped = Box<<Box<T> as GetLinks>::EntryType>;
+}
+
+impl<T: GetLinks + ?Sized> GetLinks for Box<T> {
+ type EntryType = T::EntryType;
+ fn get_links(data: &Self::EntryType) -> &Links<Self::EntryType> {
+ <T as GetLinks>::get_links(data)
+ }
+}
+
+impl<T: ?Sized> GetLinksWrapped for Arc<T>
+where
+ Arc<T>: GetLinks,
+{
+ type Wrapped = Arc<<Arc<T> as GetLinks>::EntryType>;
+}
+
+impl<T: GetLinks + ?Sized> GetLinks for Arc<T> {
+ type EntryType = T::EntryType;
+ fn get_links(data: &Self::EntryType) -> &Links<Self::EntryType> {
+ <T as GetLinks>::get_links(data)
+ }
+}
+
+/// A linked list.
+///
+/// Elements in the list are wrapped and ownership is transferred to the list while the element is
+/// in the list.
+pub struct List<G: GetLinksWrapped> {
+ list: RawList<G>,
+}
+
+impl<G: GetLinksWrapped> List<G> {
+ /// Constructs a new empty linked list.
+ pub fn new() -> Self {
+ Self {
+ list: RawList::new(),
+ }
+ }
+
+ /// Returns whether the list is empty.
+ pub fn is_empty(&self) -> bool {
+ self.list.is_empty()
+ }
+
+ /// Adds the given object to the end (back) of the list.
+ ///
+ /// It is dropped if it's already on this (or another) list; this can happen for
+ /// reference-counted objects, so dropping means decrementing the reference count.
+ pub fn push_back(&mut self, data: G::Wrapped) {
+ let ptr = data.into_pointer();
+
+ // SAFETY: We took ownership of the entry, so it is safe to insert it.
+ if !unsafe { self.list.push_back(ptr.as_ref()) } {
+ // If insertion failed, rebuild object so that it can be freed.
+ // SAFETY: We just called `into_pointer` above.
+ unsafe { G::Wrapped::from_pointer(ptr) };
+ }
+ }
+
+ /// Inserts the given object after `existing`.
+ ///
+ /// It is dropped if it's already on this (or another) list; this can happen for
+ /// reference-counted objects, so dropping means decrementing the reference count.
+ ///
+ /// # Safety
+ ///
+ /// Callers must ensure that `existing` points to a valid entry that is on the list.
+ pub unsafe fn insert_after(&mut self, existing: NonNull<G::EntryType>, data: G::Wrapped) {
+ let ptr = data.into_pointer();
+ let entry = &*existing.as_ptr();
+ if !self.list.insert_after(entry, ptr.as_ref()) {
+ // If insertion failed, rebuild object so that it can be freed.
+ G::Wrapped::from_pointer(ptr);
+ }
+ }
+
+ /// Removes the given entry.
+ ///
+ /// # Safety
+ ///
+ /// Callers must ensure that `data` is either on this list or in no list. It being on another
+ /// list leads to memory unsafety.
+ pub unsafe fn remove(&mut self, data: &G::Wrapped) -> Option<G::Wrapped> {
+ let entry_ref = Wrapper::as_ref(data);
+ if self.list.remove(entry_ref) {
+ Some(G::Wrapped::from_pointer(NonNull::from(entry_ref)))
+ } else {
+ None
+ }
+ }
+
+ /// Removes the element currently at the front of the list and returns it.
+ ///
+ /// Returns `None` if the list is empty.
+ pub fn pop_front(&mut self) -> Option<G::Wrapped> {
+ let front = self.list.pop_front()?;
+ // SAFETY: Elements on the list were inserted after a call to `into_pointer `.
+ Some(unsafe { G::Wrapped::from_pointer(front) })
+ }
+
+ /// Returns a cursor starting on the first (front) element of the list.
+ pub fn cursor_front(&self) -> Cursor<'_, G> {
+ self.list.cursor_front()
+ }
+
+ /// Returns a mutable cursor starting on the first (front) element of the list.
+ pub fn cursor_front_mut(&mut self) -> CursorMut<'_, G> {
+ CursorMut::new(self.list.cursor_front_mut())
+ }
+}
+
+impl<G: GetLinksWrapped> Default for List<G> {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl<G: GetLinksWrapped> Drop for List<G> {
+ fn drop(&mut self) {
+ while self.pop_front().is_some() {}
+ }
+}
+
+/// A list cursor that allows traversing a linked list and inspecting & mutating elements.
+pub struct CursorMut<'a, G: GetLinksWrapped> {
+ cursor: raw_list::CursorMut<'a, G>,
+}
+
+impl<'a, G: GetLinksWrapped> CursorMut<'a, G> {
+ fn new(cursor: raw_list::CursorMut<'a, G>) -> Self {
+ Self { cursor }
+ }
+
+ /// Returns the element the cursor is currently positioned on.
+ pub fn current(&mut self) -> Option<&mut G::EntryType> {
+ self.cursor.current()
+ }
+
+ /// Removes the element the cursor is currently positioned on.
+ ///
+ /// After removal, it advances the cursor to the next element.
+ pub fn remove_current(&mut self) -> Option<G::Wrapped> {
+ let ptr = self.cursor.remove_current()?;
+
+ // SAFETY: Elements on the list were inserted after a call to `into_pointer `.
+ Some(unsafe { G::Wrapped::from_pointer(ptr) })
+ }
+
+ /// Returns the element immediately after the one the cursor is positioned on.
+ pub fn peek_next(&mut self) -> Option<&mut G::EntryType> {
+ self.cursor.peek_next()
+ }
+
+ /// Returns the element immediately before the one the cursor is positioned on.
+ pub fn peek_prev(&mut self) -> Option<&mut G::EntryType> {
+ self.cursor.peek_prev()
+ }
+
+ /// Moves the cursor to the next element.
+ pub fn move_next(&mut self) {
+ self.cursor.move_next();
+ }
+}
diff --git a/rust/kernel/miscdev.rs b/rust/kernel/miscdev.rs
new file mode 100644
index 0000000000000..92c2181f30536
--- /dev/null
+++ b/rust/kernel/miscdev.rs
@@ -0,0 +1,109 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Miscellaneous devices.
+//!
+//! C header: [`include/linux/miscdevice.h`](../../../../include/linux/miscdevice.h)
+//!
+//! Reference: <https://www.kernel.org/doc/html/latest/driver-api/misc_devices.html>
+
+use crate::error::{Error, KernelResult};
+use crate::file_operations::{FileOpenAdapter, FileOpener, FileOperationsVtable};
+use crate::{bindings, c_types, CStr};
+use alloc::boxed::Box;
+use core::marker::PhantomPinned;
+use core::pin::Pin;
+
+/// A registration of a miscellaneous device.
+pub struct Registration<T: Sync = ()> {
+ registered: bool,
+ mdev: bindings::miscdevice,
+ _pin: PhantomPinned,
+
+ /// Context initialised on construction and made available to all file instances on
+ /// [`FileOpener::open`].
+ pub context: T,
+}
+
+impl<T: Sync> Registration<T> {
+ /// Creates a new [`Registration`] but does not register it yet.
+ ///
+ /// It is allowed to move.
+ pub fn new(context: T) -> Self {
+ Self {
+ registered: false,
+ mdev: bindings::miscdevice::default(),
+ _pin: PhantomPinned,
+ context,
+ }
+ }
+
+ /// Registers a miscellaneous device.
+ ///
+ /// Returns a pinned heap-allocated representation of the registration.
+ pub fn new_pinned<F: FileOpener<T>>(
+ name: CStr<'static>,
+ minor: Option<i32>,
+ context: T,
+ ) -> KernelResult<Pin<Box<Self>>> {
+ let mut r = Pin::from(Box::try_new(Self::new(context))?);
+ r.as_mut().register::<F>(name, minor)?;
+ Ok(r)
+ }
+
+ /// Registers a miscellaneous device with the rest of the kernel.
+ ///
+ /// It must be pinned because the memory block that represents the registration is
+ /// self-referential. If a minor is not given, the kernel allocates a new one if possible.
+ pub fn register<F: FileOpener<T>>(
+ self: Pin<&mut Self>,
+ name: CStr<'static>,
+ minor: Option<i32>,
+ ) -> KernelResult {
+ // SAFETY: We must ensure that we never move out of `this`.
+ let this = unsafe { self.get_unchecked_mut() };
+ if this.registered {
+ // Already registered.
+ return Err(Error::EINVAL);
+ }
+
+ // SAFETY: The adapter is compatible with `misc_register`.
+ this.mdev.fops = unsafe { FileOperationsVtable::<Self, F>::build() };
+ this.mdev.name = name.as_ptr() as *const c_types::c_char;
+ this.mdev.minor = minor.unwrap_or(bindings::MISC_DYNAMIC_MINOR as i32);
+
+ let ret = unsafe { bindings::misc_register(&mut this.mdev) };
+ if ret < 0 {
+ return Err(Error::from_kernel_errno(ret));
+ }
+ this.registered = true;
+ Ok(())
+ }
+}
+
+impl<T: Sync> FileOpenAdapter for Registration<T> {
+ type Arg = T;
+
+ unsafe fn convert(_inode: *mut bindings::inode, file: *mut bindings::file) -> *const Self::Arg {
+ let reg = crate::container_of!((*file).private_data, Self, mdev);
+ &(*reg).context
+ }
+}
+
+// SAFETY: The only method is `register()`, which requires a (pinned) mutable `Registration`, so it
+// is safe to pass `&Registration` to multiple threads because it offers no interior mutability,
+// except maybe through `Registration::context`, but it is itself `Sync`.
+unsafe impl<T: Sync> Sync for Registration<T> {}
+
+// SAFETY: All functions work from any thread. So as long as the `Registration::context` is
+// `Send`, so is `Registration<T>`. `T` needs to be `Sync` because it's a requirement of
+// `Registration<T>`.
+unsafe impl<T: Send + Sync> Send for Registration<T> {}
+
+impl<T: Sync> Drop for Registration<T> {
+ /// Removes the registration from the kernel if it has completed successfully before.
+ fn drop(&mut self) {
+ if self.registered {
+ unsafe { bindings::misc_deregister(&mut self.mdev) }
+ }
+ }
+}
diff --git a/rust/kernel/module_param.rs b/rust/kernel/module_param.rs
new file mode 100644
index 0000000000000..e8d51fe613f5b
--- /dev/null
+++ b/rust/kernel/module_param.rs
@@ -0,0 +1,497 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Types for module parameters.
+//!
+//! C header: [`include/linux/moduleparam.h`](../../../include/linux/moduleparam.h)
+
+use core::fmt::Write;
+
+/// Types that can be used for module parameters.
+///
+/// Note that displaying the type in `sysfs` will fail if
+/// [`alloc::string::ToString::to_string`] (as implemented through the
+/// [`core::fmt::Display`] trait) writes more than [`PAGE_SIZE`]
+/// bytes (including an additional null terminator).
+///
+/// [`PAGE_SIZE`]: `crate::PAGE_SIZE`
+pub trait ModuleParam: core::fmt::Display + core::marker::Sized {
+ /// The `ModuleParam` will be used by the kernel module through this type.
+ ///
+ /// This may differ from `Self` if, for example, `Self` needs to track
+ /// ownership without exposing it or allocate extra space for other possible
+ /// parameter values. See [`StringParam`] or [`ArrayParam`] for examples.
+ type Value: ?Sized;
+
+ /// Whether the parameter is allowed to be set without an argument.
+ ///
+ /// Setting this to `true` allows the parameter to be passed without an
+ /// argument (e.g. just `module.param` instead of `module.param=foo`).
+ const NOARG_ALLOWED: bool;
+
+ /// Convert a parameter argument into the parameter value.
+ ///
+ /// `None` should be returned when parsing of the argument fails.
+ /// `arg == None` indicates that the parameter was passed without an
+ /// argument. If `NOARG_ALLOWED` is set to `false` then `arg` is guaranteed
+ /// to always be `Some(_)`.
+ ///
+ /// Parameters passed at boot time will be set before [`kmalloc`] is
+ /// available (even if the module is loaded at a later time). However, in
+ /// this case, the argument buffer will be valid for the entire lifetime of
+ /// the kernel. So implementations of this method which need to allocate
+ /// should first check that the allocator is available (with
+ /// [`crate::bindings::slab_is_available`]) and when it is not available
+ /// provide an alternative implementation which doesn't allocate. In cases
+ /// where the allocator is not available it is safe to save references to
+ /// `arg` in `Self`, but in other cases a copy should be made.
+ ///
+ /// [`kmalloc`]: ../../../include/linux/slab.h
+ fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option<Self>;
+
+ /// Get the current value of the parameter for use in the kernel module.
+ ///
+ /// This function should not be used directly. Instead use the wrapper
+ /// `read` which will be generated by [`module::module`].
+ fn value(&self) -> &Self::Value;
+
+ /// Set the module parameter from a string.
+ ///
+ /// Used to set the parameter value when loading the module or when set
+ /// through `sysfs`.
+ ///
+ /// # Safety
+ ///
+ /// If `val` is non-null then it must point to a valid null-terminated
+ /// string. The `arg` field of `param` must be an instance of `Self`.
+ unsafe extern "C" fn set_param(
+ val: *const crate::c_types::c_char,
+ param: *const crate::bindings::kernel_param,
+ ) -> crate::c_types::c_int {
+ let arg = if val.is_null() {
+ None
+ } else {
+ Some(crate::c_types::c_string_bytes(val))
+ };
+ match Self::try_from_param_arg(arg) {
+ Some(new_value) => {
+ let old_value = (*param).__bindgen_anon_1.arg as *mut Self;
+ let _ = core::ptr::replace(old_value, new_value);
+ 0
+ }
+ None => crate::error::Error::EINVAL.to_kernel_errno(),
+ }
+ }
+
+ /// Write a string representation of the current parameter value to `buf`.
+ ///
+ /// Used for displaying the current parameter value in `sysfs`.
+ ///
+ /// # Safety
+ ///
+ /// `buf` must be a buffer of length at least `kernel::PAGE_SIZE` that is
+ /// writeable. The `arg` field of `param` must be an instance of `Self`.
+ unsafe extern "C" fn get_param(
+ buf: *mut crate::c_types::c_char,
+ param: *const crate::bindings::kernel_param,
+ ) -> crate::c_types::c_int {
+ let slice = core::slice::from_raw_parts_mut(buf as *mut u8, crate::PAGE_SIZE);
+ let mut buf = crate::buffer::Buffer::new(slice);
+ match write!(buf, "{}\0", *((*param).__bindgen_anon_1.arg as *mut Self)) {
+ Err(_) => crate::error::Error::EINVAL.to_kernel_errno(),
+ Ok(()) => buf.bytes_written() as crate::c_types::c_int,
+ }
+ }
+
+ /// Drop the parameter.
+ ///
+ /// Called when unloading a module.
+ ///
+ /// # Safety
+ ///
+ /// The `arg` field of `param` must be an instance of `Self`.
+ unsafe extern "C" fn free(arg: *mut crate::c_types::c_void) {
+ core::ptr::drop_in_place(arg as *mut Self);
+ }
+}
+
+/// Trait for parsing integers.
+///
+/// Strings begining with `0x`, `0o`, or `0b` are parsed as hex, octal, or
+/// binary respectively. Strings beginning with `0` otherwise are parsed as
+/// octal. Anything else is parsed as decimal. A leading `+` or `-` is also
+/// permitted. Any string parsed by [`kstrtol()`] or [`kstrtoul()`] will be
+/// successfully parsed.
+///
+/// [`kstrtol()`]: https://www.kernel.org/doc/html/latest/core-api/kernel-api.html#c.kstrtol
+/// [`kstrtoul()`]: https://www.kernel.org/doc/html/latest/core-api/kernel-api.html#c.kstrtoul
+trait ParseInt: Sized {
+ fn from_str_radix(src: &str, radix: u32) -> Result<Self, core::num::ParseIntError>;
+ fn checked_neg(self) -> Option<Self>;
+
+ fn from_str_unsigned(src: &str) -> Result<Self, core::num::ParseIntError> {
+ let (radix, digits) = if let Some(n) = src.strip_prefix("0x") {
+ (16, n)
+ } else if let Some(n) = src.strip_prefix("0X") {
+ (16, n)
+ } else if let Some(n) = src.strip_prefix("0o") {
+ (8, n)
+ } else if let Some(n) = src.strip_prefix("0O") {
+ (8, n)
+ } else if let Some(n) = src.strip_prefix("0b") {
+ (2, n)
+ } else if let Some(n) = src.strip_prefix("0B") {
+ (2, n)
+ } else if src.starts_with('0') {
+ (8, src)
+ } else {
+ (10, src)
+ };
+ Self::from_str_radix(digits, radix)
+ }
+
+ fn from_str(src: &str) -> Option<Self> {
+ match src.bytes().next() {
+ None => None,
+ Some(b'-') => Self::from_str_unsigned(&src[1..]).ok()?.checked_neg(),
+ Some(b'+') => Some(Self::from_str_unsigned(&src[1..]).ok()?),
+ Some(_) => Some(Self::from_str_unsigned(src).ok()?),
+ }
+ }
+}
+
+macro_rules! impl_parse_int {
+ ($ty:ident) => {
+ impl ParseInt for $ty {
+ fn from_str_radix(src: &str, radix: u32) -> Result<Self, core::num::ParseIntError> {
+ $ty::from_str_radix(src, radix)
+ }
+
+ fn checked_neg(self) -> Option<Self> {
+ self.checked_neg()
+ }
+ }
+ };
+}
+
+impl_parse_int!(i8);
+impl_parse_int!(u8);
+impl_parse_int!(i16);
+impl_parse_int!(u16);
+impl_parse_int!(i32);
+impl_parse_int!(u32);
+impl_parse_int!(i64);
+impl_parse_int!(u64);
+impl_parse_int!(isize);
+impl_parse_int!(usize);
+
+macro_rules! impl_module_param {
+ ($ty:ident) => {
+ impl ModuleParam for $ty {
+ type Value = $ty;
+
+ const NOARG_ALLOWED: bool = false;
+
+ fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option<Self> {
+ let bytes = arg?;
+ let utf8 = core::str::from_utf8(bytes).ok()?;
+ <$ty as crate::module_param::ParseInt>::from_str(utf8)
+ }
+
+ fn value(&self) -> &Self::Value {
+ self
+ }
+ }
+ };
+}
+
+#[doc(hidden)]
+#[macro_export]
+/// Generate a static [`kernel_param_ops`](../../../include/linux/moduleparam.h) struct.
+///
+/// # Example
+/// ```rust
+/// make_param_ops!(
+/// /// Documentation for new param ops.
+/// PARAM_OPS_MYTYPE, // Name for the static.
+/// MyType // A type which implements [`ModuleParam`].
+/// );
+/// ```
+macro_rules! make_param_ops {
+ ($ops:ident, $ty:ty) => {
+ $crate::make_param_ops!(
+ #[doc=""]
+ $ops,
+ $ty
+ );
+ };
+ ($(#[$meta:meta])* $ops:ident, $ty:ty) => {
+ $(#[$meta])*
+ ///
+ /// Static [`kernel_param_ops`](../../../include/linux/moduleparam.h)
+ /// struct generated by [`make_param_ops`].
+ pub static $ops: $crate::bindings::kernel_param_ops = $crate::bindings::kernel_param_ops {
+ flags: if <$ty as $crate::module_param::ModuleParam>::NOARG_ALLOWED {
+ $crate::bindings::KERNEL_PARAM_OPS_FL_NOARG
+ } else {
+ 0
+ },
+ set: Some(<$ty as $crate::module_param::ModuleParam>::set_param),
+ get: Some(<$ty as $crate::module_param::ModuleParam>::get_param),
+ free: Some(<$ty as $crate::module_param::ModuleParam>::free),
+ };
+ };
+}
+
+impl_module_param!(i8);
+impl_module_param!(u8);
+impl_module_param!(i16);
+impl_module_param!(u16);
+impl_module_param!(i32);
+impl_module_param!(u32);
+impl_module_param!(i64);
+impl_module_param!(u64);
+impl_module_param!(isize);
+impl_module_param!(usize);
+
+make_param_ops!(
+ /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h)
+ /// for [`i8`].
+ PARAM_OPS_I8,
+ i8
+);
+make_param_ops!(
+ /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h)
+ /// for [`u8`].
+ PARAM_OPS_U8,
+ u8
+);
+make_param_ops!(
+ /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h)
+ /// for [`i16`].
+ PARAM_OPS_I16,
+ i16
+);
+make_param_ops!(
+ /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h)
+ /// for [`u16`].
+ PARAM_OPS_U16,
+ u16
+);
+make_param_ops!(
+ /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h)
+ /// for [`i32`].
+ PARAM_OPS_I32,
+ i32
+);
+make_param_ops!(
+ /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h)
+ /// for [`u32`].
+ PARAM_OPS_U32,
+ u32
+);
+make_param_ops!(
+ /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h)
+ /// for [`i64`].
+ PARAM_OPS_I64,
+ i64
+);
+make_param_ops!(
+ /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h)
+ /// for [`u64`].
+ PARAM_OPS_U64,
+ u64
+);
+make_param_ops!(
+ /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h)
+ /// for [`isize`].
+ PARAM_OPS_ISIZE,
+ isize
+);
+make_param_ops!(
+ /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h)
+ /// for [`usize`].
+ PARAM_OPS_USIZE,
+ usize
+);
+
+impl ModuleParam for bool {
+ type Value = bool;
+
+ const NOARG_ALLOWED: bool = true;
+
+ fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option<Self> {
+ match arg {
+ None => Some(true),
+ Some(b"y") | Some(b"Y") | Some(b"1") | Some(b"true") => Some(true),
+ Some(b"n") | Some(b"N") | Some(b"0") | Some(b"false") => Some(false),
+ _ => None,
+ }
+ }
+
+ fn value(&self) -> &Self::Value {
+ self
+ }
+}
+
+make_param_ops!(
+ /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h)
+ /// for [`bool`].
+ PARAM_OPS_BOOL,
+ bool
+);
+
+/// An array of at __most__ `N` values.
+///
+/// # Invariant
+///
+/// The first `self.used` elements of `self.values` are initialized.
+pub struct ArrayParam<T, const N: usize> {
+ values: [core::mem::MaybeUninit<T>; N],
+ used: usize,
+}
+
+impl<T, const N: usize> ArrayParam<T, { N }> {
+ fn values(&self) -> &[T] {
+ // SAFETY: The invariant maintained by `ArrayParam` allows us to cast
+ // the first `self.used` elements to `T`.
+ unsafe {
+ &*(&self.values[0..self.used] as *const [core::mem::MaybeUninit<T>] as *const [T])
+ }
+ }
+}
+
+impl<T: Copy, const N: usize> ArrayParam<T, { N }> {
+ const fn new() -> Self {
+ // INVARIANT: The first `self.used` elements of `self.values` are
+ // initialized.
+ ArrayParam {
+ values: [core::mem::MaybeUninit::uninit(); N],
+ used: 0,
+ }
+ }
+
+ const fn push(&mut self, val: T) {
+ if self.used < N {
+ // INVARIANT: The first `self.used` elements of `self.values` are
+ // initialized.
+ self.values[self.used] = core::mem::MaybeUninit::new(val);
+ self.used += 1;
+ }
+ }
+
+ /// Create an instance of `ArrayParam` initialized with `vals`.
+ ///
+ /// This function is only meant to be used in the [`module::module`] macro.
+ pub const fn create(vals: &[T]) -> Self {
+ let mut result = ArrayParam::new();
+ let mut i = 0;
+ while i < vals.len() {
+ result.push(vals[i]);
+ i += 1;
+ }
+ result
+ }
+}
+
+impl<T: core::fmt::Display, const N: usize> core::fmt::Display for ArrayParam<T, { N }> {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ for val in self.values() {
+ write!(f, "{},", val)?;
+ }
+ Ok(())
+ }
+}
+
+impl<T: Copy + core::fmt::Display + ModuleParam, const N: usize> ModuleParam
+ for ArrayParam<T, { N }>
+{
+ type Value = [T];
+
+ const NOARG_ALLOWED: bool = false;
+
+ fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option<Self> {
+ arg.and_then(|args| {
+ let mut result = Self::new();
+ for arg in args.split(|b| *b == b',') {
+ result.push(T::try_from_param_arg(Some(arg))?);
+ }
+ Some(result)
+ })
+ }
+
+ fn value(&self) -> &Self::Value {
+ self.values()
+ }
+}
+
+/// A C-style string parameter.
+///
+/// The Rust version of the [`charp`] parameter. This type is meant to be
+/// used by the [`module::module`] macro, not handled directly. Instead use the
+/// `read` method generated by that macro.
+///
+/// [`charp`]: ../../../include/linux/moduleparam.h
+pub enum StringParam {
+ /// A borrowed parameter value.
+ ///
+ /// Either the default value (which is static in the module) or borrowed
+ /// from the original argument buffer used to set the value.
+ Ref(&'static [u8]),
+
+ /// A value that was allocated when the parameter was set.
+ ///
+ /// The value needs to be freed when the parameter is reset or the module is
+ /// unloaded.
+ Owned(alloc::vec::Vec<u8>),
+}
+
+impl StringParam {
+ fn bytes(&self) -> &[u8] {
+ match self {
+ StringParam::Ref(bytes) => *bytes,
+ StringParam::Owned(vec) => &vec[..],
+ }
+ }
+}
+
+impl core::fmt::Display for StringParam {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ let bytes = self.bytes();
+ match core::str::from_utf8(bytes) {
+ Ok(utf8) => write!(f, "{}", utf8),
+ Err(_) => write!(f, "{:?}", bytes),
+ }
+ }
+}
+
+impl ModuleParam for StringParam {
+ type Value = [u8];
+
+ const NOARG_ALLOWED: bool = false;
+
+ fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option<Self> {
+ // SAFETY: It is always safe to call [`slab_is_available`](../../../include/linux/slab.h).
+ let slab_available = unsafe { crate::bindings::slab_is_available() };
+ arg.and_then(|arg| {
+ if slab_available {
+ let mut vec = alloc::vec::Vec::new();
+ vec.try_reserve_exact(arg.len()).ok()?;
+ vec.extend_from_slice(arg);
+ Some(StringParam::Owned(vec))
+ } else {
+ Some(StringParam::Ref(arg))
+ }
+ })
+ }
+
+ fn value(&self) -> &Self::Value {
+ self.bytes()
+ }
+}
+
+make_param_ops!(
+ /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h)
+ /// for [`StringParam`].
+ PARAM_OPS_STR,
+ StringParam
+);
diff --git a/rust/kernel/pages.rs b/rust/kernel/pages.rs
new file mode 100644
index 0000000000000..0426d411470da
--- /dev/null
+++ b/rust/kernel/pages.rs
@@ -0,0 +1,173 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Kernel page allocation and management.
+//!
+//! TODO: This module is a work in progress.
+
+use crate::{bindings, c_types, user_ptr::UserSlicePtrReader, Error, KernelResult, PAGE_SIZE};
+use core::{marker::PhantomData, ptr};
+
+extern "C" {
+ #[allow(improper_ctypes)]
+ fn rust_helper_alloc_pages(
+ gfp_mask: bindings::gfp_t,
+ order: c_types::c_uint,
+ ) -> *mut bindings::page;
+
+ #[allow(improper_ctypes)]
+ fn rust_helper_kmap(page: *mut bindings::page) -> *mut c_types::c_void;
+
+ #[allow(improper_ctypes)]
+ fn rust_helper_kunmap(page: *mut bindings::page);
+}
+
+/// A set of physical pages.
+///
+/// `Pages` holds a reference to a set of pages of order `ORDER`. Having the order as a generic
+/// const allows the struct to have the same size as a pointer.
+///
+/// # Invariants
+///
+/// The pointer [`Pages::pages`] is valid and points to 2^ORDER pages.
+pub struct Pages<const ORDER: u32> {
+ pages: *mut bindings::page,
+}
+
+impl<const ORDER: u32> Pages<ORDER> {
+ /// Allocates a new set of contiguous pages.
+ pub fn new() -> KernelResult<Self> {
+ // TODO: Consider whether we want to allow callers to specify flags.
+ // SAFETY: This only allocates pages. We check that it succeeds in the next statement.
+ let pages = unsafe {
+ rust_helper_alloc_pages(
+ bindings::GFP_KERNEL | bindings::__GFP_ZERO | bindings::__GFP_HIGHMEM,
+ ORDER,
+ )
+ };
+ if pages.is_null() {
+ return Err(Error::ENOMEM);
+ }
+ // INVARIANTS: We checked that the allocation above succeeded>
+ Ok(Self { pages })
+ }
+
+ /// Maps a single page at the given address in the given VM area.
+ ///
+ /// This is only meant to be used by pages of order 0.
+ pub fn insert_page(&self, vma: &mut bindings::vm_area_struct, address: usize) -> KernelResult {
+ if ORDER != 0 {
+ return Err(Error::EINVAL);
+ }
+
+ // SAFETY: We check above that the allocation is of order 0. The range of `address` is
+ // already checked by `vm_insert_page`.
+ let ret = unsafe { bindings::vm_insert_page(vma, address as _, self.pages) };
+ if ret != 0 {
+ Err(Error::from_kernel_errno(ret))
+ } else {
+ Ok(())
+ }
+ }
+
+ /// Copies data from the given [`UserSlicePtrReader`] into the pages.
+ pub fn copy_into_page(
+ &self,
+ reader: &mut UserSlicePtrReader,
+ offset: usize,
+ len: usize,
+ ) -> KernelResult {
+ // TODO: For now this only works on the first page.
+ let end = offset.checked_add(len).ok_or(Error::EINVAL)?;
+ if end > PAGE_SIZE {
+ return Err(Error::EINVAL);
+ }
+
+ let mapping = self.kmap(0).ok_or(Error::EINVAL)?;
+
+ // SAFETY: We ensured that the buffer was valid with the check above.
+ unsafe { reader.read_raw((mapping.ptr as usize + offset) as _, len) }?;
+ Ok(())
+ }
+
+ /// Maps the pages and reads from them into the given buffer.
+ ///
+ /// # Safety
+ ///
+ /// Callers must ensure that the destination buffer is valid for the given length.
+ /// Additionally, if the raw buffer is intended to be recast, they must ensure that the data
+ /// can be safely cast; [`crate::user_ptr::ReadableFromBytes`] has more details about it.
+ pub unsafe fn read(&self, dest: *mut u8, offset: usize, len: usize) -> KernelResult {
+ // TODO: For now this only works on the first page.
+ let end = offset.checked_add(len).ok_or(Error::EINVAL)?;
+ if end > PAGE_SIZE {
+ return Err(Error::EINVAL);
+ }
+
+ let mapping = self.kmap(0).ok_or(Error::EINVAL)?;
+ ptr::copy((mapping.ptr as *mut u8).add(offset), dest, len);
+ Ok(())
+ }
+
+ /// Maps the pages and writes into them from the given bufer.
+ ///
+ /// # Safety
+ ///
+ /// Callers must ensure that the buffer is valid for the given length. Additionally, if the
+ /// page is (or will be) mapped by userspace, they must ensure that no kernel data is leaked
+ /// through padding if it was cast from another type; [`crate::user_ptr::WritableToBytes`] has
+ /// more details about it.
+ pub unsafe fn write(&self, src: *const u8, offset: usize, len: usize) -> KernelResult {
+ // TODO: For now this only works on the first page.
+ let end = offset.checked_add(len).ok_or(Error::EINVAL)?;
+ if end > PAGE_SIZE {
+ return Err(Error::EINVAL);
+ }
+
+ let mapping = self.kmap(0).ok_or(Error::EINVAL)?;
+ ptr::copy(src, (mapping.ptr as *mut u8).add(offset), len);
+ Ok(())
+ }
+
+ /// Maps the page at index `index`.
+ fn kmap(&self, index: usize) -> Option<PageMapping> {
+ if index >= 1usize << ORDER {
+ return None;
+ }
+
+ // SAFETY: We checked above that `index` is within range.
+ let page = unsafe { self.pages.add(index) };
+
+ // SAFETY: `page` is valid based on the checks above.
+ let ptr = unsafe { rust_helper_kmap(page) };
+ if ptr.is_null() {
+ return None;
+ }
+
+ Some(PageMapping {
+ page,
+ ptr,
+ _phantom: PhantomData,
+ })
+ }
+}
+
+impl<const ORDER: u32> Drop for Pages<ORDER> {
+ fn drop(&mut self) {
+ // SAFETY: By the type invariants, we know the pages are allocated with the given order.
+ unsafe { bindings::__free_pages(self.pages, ORDER) };
+ }
+}
+
+struct PageMapping<'a> {
+ page: *mut bindings::page,
+ ptr: *mut c_types::c_void,
+ _phantom: PhantomData<&'a i32>,
+}
+
+impl Drop for PageMapping<'_> {
+ fn drop(&mut self) {
+ // SAFETY: An instance of `PageMapping` is created only when `kmap` succeeded for the given
+ // page, so it is safe to unmap it here.
+ unsafe { rust_helper_kunmap(self.page) };
+ }
+}
diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs
new file mode 100644
index 0000000000000..2f8215bc23525
--- /dev/null
+++ b/rust/kernel/prelude.rs
@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! The `kernel` prelude.
+//!
+//! These are most common items used by Rust code in the kernel, intended to
+//! be imported by all Rust code, for convenience.
+//!
+//! # Examples
+//!
+//! ```rust,no_run
+//! use kernel::prelude::*;
+//! ```
+
+pub use alloc::{borrow::ToOwned, string::String};
+
+pub use module::{module, module_misc_device};
+
+pub use super::{pr_alert, pr_cont, pr_crit, pr_emerg, pr_err, pr_info, pr_notice, pr_warn};
+
+pub use super::static_assert;
+
+pub use super::{KernelModule, KernelResult};
diff --git a/rust/kernel/print.rs b/rust/kernel/print.rs
new file mode 100644
index 0000000000000..ff9d1269cdae6
--- /dev/null
+++ b/rust/kernel/print.rs
@@ -0,0 +1,463 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Printing facilities.
+//!
+//! C header: [`include/linux/printk.h`](../../../../include/linux/printk.h)
+//!
+//! Reference: <https://www.kernel.org/doc/html/latest/core-api/printk-basics.html>
+
+use core::cmp;
+use core::fmt;
+
+use crate::bindings;
+use crate::c_types::c_int;
+
+/// Format strings.
+///
+/// Public but hidden since it should only be used from public macros.
+#[doc(hidden)]
+pub mod format_strings {
+ use crate::bindings;
+
+ /// The length we copy from the `KERN_*` kernel prefixes.
+ const LENGTH_PREFIX: usize = 2;
+
+ /// The length of the fixed format strings.
+ pub const LENGTH: usize = 11;
+
+ /// Generates a fixed format string for the kernel's [`printk`].
+ ///
+ /// The format string is always the same for a given level, i.e. for a
+ /// given `prefix`, which are the kernel's `KERN_*` constants.
+ ///
+ /// [`printk`]: ../../../../include/linux/printk.h
+ const fn generate(is_cont: bool, prefix: &[u8; 3]) -> [u8; LENGTH] {
+ // Ensure the `KERN_*` macros are what we expect.
+ assert!(prefix[0] == b'\x01');
+ if is_cont {
+ assert!(prefix[1] == b'c');
+ } else {
+ assert!(prefix[1] >= b'0' && prefix[1] <= b'7');
+ }
+ assert!(prefix[2] == b'\x00');
+
+ let suffix: &[u8; LENGTH - LENGTH_PREFIX] = if is_cont {
+ b"%.*s\0\0\0\0\0"
+ } else {
+ b"%s: %.*s\0"
+ };
+
+ [
+ prefix[0], prefix[1], suffix[0], suffix[1], suffix[2], suffix[3], suffix[4], suffix[5],
+ suffix[6], suffix[7], suffix[8],
+ ]
+ }
+
+ // Generate the format strings at compile-time.
+ //
+ // This avoids the compiler generating the contents on the fly in the stack.
+ //
+ // Furthermore, `static` instead of `const` is used to share the strings
+ // for all the kernel.
+ pub static EMERG: [u8; LENGTH] = generate(false, bindings::KERN_EMERG);
+ pub static ALERT: [u8; LENGTH] = generate(false, bindings::KERN_ALERT);
+ pub static CRIT: [u8; LENGTH] = generate(false, bindings::KERN_CRIT);
+ pub static ERR: [u8; LENGTH] = generate(false, bindings::KERN_ERR);
+ pub static WARNING: [u8; LENGTH] = generate(false, bindings::KERN_WARNING);
+ pub static NOTICE: [u8; LENGTH] = generate(false, bindings::KERN_NOTICE);
+ pub static INFO: [u8; LENGTH] = generate(false, bindings::KERN_INFO);
+ pub static DEBUG: [u8; LENGTH] = generate(false, bindings::KERN_DEBUG);
+ pub static CONT: [u8; LENGTH] = generate(true, bindings::KERN_CONT);
+}
+
+/// Prints a message via the kernel's [`printk`].
+///
+/// Public but hidden since it should only be used from public macros.
+///
+/// # Safety
+///
+/// The format string must be one of the ones in [`format_strings`], and
+/// the module name must be null-terminated.
+///
+/// [`printk`]: ../../../../include/linux/printk.h
+#[doc(hidden)]
+pub unsafe fn call_printk(
+ format_string: &[u8; format_strings::LENGTH],
+ module_name: &[u8],
+ string: &[u8],
+) {
+ // `printk` does not seem to fail in any path.
+ bindings::printk(
+ format_string.as_ptr() as _,
+ module_name.as_ptr(),
+ string.len() as c_int,
+ string.as_ptr(),
+ );
+}
+
+/// Prints a message via the kernel's [`printk`] for the `CONT` level.
+///
+/// Public but hidden since it should only be used from public macros.
+///
+/// [`printk`]: ../../../../include/linux/printk.h
+#[doc(hidden)]
+pub fn call_printk_cont(string: &[u8]) {
+ // `printk` does not seem to fail in any path.
+ //
+ // SAFETY: The format string is fixed.
+ unsafe {
+ bindings::printk(
+ format_strings::CONT.as_ptr() as _,
+ string.len() as c_int,
+ string.as_ptr(),
+ );
+ }
+}
+
+/// The maximum size of a log line in the kernel.
+///
+/// From `kernel/printk/printk.c`.
+const LOG_LINE_MAX: usize = 1024 - 32;
+
+/// The maximum size of a log line in our side.
+///
+/// FIXME: We should be smarter than this, but for the moment, to reduce stack
+/// usage, we only allow this much which should work for most purposes.
+const LOG_LINE_SIZE: usize = 300;
+crate::static_assert!(LOG_LINE_SIZE <= LOG_LINE_MAX);
+
+/// Public but hidden since it should only be used from public macros.
+#[doc(hidden)]
+pub struct LogLineWriter {
+ data: [u8; LOG_LINE_SIZE],
+ pos: usize,
+}
+
+impl LogLineWriter {
+ /// Creates a new [`LogLineWriter`].
+ pub fn new() -> LogLineWriter {
+ LogLineWriter {
+ data: [0u8; LOG_LINE_SIZE],
+ pos: 0,
+ }
+ }
+
+ /// Returns the internal buffer as a byte slice.
+ pub fn as_bytes(&self) -> &[u8] {
+ &self.data[..self.pos]
+ }
+}
+
+impl Default for LogLineWriter {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl fmt::Write for LogLineWriter {
+ fn write_str(&mut self, s: &str) -> fmt::Result {
+ let copy_len = cmp::min(LOG_LINE_SIZE - self.pos, s.as_bytes().len());
+ self.data[self.pos..self.pos + copy_len].copy_from_slice(&s.as_bytes()[..copy_len]);
+ self.pos += copy_len;
+ Ok(())
+ }
+}
+
+/// Helper function for the [`print_macro!`] to reduce stack usage.
+///
+/// Public but hidden since it should only be used from public macros.
+///
+/// # Safety
+///
+/// The format string must be one of the ones in [`format_strings`], and
+/// the module name must be null-terminated.
+#[doc(hidden)]
+pub unsafe fn format_and_call<const CONT: bool>(
+ format_string: &[u8; format_strings::LENGTH],
+ module_name: &[u8],
+ args: fmt::Arguments,
+) {
+ // Careful: this object takes quite a bit of stack.
+ let mut writer = LogLineWriter::new();
+
+ match fmt::write(&mut writer, args) {
+ Ok(_) => {
+ if CONT {
+ call_printk_cont(writer.as_bytes());
+ } else {
+ call_printk(format_string, module_name, writer.as_bytes());
+ }
+ }
+
+ Err(_) => {
+ call_printk(
+ &format_strings::CRIT,
+ module_name,
+ b"Failure to format string.\n",
+ );
+ }
+ };
+}
+
+/// Performs formatting and forwards the string to [`call_printk`].
+///
+/// Public but hidden since it should only be used from public macros.
+#[doc(hidden)]
+#[macro_export]
+macro_rules! print_macro (
+ // Without extra arguments: no need to format anything.
+ ($format_string:path, false, $fmt:expr) => (
+ // SAFETY: This hidden macro should only be called by the documented
+ // printing macros which ensure the format string is one of the fixed
+ // ones. All `__LOG_PREFIX`s are null-terminated as they are generated
+ // by the `module!` proc macro or fixed values defined in a kernel
+ // crate.
+ unsafe {
+ $crate::print::call_printk(
+ &$format_string,
+ crate::__LOG_PREFIX,
+ $fmt.as_bytes(),
+ );
+ }
+ );
+
+ // Without extra arguments: no need to format anything (`CONT` case).
+ ($format_string:path, true, $fmt:expr) => (
+ $crate::print::call_printk_cont(
+ $fmt.as_bytes(),
+ );
+ );
+
+ // With extra arguments: we need to perform formatting.
+ ($format_string:path, $cont:literal, $fmt:expr, $($arg:tt)*) => (
+ // Forwarding the call to a function to perform the formatting
+ // is needed here to avoid stack overflows in non-optimized builds when
+ // invoking the printing macros a lot of times in the same function.
+ // Without it, the compiler reserves one `LogLineWriter` per macro
+ // invocation, which is a huge type.
+ //
+ // We could use an immediately-invoked closure for this, which
+ // seems to lower even more the stack usage at `opt-level=0` because
+ // `fmt::Arguments` objects do not pile up. However, that breaks
+ // the `?` operator if used in one of the arguments.
+ //
+ // At `opt-level=2`, the generated code is basically the same for
+ // all alternatives.
+ //
+ // SAFETY: This hidden macro should only be called by the documented
+ // printing macros which ensure the format string is one of the fixed
+ // ones. All `__LOG_PREFIX`s are null-terminated as they are generated
+ // by the `module!` proc macro or fixed values defined in a kernel
+ // crate.
+ unsafe {
+ $crate::print::format_and_call::<$cont>(
+ &$format_string,
+ crate::__LOG_PREFIX,
+ format_args!($fmt, $($arg)*),
+ );
+ }
+ );
+);
+
+// We could use a macro to generate these macros. However, doing so ends
+// up being a bit ugly: it requires the dollar token trick to escape `$` as
+// well as playing with the `doc` attribute. Furthermore, they cannot be easily
+// imported in the prelude due to [1]. So, for the moment, we just write them
+// manually, like in the C side; while keeping most of the logic in another
+// macro, i.e. [`print_macro`].
+//
+// [1]: https://github.com/rust-lang/rust/issues/52234
+
+/// Prints an emergency-level message (level 0).
+///
+/// Use this level if the system is unusable.
+///
+/// Equivalent to the kernel's [`pr_emerg`] macro.
+///
+/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
+/// [`alloc::format!`] for information about the formatting syntax.
+///
+/// [`pr_emerg`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_emerg
+/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+///
+/// # Examples
+///
+/// ```
+/// pr_emerg!("hello {}\n", "there");
+/// ```
+#[macro_export]
+macro_rules! pr_emerg (
+ ($($arg:tt)*) => (
+ $crate::print_macro!($crate::print::format_strings::EMERG, false, $($arg)*)
+ )
+);
+
+/// Prints an alert-level message (level 1).
+///
+/// Use this level if action must be taken immediately.
+///
+/// Equivalent to the kernel's [`pr_alert`] macro.
+///
+/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
+/// [`alloc::format!`] for information about the formatting syntax.
+///
+/// [`pr_alert`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_alert
+/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+///
+/// # Examples
+///
+/// ```
+/// pr_alert!("hello {}\n", "there");
+/// ```
+#[macro_export]
+macro_rules! pr_alert (
+ ($($arg:tt)*) => (
+ $crate::print_macro!($crate::print::format_strings::ALERT, false, $($arg)*)
+ )
+);
+
+/// Prints a critical-level message (level 2).
+///
+/// Use this level for critical conditions.
+///
+/// Equivalent to the kernel's [`pr_crit`] macro.
+///
+/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
+/// [`alloc::format!`] for information about the formatting syntax.
+///
+/// [`pr_crit`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_crit
+/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+///
+/// # Examples
+///
+/// ```
+/// pr_crit!("hello {}\n", "there");
+/// ```
+#[macro_export]
+macro_rules! pr_crit (
+ ($($arg:tt)*) => (
+ $crate::print_macro!($crate::print::format_strings::CRIT, false, $($arg)*)
+ )
+);
+
+/// Prints an error-level message (level 3).
+///
+/// Use this level for error conditions.
+///
+/// Equivalent to the kernel's [`pr_err`] macro.
+///
+/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
+/// [`alloc::format!`] for information about the formatting syntax.
+///
+/// [`pr_err`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_err
+/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+///
+/// # Examples
+///
+/// ```
+/// pr_err!("hello {}\n", "there");
+/// ```
+#[macro_export]
+macro_rules! pr_err (
+ ($($arg:tt)*) => (
+ $crate::print_macro!($crate::print::format_strings::ERR, false, $($arg)*)
+ )
+);
+
+/// Prints a warning-level message (level 4).
+///
+/// Use this level for warning conditions.
+///
+/// Equivalent to the kernel's [`pr_warn`] macro.
+///
+/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
+/// [`alloc::format!`] for information about the formatting syntax.
+///
+/// [`pr_warn`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_warn
+/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+///
+/// # Examples
+///
+/// ```
+/// pr_warn!("hello {}\n", "there");
+/// ```
+#[macro_export]
+macro_rules! pr_warn (
+ ($($arg:tt)*) => (
+ $crate::print_macro!($crate::print::format_strings::WARNING, false, $($arg)*)
+ )
+);
+
+/// Prints a notice-level message (level 5).
+///
+/// Use this level for normal but significant conditions.
+///
+/// Equivalent to the kernel's [`pr_notice`] macro.
+///
+/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
+/// [`alloc::format!`] for information about the formatting syntax.
+///
+/// [`pr_notice`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_notice
+/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+///
+/// # Examples
+///
+/// ```
+/// pr_notice!("hello {}\n", "there");
+/// ```
+#[macro_export]
+macro_rules! pr_notice (
+ ($($arg:tt)*) => (
+ $crate::print_macro!($crate::print::format_strings::NOTICE, false, $($arg)*)
+ )
+);
+
+/// Prints an info-level message (level 6).
+///
+/// Use this level for informational messages.
+///
+/// Equivalent to the kernel's [`pr_info`] macro.
+///
+/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
+/// [`alloc::format!`] for information about the formatting syntax.
+///
+/// [`pr_info`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_info
+/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+///
+/// # Examples
+///
+/// ```
+/// pr_info!("hello {}\n", "there");
+/// ```
+#[macro_export]
+#[doc(alias = "print")]
+macro_rules! pr_info (
+ ($($arg:tt)*) => (
+ $crate::print_macro!($crate::print::format_strings::INFO, false, $($arg)*)
+ )
+);
+
+/// Continues a previous log message in the same line.
+///
+/// Use only when continuing a previous `pr_*!` macro (e.g. [`pr_info!`]).
+///
+/// Equivalent to the kernel's [`pr_cont`] macro.
+///
+/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
+/// [`alloc::format!`] for information about the formatting syntax.
+///
+/// [`pr_cont`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_cont
+/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+///
+/// # Examples
+///
+/// ```
+/// pr_info!("hello");
+/// pr_cont!(" {}\n", "there");
+/// ```
+#[macro_export]
+macro_rules! pr_cont (
+ ($($arg:tt)*) => (
+ $crate::print_macro!($crate::print::format_strings::CONT, true, $($arg)*)
+ )
+);
diff --git a/rust/kernel/random.rs b/rust/kernel/random.rs
new file mode 100644
index 0000000000000..a7df79c1f7bf7
--- /dev/null
+++ b/rust/kernel/random.rs
@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Random numbers.
+//!
+//! C header: [`include/linux/random.h`](../../../../include/linux/random.h)
+
+use core::convert::TryInto;
+
+use crate::{bindings, c_types, error};
+
+/// Fills a byte slice with random bytes generated from the kernel's CSPRNG.
+///
+/// Ensures that the CSPRNG has been seeded before generating any random bytes,
+/// and will block until it is ready.
+pub fn getrandom(dest: &mut [u8]) -> error::KernelResult {
+ let res = unsafe { bindings::wait_for_random_bytes() };
+ if res != 0 {
+ return Err(error::Error::from_kernel_errno(res));
+ }
+
+ unsafe {
+ bindings::get_random_bytes(
+ dest.as_mut_ptr() as *mut c_types::c_void,
+ dest.len().try_into()?,
+ );
+ }
+ Ok(())
+}
+
+/// Fills a byte slice with random bytes generated from the kernel's CSPRNG.
+///
+/// If the CSPRNG is not yet seeded, returns an `Err(EAGAIN)` immediately.
+pub fn getrandom_nonblock(dest: &mut [u8]) -> error::KernelResult {
+ if !unsafe { bindings::rng_is_initialized() } {
+ return Err(error::Error::EAGAIN);
+ }
+ getrandom(dest)
+}
+
+/// Contributes the contents of a byte slice to the kernel's entropy pool.
+///
+/// Does *not* credit the kernel entropy counter though.
+pub fn add_randomness(data: &[u8]) {
+ unsafe {
+ bindings::add_device_randomness(
+ data.as_ptr() as *const c_types::c_void,
+ data.len().try_into().unwrap(),
+ );
+ }
+}
diff --git a/rust/kernel/raw_list.rs b/rust/kernel/raw_list.rs
new file mode 100644
index 0000000000000..0202b44d560ac
--- /dev/null
+++ b/rust/kernel/raw_list.rs
@@ -0,0 +1,361 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Raw lists.
+//!
+//! TODO: This module is a work in progress.
+
+use core::{
+ cell::UnsafeCell,
+ ptr,
+ ptr::NonNull,
+ sync::atomic::{AtomicBool, Ordering},
+};
+
+/// A descriptor of list elements.
+///
+/// It describes the type of list elements and provides a function to determine how to get the
+/// links to be used on a list.
+///
+/// A type that may be in multiple lists simultaneously neneds to implement one of these for each
+/// simultaneous list.
+pub trait GetLinks {
+ /// The type of the entries in the list.
+ type EntryType: ?Sized;
+
+ /// Returns the links to be used when linking an entry within a list.
+ fn get_links(data: &Self::EntryType) -> &Links<Self::EntryType>;
+}
+
+/// The links used to link an object on a linked list.
+///
+/// Instances of this type are usually embedded in structures and returned in calls to
+/// [`GetLinks::get_links`].
+pub struct Links<T: ?Sized> {
+ inserted: AtomicBool,
+ entry: UnsafeCell<ListEntry<T>>,
+}
+
+impl<T: ?Sized> Links<T> {
+ /// Constructs a new [`Links`] instance that isn't inserted on any lists yet.
+ pub fn new() -> Self {
+ Self {
+ inserted: AtomicBool::new(false),
+ entry: UnsafeCell::new(ListEntry::new()),
+ }
+ }
+
+ fn acquire_for_insertion(&self) -> bool {
+ self.inserted
+ .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
+ .is_ok()
+ }
+
+ fn release_after_removal(&self) {
+ self.inserted.store(false, Ordering::Release);
+ }
+}
+
+impl<T: ?Sized> Default for Links<T> {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+struct ListEntry<T: ?Sized> {
+ next: Option<NonNull<T>>,
+ prev: Option<NonNull<T>>,
+}
+
+impl<T: ?Sized> ListEntry<T> {
+ fn new() -> Self {
+ Self {
+ next: None,
+ prev: None,
+ }
+ }
+}
+
+/// A linked list.
+///
+/// # Invariants
+///
+/// The links of objects added to a list are owned by the list.
+pub(crate) struct RawList<G: GetLinks> {
+ head: Option<NonNull<G::EntryType>>,
+}
+
+impl<G: GetLinks> RawList<G> {
+ pub(crate) fn new() -> Self {
+ Self { head: None }
+ }
+
+ pub(crate) fn is_empty(&self) -> bool {
+ self.head.is_none()
+ }
+
+ fn insert_after_priv(
+ &mut self,
+ existing: &G::EntryType,
+ new_entry: &mut ListEntry<G::EntryType>,
+ new_ptr: Option<NonNull<G::EntryType>>,
+ ) {
+ {
+ // SAFETY: It's safe to get the previous entry of `existing` because the list cannot
+ // change.
+ let existing_links = unsafe { &mut *G::get_links(existing).entry.get() };
+ new_entry.next = existing_links.next;
+ existing_links.next = new_ptr;
+ }
+
+ new_entry.prev = Some(NonNull::from(existing));
+
+ // SAFETY: It's safe to get the next entry of `existing` because the list cannot change.
+ let next_links =
+ unsafe { &mut *G::get_links(new_entry.next.unwrap().as_ref()).entry.get() };
+ next_links.prev = new_ptr;
+ }
+
+ /// Inserts the given object after `existing`.
+ ///
+ /// # Safety
+ ///
+ /// Callers must ensure that `existing` points to a valid entry that is on the list.
+ pub(crate) unsafe fn insert_after(
+ &mut self,
+ existing: &G::EntryType,
+ new: &G::EntryType,
+ ) -> bool {
+ let links = G::get_links(new);
+ if !links.acquire_for_insertion() {
+ // Nothing to do if already inserted.
+ return false;
+ }
+
+ // SAFETY: The links are now owned by the list, so it is safe to get a mutable reference.
+ let new_entry = &mut *links.entry.get();
+ self.insert_after_priv(existing, new_entry, Some(NonNull::from(new)));
+ true
+ }
+
+ fn push_back_internal(&mut self, new: &G::EntryType) -> bool {
+ let links = G::get_links(new);
+ if !links.acquire_for_insertion() {
+ // Nothing to do if already inserted.
+ return false;
+ }
+
+ // SAFETY: The links are now owned by the list, so it is safe to get a mutable reference.
+ let new_entry = unsafe { &mut *links.entry.get() };
+ let new_ptr = Some(NonNull::from(new));
+ match self.back() {
+ // SAFETY: `back` is valid as the list cannot change.
+ Some(back) => self.insert_after_priv(unsafe { back.as_ref() }, new_entry, new_ptr),
+ None => {
+ self.head = new_ptr;
+ new_entry.next = new_ptr;
+ new_entry.prev = new_ptr;
+ }
+ }
+ true
+ }
+
+ pub(crate) unsafe fn push_back(&mut self, new: &G::EntryType) -> bool {
+ self.push_back_internal(new)
+ }
+
+ fn remove_internal(&mut self, data: &G::EntryType) -> bool {
+ let links = G::get_links(data);
+
+ // SAFETY: The links are now owned by the list, so it is safe to get a mutable reference.
+ let entry = unsafe { &mut *links.entry.get() };
+ let next = if let Some(next) = entry.next {
+ next
+ } else {
+ // Nothing to do if the entry is not on the list.
+ return false;
+ };
+
+ if ptr::eq(data, next.as_ptr()) {
+ // We're removing the only element.
+ self.head = None
+ } else {
+ // Update the head if we're removing it.
+ if let Some(raw_head) = self.head {
+ if ptr::eq(data, raw_head.as_ptr()) {
+ self.head = Some(next);
+ }
+ }
+
+ // SAFETY: It's safe to get the previous entry because the list cannot change.
+ unsafe { &mut *G::get_links(entry.prev.unwrap().as_ref()).entry.get() }.next =
+ entry.next;
+
+ // SAFETY: It's safe to get the next entry because the list cannot change.
+ unsafe { &mut *G::get_links(next.as_ref()).entry.get() }.prev = entry.prev;
+ }
+
+ // Reset the links of the element we're removing so that we know it's not on any list.
+ entry.next = None;
+ entry.prev = None;
+ links.release_after_removal();
+ true
+ }
+
+ /// Removes the given entry.
+ ///
+ /// # Safety
+ ///
+ /// Callers must ensure that `data` is either on this list or in no list. It being on another
+ /// list leads to memory unsafety.
+ pub(crate) unsafe fn remove(&mut self, data: &G::EntryType) -> bool {
+ self.remove_internal(data)
+ }
+
+ fn pop_front_internal(&mut self) -> Option<NonNull<G::EntryType>> {
+ let head = self.head?;
+ // SAFETY: The head is on the list as we just got it from there and it cannot change.
+ unsafe { self.remove(head.as_ref()) };
+ Some(head)
+ }
+
+ pub(crate) fn pop_front(&mut self) -> Option<NonNull<G::EntryType>> {
+ self.pop_front_internal()
+ }
+
+ pub(crate) fn front(&self) -> Option<NonNull<G::EntryType>> {
+ self.head
+ }
+
+ pub(crate) fn back(&self) -> Option<NonNull<G::EntryType>> {
+ // SAFETY: The links of head are owned by the list, so it is safe to get a reference.
+ unsafe { &*G::get_links(self.head?.as_ref()).entry.get() }.prev
+ }
+
+ pub(crate) fn cursor_front(&self) -> Cursor<'_, G> {
+ Cursor::new(self, self.front())
+ }
+
+ pub(crate) fn cursor_front_mut(&mut self) -> CursorMut<'_, G> {
+ CursorMut::new(self, self.front())
+ }
+}
+
+struct CommonCursor<G: GetLinks> {
+ cur: Option<NonNull<G::EntryType>>,
+}
+
+impl<G: GetLinks> CommonCursor<G> {
+ fn new(cur: Option<NonNull<G::EntryType>>) -> Self {
+ Self { cur }
+ }
+
+ fn move_next(&mut self, list: &RawList<G>) {
+ match self.cur.take() {
+ None => self.cur = list.head,
+ Some(cur) => {
+ if let Some(head) = list.head {
+ // SAFETY: We have a shared ref to the linked list, so the links can't change.
+ let links = unsafe { &*G::get_links(cur.as_ref()).entry.get() };
+ if links.next.unwrap() != head {
+ self.cur = links.next;
+ }
+ }
+ }
+ }
+ }
+
+ fn move_prev(&mut self, list: &RawList<G>) {
+ match list.head {
+ None => self.cur = None,
+ Some(head) => {
+ let next = match self.cur.take() {
+ None => head,
+ Some(cur) => {
+ if cur == head {
+ return;
+ }
+ cur
+ }
+ };
+ // SAFETY: There's a shared ref to the list, so the links can't change.
+ let links = unsafe { &*G::get_links(next.as_ref()).entry.get() };
+ self.cur = links.prev;
+ }
+ }
+ }
+}
+
+/// A list cursor that allows traversing a linked list and inspecting elements.
+pub struct Cursor<'a, G: GetLinks> {
+ cursor: CommonCursor<G>,
+ list: &'a RawList<G>,
+}
+
+impl<'a, G: GetLinks> Cursor<'a, G> {
+ fn new(list: &'a RawList<G>, cur: Option<NonNull<G::EntryType>>) -> Self {
+ Self {
+ list,
+ cursor: CommonCursor::new(cur),
+ }
+ }
+
+ /// Returns the element the cursor is currently positioned on.
+ pub fn current(&self) -> Option<&'a G::EntryType> {
+ let cur = self.cursor.cur?;
+ // SAFETY: Objects must be kept alive while on the list.
+ Some(unsafe { &*cur.as_ptr() })
+ }
+
+ /// Moves the cursor to the next element.
+ pub fn move_next(&mut self) {
+ self.cursor.move_next(self.list);
+ }
+}
+
+pub(crate) struct CursorMut<'a, G: GetLinks> {
+ cursor: CommonCursor<G>,
+ list: &'a mut RawList<G>,
+}
+
+impl<'a, G: GetLinks> CursorMut<'a, G> {
+ fn new(list: &'a mut RawList<G>, cur: Option<NonNull<G::EntryType>>) -> Self {
+ Self {
+ list,
+ cursor: CommonCursor::new(cur),
+ }
+ }
+
+ pub(crate) fn current(&mut self) -> Option<&mut G::EntryType> {
+ let cur = self.cursor.cur?;
+ // SAFETY: Objects must be kept alive while on the list.
+ Some(unsafe { &mut *cur.as_ptr() })
+ }
+
+ /// Removes the entry the cursor is pointing to and advances the cursor to the next entry. It
+ /// returns a raw pointer to the removed element (if one is removed).
+ pub(crate) fn remove_current(&mut self) -> Option<NonNull<G::EntryType>> {
+ let entry = self.cursor.cur?;
+ self.cursor.move_next(self.list);
+ // SAFETY: The entry is on the list as we just got it from there and it cannot change.
+ unsafe { self.list.remove(entry.as_ref()) };
+ Some(entry)
+ }
+
+ pub(crate) fn peek_next(&mut self) -> Option<&mut G::EntryType> {
+ let mut new = CommonCursor::new(self.cursor.cur);
+ new.move_next(self.list);
+ // SAFETY: Objects must be kept alive while on the list.
+ Some(unsafe { &mut *new.cur?.as_ptr() })
+ }
+
+ pub(crate) fn peek_prev(&mut self) -> Option<&mut G::EntryType> {
+ let mut new = CommonCursor::new(self.cursor.cur);
+ new.move_prev(self.list);
+ // SAFETY: Objects must be kept alive while on the list.
+ Some(unsafe { &mut *new.cur?.as_ptr() })
+ }
+
+ pub(crate) fn move_next(&mut self) {
+ self.cursor.move_next(self.list);
+ }
+}
diff --git a/rust/kernel/static_assert.rs b/rust/kernel/static_assert.rs
new file mode 100644
index 0000000000000..1d8f137155c68
--- /dev/null
+++ b/rust/kernel/static_assert.rs
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Static assert.
+
+/// Static assert (i.e. compile-time assert).
+///
+/// Similar to C11 [`_Static_assert`] and C++11 [`static_assert`].
+///
+/// The feature may be added to Rust in the future: see [RFC 2790].
+///
+/// [`_Static_assert`]: https://en.cppreference.com/w/c/language/_Static_assert
+/// [`static_assert`]: https://en.cppreference.com/w/cpp/language/static_assert
+/// [RFC 2790]: https://github.com/rust-lang/rfcs/issues/2790
+///
+/// # Examples
+///
+/// ```
+/// static_assert!(42 > 24);
+/// static_assert!(core::mem::size_of::<u8>() == 1);
+///
+/// const X: &[u8] = b"bar";
+/// static_assert!(X[1] == 'a' as u8);
+///
+/// const fn f(x: i32) -> i32 {
+/// x + 2
+/// }
+/// static_assert!(f(40) == 42);
+/// ```
+#[macro_export]
+macro_rules! static_assert {
+ ($condition:expr) => {
+ // Based on the latest one in `rustc`'s one before it was [removed].
+ //
+ // [removed]: https://github.com/rust-lang/rust/commit/c2dad1c6b9f9636198d7c561b47a2974f5103f6d
+ #[allow(dead_code)]
+ const _: () = [()][!($condition) as usize];
+ };
+}
diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs
new file mode 100644
index 0000000000000..9b3ae49514436
--- /dev/null
+++ b/rust/kernel/sync/arc.rs
@@ -0,0 +1,184 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! A reference-counted pointer.
+//!
+//! This module implements a way for users to create reference-counted objects and pointers to
+//! them. Such a pointer automatically increments and decrements the count, and drops the
+//! underlying object when it reaches zero. It is also safe to use concurrently from multiple
+//! threads.
+//!
+//! It is different from the standard library's [`Arc`] in two ways: it does not support weak
+//! references, which allows it to be smaller -- a single pointer-sized integer; it allows users to
+//! safely increment the reference count from a single reference to the underlying object.
+//!
+//! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html
+
+use crate::KernelResult;
+use alloc::boxed::Box;
+use core::{
+ mem::ManuallyDrop,
+ ops::Deref,
+ ptr::NonNull,
+ sync::atomic::{fence, AtomicUsize, Ordering},
+};
+
+/// A reference-counted pointer to an instance of `T`.
+///
+/// The reference count is incremented when new instances of [`Ref`] are created, and decremented
+/// when they are dropped. When the count reaches zero, the underlying `T` is also dropped.
+///
+/// # Invariants
+///
+/// The value stored in [`RefCounted::get_count`] corresponds to the number of instances of [`Ref`]
+/// that point to that instance of `T`.
+pub struct Ref<T: RefCounted + ?Sized> {
+ ptr: NonNull<T>,
+}
+
+// SAFETY: It is safe to send `Ref<T>` to another thread when the underlying `T` is `Sync` because
+// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs
+// `T` to be `Send` because any thread that has a `Ref<T>` may ultimately access `T` directly, for
+// example, when the reference count reaches zero and `T` is dropped.
+unsafe impl<T: RefCounted + ?Sized + Sync + Send> Send for Ref<T> {}
+
+// SAFETY: It is safe to send `&Ref<T>` to another thread when the underlying `T` is `Sync` for
+// the same reason as above. `T` needs to be `Send` as well because a thread can clone a `&Ref<T>`
+// into a `Ref<T>`, which may lead to `T` being accessed by the same reasoning as above.
+unsafe impl<T: RefCounted + ?Sized + Sync + Send> Sync for Ref<T> {}
+
+impl<T: RefCounted> Ref<T> {
+ /// Constructs a new reference counted instance of `T`.
+ pub fn try_new(contents: T) -> KernelResult<Self> {
+ let boxed = Box::try_new(contents)?;
+ boxed.get_count().count.store(1, Ordering::Relaxed);
+ let ptr = NonNull::from(Box::leak(boxed));
+ Ok(Ref { ptr })
+ }
+}
+
+impl<T: RefCounted + ?Sized> Ref<T> {
+ /// Creates a new reference-counted pointer to the given instance of `T`.
+ ///
+ /// It works by incrementing the current reference count as part of constructing the new
+ /// pointer.
+ pub fn new_from(obj: &T) -> Self {
+ let ref_count = obj.get_count();
+ let cur = ref_count.count.fetch_add(1, Ordering::Relaxed);
+ if cur == usize::MAX {
+ panic!("Reference count overflowed");
+ }
+ Self {
+ ptr: NonNull::from(obj),
+ }
+ }
+
+ /// Returns a mutable reference to `T` iff the reference count is one. Otherwise returns
+ /// [`None`].
+ pub fn get_mut(&mut self) -> Option<&mut T> {
+ // Synchronises with the decrement in `drop`.
+ if self.get_count().count.load(Ordering::Acquire) != 1 {
+ return None;
+ }
+ // SAFETY: Since there is only one reference, we know it isn't possible for another thread
+ // to concurrently call this.
+ Some(unsafe { self.ptr.as_mut() })
+ }
+
+ /// Determines if two reference-counted pointers point to the same underlying instance of `T`.
+ pub fn ptr_eq(a: &Self, b: &Self) -> bool {
+ core::ptr::eq(a.ptr.as_ptr(), b.ptr.as_ptr())
+ }
+
+ /// Deconstructs a [`Ref`] object into a raw pointer.
+ ///
+ /// It can be reconstructed once via [`Ref::from_raw`].
+ pub fn into_raw(obj: Self) -> *const T {
+ let no_drop = ManuallyDrop::new(obj);
+ no_drop.ptr.as_ptr()
+ }
+
+ /// Recreates a [`Ref`] instance previously deconstructed via [`Ref::into_raw`].
+ ///
+ /// # Safety
+ ///
+ /// `ptr` must have been returned by a previous call to [`Ref::into_raw`]. Additionally, it
+ /// can only be called once for each previous call to [``Ref::into_raw`].
+ pub unsafe fn from_raw(ptr: *const T) -> Self {
+ Ref {
+ ptr: NonNull::new(ptr as _).unwrap(),
+ }
+ }
+}
+
+impl<T: RefCounted + ?Sized> Deref for Ref<T> {
+ type Target = T;
+
+ fn deref(&self) -> &Self::Target {
+ // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
+ // safe to dereference it.
+ unsafe { self.ptr.as_ref() }
+ }
+}
+
+impl<T: RefCounted + ?Sized> Clone for Ref<T> {
+ fn clone(&self) -> Self {
+ Self::new_from(self)
+ }
+}
+
+impl<T: RefCounted + ?Sized> Drop for Ref<T> {
+ fn drop(&mut self) {
+ {
+ // SAFETY: By the type invariant, there is necessarily a reference to the object.
+ let obj = unsafe { self.ptr.as_ref() };
+
+ // Synchronises with the acquire below or with the acquire in `get_mut`.
+ if obj.get_count().count.fetch_sub(1, Ordering::Release) != 1 {
+ return;
+ }
+ }
+
+ // Synchronises with the release when decrementing above. This ensures that modifications
+ // from all previous threads/CPUs are visible to the underlying object's `drop`.
+ fence(Ordering::Acquire);
+
+ // The count reached zero, we must free the memory.
+ //
+ // SAFETY: The pointer was initialised from the result of `Box::into_raw`.
+ unsafe { Box::from_raw(self.ptr.as_ptr()) };
+ }
+}
+
+/// Trait for reference counted objects.
+///
+/// # Safety
+///
+/// Implementers of [`RefCounted`] must ensure that all of their constructors call
+/// [`Ref::try_new`].
+pub unsafe trait RefCounted {
+ /// Returns a pointer to the object field holds the reference count.
+ fn get_count(&self) -> &RefCount;
+}
+
+/// Holds the reference count of an object.
+///
+/// It is meant to be embedded in objects to be reference-counted, with [`RefCounted::get_count`]
+/// returning a reference to it.
+pub struct RefCount {
+ count: AtomicUsize,
+}
+
+impl RefCount {
+ /// Constructs a new instance of [`RefCount`].
+ pub fn new() -> Self {
+ Self {
+ count: AtomicUsize::new(1),
+ }
+ }
+}
+
+impl Default for RefCount {
+ fn default() -> Self {
+ Self::new()
+ }
+}
diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs
new file mode 100644
index 0000000000000..be6012794777c
--- /dev/null
+++ b/rust/kernel/sync/condvar.rs
@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! A condition variable.
+//!
+//! This module allows Rust code to use the kernel's [`struct wait_queue_head`] as a condition
+//! variable.
+
+use super::{Guard, Lock, NeedsLockClass};
+use crate::{bindings, CStr};
+use core::{cell::UnsafeCell, marker::PhantomPinned, mem::MaybeUninit, pin::Pin};
+
+extern "C" {
+ fn rust_helper_init_wait(wq: *mut bindings::wait_queue_entry);
+}
+
+/// Safely initialises a [`CondVar`] with the given name, generating a new lock class.
+#[macro_export]
+macro_rules! condvar_init {
+ ($condvar:expr, $name:literal) => {
+ $crate::init_with_lockdep!($condvar, $name)
+ };
+}
+
+// TODO: `bindgen` is not generating this constant. Figure out why.
+const POLLFREE: u32 = 0x4000;
+
+/// Exposes the kernel's [`struct wait_queue_head`] as a condition variable. It allows the caller to
+/// atomically release the given lock and go to sleep. It reacquires the lock when it wakes up. And
+/// it wakes up when notified by another thread (via [`CondVar::notify_one`] or
+/// [`CondVar::notify_all`]) or because the thread received a signal.
+///
+/// [`struct wait_queue_head`]: ../../../include/linux/wait.h
+pub struct CondVar {
+ pub(crate) wait_list: UnsafeCell<bindings::wait_queue_head>,
+
+ /// A condvar needs to be pinned because it contains a [`struct list_head`] that is
+ /// self-referential, so it cannot be safely moved once it is initialised.
+ _pin: PhantomPinned,
+}
+
+// SAFETY: `CondVar` only uses a `struct wait_queue_head`, which is safe to use on any thread.
+unsafe impl Send for CondVar {}
+
+// SAFETY: `CondVar` only uses a `struct wait_queue_head`, which is safe to use on multiple threads
+// concurrently.
+unsafe impl Sync for CondVar {}
+
+impl CondVar {
+ /// Constructs a new conditional variable.
+ ///
+ /// # Safety
+ ///
+ /// The caller must call `CondVar::init` before using the conditional variable.
+ pub unsafe fn new() -> Self {
+ Self {
+ wait_list: UnsafeCell::new(bindings::wait_queue_head::default()),
+ _pin: PhantomPinned,
+ }
+ }
+
+ /// Atomically releases the given lock (whose ownership is proven by the guard) and puts the
+ /// thread to sleep. It wakes up when notified by [`CondVar::notify_one`] or
+ /// [`CondVar::notify_all`], or when the thread receives a signal.
+ ///
+ /// Returns whether there is a signal pending.
+ #[must_use = "wait returns if a signal is pending, so the caller must check the return value"]
+ pub fn wait<L: Lock>(&self, guard: &mut Guard<L>) -> bool {
+ let lock = guard.lock;
+ let mut wait = MaybeUninit::<bindings::wait_queue_entry>::uninit();
+
+ // SAFETY: `wait` points to valid memory.
+ unsafe { rust_helper_init_wait(wait.as_mut_ptr()) };
+
+ // SAFETY: Both `wait` and `wait_list` point to valid memory.
+ unsafe {
+ bindings::prepare_to_wait_exclusive(
+ self.wait_list.get(),
+ wait.as_mut_ptr(),
+ bindings::TASK_INTERRUPTIBLE as _,
+ );
+ }
+
+ // SAFETY: The guard is evidence that the caller owns the lock.
+ unsafe { lock.unlock() };
+
+ // SAFETY: No arguments, switches to another thread.
+ unsafe { bindings::schedule() };
+
+ lock.lock_noguard();
+
+ // SAFETY: Both `wait` and `wait_list` point to valid memory.
+ unsafe { bindings::finish_wait(self.wait_list.get(), wait.as_mut_ptr()) };
+
+ super::signal_pending()
+ }
+
+ /// Calls the kernel function to notify the appropriate number of threads with the given flags.
+ fn notify(&self, count: i32, flags: u32) {
+ // SAFETY: `wait_list` points to valid memory.
+ unsafe {
+ bindings::__wake_up(
+ self.wait_list.get(),
+ bindings::TASK_NORMAL,
+ count,
+ flags as _,
+ )
+ };
+ }
+
+ /// Wakes a single waiter up, if any. This is not 'sticky' in the sense that if no thread is
+ /// waiting, the notification is lost completely (as opposed to automatically waking up the
+ /// next waiter).
+ pub fn notify_one(&self) {
+ self.notify(1, 0);
+ }
+
+ /// Wakes all waiters up, if any. This is not 'sticky' in the sense that if no thread is
+ /// waiting, the notification is lost completely (as opposed to automatically waking up the
+ /// next waiter).
+ pub fn notify_all(&self) {
+ self.notify(0, 0);
+ }
+
+ /// Wakes all waiters up. If they were added by `epoll`, they are also removed from the list of
+ /// waiters. This is useful when cleaning up a condition variable that may be waited on by
+ /// threads that use `epoll`.
+ pub fn free_waiters(&self) {
+ self.notify(1, bindings::POLLHUP | POLLFREE);
+ }
+}
+
+impl NeedsLockClass for CondVar {
+ unsafe fn init(self: Pin<&Self>, name: CStr<'static>, key: *mut bindings::lock_class_key) {
+ bindings::__init_waitqueue_head(self.wait_list.get(), name.as_ptr() as _, key);
+ }
+}
diff --git a/rust/kernel/sync/guard.rs b/rust/kernel/sync/guard.rs
new file mode 100644
index 0000000000000..84e5d319a5fd8
--- /dev/null
+++ b/rust/kernel/sync/guard.rs
@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! A generic lock guard and trait.
+//!
+//! This module contains a lock guard that can be used with any locking primitive that implements
+//! the ([`Lock`]) trait. It also contains the definition of the trait, which can be leveraged by
+//! other constructs to work on generic locking primitives.
+
+/// Allows mutual exclusion primitives that implement the [`Lock`] trait to automatically unlock
+/// when a guard goes out of scope. It also provides a safe and convenient way to access the data
+/// protected by the lock.
+#[must_use = "the lock unlocks immediately when the guard is unused"]
+pub struct Guard<'a, L: Lock + ?Sized> {
+ pub(crate) lock: &'a L,
+}
+
+// SAFETY: `Guard` is sync when the data protected by the lock is also sync. This is more
+// conservative than the default compiler implementation; more details can be found on
+// https://github.com/rust-lang/rust/issues/41622 -- it refers to `MutexGuard` from the standard
+// library.
+unsafe impl<L> Sync for Guard<'_, L>
+where
+ L: Lock + ?Sized,
+ L::Inner: Sync,
+{
+}
+
+impl<L: Lock + ?Sized> core::ops::Deref for Guard<'_, L> {
+ type Target = L::Inner;
+
+ fn deref(&self) -> &Self::Target {
+ // SAFETY: The caller owns the lock, so it is safe to deref the protected data.
+ unsafe { &*self.lock.locked_data().get() }
+ }
+}
+
+impl<L: Lock + ?Sized> core::ops::DerefMut for Guard<'_, L> {
+ fn deref_mut(&mut self) -> &mut L::Inner {
+ // SAFETY: The caller owns the lock, so it is safe to deref the protected data.
+ unsafe { &mut *self.lock.locked_data().get() }
+ }
+}
+
+impl<L: Lock + ?Sized> Drop for Guard<'_, L> {
+ fn drop(&mut self) {
+ // SAFETY: The caller owns the lock, so it is safe to unlock it.
+ unsafe { self.lock.unlock() };
+ }
+}
+
+impl<'a, L: Lock + ?Sized> Guard<'a, L> {
+ /// Constructs a new lock guard.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that it owns the lock.
+ pub(crate) unsafe fn new(lock: &'a L) -> Self {
+ Self { lock }
+ }
+}
+
+/// A generic mutual exclusion primitive.
+///
+/// [`Guard`] is written such that any mutual exclusion primitive that can implement this trait can
+/// also benefit from having an automatic way to unlock itself.
+pub trait Lock {
+ /// The type of the data protected by the lock.
+ type Inner: ?Sized;
+
+ /// Acquires the lock, making the caller its owner.
+ fn lock_noguard(&self);
+
+ /// Releases the lock, giving up ownership of the lock.
+ ///
+ /// # Safety
+ ///
+ /// It must only be called by the current owner of the lock.
+ unsafe fn unlock(&self);
+
+ /// Returns the data protected by the lock.
+ fn locked_data(&self) -> &core::cell::UnsafeCell<Self::Inner>;
+}
diff --git a/rust/kernel/sync/locked_by.rs b/rust/kernel/sync/locked_by.rs
new file mode 100644
index 0000000000000..fc540b35c53ae
--- /dev/null
+++ b/rust/kernel/sync/locked_by.rs
@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! A wrapper for data protected by a lock that does not wrap it.
+
+use super::{Guard, Lock};
+use core::{cell::UnsafeCell, ops::Deref, ptr};
+
+/// Allows access to some data to be serialised by a lock that does not wrap it.
+///
+/// In most cases, data protected by a lock is wrapped by the appropriate lock type, e.g.,
+/// [`super::Mutex`] or [`super::SpinLock`]. [`LockedBy`] is meant for cases when this is not
+/// possible. For example, if a container has a lock and some data in the contained elements needs
+/// to be protected by the same lock.
+///
+/// [`LockedBy`] wraps the data in lieu of another locking primitive, and only allows access to it
+/// when the caller shows evidence that 'external' lock is locked.
+///
+/// # Example
+///
+/// The following is an example for illustrative purposes: `InnerDirectory::bytes_used` is an
+/// aggregate of all `InnerFile::bytes_used` and must be kept consistent; so we wrap `InnerFile` in
+/// a `LockedBy` so that it shares a lock with `InnerDirectory`. This allows us to enforce at
+/// compile-time that access to `InnerFile` is only granted when an `InnerDirectory` is also
+/// locked; we enforce at run time that the right `InnerDirectory` is locked.
+///
+/// ```
+/// use super::Mutex;
+/// use alloc::{string::String, vec::Vec};
+///
+/// struct InnerFile {
+/// bytes_used: u64,
+/// }
+///
+/// struct File {
+/// name: String,
+/// inner: LockedBy<InnerFile, Mutex<InnerDirectory>>,
+/// }
+///
+/// struct InnerDirectory {
+/// /// The sum of the bytes used by all files.
+/// bytes_used: u64,
+/// files: Vec<File>,
+/// }
+///
+/// struct Directory {
+/// name: String,
+/// inner: Mutex<InnerDirectory>,
+/// }
+/// ```
+pub struct LockedBy<T: ?Sized, L: Lock + ?Sized> {
+ owner: *const L::Inner,
+ data: UnsafeCell<T>,
+}
+
+// SAFETY: `LockedBy` can be transferred across thread boundaries iff the data it protects can.
+unsafe impl<T: ?Sized + Send, L: Lock + ?Sized> Send for LockedBy<T, L> {}
+
+// SAFETY: `LockedBy` serialises the interior mutability it provides, so it is `Sync` as long as the
+// data it protects is `Send`.
+unsafe impl<T: ?Sized + Send, L: Lock + ?Sized> Sync for LockedBy<T, L> {}
+
+impl<T, L: Lock + ?Sized> LockedBy<T, L> {
+ /// Constructs a new instance of [`LockedBy`].
+ ///
+ /// It stores a raw pointer to the owner that is never dereferenced. It is only used to ensure
+ /// that the right owner is being used to access the protected data. If the owner is freed, the
+ /// data becomes inaccessible; if another instance of the owner is allocated *on the same
+ /// memory location*, the data becomes accessible again: none of this affects memory safety
+ /// because in any case at most one thread (or CPU) can access the protected data at a time.
+ pub fn new(owner: &L, data: T) -> Self {
+ Self {
+ owner: owner.locked_data().get(),
+ data: UnsafeCell::new(data),
+ }
+ }
+}
+
+impl<T: ?Sized, L: Lock + ?Sized> LockedBy<T, L> {
+ /// Returns a reference to the protected data when the caller provides evidence (via a
+ /// [`Guard`]) that the owner is locked.
+ pub fn access<'a>(&'a self, guard: &'a Guard<L>) -> &'a T {
+ if !ptr::eq(guard.deref(), self.owner) {
+ panic!("guard does not match owner");
+ }
+
+ // SAFETY: `guard` is evidence that the owner is locked.
+ unsafe { &mut *self.data.get() }
+ }
+
+ /// Returns a mutable reference to the protected data when the caller provides evidence (via a
+ /// mutable [`Guard`]) that the owner is locked mutably.
+ pub fn access_mut<'a>(&'a self, guard: &'a mut Guard<L>) -> &'a mut T {
+ if !ptr::eq(guard.deref().deref(), self.owner) {
+ panic!("guard does not match owner");
+ }
+
+ // SAFETY: `guard` is evidence that the owner is locked.
+ unsafe { &mut *self.data.get() }
+ }
+
+ /// Returns a mutable reference to the protected data when the caller provides evidence (via a
+ /// mutable owner) that the owner is locked mutably. Showing a mutable reference to the owner
+ /// is sufficient because we know no other references can exist to it.
+ pub fn access_from_mut<'a>(&'a self, owner: &'a mut L::Inner) -> &'a mut T {
+ if !ptr::eq(owner, self.owner) {
+ panic!("mismatched owners");
+ }
+
+ // SAFETY: `owner` is evidence that there is only one reference to the owner.
+ unsafe { &mut *self.data.get() }
+ }
+}
diff --git a/rust/kernel/sync/mod.rs b/rust/kernel/sync/mod.rs
new file mode 100644
index 0000000000000..0a8cec021e5c5
--- /dev/null
+++ b/rust/kernel/sync/mod.rs
@@ -0,0 +1,85 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Synchronisation primitives.
+//!
+//! This module contains the kernel APIs related to synchronisation that have been ported or
+//! wrapped for usage by Rust code in the kernel and is shared by all of them.
+//!
+//! # Example
+//!
+//! ```
+//! fn test() {
+//! // SAFETY: `init` is called below.
+//! let data = alloc::sync::Arc::pin(unsafe { Mutex::new(0) });
+//! mutex_init!(data.as_ref(), "test::data");
+//! *data.lock() = 10;
+//! pr_info!("{}\n", *data.lock());
+//! }
+//! ```
+
+use crate::{bindings, c_types, CStr};
+use core::pin::Pin;
+
+mod arc;
+mod condvar;
+mod guard;
+mod locked_by;
+mod mutex;
+mod spinlock;
+
+pub use arc::{Ref, RefCount, RefCounted};
+pub use condvar::CondVar;
+pub use guard::{Guard, Lock};
+pub use locked_by::LockedBy;
+pub use mutex::Mutex;
+pub use spinlock::SpinLock;
+
+extern "C" {
+ fn rust_helper_signal_pending() -> c_types::c_int;
+ fn rust_helper_cond_resched() -> c_types::c_int;
+}
+
+/// Safely initialises an object that has an `init` function that takes a name and a lock class as
+/// arguments, examples of these are [`Mutex`] and [`SpinLock`]. Each of them also provides a more
+/// specialised name that uses this macro.
+#[doc(hidden)]
+#[macro_export]
+macro_rules! init_with_lockdep {
+ ($obj:expr, $name:literal) => {{
+ static mut CLASS: core::mem::MaybeUninit<$crate::bindings::lock_class_key> =
+ core::mem::MaybeUninit::uninit();
+ // SAFETY: `CLASS` is never used by Rust code directly; the kernel may change it though.
+ #[allow(unused_unsafe)]
+ unsafe {
+ $crate::sync::NeedsLockClass::init($obj, $crate::cstr!($name), CLASS.as_mut_ptr())
+ };
+ }};
+}
+
+/// A trait for types that need a lock class during initialisation.
+///
+/// Implementers of this trait benefit from the [`init_with_lockdep`] macro that generates a new
+/// class for each initialisation call site.
+pub trait NeedsLockClass {
+ /// Initialises the type instance so that it can be safely used.
+ ///
+ /// Callers are encouraged to use the [`init_with_lockdep`] macro as it automatically creates a
+ /// new lock class on each usage.
+ ///
+ /// # Safety
+ ///
+ /// `key` must point to a valid memory location as it will be used by the kernel.
+ unsafe fn init(self: Pin<&Self>, name: CStr<'static>, key: *mut bindings::lock_class_key);
+}
+
+/// Determines if a signal is pending on the current process.
+pub fn signal_pending() -> bool {
+ // SAFETY: No arguments, just checks `current` for pending signals.
+ unsafe { rust_helper_signal_pending() != 0 }
+}
+
+/// Reschedules the caller's task if needed.
+pub fn cond_resched() -> bool {
+ // SAFETY: No arguments, reschedules `current` if needed.
+ unsafe { rust_helper_cond_resched() != 0 }
+}
diff --git a/rust/kernel/sync/mutex.rs b/rust/kernel/sync/mutex.rs
new file mode 100644
index 0000000000000..e528228d16c10
--- /dev/null
+++ b/rust/kernel/sync/mutex.rs
@@ -0,0 +1,101 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! A kernel mutex.
+//!
+//! This module allows Rust code to use the kernel's [`struct mutex`].
+
+use super::{Guard, Lock, NeedsLockClass};
+use crate::{bindings, CStr};
+use core::{cell::UnsafeCell, marker::PhantomPinned, pin::Pin};
+
+/// Safely initialises a [`Mutex`] with the given name, generating a new lock class.
+#[macro_export]
+macro_rules! mutex_init {
+ ($mutex:expr, $name:literal) => {
+ $crate::init_with_lockdep!($mutex, $name)
+ };
+}
+
+/// Exposes the kernel's [`struct mutex`]. When multiple threads attempt to lock the same mutex,
+/// only one at a time is allowed to progress, the others will block (sleep) until the mutex is
+/// unlocked, at which point another thread will be allowed to wake up and make progress.
+///
+/// A [`Mutex`] must first be initialised with a call to [`Mutex::init`] before it can be used. The
+/// [`mutex_init`] macro is provided to automatically assign a new lock class to a mutex instance.
+///
+/// Since it may block, [`Mutex`] needs to be used with care in atomic contexts.
+///
+/// [`struct mutex`]: ../../../include/linux/mutex.h
+pub struct Mutex<T: ?Sized> {
+ /// The kernel `struct mutex` object.
+ mutex: UnsafeCell<bindings::mutex>,
+
+ /// A mutex needs to be pinned because it contains a [`struct list_head`] that is
+ /// self-referential, so it cannot be safely moved once it is initialised.
+ _pin: PhantomPinned,
+
+ /// The data protected by the mutex.
+ data: UnsafeCell<T>,
+}
+
+// SAFETY: `Mutex` can be transferred across thread boundaries iff the data it protects can.
+unsafe impl<T: ?Sized + Send> Send for Mutex<T> {}
+
+// SAFETY: `Mutex` serialises the interior mutability it provides, so it is `Sync` as long as the
+// data it protects is `Send`.
+unsafe impl<T: ?Sized + Send> Sync for Mutex<T> {}
+
+impl<T> Mutex<T> {
+ /// Constructs a new mutex.
+ ///
+ /// # Safety
+ ///
+ /// The caller must call [`Mutex::init`] before using the mutex.
+ pub unsafe fn new(t: T) -> Self {
+ Self {
+ mutex: UnsafeCell::new(bindings::mutex::default()),
+ data: UnsafeCell::new(t),
+ _pin: PhantomPinned,
+ }
+ }
+}
+
+impl<T: ?Sized> Mutex<T> {
+ /// Locks the mutex and gives the caller access to the data protected by it. Only one thread at
+ /// a time is allowed to access the protected data.
+ pub fn lock(&self) -> Guard<Self> {
+ self.lock_noguard();
+ // SAFETY: The mutex was just acquired.
+ unsafe { Guard::new(self) }
+ }
+}
+
+impl<T: ?Sized> NeedsLockClass for Mutex<T> {
+ unsafe fn init(self: Pin<&Self>, name: CStr<'static>, key: *mut bindings::lock_class_key) {
+ bindings::__mutex_init(self.mutex.get(), name.as_ptr() as _, key);
+ }
+}
+
+impl<T: ?Sized> Lock for Mutex<T> {
+ type Inner = T;
+
+ #[cfg(not(CONFIG_DEBUG_LOCK_ALLOC))]
+ fn lock_noguard(&self) {
+ // SAFETY: `mutex` points to valid memory.
+ unsafe { bindings::mutex_lock(self.mutex.get()) };
+ }
+
+ #[cfg(CONFIG_DEBUG_LOCK_ALLOC)]
+ fn lock_noguard(&self) {
+ // SAFETY: `mutex` points to valid memory.
+ unsafe { bindings::mutex_lock_nested(self.mutex.get(), 0) };
+ }
+
+ unsafe fn unlock(&self) {
+ bindings::mutex_unlock(self.mutex.get());
+ }
+
+ fn locked_data(&self) -> &UnsafeCell<T> {
+ &self.data
+ }
+}
diff --git a/rust/kernel/sync/spinlock.rs b/rust/kernel/sync/spinlock.rs
new file mode 100644
index 0000000000000..49a7d5fd837b2
--- /dev/null
+++ b/rust/kernel/sync/spinlock.rs
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! A kernel spinlock.
+//!
+//! This module allows Rust code to use the kernel's [`struct spinlock`].
+//!
+//! See <https://www.kernel.org/doc/Documentation/locking/spinlocks.txt>.
+
+use super::{Guard, Lock, NeedsLockClass};
+use crate::{bindings, c_types, CStr};
+use core::{cell::UnsafeCell, marker::PhantomPinned, pin::Pin};
+
+extern "C" {
+ #[allow(improper_ctypes)]
+ fn rust_helper_spin_lock_init(
+ lock: *mut bindings::spinlock_t,
+ name: *const c_types::c_char,
+ key: *mut bindings::lock_class_key,
+ );
+ fn rust_helper_spin_lock(lock: *mut bindings::spinlock);
+ fn rust_helper_spin_unlock(lock: *mut bindings::spinlock);
+}
+
+/// Safely initialises a [`SpinLock`] with the given name, generating a new lock class.
+#[macro_export]
+macro_rules! spinlock_init {
+ ($spinlock:expr, $name:literal) => {
+ $crate::init_with_lockdep!($spinlock, $name)
+ };
+}
+
+/// Exposes the kernel's [`spinlock_t`]. When multiple CPUs attempt to lock the same spinlock, only
+/// one at a time is allowed to progress, the others will block (spinning) until the spinlock is
+/// unlocked, at which point another CPU will be allowed to make progress.
+///
+/// A [`SpinLock`] must first be initialised with a call to [`SpinLock::init`] before it can be
+/// used. The [`spinlock_init`] macro is provided to automatically assign a new lock class to a
+/// spinlock instance.
+///
+/// [`SpinLock`] does not manage the interrupt state, so it can be used in only two cases: (a) when
+/// the caller knows that interrupts are disabled, or (b) when callers never use it in interrupt
+/// handlers (in which case it is ok for interrupts to be enabled).
+///
+/// [`spinlock_t`]: ../../../include/linux/spinlock.h
+pub struct SpinLock<T: ?Sized> {
+ spin_lock: UnsafeCell<bindings::spinlock>,
+
+ /// Spinlocks are architecture-defined. So we conservatively require them to be pinned in case
+ /// some architecture uses self-references now or in the future.
+ _pin: PhantomPinned,
+
+ data: UnsafeCell<T>,
+}
+
+// SAFETY: `SpinLock` can be transferred across thread boundaries iff the data it protects can.
+unsafe impl<T: ?Sized + Send> Send for SpinLock<T> {}
+
+// SAFETY: `SpinLock` serialises the interior mutability it provides, so it is `Sync` as long as the
+// data it protects is `Send`.
+unsafe impl<T: ?Sized + Send> Sync for SpinLock<T> {}
+
+impl<T> SpinLock<T> {
+ /// Constructs a new spinlock.
+ ///
+ /// # Safety
+ ///
+ /// The caller must call [`SpinLock::init`] before using the spinlock.
+ pub unsafe fn new(t: T) -> Self {
+ Self {
+ spin_lock: UnsafeCell::new(bindings::spinlock::default()),
+ data: UnsafeCell::new(t),
+ _pin: PhantomPinned,
+ }
+ }
+}
+
+impl<T: ?Sized> SpinLock<T> {
+ /// Locks the spinlock and gives the caller access to the data protected by it. Only one thread
+ /// at a time is allowed to access the protected data.
+ pub fn lock(&self) -> Guard<Self> {
+ self.lock_noguard();
+ // SAFETY: The spinlock was just acquired.
+ unsafe { Guard::new(self) }
+ }
+}
+
+impl<T: ?Sized> NeedsLockClass for SpinLock<T> {
+ unsafe fn init(self: Pin<&Self>, name: CStr<'static>, key: *mut bindings::lock_class_key) {
+ rust_helper_spin_lock_init(self.spin_lock.get(), name.as_ptr() as _, key);
+ }
+}
+
+impl<T: ?Sized> Lock for SpinLock<T> {
+ type Inner = T;
+
+ fn lock_noguard(&self) {
+ // SAFETY: `spin_lock` points to valid memory.
+ unsafe { rust_helper_spin_lock(self.spin_lock.get()) };
+ }
+
+ unsafe fn unlock(&self) {
+ rust_helper_spin_unlock(self.spin_lock.get());
+ }
+
+ fn locked_data(&self) -> &UnsafeCell<T> {
+ &self.data
+ }
+}
diff --git a/rust/kernel/sysctl.rs b/rust/kernel/sysctl.rs
new file mode 100644
index 0000000000000..a1928a8523db8
--- /dev/null
+++ b/rust/kernel/sysctl.rs
@@ -0,0 +1,185 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! System control.
+//!
+//! C header: [`include/linux/sysctl.h`](../../../../include/linux/sysctl.h)
+//!
+//! Reference: <https://www.kernel.org/doc/Documentation/sysctl/README>
+
+use alloc::boxed::Box;
+use alloc::vec;
+use core::mem;
+use core::ptr;
+use core::sync::atomic;
+
+use crate::bindings;
+use crate::c_types;
+use crate::error;
+use crate::types;
+use crate::user_ptr::{UserSlicePtr, UserSlicePtrWriter};
+
+/// Sysctl storage.
+pub trait SysctlStorage: Sync {
+ /// Writes a byte slice.
+ fn store_value(&self, data: &[u8]) -> (usize, error::KernelResult);
+
+ /// Reads via a [`UserSlicePtrWriter`].
+ fn read_value(&self, data: &mut UserSlicePtrWriter) -> (usize, error::KernelResult);
+}
+
+fn trim_whitespace(mut data: &[u8]) -> &[u8] {
+ while !data.is_empty() && (data[0] == b' ' || data[0] == b'\t' || data[0] == b'\n') {
+ data = &data[1..];
+ }
+ while !data.is_empty()
+ && (data[data.len() - 1] == b' '
+ || data[data.len() - 1] == b'\t'
+ || data[data.len() - 1] == b'\n')
+ {
+ data = &data[..data.len() - 1];
+ }
+ data
+}
+
+impl<T> SysctlStorage for &T
+where
+ T: SysctlStorage,
+{
+ fn store_value(&self, data: &[u8]) -> (usize, error::KernelResult) {
+ (*self).store_value(data)
+ }
+
+ fn read_value(&self, data: &mut UserSlicePtrWriter) -> (usize, error::KernelResult) {
+ (*self).read_value(data)
+ }
+}
+
+impl SysctlStorage for atomic::AtomicBool {
+ fn store_value(&self, data: &[u8]) -> (usize, error::KernelResult) {
+ let result = match trim_whitespace(data) {
+ b"0" => {
+ self.store(false, atomic::Ordering::Relaxed);
+ Ok(())
+ }
+ b"1" => {
+ self.store(true, atomic::Ordering::Relaxed);
+ Ok(())
+ }
+ _ => Err(error::Error::EINVAL),
+ };
+ (data.len(), result)
+ }
+
+ fn read_value(&self, data: &mut UserSlicePtrWriter) -> (usize, error::KernelResult) {
+ let value = if self.load(atomic::Ordering::Relaxed) {
+ b"1\n"
+ } else {
+ b"0\n"
+ };
+ (value.len(), data.write_slice(value))
+ }
+}
+
+/// Holds a single `sysctl` entry (and its table).
+pub struct Sysctl<T: SysctlStorage> {
+ inner: Box<T>,
+ // Responsible for keeping the `ctl_table` alive.
+ _table: Box<[bindings::ctl_table]>,
+ header: *mut bindings::ctl_table_header,
+}
+
+// SAFETY: The only public method we have is `get()`, which returns `&T`, and
+// `T: Sync`. Any new methods must adhere to this requirement.
+unsafe impl<T: SysctlStorage> Sync for Sysctl<T> {}
+
+unsafe extern "C" fn proc_handler<T: SysctlStorage>(
+ ctl: *mut bindings::ctl_table,
+ write: c_types::c_int,
+ buffer: *mut c_types::c_void,
+ len: *mut usize,
+ ppos: *mut bindings::loff_t,
+) -> c_types::c_int {
+ // If we are reading from some offset other than the beginning of the file,
+ // return an empty read to signal EOF.
+ if *ppos != 0 && write == 0 {
+ *len = 0;
+ return 0;
+ }
+
+ let data = UserSlicePtr::new(buffer, *len);
+ let storage = &*((*ctl).data as *const T);
+ let (bytes_processed, result) = if write != 0 {
+ let data = match data.read_all() {
+ Ok(r) => r,
+ Err(e) => return e.to_kernel_errno(),
+ };
+ storage.store_value(&data)
+ } else {
+ let mut writer = data.writer();
+ storage.read_value(&mut writer)
+ };
+ *len = bytes_processed;
+ *ppos += *len as bindings::loff_t;
+ match result {
+ Ok(()) => 0,
+ Err(e) => e.to_kernel_errno(),
+ }
+}
+
+impl<T: SysctlStorage> Sysctl<T> {
+ /// Registers a single entry in `sysctl`.
+ pub fn register(
+ path: types::CStr<'static>,
+ name: types::CStr<'static>,
+ storage: T,
+ mode: types::Mode,
+ ) -> error::KernelResult<Sysctl<T>> {
+ if name.contains('/') {
+ return Err(error::Error::EINVAL);
+ }
+
+ let storage = Box::try_new(storage)?;
+ let mut table = vec![
+ bindings::ctl_table {
+ procname: name.as_ptr() as *const i8,
+ mode: mode.as_int(),
+ data: &*storage as *const T as *mut c_types::c_void,
+ proc_handler: Some(proc_handler::<T>),
+
+ maxlen: 0,
+ child: ptr::null_mut(),
+ poll: ptr::null_mut(),
+ extra1: ptr::null_mut(),
+ extra2: ptr::null_mut(),
+ },
+ unsafe { mem::zeroed() },
+ ]
+ .into_boxed_slice();
+
+ let result =
+ unsafe { bindings::register_sysctl(path.as_ptr() as *const i8, table.as_mut_ptr()) };
+ if result.is_null() {
+ return Err(error::Error::ENOMEM);
+ }
+
+ Ok(Sysctl {
+ inner: storage,
+ _table: table,
+ header: result,
+ })
+ }
+
+ /// Gets the storage.
+ pub fn get(&self) -> &T {
+ &self.inner
+ }
+}
+
+impl<T: SysctlStorage> Drop for Sysctl<T> {
+ fn drop(&mut self) {
+ unsafe {
+ bindings::unregister_sysctl_table(self.header);
+ }
+ self.header = ptr::null_mut();
+ }
+}
diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs
new file mode 100644
index 0000000000000..6207670c32904
--- /dev/null
+++ b/rust/kernel/types.rs
@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Kernel types.
+//!
+//! C header: [`include/linux/types.h`](../../../../include/linux/types.h)
+
+use core::ops::Deref;
+
+use crate::bindings;
+
+/// Permissions.
+///
+/// C header: [`include/uapi/linux/stat.h`](../../../../include/uapi/linux/stat.h)
+///
+/// C header: [`include/linux/stat.h`](../../../../include/linux/stat.h)
+pub struct Mode(bindings::umode_t);
+
+impl Mode {
+ /// Creates a [`Mode`] from an integer.
+ pub fn from_int(m: u16) -> Mode {
+ Mode(m)
+ }
+
+ /// Returns the mode as an integer.
+ pub fn as_int(&self) -> u16 {
+ self.0
+ }
+}
+
+/// A string that is guaranteed to have exactly one `NUL` byte, which is at the
+/// end.
+///
+/// Used for interoperability with kernel APIs that take C strings.
+#[repr(transparent)]
+pub struct CStr<'a>(&'a str);
+
+impl CStr<'_> {
+ /// Creates a [`CStr`] from a [`str`] without performing any additional
+ /// checks.
+ ///
+ /// # Safety
+ ///
+ /// `data` *must* end with a `NUL` byte, and should only have only a single
+ /// `NUL` byte (or the string will be truncated).
+ pub const unsafe fn new_unchecked(data: &str) -> CStr {
+ CStr(data)
+ }
+}
+
+impl Deref for CStr<'_> {
+ type Target = str;
+
+ fn deref(&self) -> &str {
+ self.0
+ }
+}
+
+/// Creates a new `CStr` from a string literal.
+///
+/// The string literal should not contain any `NUL` bytes.
+///
+/// # Examples
+///
+/// ```rust,no_run
+/// const MY_CSTR: CStr<'static> = cstr!("My awesome CStr!");
+/// ```
+#[macro_export]
+macro_rules! cstr {
+ ($str:expr) => {{
+ let s = concat!($str, "\x00");
+ unsafe { $crate::CStr::new_unchecked(s) }
+ }};
+}
diff --git a/rust/kernel/user_ptr.rs b/rust/kernel/user_ptr.rs
new file mode 100644
index 0000000000000..d9304d269d06d
--- /dev/null
+++ b/rust/kernel/user_ptr.rs
@@ -0,0 +1,282 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! User pointers.
+//!
+//! C header: [`include/linux/uaccess.h`](../../../../include/linux/uaccess.h)
+
+use crate::{c_types, error::Error, KernelResult};
+use alloc::vec::Vec;
+use core::mem::{size_of, MaybeUninit};
+
+extern "C" {
+ fn rust_helper_copy_from_user(
+ to: *mut c_types::c_void,
+ from: *const c_types::c_void,
+ n: c_types::c_ulong,
+ ) -> c_types::c_ulong;
+
+ fn rust_helper_copy_to_user(
+ to: *mut c_types::c_void,
+ from: *const c_types::c_void,
+ n: c_types::c_ulong,
+ ) -> c_types::c_ulong;
+}
+
+/// Specifies that a type is safely readable from byte slices.
+///
+/// Not all types can be safely read from byte slices; examples from
+/// <https://doc.rust-lang.org/reference/behavior-considered-undefined.html> include `bool`
+/// that must be either `0` or `1`, and `char` that cannot be a surrogate or above `char::MAX`.
+///
+/// # Safety
+///
+/// Implementers must ensure that the type is made up only of types that can be safely read from
+/// arbitrary byte sequences (e.g., `u32`, `u64`, etc.).
+pub unsafe trait ReadableFromBytes {}
+
+// SAFETY: All bit patterns are acceptable values of the types below.
+unsafe impl ReadableFromBytes for u8 {}
+unsafe impl ReadableFromBytes for u16 {}
+unsafe impl ReadableFromBytes for u32 {}
+unsafe impl ReadableFromBytes for u64 {}
+unsafe impl ReadableFromBytes for usize {}
+unsafe impl ReadableFromBytes for i8 {}
+unsafe impl ReadableFromBytes for i16 {}
+unsafe impl ReadableFromBytes for i32 {}
+unsafe impl ReadableFromBytes for i64 {}
+unsafe impl ReadableFromBytes for isize {}
+
+/// Specifies that a type is safely writable to byte slices.
+///
+/// This means that we don't read undefined values (which leads to UB) in preparation for writing
+/// to the byte slice. It also ensures that no potentially sensitive information is leaked into the
+/// byte slices.
+///
+/// # Safety
+///
+/// A type must not include padding bytes and must be fully initialised to safely implement
+/// [`WritableToBytes`] (i.e., it doesn't contain [`MaybeUninit`] fields). A composition of
+/// writable types in a structure is not necessarily writable because it may result in padding
+/// bytes.
+pub unsafe trait WritableToBytes {}
+
+// SAFETY: Initialised instances of the following types have no uninitialised portions.
+unsafe impl WritableToBytes for u8 {}
+unsafe impl WritableToBytes for u16 {}
+unsafe impl WritableToBytes for u32 {}
+unsafe impl WritableToBytes for u64 {}
+unsafe impl WritableToBytes for usize {}
+unsafe impl WritableToBytes for i8 {}
+unsafe impl WritableToBytes for i16 {}
+unsafe impl WritableToBytes for i32 {}
+unsafe impl WritableToBytes for i64 {}
+unsafe impl WritableToBytes for isize {}
+
+/// A reference to an area in userspace memory, which can be either
+/// read-only or read-write.
+///
+/// All methods on this struct are safe: invalid pointers return
+/// `EFAULT`. Concurrent access, *including data races to/from userspace
+/// memory*, is permitted, because fundamentally another userspace
+/// thread/process could always be modifying memory at the same time
+/// (in the same way that userspace Rust's [`std::io`] permits data races
+/// with the contents of files on disk). In the presence of a race, the
+/// exact byte values read/written are unspecified but the operation is
+/// well-defined. Kernelspace code should validate its copy of data
+/// after completing a read, and not expect that multiple reads of the
+/// same address will return the same value.
+///
+/// All APIs enforce the invariant that a given byte of memory from userspace
+/// may only be read once. By preventing double-fetches we avoid TOCTOU
+/// vulnerabilities. This is accomplished by taking `self` by value to prevent
+/// obtaining multiple readers on a given [`UserSlicePtr`], and the readers
+/// only permitting forward reads.
+///
+/// Constructing a [`UserSlicePtr`] performs no checks on the provided
+/// address and length, it can safely be constructed inside a kernel thread
+/// with no current userspace process. Reads and writes wrap the kernel APIs
+/// `copy_from_user` and `copy_to_user`, which check the memory map of the
+/// current process and enforce that the address range is within the user
+/// range (no additional calls to `access_ok` are needed).
+///
+/// [`std::io`]: https://doc.rust-lang.org/std/io/index.html
+pub struct UserSlicePtr(*mut c_types::c_void, usize);
+
+impl UserSlicePtr {
+ /// Constructs a user slice from a raw pointer and a length in bytes.
+ ///
+ /// # Safety
+ ///
+ /// Callers must be careful to avoid time-of-check-time-of-use
+ /// (TOCTOU) issues. The simplest way is to create a single instance of
+ /// [`UserSlicePtr`] per user memory block as it reads each byte at
+ /// most once.
+ pub unsafe fn new(ptr: *mut c_types::c_void, length: usize) -> Self {
+ UserSlicePtr(ptr, length)
+ }
+
+ /// Reads the entirety of the user slice.
+ ///
+ /// Returns `EFAULT` if the address does not currently point to
+ /// mapped, readable memory.
+ pub fn read_all(self) -> KernelResult<Vec<u8>> {
+ self.reader().read_all()
+ }
+
+ /// Constructs a [`UserSlicePtrReader`].
+ pub fn reader(self) -> UserSlicePtrReader {
+ UserSlicePtrReader(self.0, self.1)
+ }
+
+ /// Writes the provided slice into the user slice.
+ ///
+ /// Returns `EFAULT` if the address does not currently point to
+ /// mapped, writable memory (in which case some data from before the
+ /// fault may be written), or `data` is larger than the user slice
+ /// (in which case no data is written).
+ pub fn write_all(self, data: &[u8]) -> KernelResult {
+ self.writer().write_slice(data)
+ }
+
+ /// Constructs a [`UserSlicePtrWriter`].
+ pub fn writer(self) -> UserSlicePtrWriter {
+ UserSlicePtrWriter(self.0, self.1)
+ }
+
+ /// Constructs both a [`UserSlicePtrReader`] and a [`UserSlicePtrWriter`].
+ pub fn reader_writer(self) -> (UserSlicePtrReader, UserSlicePtrWriter) {
+ (
+ UserSlicePtrReader(self.0, self.1),
+ UserSlicePtrWriter(self.0, self.1),
+ )
+ }
+}
+
+/// A reader for [`UserSlicePtr`].
+///
+/// Used to incrementally read from the user slice.
+pub struct UserSlicePtrReader(*mut c_types::c_void, usize);
+
+impl UserSlicePtrReader {
+ /// Returns the number of bytes left to be read from this.
+ ///
+ /// Note that even reading less than this number of bytes may fail.
+ pub fn len(&self) -> usize {
+ self.1
+ }
+
+ /// Returns `true` if `self.len()` is 0.
+ pub fn is_empty(&self) -> bool {
+ self.len() == 0
+ }
+
+ /// Reads all data remaining in the user slice.
+ ///
+ /// Returns `EFAULT` if the address does not currently point to
+ /// mapped, readable memory.
+ pub fn read_all(&mut self) -> KernelResult<Vec<u8>> {
+ let mut data = Vec::<u8>::new();
+ data.try_reserve_exact(self.1)?;
+ data.resize(self.1, 0);
+ // SAFETY: The output buffer is valid as we just allocated it.
+ unsafe { self.read_raw(data.as_mut_ptr(), data.len())? };
+ Ok(data)
+ }
+
+ /// Reads a byte slice from the user slice.
+ ///
+ /// Returns `EFAULT` if the byte slice is bigger than the remaining size
+ /// of the user slice or if the address does not currently point to mapped,
+ /// readable memory.
+ pub fn read_slice(&mut self, data: &mut [u8]) -> KernelResult {
+ // SAFETY: The output buffer is valid as it's coming from a live reference.
+ unsafe { self.read_raw(data.as_mut_ptr(), data.len()) }
+ }
+
+ /// Reads raw data from the user slice into a raw kernel buffer.
+ ///
+ /// # Safety
+ ///
+ /// The output buffer must be valid.
+ pub unsafe fn read_raw(&mut self, out: *mut u8, len: usize) -> KernelResult {
+ if len > self.1 || len > u32::MAX as usize {
+ return Err(Error::EFAULT);
+ }
+ let res = rust_helper_copy_from_user(out as _, self.0, len as _);
+ if res != 0 {
+ return Err(Error::EFAULT);
+ }
+ // Since this is not a pointer to a valid object in our program,
+ // we cannot use `add`, which has C-style rules for defined
+ // behavior.
+ self.0 = self.0.wrapping_add(len);
+ self.1 -= len;
+ Ok(())
+ }
+
+ /// Reads the contents of a plain old data (POD) type from the user slice.
+ pub fn read<T: ReadableFromBytes>(&mut self) -> KernelResult<T> {
+ let mut out = MaybeUninit::<T>::uninit();
+ // SAFETY: The buffer is valid as it was just allocated.
+ unsafe { self.read_raw(out.as_mut_ptr() as _, size_of::<T>()) }?;
+ // SAFETY: We just initialised the data.
+ Ok(unsafe { out.assume_init() })
+ }
+}
+
+/// A writer for [`UserSlicePtr`].
+///
+/// Used to incrementally write into the user slice.
+pub struct UserSlicePtrWriter(*mut c_types::c_void, usize);
+
+impl UserSlicePtrWriter {
+ /// Returns the number of bytes left to be written from this.
+ ///
+ /// Note that even writing less than this number of bytes may fail.
+ pub fn len(&self) -> usize {
+ self.1
+ }
+
+ /// Returns `true` if `self.len()` is 0.
+ pub fn is_empty(&self) -> bool {
+ self.len() == 0
+ }
+
+ /// Writes a byte slice to the user slice.
+ ///
+ /// Returns `EFAULT` if the byte slice is bigger than the remaining size
+ /// of the user slice or if the address does not currently point to mapped,
+ /// writable memory.
+ pub fn write_slice(&mut self, data: &[u8]) -> KernelResult {
+ // SAFETY: The input buffer is valid as it's coming from a live reference.
+ unsafe { self.write_raw(data.as_ptr(), data.len()) }
+ }
+
+ /// Writes raw data to the user slice from a raw kernel buffer.
+ ///
+ /// # Safety
+ ///
+ /// The input buffer must be valid.
+ unsafe fn write_raw(&mut self, data: *const u8, len: usize) -> KernelResult {
+ if len > self.1 || len > u32::MAX as usize {
+ return Err(Error::EFAULT);
+ }
+ let res = rust_helper_copy_to_user(self.0, data as _, len as _);
+ if res != 0 {
+ return Err(Error::EFAULT);
+ }
+ // Since this is not a pointer to a valid object in our program,
+ // we cannot use `add`, which has C-style rules for defined
+ // behavior.
+ self.0 = self.0.wrapping_add(len);
+ self.1 -= len;
+ Ok(())
+ }
+
+ /// Writes the contents of the given data into the user slice.
+ pub fn write<T: WritableToBytes>(&mut self, data: &T) -> KernelResult<()> {
+ // SAFETY: The input buffer is valid as it's coming from a live
+ // reference to a type that implements `WritableToBytes`.
+ unsafe { self.write_raw(data as *const T as _, size_of::<T>()) }
+ }
+}
diff --git a/rust/module.rs b/rust/module.rs
new file mode 100644
index 0000000000000..bddb90d4822d4
--- /dev/null
+++ b/rust/module.rs
@@ -0,0 +1,764 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Proc macro crate implementing the [`module!`] magic.
+//!
+//! C header: [`include/linux/moduleparam.h`](../../../include/linux/moduleparam.h)
+
+#![deny(clippy::complexity)]
+#![deny(clippy::correctness)]
+#![deny(clippy::perf)]
+#![deny(clippy::style)]
+
+use proc_macro::{token_stream, Delimiter, Group, TokenStream, TokenTree};
+
+fn try_ident(it: &mut token_stream::IntoIter) -> Option<String> {
+ if let Some(TokenTree::Ident(ident)) = it.next() {
+ Some(ident.to_string())
+ } else {
+ None
+ }
+}
+
+fn try_literal(it: &mut token_stream::IntoIter) -> Option<String> {
+ if let Some(TokenTree::Literal(literal)) = it.next() {
+ Some(literal.to_string())
+ } else {
+ None
+ }
+}
+
+fn try_byte_string(it: &mut token_stream::IntoIter) -> Option<String> {
+ try_literal(it).and_then(|byte_string| {
+ if byte_string.starts_with("b\"") && byte_string.ends_with('\"') {
+ Some(byte_string[2..byte_string.len() - 1].to_string())
+ } else {
+ None
+ }
+ })
+}
+
+fn expect_ident(it: &mut token_stream::IntoIter) -> String {
+ try_ident(it).expect("Expected Ident")
+}
+
+fn expect_punct(it: &mut token_stream::IntoIter) -> char {
+ if let TokenTree::Punct(punct) = it.next().expect("Reached end of token stream for Punct") {
+ punct.as_char()
+ } else {
+ panic!("Expected Punct");
+ }
+}
+
+fn expect_literal(it: &mut token_stream::IntoIter) -> String {
+ try_literal(it).expect("Expected Literal")
+}
+
+fn expect_group(it: &mut token_stream::IntoIter) -> Group {
+ if let TokenTree::Group(group) = it.next().expect("Reached end of token stream for Group") {
+ group
+ } else {
+ panic!("Expected Group");
+ }
+}
+
+fn expect_byte_string(it: &mut token_stream::IntoIter) -> String {
+ try_byte_string(it).expect("Expected byte string")
+}
+
+#[derive(Clone, PartialEq)]
+enum ParamType {
+ Ident(String),
+ Array { vals: String, max_length: usize },
+}
+
+fn expect_array_fields(it: &mut token_stream::IntoIter) -> ParamType {
+ assert_eq!(expect_punct(it), '<');
+ let vals = expect_ident(it);
+ assert_eq!(expect_punct(it), ',');
+ let max_length_str = expect_literal(it);
+ let max_length = max_length_str
+ .parse::<usize>()
+ .expect("Expected usize length");
+ assert_eq!(expect_punct(it), '>');
+ ParamType::Array { vals, max_length }
+}
+
+fn expect_type(it: &mut token_stream::IntoIter) -> ParamType {
+ if let TokenTree::Ident(ident) = it
+ .next()
+ .expect("Reached end of token stream for param type")
+ {
+ match ident.to_string().as_ref() {
+ "ArrayParam" => expect_array_fields(it),
+ _ => ParamType::Ident(ident.to_string()),
+ }
+ } else {
+ panic!("Expected Param Type")
+ }
+}
+
+fn expect_end(it: &mut token_stream::IntoIter) {
+ if it.next().is_some() {
+ panic!("Expected end");
+ }
+}
+
+fn get_ident(it: &mut token_stream::IntoIter, expected_name: &str) -> String {
+ assert_eq!(expect_ident(it), expected_name);
+ assert_eq!(expect_punct(it), ':');
+ let ident = expect_ident(it);
+ assert_eq!(expect_punct(it), ',');
+ ident
+}
+
+fn get_literal(it: &mut token_stream::IntoIter, expected_name: &str) -> String {
+ assert_eq!(expect_ident(it), expected_name);
+ assert_eq!(expect_punct(it), ':');
+ let literal = expect_literal(it);
+ assert_eq!(expect_punct(it), ',');
+ literal
+}
+
+fn get_group(it: &mut token_stream::IntoIter, expected_name: &str) -> Group {
+ assert_eq!(expect_ident(it), expected_name);
+ assert_eq!(expect_punct(it), ':');
+ let group = expect_group(it);
+ assert_eq!(expect_punct(it), ',');
+ group
+}
+
+fn get_byte_string(it: &mut token_stream::IntoIter, expected_name: &str) -> String {
+ assert_eq!(expect_ident(it), expected_name);
+ assert_eq!(expect_punct(it), ':');
+ let byte_string = expect_byte_string(it);
+ assert_eq!(expect_punct(it), ',');
+ byte_string
+}
+
+fn __build_modinfo_string_base(
+ module: &str,
+ field: &str,
+ content: &str,
+ variable: &str,
+ builtin: bool,
+) -> String {
+ let string = if builtin {
+ // Built-in modules prefix their modinfo strings by `module.`.
+ format!(
+ "{module}.{field}={content}",
+ module = module,
+ field = field,
+ content = content
+ )
+ } else {
+ // Loadable modules' modinfo strings go as-is.
+ format!("{field}={content}", field = field, content = content)
+ };
+
+ format!(
+ "
+ {cfg}
+ #[link_section = \".modinfo\"]
+ #[used]
+ pub static {variable}: [u8; {length}] = *b\"{string}\\0\";
+ ",
+ cfg = if builtin {
+ "#[cfg(not(MODULE))]"
+ } else {
+ "#[cfg(MODULE)]"
+ },
+ variable = variable,
+ length = string.len() + 1,
+ string = string,
+ )
+}
+
+fn __build_modinfo_string_variable(module: &str, field: &str) -> String {
+ format!("__{module}_{field}", module = module, field = field)
+}
+
+fn build_modinfo_string_only_builtin(module: &str, field: &str, content: &str) -> String {
+ __build_modinfo_string_base(
+ module,
+ field,
+ content,
+ &__build_modinfo_string_variable(module, field),
+ true,
+ )
+}
+
+fn build_modinfo_string_only_loadable(module: &str, field: &str, content: &str) -> String {
+ __build_modinfo_string_base(
+ module,
+ field,
+ content,
+ &__build_modinfo_string_variable(module, field),
+ false,
+ )
+}
+
+fn build_modinfo_string(module: &str, field: &str, content: &str) -> String {
+ build_modinfo_string_only_builtin(module, field, content)
+ + &build_modinfo_string_only_loadable(module, field, content)
+}
+
+fn build_modinfo_string_param(module: &str, field: &str, param: &str, content: &str) -> String {
+ let variable = format!(
+ "__{module}_{field}_{param}",
+ module = module,
+ field = field,
+ param = param
+ );
+ let content = format!("{param}:{content}", param = param, content = content);
+ __build_modinfo_string_base(module, field, &content, &variable, true)
+ + &__build_modinfo_string_base(module, field, &content, &variable, false)
+}
+
+fn permissions_are_readonly(perms: &str) -> bool {
+ let (radix, digits) = if let Some(n) = perms.strip_prefix("0x") {
+ (16, n)
+ } else if let Some(n) = perms.strip_prefix("0o") {
+ (8, n)
+ } else if let Some(n) = perms.strip_prefix("0b") {
+ (2, n)
+ } else {
+ (10, perms)
+ };
+ match u32::from_str_radix(digits, radix) {
+ Ok(perms) => perms & 0o222 == 0,
+ Err(_) => false,
+ }
+}
+
+fn param_ops_path(param_type: &str) -> &'static str {
+ match param_type {
+ "bool" => "kernel::module_param::PARAM_OPS_BOOL",
+ "i8" => "kernel::module_param::PARAM_OPS_I8",
+ "u8" => "kernel::module_param::PARAM_OPS_U8",
+ "i16" => "kernel::module_param::PARAM_OPS_I16",
+ "u16" => "kernel::module_param::PARAM_OPS_U16",
+ "i32" => "kernel::module_param::PARAM_OPS_I32",
+ "u32" => "kernel::module_param::PARAM_OPS_U32",
+ "i64" => "kernel::module_param::PARAM_OPS_I64",
+ "u64" => "kernel::module_param::PARAM_OPS_U64",
+ "isize" => "kernel::module_param::PARAM_OPS_ISIZE",
+ "usize" => "kernel::module_param::PARAM_OPS_USIZE",
+ "str" => "kernel::module_param::PARAM_OPS_STR",
+ t => panic!("Unrecognized type {}", t),
+ }
+}
+
+fn try_simple_param_val(
+ param_type: &str,
+) -> Box<dyn Fn(&mut token_stream::IntoIter) -> Option<String>> {
+ match param_type {
+ "bool" => Box::new(|param_it| try_ident(param_it)),
+ "str" => Box::new(|param_it| {
+ try_byte_string(param_it)
+ .map(|s| format!("kernel::module_param::StringParam::Ref(b\"{}\")", s))
+ }),
+ _ => Box::new(|param_it| try_literal(param_it)),
+ }
+}
+
+fn get_default(param_type: &ParamType, param_it: &mut token_stream::IntoIter) -> String {
+ let try_param_val = match param_type {
+ ParamType::Ident(ref param_type)
+ | ParamType::Array {
+ vals: ref param_type,
+ max_length: _,
+ } => try_simple_param_val(param_type),
+ };
+ assert_eq!(expect_ident(param_it), "default");
+ assert_eq!(expect_punct(param_it), ':');
+ let default = match param_type {
+ ParamType::Ident(_) => try_param_val(param_it).expect("Expected default param value"),
+ ParamType::Array {
+ vals: _,
+ max_length: _,
+ } => {
+ let group = expect_group(param_it);
+ assert_eq!(group.delimiter(), Delimiter::Bracket);
+ let mut default_vals = Vec::new();
+ let mut it = group.stream().into_iter();
+
+ while let Some(default_val) = try_param_val(&mut it) {
+ default_vals.push(default_val);
+ match it.next() {
+ Some(TokenTree::Punct(punct)) => assert_eq!(punct.as_char(), ','),
+ None => break,
+ _ => panic!("Expected ',' or end of array default values"),
+ }
+ }
+
+ let mut default_array = "kernel::module_param::ArrayParam::create(&[".to_string();
+ default_array.push_str(
+ &default_vals
+ .iter()
+ .map(|val| val.to_string())
+ .collect::<Vec<String>>()
+ .join(","),
+ );
+ default_array.push_str("])");
+ default_array
+ }
+ };
+ assert_eq!(expect_punct(param_it), ',');
+ default
+}
+
+fn generated_array_ops_name(vals: &str, max_length: usize) -> String {
+ format!(
+ "__generated_array_ops_{vals}_{max_length}",
+ vals = vals,
+ max_length = max_length
+ )
+}
+
+/// Declares a kernel module.
+///
+/// The `type` argument should be a type which implements the [`KernelModule`]
+/// trait. Also accepts various forms of kernel metadata.
+///
+/// [`KernelModule`]: ../kernel/trait.KernelModule.html
+///
+/// # Examples
+///
+/// ```rust,no_run
+/// use kernel::prelude::*;
+///
+/// module!{
+/// type: MyKernelModule,
+/// name: b"my_kernel_module",
+/// author: b"Rust for Linux Contributors",
+/// description: b"My very own kernel module!",
+/// license: b"GPL v2",
+/// params: {
+/// my_i32: i32 {
+/// default: 42,
+/// permissions: 0o000,
+/// description: b"Example of i32",
+/// },
+/// writeable_i32: i32 {
+/// default: 42,
+/// permissions: 0o644,
+/// description: b"Example of i32",
+/// },
+/// },
+/// }
+///
+/// struct MyKernelModule;
+///
+/// impl KernelModule for MyKernelModule {
+/// fn init() -> KernelResult<Self> {
+/// // If the parameter is writeable, then the kparam lock must be
+/// // taken to read the parameter:
+/// {
+/// let lock = THIS_MODULE.kernel_param_lock();
+/// pr_info!("i32 param is: {}\n", writeable_i32.read(&lock));
+/// }
+/// // If the parameter is read only, it can be read without locking
+/// // the kernel parameters:
+/// pr_info!("i32 param is: {}\n", my_i32.read());
+/// Ok(MyKernelModule)
+/// }
+/// }
+/// ```
+///
+/// # Supported parameter types
+///
+/// - `bool`: Corresponds to C `bool` param type.
+/// - `i8`: No equivalent C param type.
+/// - `u8`: Corresponds to C `char` param type.
+/// - `i16`: Corresponds to C `short` param type.
+/// - `u16`: Corresponds to C `ushort` param type.
+/// - `i32`: Corresponds to C `int` param type.
+/// - `u32`: Corresponds to C `uint` param type.
+/// - `i64`: No equivalent C param type.
+/// - `u64`: Corresponds to C `ullong` param type.
+/// - `isize`: No equivalent C param type.
+/// - `usize`: No equivalent C param type.
+/// - `str`: Corresponds to C `charp` param type. Reading returns a byte slice.
+/// - `ArrayParam<T,N>`: Corresponds to C parameters created using `module_param_array`. An array
+/// of `T`'s of length at **most** `N`.
+///
+/// `invbool` is unsupported: it was only ever used in a few modules.
+/// Consider using a `bool` and inverting the logic instead.
+#[proc_macro]
+pub fn module(ts: TokenStream) -> TokenStream {
+ let mut it = ts.into_iter();
+
+ let type_ = get_ident(&mut it, "type");
+ let name = get_byte_string(&mut it, "name");
+ let author = get_byte_string(&mut it, "author");
+ let description = get_byte_string(&mut it, "description");
+ let license = get_byte_string(&mut it, "license");
+ let params = get_group(&mut it, "params");
+
+ expect_end(&mut it);
+
+ assert_eq!(params.delimiter(), Delimiter::Brace);
+
+ let mut it = params.stream().into_iter();
+
+ let mut params_modinfo = String::new();
+
+ let mut array_types_to_generate = Vec::new();
+
+ loop {
+ let param_name = match it.next() {
+ Some(TokenTree::Ident(ident)) => ident.to_string(),
+ Some(_) => panic!("Expected Ident or end"),
+ None => break,
+ };
+
+ assert_eq!(expect_punct(&mut it), ':');
+ let param_type = expect_type(&mut it);
+ let group = expect_group(&mut it);
+ assert_eq!(expect_punct(&mut it), ',');
+
+ assert_eq!(group.delimiter(), Delimiter::Brace);
+
+ let mut param_it = group.stream().into_iter();
+ let param_default = get_default(&param_type, &mut param_it);
+ let param_permissions = get_literal(&mut param_it, "permissions");
+ let param_description = get_byte_string(&mut param_it, "description");
+ expect_end(&mut param_it);
+
+ // TODO: more primitive types
+ // TODO: other kinds: unsafes, etc.
+ let (param_kernel_type, ops): (String, _) = match param_type {
+ ParamType::Ident(ref param_type) => (
+ param_type.to_string(),
+ param_ops_path(&param_type).to_string(),
+ ),
+ ParamType::Array {
+ ref vals,
+ max_length,
+ } => {
+ array_types_to_generate.push((vals.clone(), max_length));
+ (
+ format!("__rust_array_param_{}_{}", vals, max_length),
+ generated_array_ops_name(vals, max_length),
+ )
+ }
+ };
+
+ params_modinfo.push_str(&build_modinfo_string_param(
+ &name,
+ "parmtype",
+ &param_name,
+ &param_kernel_type,
+ ));
+ params_modinfo.push_str(&build_modinfo_string_param(
+ &name,
+ "parm",
+ &param_name,
+ &param_description,
+ ));
+ let param_type_internal = match param_type {
+ ParamType::Ident(ref param_type) => match param_type.as_ref() {
+ "str" => "kernel::module_param::StringParam".to_string(),
+ other => other.to_string(),
+ },
+ ParamType::Array {
+ ref vals,
+ max_length,
+ } => format!(
+ "kernel::module_param::ArrayParam<{vals}, {max_length}>",
+ vals = vals,
+ max_length = max_length
+ ),
+ };
+ let read_func = if permissions_are_readonly(&param_permissions) {
+ format!(
+ "
+ fn read(&self) -> &<{param_type_internal} as kernel::module_param::ModuleParam>::Value {{
+ // SAFETY: Parameters do not need to be locked because they are read only or sysfs is not enabled.
+ unsafe {{ <{param_type_internal} as kernel::module_param::ModuleParam>::value(&__{name}_{param_name}_value) }}
+ }}
+ ",
+ name = name,
+ param_name = param_name,
+ param_type_internal = param_type_internal,
+ )
+ } else {
+ format!(
+ "
+ fn read<'lck>(&self, lock: &'lck kernel::KParamGuard) -> &'lck <{param_type_internal} as kernel::module_param::ModuleParam>::Value {{
+ // SAFETY: Parameters are locked by `KParamGuard`.
+ unsafe {{ <{param_type_internal} as kernel::module_param::ModuleParam>::value(&__{name}_{param_name}_value) }}
+ }}
+ ",
+ name = name,
+ param_name = param_name,
+ param_type_internal = param_type_internal,
+ )
+ };
+ let kparam = format!(
+ "
+ kernel::bindings::kernel_param__bindgen_ty_1 {{
+ arg: unsafe {{ &__{name}_{param_name}_value }} as *const _ as *mut kernel::c_types::c_void,
+ }},
+ ",
+ name = name,
+ param_name = param_name,
+ );
+ params_modinfo.push_str(
+ &format!(
+ "
+ static mut __{name}_{param_name}_value: {param_type_internal} = {param_default};
+
+ struct __{name}_{param_name};
+
+ impl __{name}_{param_name} {{ {read_func} }}
+
+ const {param_name}: __{name}_{param_name} = __{name}_{param_name};
+
+ // Note: the C macro that generates the static structs for the `__param` section
+ // asks for them to be `aligned(sizeof(void *))`. However, that was put in place
+ // in 2003 in commit 38d5b085d2 (\"[PATCH] Fix over-alignment problem on x86-64\")
+ // to undo GCC over-alignment of static structs of >32 bytes. It seems that is
+ // not the case anymore, so we simplify to a transparent representation here
+ // in the expectation that it is not needed anymore.
+ // TODO: revisit this to confirm the above comment and remove it if it happened
+ #[repr(transparent)]
+ struct __{name}_{param_name}_RacyKernelParam(kernel::bindings::kernel_param);
+
+ unsafe impl Sync for __{name}_{param_name}_RacyKernelParam {{
+ }}
+
+ #[cfg(not(MODULE))]
+ const __{name}_{param_name}_name: *const kernel::c_types::c_char = b\"{name}.{param_name}\\0\" as *const _ as *const kernel::c_types::c_char;
+
+ #[cfg(MODULE)]
+ const __{name}_{param_name}_name: *const kernel::c_types::c_char = b\"{param_name}\\0\" as *const _ as *const kernel::c_types::c_char;
+
+ #[link_section = \"__param\"]
+ #[used]
+ static __{name}_{param_name}_struct: __{name}_{param_name}_RacyKernelParam = __{name}_{param_name}_RacyKernelParam(kernel::bindings::kernel_param {{
+ name: __{name}_{param_name}_name,
+ // SAFETY: `__this_module` is constructed by the kernel at load time and will not be freed until the module is unloaded.
+ #[cfg(MODULE)]
+ mod_: unsafe {{ &kernel::bindings::__this_module as *const _ as *mut _ }},
+ #[cfg(not(MODULE))]
+ mod_: core::ptr::null_mut(),
+ ops: unsafe {{ &{ops} }} as *const kernel::bindings::kernel_param_ops,
+ perm: {permissions},
+ level: -1,
+ flags: 0,
+ __bindgen_anon_1: {kparam}
+ }});
+ ",
+ name = name,
+ param_type_internal = param_type_internal,
+ read_func = read_func,
+ param_default = param_default,
+ param_name = param_name,
+ ops = ops,
+ permissions = param_permissions,
+ kparam = kparam,
+ )
+ );
+ }
+
+ let mut generated_array_types = String::new();
+
+ for (vals, max_length) in array_types_to_generate {
+ let ops_name = generated_array_ops_name(&vals, max_length);
+ generated_array_types.push_str(&format!(
+ "
+ kernel::make_param_ops!(
+ {ops_name},
+ kernel::module_param::ArrayParam<{vals}, {{ {max_length} }}>
+ );
+ ",
+ ops_name = ops_name,
+ vals = vals,
+ max_length = max_length,
+ ));
+ }
+
+ let file =
+ std::env::var("RUST_MODFILE").expect("Unable to fetch RUST_MODFILE environmental variable");
+
+ format!(
+ "
+ /// The module name.
+ ///
+ /// Used by the printing macros, e.g. [`info!`].
+ const __LOG_PREFIX: &[u8] = b\"{name}\\0\";
+
+ static mut __MOD: Option<{type_}> = None;
+
+ // SAFETY: `__this_module` is constructed by the kernel at load time and will not be freed until the module is unloaded.
+ #[cfg(MODULE)]
+ static THIS_MODULE: kernel::ThisModule = unsafe {{ kernel::ThisModule::from_ptr(&kernel::bindings::__this_module as *const _ as *mut _) }};
+ #[cfg(not(MODULE))]
+ static THIS_MODULE: kernel::ThisModule = unsafe {{ kernel::ThisModule::from_ptr(core::ptr::null_mut()) }};
+
+ // Loadable modules need to export the `{{init,cleanup}}_module` identifiers
+ #[cfg(MODULE)]
+ #[no_mangle]
+ pub extern \"C\" fn init_module() -> kernel::c_types::c_int {{
+ __init()
+ }}
+
+ #[cfg(MODULE)]
+ #[no_mangle]
+ pub extern \"C\" fn cleanup_module() {{
+ __exit()
+ }}
+
+ // Built-in modules are initialized through an initcall pointer
+ // and the identifiers need to be unique
+ #[cfg(not(MODULE))]
+ #[cfg(not(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS))]
+ #[link_section = \"{initcall_section}\"]
+ #[used]
+ pub static __{name}_initcall: extern \"C\" fn() -> kernel::c_types::c_int = __{name}_init;
+
+ #[cfg(not(MODULE))]
+ #[cfg(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS)]
+ global_asm!(
+ r#\".section \"{initcall_section}\", \"a\"
+ __{name}_initcall:
+ .long __{name}_init - .
+ .previous
+ \"#
+ );
+
+ #[cfg(not(MODULE))]
+ #[no_mangle]
+ pub extern \"C\" fn __{name}_init() -> kernel::c_types::c_int {{
+ __init()
+ }}
+
+ #[cfg(not(MODULE))]
+ #[no_mangle]
+ pub extern \"C\" fn __{name}_exit() {{
+ __exit()
+ }}
+
+ fn __init() -> kernel::c_types::c_int {{
+ match <{type_} as kernel::KernelModule>::init() {{
+ Ok(m) => {{
+ unsafe {{
+ __MOD = Some(m);
+ }}
+ return 0;
+ }}
+ Err(e) => {{
+ return e.to_kernel_errno();
+ }}
+ }}
+ }}
+
+ fn __exit() {{
+ unsafe {{
+ // Invokes `drop()` on `__MOD`, which should be used for cleanup.
+ __MOD = None;
+ }}
+ }}
+
+ {author}
+ {description}
+ {license}
+
+ // Built-in modules also export the `file` modinfo string
+ {file}
+
+ {params_modinfo}
+
+ {generated_array_types}
+ ",
+ type_ = type_,
+ name = name,
+ author = &build_modinfo_string(&name, "author", &author),
+ description = &build_modinfo_string(&name, "description", &description),
+ license = &build_modinfo_string(&name, "license", &license),
+ file = &build_modinfo_string_only_builtin(&name, "file", &file),
+ params_modinfo = params_modinfo,
+ generated_array_types = generated_array_types,
+ initcall_section = ".initcall6.init"
+ ).parse().expect("Error parsing formatted string into token stream.")
+}
+
+/// Declares a kernel module that exposes a single misc device.
+///
+/// The `type` argument should be a type which implements the [`FileOpener`] trait. Also accepts
+/// various forms of kernel metadata.
+///
+/// [`FileOpener`]: ../kernel/file_operations/trait.FileOpener.html
+///
+/// # Examples
+///
+/// ```rust,no_run
+/// use kernel::prelude::*;
+///
+/// module_misc_device! {
+/// type: MyFile,
+/// name: b"my_miscdev_kernel_module",
+/// author: b"Rust for Linux Contributors",
+/// description: b"My very own misc device kernel module!",
+/// license: b"GPL v2",
+/// }
+///
+/// #[derive(Default)]
+/// struct MyFile;
+///
+/// impl kernel::file_operations::FileOperations for MyFile {
+/// kernel::declare_file_operations!();
+/// }
+/// ```
+#[proc_macro]
+pub fn module_misc_device(ts: TokenStream) -> TokenStream {
+ let mut it = ts.into_iter();
+
+ let type_ = get_ident(&mut it, "type");
+ let name = get_byte_string(&mut it, "name");
+ let author = get_byte_string(&mut it, "author");
+ let description = get_byte_string(&mut it, "description");
+ let license = get_byte_string(&mut it, "license");
+ expect_end(&mut it);
+
+ let module = format!("__internal_ModuleFor{}", type_);
+
+ format!(
+ "
+ #[doc(hidden)]
+ struct {module} {{
+ _dev: core::pin::Pin<alloc::boxed::Box<kernel::miscdev::Registration>>,
+ }}
+
+ impl kernel::KernelModule for {module} {{
+ fn init() -> kernel::KernelResult<Self> {{
+ Ok(Self {{
+ _dev: kernel::miscdev::Registration::new_pinned::<{type_}>(
+ kernel::cstr!(\"{name}\"),
+ None,
+ (),
+ )?,
+ }})
+ }}
+ }}
+
+ kernel::prelude::module! {{
+ type: {module},
+ name: b\"{name}\",
+ author: b\"{author}\",
+ description: b\"{description}\",
+ license: b\"{license}\",
+ params: {{}},
+ }}
+ ",
+ module = module,
+ type_ = type_,
+ name = name,
+ author = author,
+ description = description,
+ license = license
+ )
+ .parse()
+ .expect("Error parsing formatted string into token stream.")
+}
diff --git a/samples/Kconfig b/samples/Kconfig
index b5a1a7aa7e23a..eaa06c05d37f1 100644
--- a/samples/Kconfig
+++ b/samples/Kconfig
@@ -223,4 +223,6 @@ config SAMPLE_WATCH_QUEUE
Build example userspace program to use the new mount_notify(),
sb_notify() syscalls and the KEYCTL_WATCH_KEY keyctl() function.
+source "samples/rust/Kconfig"
+
endif # SAMPLES
diff --git a/samples/Makefile b/samples/Makefile
index 087e0988ccc56..291663e56a3cd 100644
--- a/samples/Makefile
+++ b/samples/Makefile
@@ -30,3 +30,4 @@ obj-$(CONFIG_SAMPLE_INTEL_MEI) += mei/
subdir-$(CONFIG_SAMPLE_WATCHDOG) += watchdog
subdir-$(CONFIG_SAMPLE_WATCH_QUEUE) += watch_queue
obj-$(CONFIG_DEBUG_KMEMLEAK_TEST) += kmemleak/
+obj-$(CONFIG_SAMPLES_RUST) += rust/
diff --git a/samples/rust/Kconfig b/samples/rust/Kconfig
new file mode 100644
index 0000000000000..183a3c4dc80cd
--- /dev/null
+++ b/samples/rust/Kconfig
@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: GPL-2.0
+
+menuconfig SAMPLES_RUST
+ bool "Rust samples"
+ depends on RUST
+ help
+ You can build sample Rust kernel code here.
+
+ If unsure, say N.
+
+if SAMPLES_RUST
+
+config SAMPLE_RUST_MINIMAL
+ tristate "Minimal"
+ help
+ This option builds the Rust minimal module sample.
+
+ To compile this as a module, choose M here:
+ the module will be called rust_minimal.
+
+ If unsure, say N.
+
+config SAMPLE_RUST_PRINT
+ tristate "Printing macros"
+ help
+ This option builds the Rust printing macros sample.
+
+ To compile this as a module, choose M here:
+ the module will be called rust_print.
+
+ If unsure, say N.
+
+config SAMPLE_RUST_MODULE_PARAMETERS
+ tristate "Module parameters"
+ help
+ This option builds the Rust module parameters sample.
+
+ To compile this as a module, choose M here:
+ the module will be called rust_module_parameters.
+
+ If unsure, say N.
+
+config SAMPLE_RUST_SYNC
+ tristate "Synchronisation primitives"
+ help
+ This option builds the Rust synchronisation primitives sample.
+
+ To compile this as a module, choose M here:
+ the module will be called rust_sync.
+
+ If unsure, say N.
+
+config SAMPLE_RUST_CHRDEV
+ tristate "Character device"
+ help
+ This option builds the Rust character device sample.
+
+ To compile this as a module, choose M here:
+ the module will be called rust_chrdev.
+
+ If unsure, say N.
+
+config SAMPLE_RUST_MISCDEV
+ tristate "Miscellaneous device"
+ help
+ This option builds the Rust miscellaneous device sample.
+
+ To compile this as a module, choose M here:
+ the module will be called rust_miscdev.
+
+ If unsure, say N.
+
+config SAMPLE_RUST_STACK_PROBING
+ tristate "Stack probing"
+ help
+ This option builds the Rust stack probing sample.
+
+ To compile this as a module, choose M here:
+ the module will be called rust_stack_probing.
+
+ If unsure, say N.
+
+config SAMPLE_RUST_SEMAPHORE
+ tristate "Semaphore"
+ help
+ This option builds the Rust semaphore sample.
+
+ To compile this as a module, choose M here:
+ the module will be called rust_semaphore.
+
+ If unsure, say N.
+
+config SAMPLE_RUST_SEMAPHORE_C
+ tristate "Semaphore (in C, for comparison)"
+ help
+ This option builds the Rust semaphore sample (in C, for comparison).
+
+ To compile this as a module, choose M here:
+ the module will be called rust_semaphore_c.
+
+ If unsure, say N.
+
+config SAMPLE_RUST_RANDOM
+ tristate "Random"
+ help
+ This option builds the Rust random sample.
+
+ To compile this as a module, choose M here:
+ the module will be called rust_random.
+
+ If unsure, say N.
+
+endif # SAMPLES_RUST
diff --git a/samples/rust/Makefile b/samples/rust/Makefile
new file mode 100644
index 0000000000000..48bc871ea1f8f
--- /dev/null
+++ b/samples/rust/Makefile
@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_SAMPLE_RUST_MINIMAL) += rust_minimal.o
+obj-$(CONFIG_SAMPLE_RUST_PRINT) += rust_print.o
+obj-$(CONFIG_SAMPLE_RUST_MODULE_PARAMETERS) += rust_module_parameters.o
+obj-$(CONFIG_SAMPLE_RUST_SYNC) += rust_sync.o
+obj-$(CONFIG_SAMPLE_RUST_CHRDEV) += rust_chrdev.o
+obj-$(CONFIG_SAMPLE_RUST_MISCDEV) += rust_miscdev.o
+obj-$(CONFIG_SAMPLE_RUST_STACK_PROBING) += rust_stack_probing.o
+obj-$(CONFIG_SAMPLE_RUST_SEMAPHORE) += rust_semaphore.o
+obj-$(CONFIG_SAMPLE_RUST_SEMAPHORE_C) += rust_semaphore_c.o
+obj-$(CONFIG_SAMPLE_RUST_RANDOM) += rust_random.o
diff --git a/samples/rust/rust_chrdev.rs b/samples/rust/rust_chrdev.rs
new file mode 100644
index 0000000000000..78423b1e3d116
--- /dev/null
+++ b/samples/rust/rust_chrdev.rs
@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Rust character device sample
+
+#![no_std]
+#![feature(allocator_api, global_asm)]
+
+use alloc::boxed::Box;
+use core::pin::Pin;
+use kernel::prelude::*;
+use kernel::{chrdev, cstr, file_operations::FileOperations};
+
+module! {
+ type: RustChrdev,
+ name: b"rust_chrdev",
+ author: b"Rust for Linux Contributors",
+ description: b"Rust character device sample",
+ license: b"GPL v2",
+ params: {
+ },
+}
+
+#[derive(Default)]
+struct RustFile;
+
+impl FileOperations for RustFile {
+ kernel::declare_file_operations!();
+}
+
+struct RustChrdev {
+ _dev: Pin<Box<chrdev::Registration<2>>>,
+}
+
+impl KernelModule for RustChrdev {
+ fn init() -> KernelResult<Self> {
+ pr_info!("Rust character device sample (init)\n");
+
+ let mut chrdev_reg =
+ chrdev::Registration::new_pinned(cstr!("rust_chrdev"), 0, &THIS_MODULE)?;
+
+ // Register the same kind of device twice, we're just demonstrating
+ // that you can use multiple minors. There are two minors in this case
+ // because its type is `chrdev::Registration<2>`
+ chrdev_reg.as_mut().register::<RustFile>()?;
+ chrdev_reg.as_mut().register::<RustFile>()?;
+
+ Ok(RustChrdev { _dev: chrdev_reg })
+ }
+}
+
+impl Drop for RustChrdev {
+ fn drop(&mut self) {
+ pr_info!("Rust character device sample (exit)\n");
+ }
+}
diff --git a/samples/rust/rust_minimal.rs b/samples/rust/rust_minimal.rs
new file mode 100644
index 0000000000000..21627ce5656e2
--- /dev/null
+++ b/samples/rust/rust_minimal.rs
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Rust minimal sample
+
+#![no_std]
+#![feature(allocator_api, global_asm)]
+
+use kernel::prelude::*;
+
+module! {
+ type: RustMinimal,
+ name: b"rust_minimal",
+ author: b"Rust for Linux Contributors",
+ description: b"Rust minimal sample",
+ license: b"GPL v2",
+ params: {
+ },
+}
+
+struct RustMinimal {
+ message: String,
+}
+
+impl KernelModule for RustMinimal {
+ fn init() -> KernelResult<Self> {
+ pr_info!("Rust minimal sample (init)\n");
+ pr_info!("Am I built-in? {}\n", !cfg!(MODULE));
+
+ Ok(RustMinimal {
+ message: "on the heap!".to_owned(),
+ })
+ }
+}
+
+impl Drop for RustMinimal {
+ fn drop(&mut self) {
+ pr_info!("My message is {}\n", self.message);
+ pr_info!("Rust minimal sample (exit)\n");
+ }
+}
diff --git a/samples/rust/rust_miscdev.rs b/samples/rust/rust_miscdev.rs
new file mode 100644
index 0000000000000..3d48f1e2fad3a
--- /dev/null
+++ b/samples/rust/rust_miscdev.rs
@@ -0,0 +1,145 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Rust miscellaneous device sample
+
+#![no_std]
+#![feature(allocator_api, global_asm)]
+
+use alloc::{boxed::Box, sync::Arc};
+use core::pin::Pin;
+use kernel::prelude::*;
+use kernel::{
+ cstr,
+ file_operations::{File, FileOpener, FileOperations},
+ miscdev,
+ sync::{CondVar, Mutex},
+ user_ptr::{UserSlicePtrReader, UserSlicePtrWriter},
+ Error,
+};
+
+module! {
+ type: RustMiscdev,
+ name: b"rust_miscdev",
+ author: b"Rust for Linux Contributors",
+ description: b"Rust miscellaneous device sample",
+ license: b"GPL v2",
+ params: {
+ },
+}
+
+const MAX_TOKENS: usize = 3;
+
+struct SharedStateInner {
+ token_count: usize,
+}
+
+struct SharedState {
+ state_changed: CondVar,
+ inner: Mutex<SharedStateInner>,
+}
+
+impl SharedState {
+ fn try_new() -> KernelResult<Arc<Self>> {
+ let state = Arc::try_new(Self {
+ // SAFETY: `condvar_init!` is called below.
+ state_changed: unsafe { CondVar::new() },
+ // SAFETY: `mutex_init!` is called below.
+ inner: unsafe { Mutex::new(SharedStateInner { token_count: 0 }) },
+ })?;
+ // SAFETY: `state_changed` is pinned behind `Arc`.
+ let state_changed = unsafe { Pin::new_unchecked(&state.state_changed) };
+ kernel::condvar_init!(state_changed, "SharedState::state_changed");
+ // SAFETY: `inner` is pinned behind `Arc`.
+ let inner = unsafe { Pin::new_unchecked(&state.inner) };
+ kernel::mutex_init!(inner, "SharedState::inner");
+ Ok(state)
+ }
+}
+
+struct Token {
+ shared: Arc<SharedState>,
+}
+
+impl FileOpener<Arc<SharedState>> for Token {
+ fn open(shared: &Arc<SharedState>) -> KernelResult<Self::Wrapper> {
+ Ok(Box::try_new(Self {
+ shared: shared.clone(),
+ })?)
+ }
+}
+
+impl FileOperations for Token {
+ type Wrapper = Box<Self>;
+
+ kernel::declare_file_operations!(read, write);
+
+ fn read(&self, _: &File, data: &mut UserSlicePtrWriter, offset: u64) -> KernelResult<usize> {
+ // Succeed if the caller doesn't provide a buffer or if not at the start.
+ if data.is_empty() || offset != 0 {
+ return Ok(0);
+ }
+
+ {
+ let mut inner = self.shared.inner.lock();
+
+ // Wait until we are allowed to decrement the token count or a signal arrives.
+ while inner.token_count == 0 {
+ if self.shared.state_changed.wait(&mut inner) {
+ return Err(Error::EINTR);
+ }
+ }
+
+ // Consume a token.
+ inner.token_count -= 1;
+ }
+
+ // Notify a possible writer waiting.
+ self.shared.state_changed.notify_all();
+
+ // Write a one-byte 1 to the reader.
+ data.write_slice(&[1u8; 1])?;
+ Ok(1)
+ }
+
+ fn write(&self, data: &mut UserSlicePtrReader, _offset: u64) -> KernelResult<usize> {
+ {
+ let mut inner = self.shared.inner.lock();
+
+ // Wait until we are allowed to increment the token count or a signal arrives.
+ while inner.token_count == MAX_TOKENS {
+ if self.shared.state_changed.wait(&mut inner) {
+ return Err(Error::EINTR);
+ }
+ }
+
+ // Increment the number of token so that a reader can be released.
+ inner.token_count += 1;
+ }
+
+ // Notify a possible reader waiting.
+ self.shared.state_changed.notify_all();
+ Ok(data.len())
+ }
+}
+
+struct RustMiscdev {
+ _dev: Pin<Box<miscdev::Registration<Arc<SharedState>>>>,
+}
+
+impl KernelModule for RustMiscdev {
+ fn init() -> KernelResult<Self> {
+ pr_info!("Rust miscellaneous device sample (init)\n");
+
+ let state = SharedState::try_new()?;
+
+ Ok(RustMiscdev {
+ _dev: miscdev::Registration::new_pinned::<Token>(cstr!("rust_miscdev"), None, state)?,
+ })
+ }
+}
+
+impl Drop for RustMiscdev {
+ fn drop(&mut self) {
+ pr_info!("Rust miscellaneous device sample (exit)\n");
+ }
+}
diff --git a/samples/rust/rust_module_parameters.rs b/samples/rust/rust_module_parameters.rs
new file mode 100644
index 0000000000000..d9b6de695384f
--- /dev/null
+++ b/samples/rust/rust_module_parameters.rs
@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Rust module parameters sample
+
+#![no_std]
+#![feature(allocator_api, global_asm)]
+
+use kernel::prelude::*;
+
+module! {
+ type: RustModuleParameters,
+ name: b"rust_module_parameters",
+ author: b"Rust for Linux Contributors",
+ description: b"Rust module parameters sample",
+ license: b"GPL v2",
+ params: {
+ my_bool: bool {
+ default: true,
+ permissions: 0,
+ description: b"Example of bool",
+ },
+ my_i32: i32 {
+ default: 42,
+ permissions: 0o644,
+ description: b"Example of i32",
+ },
+ my_str: str {
+ default: b"default str val",
+ permissions: 0o644,
+ description: b"Example of a string param",
+ },
+ my_usize: usize {
+ default: 42,
+ permissions: 0o644,
+ description: b"Example of usize",
+ },
+ my_array: ArrayParam<i32, 3> {
+ default: [0, 1],
+ permissions: 0,
+ description: b"Example of array",
+ },
+ },
+}
+
+struct RustModuleParameters;
+
+impl KernelModule for RustModuleParameters {
+ fn init() -> KernelResult<Self> {
+ pr_info!("Rust module parameters sample (init)\n");
+
+ {
+ let lock = THIS_MODULE.kernel_param_lock();
+ pr_info!("Parameters:\n");
+ pr_info!(" my_bool: {}\n", my_bool.read());
+ pr_info!(" my_i32: {}\n", my_i32.read(&lock));
+ pr_info!(
+ " my_str: {}\n",
+ core::str::from_utf8(my_str.read(&lock))?
+ );
+ pr_info!(" my_usize: {}\n", my_usize.read(&lock));
+ pr_info!(" my_array: {:?}\n", my_array.read());
+ }
+
+ Ok(RustModuleParameters)
+ }
+}
+
+impl Drop for RustModuleParameters {
+ fn drop(&mut self) {
+ pr_info!("Rust module parameters sample (exit)\n");
+ }
+}
diff --git a/samples/rust/rust_print.rs b/samples/rust/rust_print.rs
new file mode 100644
index 0000000000000..ddfac800f425d
--- /dev/null
+++ b/samples/rust/rust_print.rs
@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Rust printing macros sample
+
+#![no_std]
+#![feature(allocator_api, global_asm)]
+
+use kernel::prelude::*;
+
+module! {
+ type: RustPrint,
+ name: b"rust_print",
+ author: b"Rust for Linux Contributors",
+ description: b"Rust printing macros sample",
+ license: b"GPL v2",
+ params: {
+ },
+}
+
+struct RustPrint;
+
+impl KernelModule for RustPrint {
+ fn init() -> KernelResult<Self> {
+ pr_info!("Rust printing macros sample (init)\n");
+
+ pr_emerg!("Emergency message (level 0) without args\n");
+ pr_alert!("Alert message (level 1) without args\n");
+ pr_crit!("Critical message (level 2) without args\n");
+ pr_err!("Error message (level 3) without args\n");
+ pr_warn!("Warning message (level 4) without args\n");
+ pr_notice!("Notice message (level 5) without args\n");
+ pr_info!("Info message (level 6) without args\n");
+
+ pr_info!("A line that");
+ pr_cont!(" is continued");
+ pr_cont!(" without args\n");
+
+ pr_emerg!("{} message (level {}) with args\n", "Emergency", 0);
+ pr_alert!("{} message (level {}) with args\n", "Alert", 1);
+ pr_crit!("{} message (level {}) with args\n", "Critical", 2);
+ pr_err!("{} message (level {}) with args\n", "Error", 3);
+ pr_warn!("{} message (level {}) with args\n", "Warning", 4);
+ pr_notice!("{} message (level {}) with args\n", "Notice", 5);
+ pr_info!("{} message (level {}) with args\n", "Info", 6);
+
+ pr_info!("A {} that", "line");
+ pr_cont!(" is {}", "continued");
+ pr_cont!(" with {}\n", "args");
+
+ Ok(RustPrint)
+ }
+}
+
+impl Drop for RustPrint {
+ fn drop(&mut self) {
+ pr_info!("Rust printing macros sample (exit)\n");
+ }
+}
diff --git a/samples/rust/rust_random.rs b/samples/rust/rust_random.rs
new file mode 100644
index 0000000000000..9f17b94404661
--- /dev/null
+++ b/samples/rust/rust_random.rs
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Rust random device
+//!
+//! Adapted from Alex Gaynor's original available at
+//! <https://github.com/alex/just-use/blob/master/src/lib.rs>.
+
+#![no_std]
+#![feature(allocator_api, global_asm)]
+
+use kernel::{
+ file_operations::{File, FileOperations},
+ prelude::*,
+ user_ptr::{UserSlicePtrReader, UserSlicePtrWriter},
+};
+
+#[derive(Default)]
+struct RandomFile;
+
+impl FileOperations for RandomFile {
+ kernel::declare_file_operations!(read, write);
+
+ fn read(&self, file: &File, buf: &mut UserSlicePtrWriter, _offset: u64) -> KernelResult<usize> {
+ let total_len = buf.len();
+ let mut chunkbuf = [0; 256];
+
+ while !buf.is_empty() {
+ let len = chunkbuf.len().min(buf.len());
+ let chunk = &mut chunkbuf[0..len];
+
+ if file.is_blocking() {
+ kernel::random::getrandom(chunk)?;
+ } else {
+ kernel::random::getrandom_nonblock(chunk)?;
+ }
+ buf.write_slice(chunk)?;
+ }
+ Ok(total_len)
+ }
+
+ fn write(&self, buf: &mut UserSlicePtrReader, _offset: u64) -> KernelResult<usize> {
+ let total_len = buf.len();
+ let mut chunkbuf = [0; 256];
+ while !buf.is_empty() {
+ let len = chunkbuf.len().min(buf.len());
+ let chunk = &mut chunkbuf[0..len];
+ buf.read_slice(chunk)?;
+ kernel::random::add_randomness(chunk);
+ }
+ Ok(total_len)
+ }
+}
+
+module_misc_device! {
+ type: RandomFile,
+ name: b"rust_random",
+ author: b"Rust for Linux Contributors",
+ description: b"Just use /dev/urandom: Now with early-boot safety",
+ license: b"GPL v2",
+}
diff --git a/samples/rust/rust_semaphore.rs b/samples/rust/rust_semaphore.rs
new file mode 100644
index 0000000000000..eb1dcc202e719
--- /dev/null
+++ b/samples/rust/rust_semaphore.rs
@@ -0,0 +1,176 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Rust semaphore sample
+//!
+//! A counting semaphore that can be used by userspace.
+//!
+//! The count is incremented by writes to the device. A write of `n` bytes results in an increment
+//! of `n`. It is decremented by reads; each read results in the count being decremented by 1. If
+//! the count is already zero, a read will block until another write increments it.
+//!
+//! This can be used in user space from the shell for example as follows (assuming a node called
+//! `semaphore`): `cat semaphore` decrements the count by 1 (waiting for it to become non-zero
+//! before decrementing); `echo -n 123 > semaphore` increments the semaphore by 3, potentially
+//! unblocking up to 3 blocked readers.
+
+#![no_std]
+#![feature(allocator_api, global_asm)]
+
+use alloc::{boxed::Box, sync::Arc};
+use core::{
+ pin::Pin,
+ sync::atomic::{AtomicU64, Ordering},
+};
+use kernel::{
+ condvar_init, cstr, declare_file_operations,
+ file_operations::{File, FileOpener, FileOperations, IoctlCommand, IoctlHandler},
+ miscdev::Registration,
+ mutex_init,
+ prelude::*,
+ sync::{CondVar, Mutex},
+ user_ptr::{UserSlicePtrReader, UserSlicePtrWriter},
+ Error,
+};
+
+module! {
+ type: RustSemaphore,
+ name: b"rust_semaphore",
+ author: b"Rust for Linux Contributors",
+ description: b"Rust semaphore sample",
+ license: b"GPL v2",
+ params: {},
+}
+
+struct SemaphoreInner {
+ count: usize,
+ max_seen: usize,
+}
+
+struct Semaphore {
+ changed: CondVar,
+ inner: Mutex<SemaphoreInner>,
+}
+
+struct FileState {
+ read_count: AtomicU64,
+ shared: Arc<Semaphore>,
+}
+
+impl FileState {
+ fn consume(&self) -> KernelResult {
+ let mut inner = self.shared.inner.lock();
+ while inner.count == 0 {
+ if self.shared.changed.wait(&mut inner) {
+ return Err(Error::EINTR);
+ }
+ }
+ inner.count -= 1;
+ Ok(())
+ }
+}
+
+impl FileOpener<Arc<Semaphore>> for FileState {
+ fn open(shared: &Arc<Semaphore>) -> KernelResult<Box<Self>> {
+ Ok(Box::try_new(Self {
+ read_count: AtomicU64::new(0),
+ shared: shared.clone(),
+ })?)
+ }
+}
+
+impl FileOperations for FileState {
+ type Wrapper = Box<Self>;
+
+ declare_file_operations!(read, write, ioctl);
+
+ fn read(&self, _: &File, data: &mut UserSlicePtrWriter, offset: u64) -> KernelResult<usize> {
+ if data.is_empty() || offset > 0 {
+ return Ok(0);
+ }
+ self.consume()?;
+ data.write_slice(&[0u8; 1])?;
+ self.read_count.fetch_add(1, Ordering::Relaxed);
+ Ok(1)
+ }
+
+ fn write(&self, data: &mut UserSlicePtrReader, _offset: u64) -> KernelResult<usize> {
+ {
+ let mut inner = self.shared.inner.lock();
+ inner.count = inner.count.saturating_add(data.len());
+ if inner.count > inner.max_seen {
+ inner.max_seen = inner.count;
+ }
+ }
+
+ self.shared.changed.notify_all();
+ Ok(data.len())
+ }
+
+ fn ioctl(&self, file: &File, cmd: &mut IoctlCommand) -> KernelResult<i32> {
+ cmd.dispatch(self, file)
+ }
+}
+
+struct RustSemaphore {
+ _dev: Pin<Box<Registration<Arc<Semaphore>>>>,
+}
+
+impl KernelModule for RustSemaphore {
+ fn init() -> KernelResult<Self> {
+ pr_info!("Rust semaphore sample (init)\n");
+
+ let sema = Arc::try_new(Semaphore {
+ // SAFETY: `condvar_init!` is called below.
+ changed: unsafe { CondVar::new() },
+
+ // SAFETY: `mutex_init!` is called below.
+ inner: unsafe {
+ Mutex::new(SemaphoreInner {
+ count: 0,
+ max_seen: 0,
+ })
+ },
+ })?;
+
+ // SAFETY: `changed` is pinned behind `Arc`.
+ condvar_init!(Pin::new_unchecked(&sema.changed), "Semaphore::changed");
+
+ // SAFETY: `inner` is pinned behind `Arc`.
+ mutex_init!(Pin::new_unchecked(&sema.inner), "Semaphore::inner");
+
+ Ok(Self {
+ _dev: Registration::new_pinned::<FileState>(cstr!("rust_semaphore"), None, sema)?,
+ })
+ }
+}
+
+impl Drop for RustSemaphore {
+ fn drop(&mut self) {
+ pr_info!("Rust semaphore sample (exit)\n");
+ }
+}
+
+const IOCTL_GET_READ_COUNT: u32 = 0x80086301;
+const IOCTL_SET_READ_COUNT: u32 = 0x40086301;
+
+impl IoctlHandler for FileState {
+ fn read(&self, _: &File, cmd: u32, writer: &mut UserSlicePtrWriter) -> KernelResult<i32> {
+ match cmd {
+ IOCTL_GET_READ_COUNT => {
+ writer.write(&self.read_count.load(Ordering::Relaxed))?;
+ Ok(0)
+ }
+ _ => Err(Error::EINVAL),
+ }
+ }
+
+ fn write(&self, _: &File, cmd: u32, reader: &mut UserSlicePtrReader) -> KernelResult<i32> {
+ match cmd {
+ IOCTL_SET_READ_COUNT => {
+ self.read_count.store(reader.read()?, Ordering::Relaxed);
+ Ok(0)
+ }
+ _ => Err(Error::EINVAL),
+ }
+ }
+}
diff --git a/samples/rust/rust_semaphore_c.c b/samples/rust/rust_semaphore_c.c
new file mode 100644
index 0000000000000..cdc121d4030d0
--- /dev/null
+++ b/samples/rust/rust_semaphore_c.c
@@ -0,0 +1,212 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Rust semaphore sample (in C, for comparison)
+ *
+ * This is a C implementation of `rust_semaphore.rs`. Refer to the description
+ * in that file for details on the device.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/miscdevice.h>
+#include <linux/module.h>
+#include <linux/fs.h>
+#include <linux/slab.h>
+#include <linux/refcount.h>
+#include <linux/wait.h>
+
+#define IOCTL_GET_READ_COUNT _IOR('c', 1, u64)
+#define IOCTL_SET_READ_COUNT _IOW('c', 1, u64)
+
+struct semaphore_state {
+ struct kref ref;
+ struct miscdevice miscdev;
+ wait_queue_head_t changed;
+ struct mutex mutex;
+ size_t count;
+ size_t max_seen;
+};
+
+struct file_state {
+ atomic64_t read_count;
+ struct semaphore_state *shared;
+};
+
+static int semaphore_consume(struct semaphore_state *state)
+{
+ DEFINE_WAIT(wait);
+
+ mutex_lock(&state->mutex);
+ while (state->count == 0) {
+ prepare_to_wait(&state->changed, &wait, TASK_INTERRUPTIBLE);
+ mutex_unlock(&state->mutex);
+ schedule();
+ finish_wait(&state->changed, &wait);
+ if (signal_pending(current))
+ return -EINTR;
+ mutex_lock(&state->mutex);
+ }
+
+ state->count--;
+ mutex_unlock(&state->mutex);
+
+ return 0;
+}
+
+static int semaphore_open(struct inode *nodp, struct file *filp)
+{
+ struct semaphore_state *shared =
+ container_of(filp->private_data, struct semaphore_state, miscdev);
+ struct file_state *state;
+
+ state = kzalloc(sizeof(*state), GFP_KERNEL);
+ if (!state)
+ return -ENOMEM;
+
+ kref_get(&shared->ref);
+ state->shared = shared;
+ atomic64_set(&state->read_count, 0);
+
+ filp->private_data = state;
+
+ return 0;
+}
+
+static ssize_t semaphore_write(struct file *filp, const char __user *buffer, size_t count,
+ loff_t *ppos)
+{
+ struct file_state *state = filp->private_data;
+ struct semaphore_state *shared = state->shared;
+
+ mutex_lock(&shared->mutex);
+
+ shared->count += count;
+ if (shared->count < count)
+ shared->count = SIZE_MAX;
+
+ if (shared->count > shared->max_seen)
+ shared->max_seen = shared->count;
+
+ mutex_unlock(&shared->mutex);
+
+ wake_up_all(&shared->changed);
+
+ return count;
+}
+
+static ssize_t semaphore_read(struct file *filp, char __user *buffer,
+ size_t count, loff_t *ppos)
+{
+ struct file_state *state = filp->private_data;
+ char c = 0;
+ int ret;
+
+ if (count == 0 || *ppos > 0)
+ return 0;
+
+ ret = semaphore_consume(state->shared);
+ if (ret)
+ return ret;
+
+ if (copy_to_user(buffer, &c, sizeof(c)))
+ return -EFAULT;
+
+ atomic64_add(1, &state->read_count);
+ *ppos += 1;
+ return 1;
+}
+
+static long semaphore_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+ struct file_state *state = filp->private_data;
+ void __user *buffer = (void __user *)arg;
+ u64 value;
+
+ switch (cmd) {
+ case IOCTL_GET_READ_COUNT:
+ value = atomic64_read(&state->read_count);
+ if (copy_to_user(buffer, &value, sizeof(value)))
+ return -EFAULT;
+ return 0;
+ case IOCTL_SET_READ_COUNT:
+ if (copy_from_user(&value, buffer, sizeof(value)))
+ return -EFAULT;
+ atomic64_set(&state->read_count, value);
+ return 0;
+ default:
+ return -EINVAL;
+ }
+}
+
+static void semaphore_free(struct kref *kref)
+{
+ struct semaphore_state *device;
+
+ device = container_of(kref, struct semaphore_state, ref);
+ kfree(device);
+}
+
+static int semaphore_release(struct inode *nodp, struct file *filp)
+{
+ struct file_state *state = filp->private_data;
+
+ kref_put(&state->shared->ref, semaphore_free);
+ kfree(state);
+ return 0;
+}
+
+static const struct file_operations semaphore_fops = {
+ .owner = THIS_MODULE,
+ .open = semaphore_open,
+ .read = semaphore_read,
+ .write = semaphore_write,
+ .compat_ioctl = semaphore_ioctl,
+ .release = semaphore_release,
+};
+
+static struct semaphore_state *device;
+
+static int __init semaphore_init(void)
+{
+ int ret;
+ struct semaphore_state *state;
+
+ pr_info("Rust semaphore sample (in C, for comparison) (init)\n");
+
+ state = kzalloc(sizeof(*state), GFP_KERNEL);
+ if (!state)
+ return -ENOMEM;
+
+ mutex_init(&state->mutex);
+ kref_init(&state->ref);
+ init_waitqueue_head(&state->changed);
+
+ state->miscdev.fops = &semaphore_fops;
+ state->miscdev.minor = MISC_DYNAMIC_MINOR;
+ state->miscdev.name = "semaphore";
+
+ ret = misc_register(&state->miscdev);
+ if (ret < 0) {
+ kfree(state);
+ return ret;
+ }
+
+ device = state;
+
+ return 0;
+}
+
+static void __exit semaphore_exit(void)
+{
+ pr_info("Rust semaphore sample (in C, for comparison) (exit)\n");
+
+ misc_deregister(&device->miscdev);
+ kref_put(&device->ref, semaphore_free);
+}
+
+module_init(semaphore_init);
+module_exit(semaphore_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Rust for Linux Contributors");
+MODULE_DESCRIPTION("Rust semaphore sample (in C, for comparison)");
diff --git a/samples/rust/rust_stack_probing.rs b/samples/rust/rust_stack_probing.rs
new file mode 100644
index 0000000000000..f992773545658
--- /dev/null
+++ b/samples/rust/rust_stack_probing.rs
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Rust stack probing sample
+
+#![no_std]
+#![feature(allocator_api, global_asm)]
+#![feature(test)]
+
+use kernel::prelude::*;
+
+module! {
+ type: RustStackProbing,
+ name: b"rust_stack_probing",
+ author: b"Rust for Linux Contributors",
+ description: b"Rust stack probing sample",
+ license: b"GPL v2",
+ params: {
+ },
+}
+
+struct RustStackProbing;
+
+impl KernelModule for RustStackProbing {
+ fn init() -> KernelResult<Self> {
+ pr_info!("Rust stack probing sample (init)\n");
+
+ // Including this large variable on the stack will trigger
+ // stack probing on the supported archs.
+ // This will verify that stack probing does not lead to
+ // any errors if we need to link `__rust_probestack`.
+ let x: [u64; 514] = core::hint::black_box([5; 514]);
+ pr_info!("Large array has length: {}\n", x.len());
+
+ Ok(RustStackProbing)
+ }
+}
+
+impl Drop for RustStackProbing {
+ fn drop(&mut self) {
+ pr_info!("Rust stack probing sample (exit)\n");
+ }
+}
diff --git a/samples/rust/rust_sync.rs b/samples/rust/rust_sync.rs
new file mode 100644
index 0000000000000..a921bfd7d55fb
--- /dev/null
+++ b/samples/rust/rust_sync.rs
@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Rust synchronisation primitives sample
+
+#![no_std]
+#![feature(allocator_api, global_asm)]
+
+use alloc::boxed::Box;
+use core::pin::Pin;
+use kernel::prelude::*;
+use kernel::{
+ condvar_init, mutex_init, spinlock_init,
+ sync::{CondVar, Mutex, SpinLock},
+};
+
+module! {
+ type: RustSync,
+ name: b"rust_sync",
+ author: b"Rust for Linux Contributors",
+ description: b"Rust synchronisation primitives sample",
+ license: b"GPL v2",
+ params: {
+ },
+}
+
+struct RustSync;
+
+impl KernelModule for RustSync {
+ fn init() -> KernelResult<Self> {
+ pr_info!("Rust synchronisation primitives sample (init)\n");
+
+ // Test mutexes.
+ {
+ // SAFETY: `init` is called below.
+ let data = Pin::from(Box::try_new(unsafe { Mutex::new(0) })?);
+ mutex_init!(data.as_ref(), "RustSync::init::data1");
+ *data.lock() = 10;
+ pr_info!("Value: {}\n", *data.lock());
+
+ // SAFETY: `init` is called below.
+ let cv = Pin::from(Box::try_new(unsafe { CondVar::new() })?);
+ condvar_init!(cv.as_ref(), "RustSync::init::cv1");
+ {
+ let mut guard = data.lock();
+ while *guard != 10 {
+ let _ = cv.wait(&mut guard);
+ }
+ }
+ cv.notify_one();
+ cv.notify_all();
+ cv.free_waiters();
+ }
+
+ // Test spinlocks.
+ {
+ // SAFETY: `init` is called below.
+ let data = Pin::from(Box::try_new(unsafe { SpinLock::new(0) })?);
+ spinlock_init!(data.as_ref(), "RustSync::init::data2");
+ *data.lock() = 10;
+ pr_info!("Value: {}\n", *data.lock());
+
+ // SAFETY: `init` is called below.
+ let cv = Pin::from(Box::try_new(unsafe { CondVar::new() })?);
+ condvar_init!(cv.as_ref(), "RustSync::init::cv2");
+ {
+ let mut guard = data.lock();
+ while *guard != 10 {
+ let _ = cv.wait(&mut guard);
+ }
+ }
+ cv.notify_one();
+ cv.notify_all();
+ cv.free_waiters();
+ }
+
+ Ok(RustSync)
+ }
+}
+
+impl Drop for RustSync {
+ fn drop(&mut self) {
+ pr_info!("Rust synchronisation primitives sample (exit)\n");
+ }
+}
diff --git a/scripts/Makefile.build b/scripts/Makefile.build
index 10b2f2380d6fb..d766d3daf75b1 100644
--- a/scripts/Makefile.build
+++ b/scripts/Makefile.build
@@ -26,6 +26,7 @@ EXTRA_CPPFLAGS :=
EXTRA_LDFLAGS :=
asflags-y :=
ccflags-y :=
+rustcflags-y :=
cppflags-y :=
ldflags-y :=
@@ -287,6 +288,24 @@ quiet_cmd_cc_lst_c = MKLST $@
$(obj)/%.lst: $(src)/%.c FORCE
$(call if_changed_dep,cc_lst_c)
+# Compile Rust sources (.rs)
+# ---------------------------------------------------------------------------
+
+rustc_cross_flags := --target=$(srctree)/arch/$(SRCARCH)/rust/target.json
+
+quiet_cmd_rustc_o_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@
+ cmd_rustc_o_rs = \
+ RUST_MODFILE=$(modfile) \
+ $(RUSTC_OR_CLIPPY) $(rustc_flags) $(rustc_cross_flags) \
+ --extern alloc --extern kernel \
+ --crate-type rlib --out-dir $(obj) -L $(objtree)/rust/ \
+ --crate-name $(patsubst %.o,%,$(notdir $@)) $<; \
+ mv $(obj)/$(subst .o,,$(notdir $@)).d $(depfile); \
+ sed -i '/^\#/d' $(depfile)
+
+$(obj)/%.o: $(src)/%.rs FORCE
+ $(call if_changed_dep,rustc_o_rs)
+
# Compile assembler sources (.S)
# ---------------------------------------------------------------------------
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 10950559b223b..b307b40870392 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -8,6 +8,7 @@ ldflags-y += $(EXTRA_LDFLAGS)
# flags that take effect in current and sub directories
KBUILD_AFLAGS += $(subdir-asflags-y)
KBUILD_CFLAGS += $(subdir-ccflags-y)
+KBUILD_RUSTCFLAGS += $(subdir-rustcflags-y)
# Figure out what we need to build from the various variables
# ===========================================================================
@@ -133,6 +134,10 @@ _c_flags = $(filter-out $(CFLAGS_REMOVE_$(target-stem).o), \
$(filter-out $(ccflags-remove-y), \
$(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS) $(ccflags-y)) \
$(CFLAGS_$(target-stem).o))
+_rustc_flags = $(filter-out $(RUSTCFLAGS_REMOVE_$(target-stem).o), \
+ $(filter-out $(rustcflags-remove-y), \
+ $(KBUILD_RUSTCFLAGS) $(rustcflags-y)) \
+ $(RUSTCFLAGS_$(target-stem).o))
_a_flags = $(filter-out $(AFLAGS_REMOVE_$(target-stem).o), \
$(filter-out $(asflags-remove-y), \
$(KBUILD_CPPFLAGS) $(KBUILD_AFLAGS) $(asflags-y)) \
@@ -202,6 +207,11 @@ modkern_cflags = \
$(KBUILD_CFLAGS_MODULE) $(CFLAGS_MODULE), \
$(KBUILD_CFLAGS_KERNEL) $(CFLAGS_KERNEL) $(modfile_flags))
+modkern_rustcflags = \
+ $(if $(part-of-module), \
+ $(KBUILD_RUSTCFLAGS_MODULE) $(RUSTCFLAGS_MODULE), \
+ $(KBUILD_RUSTCFLAGS_KERNEL) $(RUSTCFLAGS_KERNEL))
+
modkern_aflags = $(if $(part-of-module), \
$(KBUILD_AFLAGS_MODULE) $(AFLAGS_MODULE), \
$(KBUILD_AFLAGS_KERNEL) $(AFLAGS_KERNEL))
@@ -211,6 +221,8 @@ c_flags = -Wp,-MMD,$(depfile) $(NOSTDINC_FLAGS) $(LINUXINCLUDE) \
$(_c_flags) $(modkern_cflags) \
$(basename_flags) $(modname_flags)
+rustc_flags = $(_rustc_flags) $(modkern_rustcflags) @$(objtree)/include/generated/rustc_cfg
+
a_flags = -Wp,-MMD,$(depfile) $(NOSTDINC_FLAGS) $(LINUXINCLUDE) \
$(_a_flags) $(modkern_aflags)
diff --git a/scripts/dummy-tools/elfedit b/scripts/dummy-tools/elfedit
new file mode 120000
index 0000000000000..c0648b38dd424
--- /dev/null
+++ b/scripts/dummy-tools/elfedit
@@ -0,0 +1 @@
+ld \ No newline at end of file
diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c
index 54ad86d137849..9bab5f55ade3d 100644
--- a/scripts/kallsyms.c
+++ b/scripts/kallsyms.c
@@ -27,7 +27,7 @@
#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof(arr[0]))
-#define KSYM_NAME_LEN 128
+#define KSYM_NAME_LEN 512
struct sym_entry {
unsigned long long addr;
@@ -470,12 +470,37 @@ static void write_src(void)
if ((i & 0xFF) == 0)
markers[i >> 8] = off;
- printf("\t.byte 0x%02x", table[i]->len);
+ /*
+ * There cannot be any symbol of length zero -- we use that
+ * to mark a "big" symbol (and it doesn't make sense anyway).
+ */
+ if (table[i]->len == 0) {
+ fprintf(stderr, "kallsyms failure: "
+ "unexpected zero symbol length\n");
+ exit(EXIT_FAILURE);
+ }
+
+ /* Only lengths that fit in up to two bytes are supported. */
+ if (table[i]->len > 0xFFFF) {
+ fprintf(stderr, "kallsyms failure: "
+ "unexpected huge symbol length\n");
+ exit(EXIT_FAILURE);
+ }
+
+ if (table[i]->len <= 0xFF) {
+ /* Most symbols use a single byte for the length. */
+ printf("\t.byte 0x%02x", table[i]->len);
+ off += table[i]->len + 1;
+ } else {
+ /* "Big" symbols use a zero and then two bytes. */
+ printf("\t.byte 0x00, 0x%02x, 0x%02x",
+ (table[i]->len >> 8) & 0xFF,
+ table[i]->len & 0xFF);
+ off += table[i]->len + 3;
+ }
for (k = 0; k < table[i]->len; k++)
printf(", 0x%02x", table[i]->sym[k]);
printf("\n");
-
- off += table[i]->len + 1;
}
printf("\n");
diff --git a/scripts/kconfig/confdata.c b/scripts/kconfig/confdata.c
index cf72680cd7692..d9fc638dfa86a 100644
--- a/scripts/kconfig/confdata.c
+++ b/scripts/kconfig/confdata.c
@@ -652,6 +652,56 @@ static struct conf_printer kconfig_printer_cb =
};
/*
+ * rustc cfg printer
+ *
+ * This printer is used when generating the resulting rustc configuration
+ * after kconfig invocation and `defconfig` files.
+ */
+static void rustc_cfg_print_symbol(FILE *fp, struct symbol *sym, const char *value, void *arg)
+{
+ const char *str;
+
+ switch (sym->type) {
+ case S_INT:
+ case S_HEX:
+ case S_BOOLEAN:
+ case S_TRISTATE:
+ str = sym_escape_string_value(value);
+
+ /*
+ * We don't care about disabled ones, i.e. no need for
+ * what otherwise are "comments" in other printers.
+ */
+ if (*value == 'n')
+ return;
+
+ /*
+ * To have similar functionality to the C macro `IS_ENABLED()`
+ * we provide an empty `--cfg CONFIG_X` here in both `y`
+ * and `m` cases.
+ *
+ * Then, the common `fprintf()` below will also give us
+ * a `--cfg CONFIG_X="y"` or `--cfg CONFIG_X="m"`, which can
+ * be used as the equivalent of `IS_BUILTIN()`/`IS_MODULE()`.
+ */
+ if (*value == 'y' || *value == 'm')
+ fprintf(fp, "--cfg=%s%s\n", CONFIG_, sym->name);
+
+ break;
+ default:
+ str = value;
+ break;
+ }
+
+ fprintf(fp, "--cfg=%s%s=%s\n", CONFIG_, sym->name, str);
+}
+
+static struct conf_printer rustc_cfg_printer_cb =
+{
+ .print_symbol = rustc_cfg_print_symbol,
+};
+
+/*
* Header printer
*
* This printer is used when generating the `include/generated/autoconf.h' file.
@@ -1058,7 +1108,7 @@ int conf_write_autoconf(int overwrite)
struct symbol *sym;
const char *name;
const char *autoconf_name = conf_get_autoconfig_name();
- FILE *out, *out_h;
+ FILE *out, *out_h, *out_rustc_cfg;
int i;
if (!overwrite && is_present(autoconf_name))
@@ -1079,6 +1129,13 @@ int conf_write_autoconf(int overwrite)
return 1;
}
+ out_rustc_cfg = fopen(".tmp_rustc_cfg", "w");
+ if (!out_rustc_cfg) {
+ fclose(out);
+ fclose(out_h);
+ return 1;
+ }
+
conf_write_heading(out, &kconfig_printer_cb, NULL);
conf_write_heading(out_h, &header_printer_cb, NULL);
@@ -1090,9 +1147,11 @@ int conf_write_autoconf(int overwrite)
/* write symbols to auto.conf and autoconf.h */
conf_write_symbol(out, sym, &kconfig_printer_cb, (void *)1);
conf_write_symbol(out_h, sym, &header_printer_cb, NULL);
+ conf_write_symbol(out_rustc_cfg, sym, &rustc_cfg_printer_cb, NULL);
}
fclose(out);
fclose(out_h);
+ fclose(out_rustc_cfg);
name = getenv("KCONFIG_AUTOHEADER");
if (!name)
@@ -1111,6 +1170,12 @@ int conf_write_autoconf(int overwrite)
if (rename(".tmpconfig", autoconf_name))
return 1;
+ name = "include/generated/rustc_cfg";
+ if (make_parent_dir(name))
+ return 1;
+ if (rename(".tmp_rustc_cfg", name))
+ return 1;
+
return 0;
}
diff --git a/scripts/rust-version.sh b/scripts/rust-version.sh
new file mode 100755
index 0000000000000..67b6d31688e24
--- /dev/null
+++ b/scripts/rust-version.sh
@@ -0,0 +1,31 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0
+#
+# rust-version rust-command
+#
+# Print the compiler version of `rust-command' in a 5 or 6-digit form
+# such as `14502' for rustc-1.45.2 etc.
+#
+# Returns 0 if not found (so that Kconfig does not complain)
+compiler="$*"
+
+if [ ${#compiler} -eq 0 ]; then
+ echo "Error: No compiler specified." >&2
+ printf "Usage:\n\t$0 <rust-command>\n" >&2
+ exit 1
+fi
+
+if ! command -v $compiler >/dev/null 2>&1; then
+ echo 0
+ exit 0
+fi
+
+VERSION=$($compiler --version | cut -f2 -d' ')
+
+# Cut suffix if any (e.g. `-dev`)
+VERSION=$(echo $VERSION | cut -f1 -d'-')
+
+MAJOR=$(echo $VERSION | cut -f1 -d'.')
+MINOR=$(echo $VERSION | cut -f2 -d'.')
+PATCHLEVEL=$(echo $VERSION | cut -f3 -d'.')
+printf "%d%02d%02d\\n" $MAJOR $MINOR $PATCHLEVEL
diff --git a/tools/include/linux/kallsyms.h b/tools/include/linux/kallsyms.h
index efb6c3f5f2a9a..5a37ccbec54fb 100644
--- a/tools/include/linux/kallsyms.h
+++ b/tools/include/linux/kallsyms.h
@@ -6,7 +6,7 @@
#include <stdio.h>
#include <unistd.h>
-#define KSYM_NAME_LEN 128
+#define KSYM_NAME_LEN 512
struct module;
diff --git a/tools/include/linux/lockdep.h b/tools/include/linux/lockdep.h
index e56997288f2b0..d9c163f3ab242 100644
--- a/tools/include/linux/lockdep.h
+++ b/tools/include/linux/lockdep.h
@@ -47,7 +47,7 @@ static inline int debug_locks_off(void)
#define task_pid_nr(tsk) ((tsk)->pid)
-#define KSYM_NAME_LEN 128
+#define KSYM_NAME_LEN 512
#define printk(...) dprintf(STDOUT_FILENO, __VA_ARGS__)
#define pr_err(format, ...) fprintf (stderr, format, ## __VA_ARGS__)
#define pr_warn pr_err
diff --git a/tools/lib/perf/include/perf/event.h b/tools/lib/perf/include/perf/event.h
index 4d0c02ba3f7d3..095d60144a70c 100644
--- a/tools/lib/perf/include/perf/event.h
+++ b/tools/lib/perf/include/perf/event.h
@@ -95,7 +95,7 @@ struct perf_record_throttle {
};
#ifndef KSYM_NAME_LEN
-#define KSYM_NAME_LEN 256
+#define KSYM_NAME_LEN 512
#endif
struct perf_record_ksymbol {
diff --git a/tools/lib/symbol/kallsyms.h b/tools/lib/symbol/kallsyms.h
index 72ab9870454ba..542f9b059c3bd 100644
--- a/tools/lib/symbol/kallsyms.h
+++ b/tools/lib/symbol/kallsyms.h
@@ -7,7 +7,7 @@
#include <linux/types.h>
#ifndef KSYM_NAME_LEN
-#define KSYM_NAME_LEN 256
+#define KSYM_NAME_LEN 512
#endif
static inline u8 kallsyms2elf_binding(char type)