diff options
| author | Stephen Rothwell <sfr@canb.auug.org.au> | 2021-06-17 17:22:25 +1000 |
|---|---|---|
| committer | Stephen Rothwell <sfr@canb.auug.org.au> | 2021-06-17 17:22:25 +1000 |
| commit | 7cf6b8b2124fb6e3ad5c89f55170a6ce78fabd7c (patch) | |
| tree | d1574e35601f2d3cc1a8bb82822e40eccf9c566b | |
| parent | 7696cf52e1d3e25826e68cb475fab5a1a56dc578 (diff) | |
| parent | fef0214731ccdff3f674940e579903ccc9638f93 (diff) | |
| download | linux-next-7cf6b8b2124fb6e3ad5c89f55170a6ce78fabd7c.tar.gz | |
Merge remote-tracking branch 'rust/rust-next'
Notice: this object is not reachable from any branch.
# Conflicts:
# Makefile
# include/uapi/linux/android/binder.h
# kernel/printk/printk.c
Notice: this object is not reachable from any branch.
95 files changed, 11357 insertions, 48 deletions
diff --git a/.gitignore b/.gitignore index 7afd412dadd2c..48c68948f476d 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ *.o *.o.* *.patch +*.rmeta *.s *.so *.so.dbg @@ -96,6 +97,7 @@ modules.order !.gitattributes !.gitignore !.mailmap +!.rustfmt.toml # # Generated include files @@ -161,3 +163,6 @@ x509.genkey # Documentation toolchain sphinx_*/ + +# Rust analyzer configuration +/rust-project.json diff --git a/.rustfmt.toml b/.rustfmt.toml new file mode 100644 index 0000000000000..5893c0e3cbde1 --- /dev/null +++ b/.rustfmt.toml @@ -0,0 +1,12 @@ +edition = "2018" +format_code_in_doc_comments = true +newline_style = "Unix" + +# Unstable options that help catching some mistakes in formatting and that we may want to enable +# when they become stable. +# +# They are kept here since they are useful to run from time to time. +#reorder_impl_items = true +#comment_width = 100 +#wrap_comments = true +#normalize_comments = true diff --git a/Documentation/doc-guide/kernel-doc.rst b/Documentation/doc-guide/kernel-doc.rst index 79aaa55d6bcf2..724e2ffddff12 100644 --- a/Documentation/doc-guide/kernel-doc.rst +++ b/Documentation/doc-guide/kernel-doc.rst @@ -11,6 +11,9 @@ when it is embedded in source files. reasons. The kernel source contains tens of thousands of kernel-doc comments. Please stick to the style described here. +.. note:: kernel-doc does not cover Rust code: please see + Documentation/rust/docs.rst instead. + The kernel-doc structure is extracted from the comments, and proper `Sphinx C Domain`_ function and type descriptions with anchors are generated from them. The descriptions are filtered for special kernel-doc diff --git a/Documentation/index.rst b/Documentation/index.rst index 54ce34fd6fbda..1b13c2445e87b 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -82,6 +82,7 @@ merged much easier. maintainer/index fault-injection/index livepatch/index + rust/index Kernel API documentation diff --git a/Documentation/kbuild/kbuild.rst b/Documentation/kbuild/kbuild.rst index 2d1fc03d346ea..1109d18d9377d 100644 --- a/Documentation/kbuild/kbuild.rst +++ b/Documentation/kbuild/kbuild.rst @@ -57,6 +57,10 @@ CFLAGS_MODULE ------------- Additional module specific options to use for $(CC). +KRUSTCFLAGS +----------- +Additional options to the Rust compiler (for built-in and modules). + LDFLAGS_MODULE -------------- Additional options used for $(LD) when linking modules. diff --git a/Documentation/process/changes.rst b/Documentation/process/changes.rst index d3a8557b66a1a..7a1d2837d2c3d 100644 --- a/Documentation/process/changes.rst +++ b/Documentation/process/changes.rst @@ -31,6 +31,8 @@ you probably needn't concern yourself with pcmciautils. ====================== =============== ======================================== GNU C 4.9 gcc --version Clang/LLVM (optional) 10.0.1 clang --version +rustc (optional) nightly rustc --version +bindgen (optional) 0.56.0 bindgen --version GNU make 3.81 make --version binutils 2.23 ld -v flex 2.5.35 flex --version @@ -55,6 +57,7 @@ iptables 1.4.2 iptables -V openssl & libcrypto 1.0.0 openssl version bc 1.06.95 bc --version Sphinx\ [#f1]_ 1.3 sphinx-build --version +rustdoc (optional) nightly rustdoc --version ====================== =============== ======================================== .. [#f1] Sphinx is needed only to build the Kernel documentation @@ -329,6 +332,12 @@ Sphinx Please see :ref:`sphinx_install` in :ref:`Documentation/doc-guide/sphinx.rst <sphinxdoc>` for details about Sphinx requirements. +rustdoc +------- + +``rustdoc`` is used to generate Rust documentation. Please see +:ref:`Documentation/rust/docs.rst <rust_docs>` for more information. + Getting updated software ======================== diff --git a/Documentation/rust/arch-support.rst b/Documentation/rust/arch-support.rst new file mode 100644 index 0000000000000..e93a8ceb3f82d --- /dev/null +++ b/Documentation/rust/arch-support.rst @@ -0,0 +1,32 @@ +.. _rust_arch_support: + +Arch Support +============ + +Currently, the Rust compiler (``rustc``) uses LLVM for code generation, +which limits the supported architectures we can target. In addition, support +for building the kernel with LLVM/Clang varies (see :ref:`kbuild_llvm`), +which ``bindgen`` relies on through ``libclang``. + +Below is a general summary of architectures that currently work. Level of +support corresponds to ``S`` values in the ``MAINTAINERS`` file. + +.. list-table:: + :widths: 10 10 10 + :header-rows: 1 + + * - Architecture + - Level of support + - Constraints + * - ``arm`` + - Maintained + - ``armv6`` and compatible only, ``RUST_OPT_LEVEL >= 2`` + * - ``arm64`` + - Maintained + - None + * - ``powerpc`` + - Maintained + - ``ppc64le`` only, ``RUST_OPT_LEVEL < 2`` requires ``CONFIG_THREAD_SHIFT=15`` + * - ``x86`` + - Maintained + - ``x86_64`` only diff --git a/Documentation/rust/coding.rst b/Documentation/rust/coding.rst new file mode 100644 index 0000000000000..46da5fb0974cb --- /dev/null +++ b/Documentation/rust/coding.rst @@ -0,0 +1,92 @@ +.. _rust_coding: + +Coding +====== + +This document describes how to write Rust code in the kernel. + + +Coding style +------------ + +The code is automatically formatted using the ``rustfmt`` tool. This is very +good news! + +- If you contribute from time to time to the kernel, you do not need to learn + and remember one more style guide. You will also need less patch roundtrips + to land a change. + +- If you are a reviewer or a maintainer, you will not need to spend time on + pointing out style issues anymore. + +.. note:: Conventions on comments and documentation are not checked by + ``rustfmt``. Thus we still need to take care of those: please see + :ref:`Documentation/rust/docs.rst <rust_docs>`. + +We use the tool's default settings. This means we are following the idiomatic +Rust style. For instance, we use 4 spaces for indentation rather than tabs. + +Typically, you will want to instruct your editor/IDE to format while you type, +when you save or at commit time. However, if for some reason you want +to reformat the entire kernel Rust sources at some point, you may run:: + + make rustfmt + +To check if everything is formatted (printing a diff otherwise), e.g. if you +have configured a CI for a tree as a maintainer, you may run:: + + make rustfmtcheck + +Like ``clang-format`` for the rest of the kernel, ``rustfmt`` works on +individual files, and does not require a kernel configuration. Sometimes it may +even work with broken code. + + +Extra lints +----------- + +While ``rustc`` is a very helpful compiler, some extra lints and analysis are +available via ``clippy``, a Rust linter. To enable it, pass ``CLIPPY=1`` to +the same invocation you use for compilation, e.g.:: + + make ARCH=... CROSS_COMPILE=... CC=... -j... CLIPPY=1 + +At the moment, we do not enforce a "clippy-free" compilation, so you can treat +the output the same way as the extra warning levels for C, e.g. like ``W=2``. +Still, we use the default configuration, which is relatively conservative, so +it is a good idea to read any output it may produce from time to time and fix +the pointed out issues. The list of enabled lists will be likely tweaked over +time, and extra levels may end up being introduced, e.g. ``CLIPPY=2``. + + +Abstractions vs. bindings +------------------------- + +We don't have abstractions for all the kernel internal APIs and concepts, +but we would like to expand coverage as time goes on. Unless there is +a good reason not to, always use the abstractions instead of calling +the C bindings directly. + +If you are writing some code that requires a call to an internal kernel API +or concept that isn't abstracted yet, consider providing an (ideally safe) +abstraction for everyone to use. + + +Conditional compilation +----------------------- + +Rust code has access to conditional compilation based on the kernel +configuration: + +.. code-block:: rust + + #[cfg(CONFIG_X)] // `CONFIG_X` is enabled (`y` or `m`) + #[cfg(CONFIG_X="y")] // `CONFIG_X` is enabled as a built-in (`y`) + #[cfg(CONFIG_X="m")] // `CONFIG_X` is enabled as a module (`m`) + #[cfg(not(CONFIG_X))] // `CONFIG_X` is disabled + + +Documentation +------------- + +Please see :ref:`Documentation/rust/docs.rst <rust_docs>`. diff --git a/Documentation/rust/docs.rst b/Documentation/rust/docs.rst new file mode 100644 index 0000000000000..58c5f98ccb353 --- /dev/null +++ b/Documentation/rust/docs.rst @@ -0,0 +1,109 @@ +.. _rust_docs: + +Docs +==== + +Rust kernel code is not documented like C kernel code (i.e. via kernel-doc). +Instead, we use the usual system for documenting Rust code: the ``rustdoc`` +tool, which uses Markdown (a *very* lightweight markup language). + +This document describes how to make the most out of the kernel documentation +for Rust. + + +Reading the docs +---------------- + +An advantage of using Markdown is that it attempts to make text look almost as +you would have written it in plain text. This makes the documentation quite +pleasant to read even in its source form. + +However, the generated HTML docs produced by ``rustdoc`` provide a *very* nice +experience, including integrated instant search, clickable items (types, +functions, constants, etc. -- including to all the standard Rust library ones +that we use in the kernel, e.g. ``core``), categorization, links to the source +code, etc. + +Like for the rest of the kernel documentation, pregenerated HTML docs for +the libraries (crates) inside ``rust/`` that are used by the rest of the kernel +are available at `kernel.org`_. + +// TODO: link when ready + +.. _kernel.org: http://kernel.org/ + +Otherwise, you can generate them locally. This is quite fast (same order as +compiling the code itself) and you do not need any special tools or environment. +This has the added advantage that they will be tailored to your particular +kernel configuration. To generate them, simply use the ``rustdoc`` target with +the same invocation you use for compilation, e.g.:: + + make ARCH=... CROSS_COMPILE=... CC=... -j... rustdoc + + +Writing the docs +---------------- + +If you already know Markdown, learning how to write Rust documentation will be +a breeze. If not, understanding the basics is a matter of minutes reading other +code. There are also many guides available out there, a particularly nice one +is at `GitHub`_. + +.. _GitHub: https://guides.github.com/features/mastering-markdown/#syntax + +This is how a well-documented Rust function may look like (derived from the Rust +standard library):: + + /// Returns the contained [`Some`] value, consuming the `self` value, + /// without checking that the value is not [`None`]. + /// + /// # Safety + /// + /// Calling this method on [`None`] is *[undefined behavior]*. + /// + /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html + /// + /// # Examples + /// + /// ``` + /// let x = Some("air"); + /// assert_eq!(unsafe { x.unwrap_unchecked() }, "air"); + /// ``` + pub unsafe fn unwrap_unchecked(self) -> T { + match self { + Some(val) => val, + + // SAFETY: the safety contract must be upheld by the caller. + None => unsafe { hint::unreachable_unchecked() }, + } + } + +This example showcases a few ``rustdoc`` features and some common conventions +(that we also follow in the kernel): + +* The first paragraph must be a single sentence briefly describing what + the documented item does. Further explanations must go in extra paragraphs. + +* ``unsafe`` functions must document the preconditions needed for a call to be + safe under a ``Safety`` section. + +* While not shown here, if a function may panic, the conditions under which + that happens must be described under a ``Panics`` section. + +* If providing examples of usage would help readers, they must be written in + a section called ``Examples``. + +* Rust items (functions, types, constants...) will be automatically linked + (``rustdoc`` will find out the URL for you). + +* Following the Rust standard library conventions, any ``unsafe`` block must be + preceded by a ``SAFETY`` comment describing why the code inside is sound. + + While sometimes the reason might look trivial and therefore unneeded, writing + these comments is not just a good way of documenting what has been taken into + account, but also that there are no *extra* implicit constraints. + +To learn more about how to write documentation for Rust and extra features, +please take a look at the ``rustdoc`` `book`_. + +.. _book: https://doc.rust-lang.org/rustdoc/how-to-write-documentation.html diff --git a/Documentation/rust/index.rst b/Documentation/rust/index.rst new file mode 100644 index 0000000000000..257cf2b200b8a --- /dev/null +++ b/Documentation/rust/index.rst @@ -0,0 +1,20 @@ +Rust +==== + +Documentation related to Rust within the kernel. If you are starting out, +read the :ref:`Documentation/rust/quick-start.rst <rust_quick_start>` guide. + +.. toctree:: + :maxdepth: 1 + + quick-start + coding + docs + arch-support + +.. only:: subproject and html + + Indices + ======= + + * :ref:`genindex` diff --git a/Documentation/rust/quick-start.rst b/Documentation/rust/quick-start.rst new file mode 100644 index 0000000000000..e78125d9289bd --- /dev/null +++ b/Documentation/rust/quick-start.rst @@ -0,0 +1,220 @@ +.. _rust_quick_start: + +Quick Start +=========== + +This document describes how to get started with kernel development in Rust. +If you have worked previously with Rust, this will only take a moment. + +Please note that, at the moment, a very restricted subset of architectures +is supported, see :doc:`/rust/arch-support`. + + +Requirements: Building +---------------------- + +This section explains how to fetch the tools needed for building. + +Some of these requirements might be available from your Linux distribution +under names like ``rustc``, ``rust-src``, ``rust-bindgen``, etc. However, +at the time of writing, they are likely to not be recent enough. + + +rustc +***** + +A recent *nightly* Rust toolchain (with, at least, ``rustc``) is required, +e.g. ``nightly-2021-02-20``. Our goal is to use a stable toolchain as soon +as possible, but for the moment we depend on a handful of nightly features. + +If you are using ``rustup``, run:: + + rustup default nightly-2021-02-20 + +Please avoid the very latest nightlies (>= nightly-2021-03-05) until +https://github.com/Rust-for-Linux/linux/issues/135 is resolved. + +Otherwise, fetch a standalone installer or install ``rustup`` from: + + https://www.rust-lang.org + + +Rust standard library source +**************************** + +The Rust standard library source is required because the build system will +cross-compile ``core`` and ``alloc``. + +If you are using ``rustup``, run:: + + rustup component add rust-src + +Otherwise, if you used a standalone installer, you can clone the Rust +repository into the installation folder of your nightly toolchain:: + + git clone --recurse-submodules https://github.com/rust-lang/rust $(rustc --print sysroot)/lib/rustlib/src/rust + + +libclang +******** + +``libclang`` (part of LLVM) is used by ``bindgen`` to understand the C code +in the kernel, which means you will need a recent LLVM installed; like when +you compile the kernel with ``CC=clang`` or ``LLVM=1``. + +Your Linux distribution is likely to have a suitable one available, so it is +best if you check that first. + +There are also some binaries for several systems and architectures uploaded at: + + https://releases.llvm.org/download.html + +For Debian-based distributions, you can also fetch them from: + + https://apt.llvm.org + +Otherwise, building LLVM takes quite a while, but it is not a complex process: + + https://llvm.org/docs/GettingStarted.html#getting-the-source-code-and-building-llvm + + +bindgen +******* + +The bindings to the C side of the kernel are generated at build time using +the ``bindgen`` tool. A recent version should work, e.g. ``0.56.0``. + +Install it via (this will build the tool from source):: + + cargo install --locked --version 0.56.0 bindgen + + +Requirements: Developing +------------------------ + +This section explains how to fetch the tools needed for developing. That is, +if you only want to build the kernel, you do not need them. + + +rustfmt +******* + +The ``rustfmt`` tool is used to automatically format all the Rust kernel code, +including the generated C bindings (for details, please see +:ref:`Documentation/rust/coding.rst <rust_coding>`). + +If you are using ``rustup``, its ``default`` profile already installs the tool, +so you should be good to go. If you are using another profile, you can install +the component manually:: + + rustup component add rustfmt + +The standalone installers also come with ``rustfmt``. + + +clippy +****** + +``clippy`` is a Rust linter. Installing it allows you to get extra warnings +for Rust code passing ``CLIPPY=1`` to ``make`` (for details, please see +:ref:`Documentation/rust/coding.rst <rust_coding>`). + +If you are using ``rustup``, its ``default`` profile already installs the tool, +so you should be good to go. If you are using another profile, you can install +the component manually:: + + rustup component add clippy + +The standalone installers also come with ``clippy``. + + +rustdoc +******* + +If you install the ``rustdoc`` tool, then you will be able to generate pretty +HTML documentation for Rust code, including for the libraries (crates) inside +``rust/`` that are used by the rest of the kernel (for details, please see +:ref:`Documentation/rust/docs.rst <rust_docs>`). + +If you are using ``rustup``, its ``default`` profile already installs the tool, +so you should be good to go. If you are using another profile, you can install +the component manually:: + + rustup component add rustdoc + +The standalone installers also come with ``rustdoc``. + + +rust-analyzer +************* + +The `rust-analyzer <https://rust-analyzer.github.io/>`_ language server can +be used with many editors to enable syntax highlighting, completion, go to +definition, and other features. + +``rust-analyzer`` will need to be +`configured <https://rust-analyzer.github.io/manual.html#non-cargo-based-projects>`_ +to work with the kernel by adding a ``rust-project.json`` file in the root folder. +The example ``Documentation/rust/rust-project.json`` can +be used after updating ``sysroot_src`` and including the relevant modules. +The path to ``sysroot_src`` is given by:: + + $(rustc --print sysroot)/lib/rustlib/src/rust/library + + +Configuration +------------- + +``Rust support`` (``CONFIG_RUST``) needs to be enabled in the ``General setup`` +menu. The option is only shown if the build system can locate ``rustc``. +In turn, this will make visible the rest of options that depend on Rust. + +Afterwards, go to:: + + Kernel hacking + -> Sample kernel code + -> Rust samples + +And enable some sample modules either as built-in or as loadable. + + +Building +-------- + +Building a kernel with Clang or a complete LLVM toolchain is the best supported +setup at the moment. That is:: + + make ARCH=... CROSS_COMPILE=... CC=clang -j... + +or:: + + make ARCH=... CROSS_COMPILE=... LLVM=1 -j... + +Using GCC also works for some configurations, but it is *very* experimental at +the moment. + + +Hacking +------- + +If you want to dive deeper, take a look at the source code of the samples +at ``samples/rust/``, the Rust support code under ``rust/`` and +the ``Rust hacking`` menu under ``Kernel hacking``. + +If you use GDB/Binutils and Rust symbols aren't getting demangled, the reason +is your toolchain doesn't support Rust's new v0 mangling scheme yet. There are +a few ways out: + + - If you don't mind building your own tools, we provide the following fork + with the support cherry-picked from GCC on top of very recent releases: + + https://github.com/Rust-for-Linux/binutils-gdb/releases/tag/gdb-10.1-release-rust + https://github.com/Rust-for-Linux/binutils-gdb/releases/tag/binutils-2_35_1-rust + + - If you only need GDB and can enable ``CONFIG_DEBUG_INFO``, do so: + some versions of GDB (e.g. vanilla GDB 10.1) are able to use + the pre-demangled names embedded in the debug info. + + - If you don't need loadable module support, you may compile without + the ``-Zsymbol-mangling-version=v0`` flag. However, we don't maintain + support for that, so avoid it unless you are in a hurry. diff --git a/Documentation/rust/rust-project.json b/Documentation/rust/rust-project.json new file mode 100644 index 0000000000000..e0f64b8dec1e3 --- /dev/null +++ b/Documentation/rust/rust-project.json @@ -0,0 +1,35 @@ +{ + "sysroot_src": <path-to-sysroot-source>, + "crates": [ + { + "root_module": "rust/module.rs", + "edition": "2018", + "deps": [] + }, + { + "root_module": "rust/kernel/lib.rs", + "edition": "2018", + "deps": [ + { + "crate": 0, + "name": "module" + } + ] + }, + { + "root_module": <path-to-module>, + "edition": "2018", + "deps": [ + { + "crate": 0, + "name": "module" + }, + { + "crate": 1, + "name": "kernel" + } + ] + }, + <...entries for other modules...> + ] +}
\ No newline at end of file diff --git a/MAINTAINERS b/MAINTAINERS index 0c396f360c164..285e7c02e9e5a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -16061,6 +16061,20 @@ L: linux-rdma@vger.kernel.org S: Maintained F: drivers/infiniband/ulp/rtrs/ +RUST +M: Miguel Ojeda <ojeda@kernel.org> +M: Alex Gaynor <alex.gaynor@gmail.com> +M: Wedson Almeida Filho <wedsonaf@google.com> +L: rust-for-linux@vger.kernel.org +S: Maintained +W: https://github.com/Rust-for-Linux/linux +B: https://github.com/Rust-for-Linux/linux/issues +T: git https://github.com/Rust-for-Linux/linux.git rust-next +F: rust/ +F: samples/rust/ +F: Documentation/rust/ +K: \b(?i:rust)\b + RXRPC SOCKETS (AF_RXRPC) M: David Howells <dhowells@redhat.com> M: Marc Dionne <marc.dionne@auristor.com> diff --git a/Makefile b/Makefile index 2d3725f6a0393..cc416528c21d2 100644 --- a/Makefile +++ b/Makefile @@ -120,6 +120,13 @@ endif export KBUILD_CHECKSRC +# Enable "clippy" (a linter) as part of the Rust compilation. +# +# Use 'make CLIPPY=1' to enable it. +ifeq ("$(origin CLIPPY)", "command line") + KBUILD_CLIPPY := $(CLIPPY) +endif + # Use make M=dir or set the environment variable KBUILD_EXTMOD to specify the # directory of external module to build. Setting M= takes precedence. ifeq ("$(origin M)", "command line") @@ -263,7 +270,7 @@ no-dot-config-targets := $(clean-targets) \ cscope gtags TAGS tags help% %docs check% coccicheck \ $(version_h) headers headers_% archheaders archscripts \ %asm-generic kernelversion %src-pkg dt_binding_check \ - outputmakefile + outputmakefile rustfmt rustfmtcheck # Installation targets should not require compiler. Unfortunately, vdso_install # is an exception where build artifacts may be updated. This must be fixed. no-compiler-targets := $(no-dot-config-targets) install dtbs_install \ @@ -452,6 +459,10 @@ OBJDUMP = $(CROSS_COMPILE)objdump READELF = $(CROSS_COMPILE)readelf STRIP = $(CROSS_COMPILE)strip endif +RUSTC = rustc +RUSTFMT = rustfmt +CLIPPY_DRIVER = clippy-driver +BINDGEN = bindgen PAHOLE = pahole RESOLVE_BTFIDS = $(objtree)/tools/bpf/resolve_btfids/resolve_btfids LEX = flex @@ -475,9 +486,11 @@ CHECKFLAGS := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \ -Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF) NOSTDINC_FLAGS := CFLAGS_MODULE = +RUSTCFLAGS_MODULE = AFLAGS_MODULE = LDFLAGS_MODULE = CFLAGS_KERNEL = +RUSTCFLAGS_KERNEL = AFLAGS_KERNEL = LDFLAGS_vmlinux = @@ -506,15 +519,30 @@ KBUILD_CFLAGS := -Wall -Wundef -Werror=strict-prototypes -Wno-trigraphs \ -Werror=return-type -Wno-format-security \ -std=gnu89 KBUILD_CPPFLAGS := -D__KERNEL__ +KBUILD_RUSTCFLAGS := --emit=dep-info,obj,metadata --edition=2018 \ + -Cpanic=abort -Cembed-bitcode=n -Clto=n -Crpath=n \ + -Cforce-unwind-tables=n -Ccodegen-units=1 \ + -Zbinary_dep_depinfo=y -Zsymbol-mangling-version=v0 KBUILD_AFLAGS_KERNEL := KBUILD_CFLAGS_KERNEL := +KBUILD_RUSTCFLAGS_KERNEL := KBUILD_AFLAGS_MODULE := -DMODULE KBUILD_CFLAGS_MODULE := -DMODULE +KBUILD_RUSTCFLAGS_MODULE := --cfg MODULE KBUILD_LDFLAGS_MODULE := KBUILD_LDFLAGS := CLANG_FLAGS := -export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC +ifeq ($(KBUILD_CLIPPY),1) + RUSTC_OR_CLIPPY_QUIET := CLIPPY + RUSTC_OR_CLIPPY = $(CLIPPY_DRIVER) +else + RUSTC_OR_CLIPPY_QUIET := RUSTC + RUSTC_OR_CLIPPY = $(RUSTC) +endif +export RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY + +export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC RUSTC BINDGEN export CPP AR NM STRIP OBJCOPY OBJDUMP READELF PAHOLE RESOLVE_BTFIDS LEX YACC AWK INSTALLKERNEL export PERL PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ ZSTD @@ -522,9 +550,10 @@ export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE +export KBUILD_RUSTCFLAGS RUSTCFLAGS_KERNEL RUSTCFLAGS_MODULE export KBUILD_AFLAGS AFLAGS_KERNEL AFLAGS_MODULE -export KBUILD_AFLAGS_MODULE KBUILD_CFLAGS_MODULE KBUILD_LDFLAGS_MODULE -export KBUILD_AFLAGS_KERNEL KBUILD_CFLAGS_KERNEL +export KBUILD_AFLAGS_MODULE KBUILD_CFLAGS_MODULE KBUILD_RUSTCFLAGS_MODULE KBUILD_LDFLAGS_MODULE +export KBUILD_AFLAGS_KERNEL KBUILD_CFLAGS_KERNEL KBUILD_RUSTCFLAGS_KERNEL # Files to ignore in find ... statements @@ -574,24 +603,29 @@ outputmakefile: { echo "# this is build directory, ignore it"; echo "*"; } > .gitignore endif -# The expansion should be delayed until arch/$(SRCARCH)/Makefile is included. -# Some architectures define CROSS_COMPILE in arch/$(SRCARCH)/Makefile. -# CC_VERSION_TEXT is referenced from Kconfig (so it needs export), -# and from include/config/auto.conf.cmd to detect the compiler upgrade. -CC_VERSION_TEXT = $(subst $(pound),,$(shell $(CC) --version 2>/dev/null | head -n 1)) +TENTATIVE_CLANG_FLAGS := -Werror=unknown-warning-option -ifneq ($(findstring clang,$(CC_VERSION_TEXT)),) ifneq ($(CROSS_COMPILE),) -CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) +TENTATIVE_CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%)) endif ifeq ($(LLVM_IAS),1) -CLANG_FLAGS += -integrated-as +TENTATIVE_CLANG_FLAGS += -integrated-as else -CLANG_FLAGS += -no-integrated-as +TENTATIVE_CLANG_FLAGS += -no-integrated-as GCC_TOOLCHAIN_DIR := $(dir $(shell which $(CROSS_COMPILE)elfedit)) -CLANG_FLAGS += --prefix=$(GCC_TOOLCHAIN_DIR)$(notdir $(CROSS_COMPILE)) +TENTATIVE_CLANG_FLAGS += --prefix=$(GCC_TOOLCHAIN_DIR)$(notdir $(CROSS_COMPILE)) endif -CLANG_FLAGS += -Werror=unknown-warning-option + +export TENTATIVE_CLANG_FLAGS + +# The expansion should be delayed until arch/$(SRCARCH)/Makefile is included. +# Some architectures define CROSS_COMPILE in arch/$(SRCARCH)/Makefile. +# CC_VERSION_TEXT is referenced from Kconfig (so it needs export), +# and from include/config/auto.conf.cmd to detect the compiler upgrade. +CC_VERSION_TEXT = $(shell $(CC) --version 2>/dev/null | head -n 1 | sed 's/\#//g') + +ifneq ($(findstring clang,$(CC_VERSION_TEXT)),) +CLANG_FLAGS += $(TENTATIVE_CLANG_FLAGS) KBUILD_CFLAGS += $(CLANG_FLAGS) KBUILD_AFLAGS += $(CLANG_FLAGS) export CLANG_FLAGS @@ -726,7 +760,7 @@ $(KCONFIG_CONFIG): quiet_cmd_syncconfig = SYNC $@ cmd_syncconfig = $(MAKE) -f $(srctree)/Makefile syncconfig -%/config/auto.conf %/config/auto.conf.cmd %/generated/autoconf.h: $(KCONFIG_CONFIG) +%/config/auto.conf %/config/auto.conf.cmd %/generated/autoconf.h %/generated/rustc_cfg: $(KCONFIG_CONFIG) +$(call cmd,syncconfig) else # !may-sync-config # External modules and some install targets need include/generated/autoconf.h @@ -752,12 +786,43 @@ KBUILD_CFLAGS += $(call cc-disable-warning, format-truncation) KBUILD_CFLAGS += $(call cc-disable-warning, format-overflow) KBUILD_CFLAGS += $(call cc-disable-warning, address-of-packed-member) +ifdef CONFIG_RUST_DEBUG_ASSERTIONS +KBUILD_RUSTCFLAGS += -Cdebug-assertions=y +else +KBUILD_RUSTCFLAGS += -Cdebug-assertions=n +endif + +ifdef CONFIG_RUST_OVERFLOW_CHECKS +KBUILD_RUSTCFLAGS += -Coverflow-checks=y +else +KBUILD_RUSTCFLAGS += -Coverflow-checks=n +endif + ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE KBUILD_CFLAGS += -O2 +KBUILD_RUSTCFLAGS_OPT_LEVEL_MAP := 2 else ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE_O3 KBUILD_CFLAGS += -O3 +KBUILD_RUSTCFLAGS_OPT_LEVEL_MAP := 3 else ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE KBUILD_CFLAGS += -Os +KBUILD_RUSTCFLAGS_OPT_LEVEL_MAP := z +endif + +ifdef CONFIG_RUST_OPT_LEVEL_SIMILAR_AS_CHOSEN_FOR_C +KBUILD_RUSTCFLAGS += -Copt-level=$(KBUILD_RUSTCFLAGS_OPT_LEVEL_MAP) +else ifdef CONFIG_RUST_OPT_LEVEL_0 +KBUILD_RUSTCFLAGS += -Copt-level=0 +else ifdef CONFIG_RUST_OPT_LEVEL_1 +KBUILD_RUSTCFLAGS += -Copt-level=1 +else ifdef CONFIG_RUST_OPT_LEVEL_2 +KBUILD_RUSTCFLAGS += -Copt-level=2 +else ifdef CONFIG_RUST_OPT_LEVEL_3 +KBUILD_RUSTCFLAGS += -Copt-level=3 +else ifdef CONFIG_RUST_OPT_LEVEL_S +KBUILD_RUSTCFLAGS += -Copt-level=s +else ifdef CONFIG_RUST_OPT_LEVEL_Z +KBUILD_RUSTCFLAGS += -Copt-level=z endif # Tell gcc to never replace conditional load with a non-conditional one @@ -807,6 +872,7 @@ KBUILD_CFLAGS += $(call cc-disable-warning, unused-but-set-variable) KBUILD_CFLAGS += $(call cc-disable-warning, unused-const-variable) ifdef CONFIG_FRAME_POINTER KBUILD_CFLAGS += -fno-omit-frame-pointer -fno-optimize-sibling-calls +KBUILD_RUSTCFLAGS += -Cforce-frame-pointers=y else # Some targets (ARM with Thumb2, for example), can't be built with frame # pointers. For those, we don't have FUNCTION_TRACER automatically @@ -844,6 +910,8 @@ ifdef CONFIG_CC_IS_GCC DEBUG_CFLAGS += $(call cc-ifversion, -lt, 0500, $(call cc-option, -fno-var-tracking-assignments)) endif +DEBUG_RUSTCFLAGS := + ifdef CONFIG_DEBUG_INFO ifdef CONFIG_DEBUG_INFO_SPLIT @@ -854,6 +922,11 @@ endif ifneq ($(LLVM_IAS),1) KBUILD_AFLAGS += -Wa,-gdwarf-2 +ifdef CONFIG_DEBUG_INFO_REDUCED +DEBUG_RUSTCFLAGS += -Cdebuginfo=1 +else +DEBUG_RUSTCFLAGS += -Cdebuginfo=2 +endif endif ifndef CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT @@ -878,6 +951,9 @@ endif # CONFIG_DEBUG_INFO KBUILD_CFLAGS += $(DEBUG_CFLAGS) export DEBUG_CFLAGS +KBUILD_RUSTCFLAGS += $(DEBUG_RUSTCFLAGS) +export DEBUG_RUSTCFLAGS + ifdef CONFIG_FUNCTION_TRACER ifdef CONFIG_FTRACE_MCOUNT_USE_CC CC_FLAGS_FTRACE += -mrecord-mcount @@ -1033,10 +1109,11 @@ include $(addprefix $(srctree)/, $(include-y)) # Do not add $(call cc-option,...) below this line. When you build the kernel # from the clean source tree, the GCC plugins do not exist at this point. -# Add user supplied CPPFLAGS, AFLAGS and CFLAGS as the last assignments +# Add user supplied CPPFLAGS, AFLAGS, CFLAGS and RUSTCFLAGS as the last assignments KBUILD_CPPFLAGS += $(KCPPFLAGS) KBUILD_AFLAGS += $(KAFLAGS) KBUILD_CFLAGS += $(KCFLAGS) +KBUILD_RUSTCFLAGS += $(KRUSTCFLAGS) KBUILD_LDFLAGS_MODULE += --build-id=sha1 LDFLAGS_vmlinux += --build-id=sha1 @@ -1105,6 +1182,10 @@ export MODULES_NSDEPS := $(extmod_prefix)modules.nsdeps ifeq ($(KBUILD_EXTMOD),) core-y += kernel/ certs/ mm/ fs/ ipc/ security/ crypto/ block/ +ifdef CONFIG_RUST +core-y += rust/ +endif + vmlinux-dirs := $(patsubst %/,%,$(filter %/, \ $(core-y) $(core-m) $(drivers-y) $(drivers-m) \ $(libs-y) $(libs-m))) @@ -1205,6 +1286,9 @@ archprepare: outputmakefile archheaders archscripts scripts include/config/kerne prepare0: archprepare $(Q)$(MAKE) $(build)=scripts/mod $(Q)$(MAKE) $(build)=. +ifdef CONFIG_RUST + $(Q)$(MAKE) $(build)=rust +endif # All the preparing.. prepare: prepare0 @@ -1618,6 +1702,13 @@ help: @echo ' kselftest-merge - Merge all the config dependencies of' @echo ' kselftest to existing .config.' @echo '' + @echo 'Rust targets:' + @echo ' rustfmt - Reformat all the Rust code in the kernel' + @echo ' rustfmtcheck - Checks if all the Rust code in the kernel' + @echo ' is formatted, printing a diff otherwise.' + @echo ' rustdoc - Generate Rust documentation' + @echo ' (requires kernel .config)' + @echo '' @$(if $(dtstree), \ echo 'Devicetree:'; \ echo '* dtbs - Build device tree blobs for enabled boards'; \ @@ -1689,6 +1780,27 @@ PHONY += $(DOC_TARGETS) $(DOC_TARGETS): $(Q)$(MAKE) $(build)=Documentation $@ + +# Rust targets +# --------------------------------------------------------------------------- + +# Documentation target +# +# Using the singular to avoid running afoul of `no-dot-config-targets`. +PHONY += rustdoc +rustdoc: prepare0 + $(Q)$(MAKE) $(build)=rust $@ + +# Formatting targets +PHONY += rustfmt rustfmtcheck + +rustfmt: + find -name '*.rs' | xargs $(RUSTFMT) + +rustfmtcheck: + find -name '*.rs' | xargs $(RUSTFMT) --check + + # Misc # --------------------------------------------------------------------------- @@ -1846,6 +1958,7 @@ clean: $(clean-dirs) $(call cmd,rmfiles) @find $(if $(KBUILD_EXTMOD), $(KBUILD_EXTMOD), .) $(RCS_FIND_IGNORE) \ \( -name '*.[aios]' -o -name '*.ko' -o -name '.*.cmd' \ + -o -name '*.rmeta' \ -o -name '*.ko.*' \ -o -name '*.dtb' -o -name '*.dtbo' -o -name '*.dtb.S' -o -name '*.dt.yaml' \ -o -name '*.dwo' -o -name '*.lst' \ diff --git a/arch/arm/rust/target.json b/arch/arm/rust/target.json new file mode 100644 index 0000000000000..37710eb727b1c --- /dev/null +++ b/arch/arm/rust/target.json @@ -0,0 +1,28 @@ +{ + "arch": "arm", + "crt-static-respected": true, + "data-layout": "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64", + "dynamic-linking": true, + "env": "gnu", + "executables": true, + "features": "+strict-align,+v6", + "function-sections": false, + "has-elf-tls": true, + "has-rpath": true, + "is-builtin": true, + "linker-is-gnu": true, + "llvm-target": "arm-unknown-linux-gnueabi", + "max-atomic-width": 64, + "os": "linux", + "position-independent-executables": true, + "pre-link-args": { + "gcc": [ + "-Wl,--as-needed", + "-Wl,-z,noexecstack" + ] + }, + "relocation-model": "static", + "target-family": "unix", + "target-mcount": "\u0001__gnu_mcount_nc", + "target-pointer-width": "32" +} diff --git a/arch/arm64/rust/target.json b/arch/arm64/rust/target.json new file mode 100644 index 0000000000000..44953e2725c4a --- /dev/null +++ b/arch/arm64/rust/target.json @@ -0,0 +1,40 @@ +{ + "arch": "aarch64", + "data-layout": "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128", + "disable-redzone": true, + "eliminate-frame-pointer": false, + "emit-debug-gdb-scripts": false, + "env": "gnu", + "features": "+strict-align,+neon,+fp-armv8", + "function-sections": false, + "is-builtin": true, + "linker-flavor": "gcc", + "linker-is-gnu": true, + "llvm-target": "aarch64-unknown-none", + "max-atomic-width": 128, + "needs-plt": true, + "os": "none", + "panic-strategy": "abort", + "position-independent-executables": true, + "pre-link-args": { + "gcc": [ + "-Wl,--as-needed", + "-Wl,-z,noexecstack", + "-m64" + ] + }, + "relocation-model": "static", + "relro-level": "full", + "stack-probes": { + "kind": "inline-or-call", + "min-llvm-version-for-inline": [ + 11, + 0, + 1 + ] + }, + "target-c-int-width": "32", + "target-endian": "little", + "target-pointer-width": "64", + "vendor": "" +} diff --git a/arch/powerpc/rust/target.json b/arch/powerpc/rust/target.json new file mode 100644 index 0000000000000..1e53f83080927 --- /dev/null +++ b/arch/powerpc/rust/target.json @@ -0,0 +1,30 @@ +{ + "arch": "powerpc64", + "code-mode": "kernel", + "cpu": "ppc64le", + "data-layout": "e-m:e-i64:64-n32:64", + "env": "gnu", + "features": "-altivec,-vsx,-hard-float", + "function-sections": false, + "is-builtin": true, + "linker-flavor": "gcc", + "linker-is-gnu": true, + "llvm-target": "powerpc64le-elf", + "max-atomic-width": 64, + "os": "none", + "panic-strategy": "abort", + "position-independent-executables": true, + "pre-link-args": { + "gcc": [ + "-Wl,--as-needed", + "-Wl,-z,noexecstack", + "-m64" + ] + }, + "relocation-model": "static", + "relro-level": "full", + "target-family": "unix", + "target-mcount": "_mcount", + "target-endian": "little", + "target-pointer-width": "64" +} diff --git a/arch/x86/rust/target.json b/arch/x86/rust/target.json new file mode 100644 index 0000000000000..6e1759cd45bf5 --- /dev/null +++ b/arch/x86/rust/target.json @@ -0,0 +1,42 @@ +{ + "arch": "x86_64", + "code-model": "kernel", + "cpu": "x86-64", + "data-layout": "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128", + "disable-redzone": true, + "eliminate-frame-pointer": false, + "emit-debug-gdb-scripts": false, + "env": "gnu", + "features": "-mmx,-sse,-sse2,-sse3,-ssse3,-sse4.1,-sse4.2,-3dnow,-3dnowa,-avx,-avx2,+soft-float", + "function-sections": false, + "is-builtin": true, + "linker-flavor": "gcc", + "linker-is-gnu": true, + "llvm-target": "x86_64-elf", + "max-atomic-width": 64, + "needs-plt": true, + "os": "none", + "panic-strategy": "abort", + "position-independent-executables": true, + "pre-link-args": { + "gcc": [ + "-Wl,--as-needed", + "-Wl,-z,noexecstack", + "-m64" + ] + }, + "relocation-model": "static", + "relro-level": "full", + "stack-probes": { + "kind": "inline-or-call", + "min-llvm-version-for-inline": [ + 11, + 0, + 1 + ] + }, + "target-c-int-width": "32", + "target-endian": "little", + "target-pointer-width": "64", + "vendor": "unknown" +} diff --git a/drivers/android/Kconfig b/drivers/android/Kconfig index 53b22e26266c3..7aef7f38f9887 100644 --- a/drivers/android/Kconfig +++ b/drivers/android/Kconfig @@ -20,6 +20,13 @@ config ANDROID_BINDER_IPC Android process, using Binder to identify, invoke and pass arguments between said processes. +config ANDROID_BINDER_IPC_RUST + bool "Android Binder IPC Driver in Rust" + depends on MMU && RUST + default n + help + Implementation of the Binder IPC in Rust. + config ANDROID_BINDERFS bool "Android Binderfs filesystem" depends on ANDROID_BINDER_IPC diff --git a/drivers/android/Makefile b/drivers/android/Makefile index c9d3d0c99c257..c428f2ce2f05e 100644 --- a/drivers/android/Makefile +++ b/drivers/android/Makefile @@ -4,3 +4,5 @@ ccflags-y += -I$(src) # needed for trace events obj-$(CONFIG_ANDROID_BINDERFS) += binderfs.o obj-$(CONFIG_ANDROID_BINDER_IPC) += binder.o binder_alloc.o obj-$(CONFIG_ANDROID_BINDER_IPC_SELFTEST) += binder_alloc_selftest.o + +obj-$(CONFIG_ANDROID_BINDER_IPC_RUST) += rust_binder.o diff --git a/drivers/android/allocation.rs b/drivers/android/allocation.rs new file mode 100644 index 0000000000000..80b96628c778c --- /dev/null +++ b/drivers/android/allocation.rs @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: GPL-2.0 + +use alloc::sync::Arc; +use core::mem::{size_of, MaybeUninit}; +use kernel::{bindings, pages::Pages, prelude::*, user_ptr::UserSlicePtrReader, Error}; + +use crate::{ + node::NodeRef, + process::{AllocationInfo, Process}, + thread::{BinderError, BinderResult}, +}; + +pub(crate) struct Allocation<'a> { + pub(crate) offset: usize, + size: usize, + pub(crate) ptr: usize, + pages: Arc<[Pages<0>]>, + pub(crate) process: &'a Process, + allocation_info: Option<AllocationInfo>, + free_on_drop: bool, +} + +impl<'a> Allocation<'a> { + pub(crate) fn new( + process: &'a Process, + offset: usize, + size: usize, + ptr: usize, + pages: Arc<[Pages<0>]>, + ) -> Self { + Self { + process, + offset, + size, + ptr, + pages, + allocation_info: None, + free_on_drop: true, + } + } + + fn iterate<T>(&self, mut offset: usize, mut size: usize, mut cb: T) -> KernelResult + where + T: FnMut(&Pages<0>, usize, usize) -> KernelResult, + { + // Check that the request is within the buffer. + if offset.checked_add(size).ok_or(Error::EINVAL)? > self.size { + return Err(Error::EINVAL); + } + offset += self.offset; + let mut page_index = offset >> bindings::PAGE_SHIFT; + offset &= (1 << bindings::PAGE_SHIFT) - 1; + while size > 0 { + let available = core::cmp::min(size, (1 << bindings::PAGE_SHIFT) as usize - offset); + cb(&self.pages[page_index], offset, available)?; + size -= available; + page_index += 1; + offset = 0; + } + Ok(()) + } + + pub(crate) fn copy_into( + &self, + reader: &mut UserSlicePtrReader, + offset: usize, + size: usize, + ) -> KernelResult { + self.iterate(offset, size, |page, offset, to_copy| { + page.copy_into_page(reader, offset, to_copy) + }) + } + + pub(crate) fn read<T>(&self, offset: usize) -> KernelResult<T> { + let mut out = MaybeUninit::<T>::uninit(); + let mut out_offset = 0; + self.iterate(offset, size_of::<T>(), |page, offset, to_copy| { + // SAFETY: Data buffer is allocated on the stack. + unsafe { + page.read( + (out.as_mut_ptr() as *mut u8).add(out_offset), + offset, + to_copy, + ) + }?; + out_offset += to_copy; + Ok(()) + })?; + // SAFETY: We just initialised the data. + Ok(unsafe { out.assume_init() }) + } + + pub(crate) fn write<T>(&self, offset: usize, obj: &T) -> KernelResult { + let mut obj_offset = 0; + self.iterate(offset, size_of::<T>(), |page, offset, to_copy| { + // SAFETY: The sum of `offset` and `to_copy` is bounded by the size of T. + let obj_ptr = unsafe { (obj as *const T as *const u8).add(obj_offset) }; + // SAFETY: We have a reference to the object, so the pointer is valid. + unsafe { page.write(obj_ptr, offset, to_copy) }?; + obj_offset += to_copy; + Ok(()) + }) + } + + pub(crate) fn keep_alive(mut self) { + self.process + .buffer_make_freeable(self.offset, self.allocation_info.take()); + self.free_on_drop = false; + } + + pub(crate) fn set_info(&mut self, info: AllocationInfo) { + self.allocation_info = Some(info); + } + + fn cleanup_object(&self, index_offset: usize, view: &AllocationView) -> KernelResult { + let offset = self.read(index_offset)?; + let header = view.read::<bindings::binder_object_header>(offset)?; + // TODO: Handle other types. + match header.type_ { + bindings::BINDER_TYPE_WEAK_BINDER | bindings::BINDER_TYPE_BINDER => { + let obj = view.read::<bindings::flat_binder_object>(offset)?; + let strong = header.type_ == bindings::BINDER_TYPE_BINDER; + // SAFETY: The type is `BINDER_TYPE_{WEAK_}BINDER`, so the `binder` field is + // populated. + let ptr = unsafe { obj.__bindgen_anon_1.binder } as usize; + let cookie = obj.cookie as usize; + self.process.update_node(ptr, cookie, strong, false); + Ok(()) + } + bindings::BINDER_TYPE_WEAK_HANDLE | bindings::BINDER_TYPE_HANDLE => { + let obj = view.read::<bindings::flat_binder_object>(offset)?; + let strong = header.type_ == bindings::BINDER_TYPE_HANDLE; + // SAFETY: The type is `BINDER_TYPE_{WEAK_}HANDLE`, so the `handle` field is + // populated. + let handle = unsafe { obj.__bindgen_anon_1.handle } as _; + self.process.update_ref(handle, false, strong) + } + _ => Ok(()), + } + } +} + +impl Drop for Allocation<'_> { + fn drop(&mut self) { + if !self.free_on_drop { + return; + } + + if let Some(info) = &self.allocation_info { + let view = AllocationView::new(self, info.offsets.start); + for i in info.offsets.clone().step_by(size_of::<usize>()) { + if self.cleanup_object(i, &view).is_err() { + pr_warn!("Error cleaning up object at offset {}\n", i) + } + } + } + + self.process.buffer_raw_free(self.ptr); + } +} + +pub(crate) struct AllocationView<'a> { + alloc: &'a Allocation<'a>, + limit: usize, +} + +impl<'a> AllocationView<'a> { + pub(crate) fn new(alloc: &'a Allocation, limit: usize) -> Self { + AllocationView { alloc, limit } + } + + pub fn read<T>(&self, offset: usize) -> KernelResult<T> { + if offset.checked_add(size_of::<T>()).ok_or(Error::EINVAL)? > self.limit { + return Err(Error::EINVAL); + } + self.alloc.read(offset) + } + + pub fn write<T>(&self, offset: usize, obj: &T) -> KernelResult { + if offset.checked_add(size_of::<T>()).ok_or(Error::EINVAL)? > self.limit { + return Err(Error::EINVAL); + } + self.alloc.write(offset, obj) + } + + pub(crate) fn transfer_binder_object<T>( + &self, + offset: usize, + strong: bool, + get_node: T, + ) -> BinderResult + where + T: FnOnce(&bindings::flat_binder_object) -> BinderResult<NodeRef>, + { + // TODO: Do we want this function to take a &mut self? + let obj = self.read::<bindings::flat_binder_object>(offset)?; + let node_ref = get_node(&obj)?; + + if core::ptr::eq(&*node_ref.node.owner, self.alloc.process) { + // The receiving process is the owner of the node, so send it a binder object (instead + // of a handle). + let (ptr, cookie) = node_ref.node.get_id(); + let newobj = bindings::flat_binder_object { + hdr: bindings::binder_object_header { + type_: if strong { + bindings::BINDER_TYPE_BINDER + } else { + bindings::BINDER_TYPE_WEAK_BINDER + }, + }, + flags: obj.flags, + __bindgen_anon_1: bindings::flat_binder_object__bindgen_ty_1 { binder: ptr as _ }, + cookie: cookie as _, + }; + self.write(offset, &newobj)?; + + // Increment the user ref count on the node. It will be decremented as part of the + // destruction of the buffer, when we see a binder or weak-binder object. + node_ref.node.update_refcount(true, strong); + } else { + // The receiving process is different from the owner, so we need to insert a handle to + // the binder object. + let handle = self + .alloc + .process + .insert_or_update_handle(node_ref, false)?; + + let newobj = bindings::flat_binder_object { + hdr: bindings::binder_object_header { + type_: if strong { + bindings::BINDER_TYPE_HANDLE + } else { + bindings::BINDER_TYPE_WEAK_HANDLE + }, + }, + flags: obj.flags, + // TODO: To avoid padding, we write to `binder` instead of `handle` here. We need a + // better solution though. + __bindgen_anon_1: bindings::flat_binder_object__bindgen_ty_1 { + binder: handle as _, + }, + ..bindings::flat_binder_object::default() + }; + if self.write(offset, &newobj).is_err() { + // Decrement ref count on the handle we just created. + let _ = self.alloc.process.update_ref(handle, false, strong); + return Err(BinderError::new_failed()); + } + } + Ok(()) + } +} diff --git a/drivers/android/context.rs b/drivers/android/context.rs new file mode 100644 index 0000000000000..1ab2b26c8fd3a --- /dev/null +++ b/drivers/android/context.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: GPL-2.0 + +extern crate alloc; + +use alloc::sync::Arc; +use core::pin::Pin; +use kernel::{bindings, prelude::*, sync::Mutex, Error}; + +use crate::{ + node::NodeRef, + thread::{BinderError, BinderResult}, +}; + +struct Manager { + node: Option<NodeRef>, + uid: Option<bindings::kuid_t>, +} + +pub(crate) struct Context { + manager: Mutex<Manager>, +} + +unsafe impl Send for Context {} +unsafe impl Sync for Context {} + +impl Context { + pub(crate) fn new() -> KernelResult<Pin<Arc<Self>>> { + let mut ctx_ref = Arc::try_new(Self { + // SAFETY: Init is called below. + manager: unsafe { + Mutex::new(Manager { + node: None, + uid: None, + }) + }, + })?; + let ctx = Arc::get_mut(&mut ctx_ref).unwrap(); + + // SAFETY: `manager` is also pinned when `ctx` is. + let manager = unsafe { Pin::new_unchecked(&ctx.manager) }; + kernel::mutex_init!(manager, "Context::manager"); + + // SAFETY: `ctx_ref` is pinned behind the `Arc` reference. + Ok(unsafe { Pin::new_unchecked(ctx_ref) }) + } + + pub(crate) fn set_manager_node(&self, node_ref: NodeRef) -> KernelResult { + let mut manager = self.manager.lock(); + if manager.node.is_some() { + return Err(Error::EBUSY); + } + // TODO: Call security_binder_set_context_mgr. + + // TODO: Get the actual caller id. + let caller_uid = bindings::kuid_t::default(); + if let Some(ref uid) = manager.uid { + if uid.val != caller_uid.val { + return Err(Error::EPERM); + } + } + + manager.node = Some(node_ref); + manager.uid = Some(caller_uid); + Ok(()) + } + + pub(crate) fn unset_manager_node(&self) { + let node_ref = self.manager.lock().node.take(); + drop(node_ref); + } + + pub(crate) fn get_manager_node(&self, strong: bool) -> BinderResult<NodeRef> { + self.manager + .lock() + .node + .as_ref() + .ok_or_else(BinderError::new_dead)? + .clone(strong) + } +} diff --git a/drivers/android/defs.rs b/drivers/android/defs.rs new file mode 100644 index 0000000000000..619bb1a234200 --- /dev/null +++ b/drivers/android/defs.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 + +use core::ops::{Deref, DerefMut}; +use kernel::{ + bindings, + bindings::*, + user_ptr::{ReadableFromBytes, WritableToBytes}, +}; + +macro_rules! pub_no_prefix { + ($prefix:ident, $($newname:ident),+) => { + $(pub const $newname: u32 = concat_idents!($prefix, $newname);)+ + }; +} + +pub_no_prefix!( + binder_driver_return_protocol_, + BR_OK, + BR_ERROR, + BR_TRANSACTION, + BR_REPLY, + BR_DEAD_REPLY, + BR_TRANSACTION_COMPLETE, + BR_INCREFS, + BR_ACQUIRE, + BR_RELEASE, + BR_DECREFS, + BR_NOOP, + BR_SPAWN_LOOPER, + BR_DEAD_BINDER, + BR_CLEAR_DEATH_NOTIFICATION_DONE, + BR_FAILED_REPLY +); + +pub_no_prefix!( + binder_driver_command_protocol_, + BC_TRANSACTION, + BC_REPLY, + BC_FREE_BUFFER, + BC_INCREFS, + BC_ACQUIRE, + BC_RELEASE, + BC_DECREFS, + BC_INCREFS_DONE, + BC_ACQUIRE_DONE, + BC_REGISTER_LOOPER, + BC_ENTER_LOOPER, + BC_EXIT_LOOPER, + BC_REQUEST_DEATH_NOTIFICATION, + BC_CLEAR_DEATH_NOTIFICATION, + BC_DEAD_BINDER_DONE +); + +macro_rules! decl_wrapper { + ($newname:ident, $wrapped:ty) => { + #[derive(Copy, Clone, Default)] + pub(crate) struct $newname($wrapped); + + // TODO: This must be justified by inspecting the type, so should live outside the macro or + // the macro should be somehow marked unsafe. + unsafe impl ReadableFromBytes for $newname {} + unsafe impl WritableToBytes for $newname {} + + impl Deref for $newname { + type Target = $wrapped; + fn deref(&self) -> &Self::Target { + &self.0 + } + } + + impl DerefMut for $newname { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } + } + }; +} + +decl_wrapper!(BinderNodeDebugInfo, bindings::binder_node_debug_info); +decl_wrapper!(BinderNodeInfoForRef, bindings::binder_node_info_for_ref); +decl_wrapper!(FlatBinderObject, bindings::flat_binder_object); +decl_wrapper!(BinderTransactionData, bindings::binder_transaction_data); +decl_wrapper!(BinderWriteRead, bindings::binder_write_read); +decl_wrapper!(BinderVersion, bindings::binder_version); + +impl BinderVersion { + pub(crate) fn current() -> Self { + Self(bindings::binder_version { + protocol_version: bindings::BINDER_CURRENT_PROTOCOL_VERSION as _, + }) + } +} diff --git a/drivers/android/node.rs b/drivers/android/node.rs new file mode 100644 index 0000000000000..94c8adddbb17a --- /dev/null +++ b/drivers/android/node.rs @@ -0,0 +1,479 @@ +// SPDX-License-Identifier: GPL-2.0 + +use alloc::sync::Arc; +use core::{ + pin::Pin, + sync::atomic::{AtomicU64, Ordering}, +}; +use kernel::{ + linked_list::{GetLinks, Links, List}, + prelude::*, + sync::{Guard, LockedBy, Mutex, Ref, SpinLock}, + user_ptr::UserSlicePtrWriter, +}; + +use crate::{ + defs::*, + process::{Process, ProcessInner}, + thread::{BinderError, BinderResult, Thread}, + DeliverToRead, +}; + +struct CountState { + count: usize, + has_count: bool, + is_biased: bool, +} + +impl CountState { + fn new() -> Self { + Self { + count: 0, + has_count: false, + is_biased: false, + } + } + + fn add_bias(&mut self) { + self.count += 1; + self.is_biased = true; + } +} + +struct NodeInner { + strong: CountState, + weak: CountState, + death_list: List<Arc<NodeDeath>>, +} + +struct NodeDeathInner { + dead: bool, + cleared: bool, + notification_done: bool, + + /// Indicates whether the normal flow was interrupted by removing the handle. In this case, we + /// need behave as if the death notification didn't exist (i.e., we don't deliver anything to + /// the user. + aborted: bool, +} + +pub(crate) struct NodeDeath { + node: Arc<Node>, + process: Ref<Process>, + // TODO: Make this private. + pub(crate) cookie: usize, + work_links: Links<dyn DeliverToRead>, + // TODO: Add the moment we're using this for two lists, which isn't safe because we want to + // remove from the list without knowing the list it's in. We need to separate this out. + death_links: Links<NodeDeath>, + inner: SpinLock<NodeDeathInner>, +} + +impl NodeDeath { + /// Constructs a new node death notification object. + /// + /// # Safety + /// + /// The caller must call `NodeDeath::init` before using the notification object. + pub(crate) unsafe fn new(node: Arc<Node>, process: Ref<Process>, cookie: usize) -> Self { + Self { + node, + process, + cookie, + work_links: Links::new(), + death_links: Links::new(), + inner: SpinLock::new(NodeDeathInner { + dead: false, + cleared: false, + notification_done: false, + aborted: false, + }), + } + } + + pub(crate) fn init(self: Pin<&Self>) { + // SAFETY: `inner` is pinned when `self` is. + let inner = unsafe { self.map_unchecked(|s| &s.inner) }; + kernel::spinlock_init!(inner, "NodeDeath::inner"); + } + + /// Sets the cleared flag to `true`. + /// + /// It removes `self` from the node's death notification list if needed. It must only be called + /// once. + /// + /// Returns whether it needs to be queued. + pub(crate) fn set_cleared(self: &Arc<Self>, abort: bool) -> bool { + let (needs_removal, needs_queueing) = { + // Update state and determine if we need to queue a work item. We only need to do it + // when the node is not dead or if the user already completed the death notification. + let mut inner = self.inner.lock(); + inner.cleared = true; + if abort { + inner.aborted = true; + } + (!inner.dead, !inner.dead || inner.notification_done) + }; + + // Remove death notification from node. + if needs_removal { + let mut owner_inner = self.node.owner.inner.lock(); + let node_inner = self.node.inner.access_mut(&mut owner_inner); + unsafe { node_inner.death_list.remove(self) }; + } + + needs_queueing + } + + /// Sets the 'notification done' flag to `true`. + /// + /// Returns whether it needs to be queued. + pub(crate) fn set_notification_done(self: Arc<Self>, thread: &Thread) { + let needs_queueing = { + let mut inner = self.inner.lock(); + inner.notification_done = true; + inner.cleared + }; + + if needs_queueing { + let _ = thread.push_work_if_looper(self); + } + } + + /// Sets the 'dead' flag to `true` and queues work item if needed. + pub(crate) fn set_dead(self: Arc<Self>) { + let needs_queueing = { + let mut inner = self.inner.lock(); + if inner.cleared { + false + } else { + inner.dead = true; + true + } + }; + + if needs_queueing { + // Push the death notification to the target process. There is nothing else to do if + // it's already dead. + let process = self.process.clone(); + let _ = process.push_work(self); + } + } +} + +impl GetLinks for NodeDeath { + type EntryType = NodeDeath; + fn get_links(data: &NodeDeath) -> &Links<NodeDeath> { + &data.death_links + } +} + +impl DeliverToRead for NodeDeath { + fn do_work( + self: Arc<Self>, + _thread: &Thread, + writer: &mut UserSlicePtrWriter, + ) -> KernelResult<bool> { + let done = { + let inner = self.inner.lock(); + if inner.aborted { + return Ok(true); + } + inner.cleared && (!inner.dead || inner.notification_done) + }; + + let cookie = self.cookie; + let cmd = if done { + BR_CLEAR_DEATH_NOTIFICATION_DONE + } else { + let process = self.process.clone(); + let mut process_inner = process.inner.lock(); + let inner = self.inner.lock(); + if inner.aborted { + return Ok(true); + } + // We're still holding the inner lock, so it cannot be aborted while we insert it into + // the delivered list. + process_inner.death_delivered(self.clone()); + BR_DEAD_BINDER + }; + + writer.write(&cmd)?; + writer.write(&cookie)?; + + // Mimic the original code: we stop processing work items when we get to a death + // notification. + Ok(cmd != BR_DEAD_BINDER) + } + + fn get_links(&self) -> &Links<dyn DeliverToRead> { + &self.work_links + } +} + +pub(crate) struct Node { + pub(crate) global_id: u64, + ptr: usize, + cookie: usize, + pub(crate) owner: Ref<Process>, + inner: LockedBy<NodeInner, Mutex<ProcessInner>>, + links: Links<dyn DeliverToRead>, +} + +impl Node { + pub(crate) fn new(ptr: usize, cookie: usize, owner: Ref<Process>) -> Self { + static NEXT_ID: AtomicU64 = AtomicU64::new(1); + let inner = LockedBy::new( + &owner.inner, + NodeInner { + strong: CountState::new(), + weak: CountState::new(), + death_list: List::new(), + }, + ); + Self { + global_id: NEXT_ID.fetch_add(1, Ordering::Relaxed), + ptr, + cookie, + owner, + inner, + links: Links::new(), + } + } + + pub(crate) fn get_id(&self) -> (usize, usize) { + (self.ptr, self.cookie) + } + + pub(crate) fn next_death( + &self, + guard: &mut Guard<Mutex<ProcessInner>>, + ) -> Option<Arc<NodeDeath>> { + self.inner.access_mut(guard).death_list.pop_front() + } + + pub(crate) fn add_death(&self, death: Arc<NodeDeath>, guard: &mut Guard<Mutex<ProcessInner>>) { + self.inner.access_mut(guard).death_list.push_back(death); + } + + pub(crate) fn update_refcount_locked( + &self, + inc: bool, + strong: bool, + biased: bool, + owner_inner: &mut ProcessInner, + ) -> bool { + let inner = self.inner.access_from_mut(owner_inner); + + // Get a reference to the state we'll update. + let state = if strong { + &mut inner.strong + } else { + &mut inner.weak + }; + + // Update biased state: if the count is not biased, there is nothing to do; otherwise, + // we're removing the bias, so mark the state as such. + if biased { + if !state.is_biased { + return false; + } + + state.is_biased = false; + } + + // Update the count and determine whether we need to push work. + // TODO: Here we may want to check the weak count being zero but the strong count being 1, + // because in such cases, we won't deliver anything to userspace, so we shouldn't queue + // either. + if inc { + state.count += 1; + !state.has_count + } else { + state.count -= 1; + state.count == 0 && state.has_count + } + } + + pub(crate) fn update_refcount(self: &Arc<Self>, inc: bool, strong: bool) { + self.owner + .inner + .lock() + .update_node_refcount(self, inc, strong, false, None); + } + + pub(crate) fn populate_counts( + &self, + out: &mut BinderNodeInfoForRef, + guard: &Guard<Mutex<ProcessInner>>, + ) { + let inner = self.inner.access(guard); + out.strong_count = inner.strong.count as _; + out.weak_count = inner.weak.count as _; + } + + pub(crate) fn populate_debug_info( + &self, + out: &mut BinderNodeDebugInfo, + guard: &Guard<Mutex<ProcessInner>>, + ) { + out.ptr = self.ptr as _; + out.cookie = self.cookie as _; + let inner = self.inner.access(&guard); + if inner.strong.has_count { + out.has_strong_ref = 1; + } + if inner.weak.has_count { + out.has_weak_ref = 1; + } + } + + pub(crate) fn force_has_count(&self, guard: &mut Guard<Mutex<ProcessInner>>) { + let inner = self.inner.access_mut(guard); + inner.strong.has_count = true; + inner.weak.has_count = true; + } + + fn write(&self, writer: &mut UserSlicePtrWriter, code: u32) -> KernelResult { + writer.write(&code)?; + writer.write(&self.ptr)?; + writer.write(&self.cookie)?; + Ok(()) + } +} + +impl DeliverToRead for Node { + fn do_work( + self: Arc<Self>, + _thread: &Thread, + writer: &mut UserSlicePtrWriter, + ) -> KernelResult<bool> { + let mut owner_inner = self.owner.inner.lock(); + let inner = self.inner.access_mut(&mut owner_inner); + let strong = inner.strong.count > 0; + let has_strong = inner.strong.has_count; + let weak = strong || inner.weak.count > 0; + let has_weak = inner.weak.has_count; + inner.weak.has_count = weak; + inner.strong.has_count = strong; + + if !weak { + // Remove the node if there are no references to it. + owner_inner.remove_node(self.ptr); + } else { + if !has_weak { + inner.weak.add_bias(); + } + + if !has_strong && strong { + inner.strong.add_bias(); + } + } + + drop(owner_inner); + + // This could be done more compactly but we write out all the posibilities for + // compatibility with the original implementation wrt the order of events. + if weak && !has_weak { + self.write(writer, BR_INCREFS)?; + } + + if strong && !has_strong { + self.write(writer, BR_ACQUIRE)?; + } + + if !strong && has_strong { + self.write(writer, BR_RELEASE)?; + } + + if !weak && has_weak { + self.write(writer, BR_DECREFS)?; + } + + Ok(true) + } + + fn get_links(&self) -> &Links<dyn DeliverToRead> { + &self.links + } +} + +pub struct NodeRef { + pub(crate) node: Arc<Node>, + strong_count: usize, + weak_count: usize, +} + +impl NodeRef { + pub(crate) fn new(node: Arc<Node>, strong_count: usize, weak_count: usize) -> Self { + Self { + node, + strong_count, + weak_count, + } + } + + pub(crate) fn absorb(&mut self, mut other: Self) { + self.strong_count += other.strong_count; + self.weak_count += other.weak_count; + other.strong_count = 0; + other.weak_count = 0; + } + + pub(crate) fn clone(&self, strong: bool) -> BinderResult<NodeRef> { + if strong && self.strong_count == 0 { + return Err(BinderError::new_failed()); + } + + Ok(self + .node + .owner + .inner + .lock() + .new_node_ref(self.node.clone(), strong, None)) + } + + /// Updates (increments or decrements) the number of references held against the node. If the + /// count being updated transitions from 0 to 1 or from 1 to 0, the node is notified by having + /// its `update_refcount` function called. + /// + /// Returns whether `self` should be removed (when both counts are zero). + pub(crate) fn update(&mut self, inc: bool, strong: bool) -> bool { + if strong && self.strong_count == 0 { + return false; + } + + let (count, other_count) = if strong { + (&mut self.strong_count, self.weak_count) + } else { + (&mut self.weak_count, self.strong_count) + }; + + if inc { + if *count == 0 { + self.node.update_refcount(true, strong); + } + *count += 1; + } else { + *count -= 1; + if *count == 0 { + self.node.update_refcount(false, strong); + return other_count == 0; + } + } + + false + } +} + +impl Drop for NodeRef { + fn drop(&mut self) { + if self.strong_count > 0 { + self.node.update_refcount(false, true); + } + + if self.weak_count > 0 { + self.node.update_refcount(false, false); + } + } +} diff --git a/drivers/android/process.rs b/drivers/android/process.rs new file mode 100644 index 0000000000000..9d4d85646432c --- /dev/null +++ b/drivers/android/process.rs @@ -0,0 +1,950 @@ +// SPDX-License-Identifier: GPL-2.0 + +use alloc::{collections::btree_map::BTreeMap, sync::Arc, vec::Vec}; +use core::{ + mem::{swap, MaybeUninit}, + ops::Range, + pin::Pin, +}; +use kernel::{ + bindings, c_types, + file_operations::{File, FileOpener, FileOperations, IoctlCommand, IoctlHandler, PollTable}, + linked_list::List, + pages::Pages, + prelude::*, + sync::{Guard, Mutex, Ref, RefCount, RefCounted}, + user_ptr::{UserSlicePtr, UserSlicePtrReader}, + Error, +}; + +use crate::{ + allocation::Allocation, + context::Context, + defs::*, + node::{Node, NodeDeath, NodeRef}, + range_alloc::RangeAllocator, + thread::{BinderError, BinderResult, Thread}, + DeliverToRead, DeliverToReadListAdapter, Either, +}; + +// TODO: Review this: +// Lock order: Process::node_refs -> Process::inner -> Thread::inner + +extern "C" { + fn rust_helper_current_pid() -> c_types::c_int; +} + +pub(crate) struct AllocationInfo { + /// Range within the allocation where we can find the offsets to the object descriptors. + pub(crate) offsets: Range<usize>, +} + +struct Mapping { + address: usize, + alloc: RangeAllocator<AllocationInfo>, + pages: Arc<[Pages<0>]>, +} + +impl Mapping { + fn new(address: usize, size: usize, pages: Arc<[Pages<0>]>) -> KernelResult<Self> { + let alloc = RangeAllocator::new(size)?; + Ok(Self { + address, + alloc, + pages, + }) + } +} + +// TODO: Make this private. +pub(crate) struct ProcessInner { + is_manager: bool, + is_dead: bool, + threads: BTreeMap<i32, Arc<Thread>>, + ready_threads: List<Arc<Thread>>, + work: List<DeliverToReadListAdapter>, + mapping: Option<Mapping>, + nodes: BTreeMap<usize, Arc<Node>>, + + delivered_deaths: List<Arc<NodeDeath>>, + + /// The number of requested threads that haven't registered yet. + requested_thread_count: u32, + + /// The maximum number of threads used by the process thread pool. + max_threads: u32, + + /// The number of threads the started and registered with the thread pool. + started_thread_count: u32, +} + +impl ProcessInner { + fn new() -> Self { + Self { + is_manager: false, + is_dead: false, + threads: BTreeMap::new(), + ready_threads: List::new(), + work: List::new(), + mapping: None, + nodes: BTreeMap::new(), + requested_thread_count: 0, + max_threads: 0, + started_thread_count: 0, + delivered_deaths: List::new(), + } + } + + fn push_work(&mut self, work: Arc<dyn DeliverToRead>) -> BinderResult { + // Try to find a ready thread to which to push the work. + if let Some(thread) = self.ready_threads.pop_front() { + // Push to thread while holding state lock. This prevents the thread from giving up + // (for example, because of a signal) when we're about to deliver work. + thread.push_work(work) + } else if self.is_dead { + Err(BinderError::new_dead()) + } else { + // There are no ready threads. Push work to process queue. + self.work.push_back(work); + + // Wake up polling threads, if any. + for thread in self.threads.values() { + thread.notify_if_poll_ready(); + } + Ok(()) + } + } + + // TODO: Should this be private? + pub(crate) fn remove_node(&mut self, ptr: usize) { + self.nodes.remove(&ptr); + } + + /// Updates the reference count on the given node. + // TODO: Decide if this should be private. + pub(crate) fn update_node_refcount( + &mut self, + node: &Arc<Node>, + inc: bool, + strong: bool, + biased: bool, + othread: Option<&Thread>, + ) { + let push = node.update_refcount_locked(inc, strong, biased, self); + + // If we decided that we need to push work, push either to the process or to a thread if + // one is specified. + if push { + if let Some(thread) = othread { + thread.push_work_deferred(node.clone()); + } else { + let _ = self.push_work(node.clone()); + // Nothing to do: `push_work` may fail if the process is dead, but that's ok as in + // that case, it doesn't care about the notification. + } + } + } + + // TODO: Make this private. + pub(crate) fn new_node_ref( + &mut self, + node: Arc<Node>, + strong: bool, + thread: Option<&Thread>, + ) -> NodeRef { + self.update_node_refcount(&node, true, strong, false, thread); + let strong_count = if strong { 1 } else { 0 }; + NodeRef::new(node, strong_count, 1 - strong_count) + } + + /// Returns an existing node with the given pointer and cookie, if one exists. + /// + /// Returns an error if a node with the given pointer but a different cookie exists. + fn get_existing_node(&self, ptr: usize, cookie: usize) -> KernelResult<Option<Arc<Node>>> { + match self.nodes.get(&ptr) { + None => Ok(None), + Some(node) => { + let (_, node_cookie) = node.get_id(); + if node_cookie == cookie { + Ok(Some(node.clone())) + } else { + Err(Error::EINVAL) + } + } + } + } + + /// Returns a reference to an existing node with the given pointer and cookie. It requires a + /// mutable reference because it needs to increment the ref count on the node, which may + /// require pushing work to the work queue (to notify userspace of 0 to 1 transitions). + fn get_existing_node_ref( + &mut self, + ptr: usize, + cookie: usize, + strong: bool, + thread: Option<&Thread>, + ) -> KernelResult<Option<NodeRef>> { + Ok(match self.get_existing_node(ptr, cookie)? { + None => None, + Some(node) => Some(self.new_node_ref(node, strong, thread)), + }) + } + + fn register_thread(&mut self) -> bool { + if self.requested_thread_count == 0 { + return false; + } + + self.requested_thread_count -= 1; + self.started_thread_count += 1; + true + } + + /// Finds a delivered death notification with the given cookie, removes it from the thread's + /// delivered list, and returns it. + fn pull_delivered_death(&mut self, cookie: usize) -> Option<Arc<NodeDeath>> { + let mut cursor = self.delivered_deaths.cursor_front_mut(); + while let Some(death) = cursor.current() { + if death.cookie == cookie { + return cursor.remove_current(); + } + cursor.move_next(); + } + None + } + + pub(crate) fn death_delivered(&mut self, death: Arc<NodeDeath>) { + self.delivered_deaths.push_back(death); + } +} + +struct ArcReservation<T> { + mem: Arc<MaybeUninit<T>>, +} + +impl<T> ArcReservation<T> { + fn new() -> KernelResult<Self> { + Ok(Self { + mem: Arc::try_new(MaybeUninit::<T>::uninit())?, + }) + } + + fn commit(mut self, data: T) -> Arc<T> { + // SAFETY: Memory was allocated and properly aligned by using `MaybeUninit`. + unsafe { + Arc::get_mut(&mut self.mem) + .unwrap() + .as_mut_ptr() + .write(data); + } + + // SAFETY: We have just initialised the memory block, and we know it's compatible with `T` + // because we used `MaybeUninit`. + unsafe { Arc::from_raw(Arc::into_raw(self.mem) as _) } + } +} + +struct NodeRefInfo { + node_ref: NodeRef, + death: Option<Arc<NodeDeath>>, +} + +impl NodeRefInfo { + fn new(node_ref: NodeRef) -> Self { + Self { + node_ref, + death: None, + } + } +} + +struct ProcessNodeRefs { + by_handle: BTreeMap<u32, NodeRefInfo>, + by_global_id: BTreeMap<u64, u32>, +} + +impl ProcessNodeRefs { + fn new() -> Self { + Self { + by_handle: BTreeMap::new(), + by_global_id: BTreeMap::new(), + } + } +} + +pub(crate) struct Process { + ctx: Arc<Context>, + ref_count: RefCount, + + // TODO: For now this a mutex because we have allocations in BTreeMap and RangeAllocator while + // holding the lock. We may want to split up the process state at some point to use a spin lock + // for the other fields; we can also get rid of allocations in BTreeMap once we replace it. + // TODO: Make this private again. + pub(crate) inner: Mutex<ProcessInner>, + + // References are in a different mutex to avoid recursive acquisition when + // incrementing/decrementing a node in another process. + node_refs: Mutex<ProcessNodeRefs>, +} + +unsafe impl Send for Process {} +unsafe impl Sync for Process {} + +impl Process { + fn new(ctx: Arc<Context>) -> KernelResult<Ref<Self>> { + let mut proc_ref = Ref::try_new(Self { + ref_count: RefCount::new(), + ctx, + // SAFETY: `inner` is initialised in the call to `mutex_init` below. + inner: unsafe { Mutex::new(ProcessInner::new()) }, + // SAFETY: `node_refs` is initialised in the call to `mutex_init` below. + node_refs: unsafe { Mutex::new(ProcessNodeRefs::new()) }, + })?; + let process = Ref::get_mut(&mut proc_ref).ok_or(Error::EINVAL)?; + // SAFETY: `inner` is pinned behind the `Arc` reference. + let pinned = unsafe { Pin::new_unchecked(&process.inner) }; + kernel::mutex_init!(pinned, "Process::inner"); + // SAFETY: `node_refs` is pinned behind the `Arc` reference. + let pinned = unsafe { Pin::new_unchecked(&process.node_refs) }; + kernel::mutex_init!(pinned, "Process::node_refs"); + Ok(proc_ref) + } + + /// Attemps to fetch a work item from the process queue. + pub(crate) fn get_work(&self) -> Option<Arc<dyn DeliverToRead>> { + self.inner.lock().work.pop_front() + } + + /// Attemps to fetch a work item from the process queue. If none is available, it registers the + /// given thread as ready to receive work directly. + /// + /// This must only be called when the thread is not participating in a transaction chain; when + /// it is, work will always be delivered directly to the thread (and not through the process + /// queue). + pub(crate) fn get_work_or_register<'a>( + &'a self, + thread: &'a Arc<Thread>, + ) -> Either<Arc<dyn DeliverToRead>, Registration<'a>> { + let mut inner = self.inner.lock(); + + // Try to get work from the process queue. + if let Some(work) = inner.work.pop_front() { + return Either::Left(work); + } + + // Register the thread as ready. + Either::Right(Registration::new(self, thread, &mut inner)) + } + + fn get_thread(&self, id: i32) -> KernelResult<Arc<Thread>> { + // TODO: Consider using read/write locks here instead. + { + let inner = self.inner.lock(); + if let Some(thread) = inner.threads.get(&id) { + return Ok(thread.clone()); + } + } + + // Allocate a new `Thread` without holding any locks. + let ta = Thread::new(id, Ref::new_from(self))?; + + let mut inner = self.inner.lock(); + + // Recheck. It's possible the thread was create while we were not holding the lock. + if let Some(thread) = inner.threads.get(&id) { + return Ok(thread.clone()); + } + + // TODO: We need a better solution here. Since this allocates, we cannot do it while + // holding a spinlock because it could block. It could panic too. + inner.threads.insert(id, ta.clone()); + Ok(ta) + } + + pub(crate) fn push_work(&self, work: Arc<dyn DeliverToRead>) -> BinderResult { + self.inner.lock().push_work(work) + } + + fn set_as_manager(&self, info: Option<FlatBinderObject>, thread: &Thread) -> KernelResult { + let (ptr, cookie) = if let Some(obj) = info { + (unsafe { obj.__bindgen_anon_1.binder }, obj.cookie) + } else { + (0, 0) + }; + let node_ref = self.get_node(ptr as _, cookie as _, true, Some(thread))?; + let node = node_ref.node.clone(); + self.ctx.set_manager_node(node_ref)?; + self.inner.lock().is_manager = true; + + // Force the state of the node to prevent the delivery of acquire/increfs. + let mut owner_inner = node.owner.inner.lock(); + node.force_has_count(&mut owner_inner); + Ok(()) + } + + pub(crate) fn get_node( + &self, + ptr: usize, + cookie: usize, + strong: bool, + thread: Option<&Thread>, + ) -> KernelResult<NodeRef> { + // Try to find an existing node. + { + let mut inner = self.inner.lock(); + if let Some(node) = inner.get_existing_node_ref(ptr, cookie, strong, thread)? { + return Ok(node); + } + } + + // Allocate the node before reacquiring the lock. + let node = Arc::try_new(Node::new(ptr, cookie, Ref::new_from(self)))?; + + let mut inner = self.inner.lock(); + if let Some(node) = inner.get_existing_node_ref(ptr, cookie, strong, thread)? { + return Ok(node); + } + + // TODO: Avoid allocation while holding lock. + inner.nodes.insert(ptr, node.clone()); + Ok(inner.new_node_ref(node, strong, thread)) + } + + pub(crate) fn insert_or_update_handle( + &self, + node_ref: NodeRef, + is_mananger: bool, + ) -> KernelResult<u32> { + let mut refs = self.node_refs.lock(); + + // Do a lookup before inserting. + if let Some(handle_ref) = refs.by_global_id.get(&node_ref.node.global_id) { + let handle = *handle_ref; + let info = refs.by_handle.get_mut(&handle).unwrap(); + info.node_ref.absorb(node_ref); + return Ok(handle); + } + + // Find id. + let mut target = if is_mananger { 0 } else { 1 }; + for handle in refs.by_handle.keys() { + if *handle > target { + break; + } + if *handle == target { + target = target.checked_add(1).ok_or(Error::ENOMEM)?; + } + } + + // Ensure the process is still alive while we insert a new reference. + let inner = self.inner.lock(); + if inner.is_dead { + return Err(Error::ESRCH); + } + // TODO: Two allocations below. + refs.by_global_id.insert(node_ref.node.global_id, target); + refs.by_handle.insert(target, NodeRefInfo::new(node_ref)); + Ok(target) + } + + pub(crate) fn get_transaction_node(&self, handle: u32) -> BinderResult<NodeRef> { + // When handle is zero, try to get the context manager. + if handle == 0 { + self.ctx.get_manager_node(true) + } else { + self.get_node_from_handle(handle, true) + } + } + + pub(crate) fn get_node_from_handle(&self, handle: u32, strong: bool) -> BinderResult<NodeRef> { + self.node_refs + .lock() + .by_handle + .get(&handle) + .ok_or(Error::ENOENT)? + .node_ref + .clone(strong) + } + + pub(crate) fn remove_from_delivered_deaths(&self, death: &Arc<NodeDeath>) { + let mut inner = self.inner.lock(); + let removed = unsafe { inner.delivered_deaths.remove(death) }; + drop(inner); + drop(removed); + } + + pub(crate) fn update_ref(&self, handle: u32, inc: bool, strong: bool) -> KernelResult { + if inc && handle == 0 { + if let Ok(node_ref) = self.ctx.get_manager_node(strong) { + if core::ptr::eq(self, &*node_ref.node.owner) { + return Err(Error::EINVAL); + } + let _ = self.insert_or_update_handle(node_ref, true); + return Ok(()); + } + } + + // To preserve original binder behaviour, we only fail requests where the manager tries to + // increment references on itself. + let mut refs = self.node_refs.lock(); + if let Some(info) = refs.by_handle.get_mut(&handle) { + if info.node_ref.update(inc, strong) { + // Clean up death if there is one attached to this node reference. + if let Some(death) = info.death.take() { + death.set_cleared(true); + self.remove_from_delivered_deaths(&death); + } + + // Remove reference from process tables. + let id = info.node_ref.node.global_id; + refs.by_handle.remove(&handle); + refs.by_global_id.remove(&id); + } + } + Ok(()) + } + + /// Decrements the refcount of the given node, if one exists. + pub(crate) fn update_node(&self, ptr: usize, cookie: usize, strong: bool, biased: bool) { + let mut inner = self.inner.lock(); + if let Ok(Some(node)) = inner.get_existing_node(ptr, cookie) { + inner.update_node_refcount(&node, false, strong, biased, None); + } + } + + pub(crate) fn inc_ref_done( + &self, + reader: &mut UserSlicePtrReader, + strong: bool, + ) -> KernelResult { + let ptr = reader.read::<usize>()?; + let cookie = reader.read::<usize>()?; + self.update_node(ptr, cookie, strong, true); + Ok(()) + } + + pub(crate) fn buffer_alloc(&self, size: usize) -> BinderResult<Allocation> { + let mut inner = self.inner.lock(); + let mapping = inner.mapping.as_mut().ok_or_else(BinderError::new_dead)?; + + let offset = mapping.alloc.reserve_new(size)?; + Ok(Allocation::new( + self, + offset, + size, + mapping.address + offset, + mapping.pages.clone(), + )) + } + + // TODO: Review if we want an Option or a KernelResult. + pub(crate) fn buffer_get(&self, ptr: usize) -> Option<Allocation> { + let mut inner = self.inner.lock(); + let mapping = inner.mapping.as_mut()?; + let offset = ptr - mapping.address; + let (size, odata) = mapping.alloc.reserve_existing(offset).ok()?; + let mut alloc = Allocation::new(self, offset, size, ptr, mapping.pages.clone()); + if let Some(data) = odata { + alloc.set_info(data); + } + Some(alloc) + } + + pub(crate) fn buffer_raw_free(&self, ptr: usize) { + let mut inner = self.inner.lock(); + if let Some(ref mut mapping) = &mut inner.mapping { + if mapping + .alloc + .reservation_abort(ptr - mapping.address) + .is_err() + { + pr_warn!("Offset {} failed to free\n", ptr - mapping.address); + } + } + } + + pub(crate) fn buffer_make_freeable(&self, offset: usize, data: Option<AllocationInfo>) { + let mut inner = self.inner.lock(); + if let Some(ref mut mapping) = &mut inner.mapping { + if mapping.alloc.reservation_commit(offset, data).is_err() { + pr_warn!("Offset {} failed to be marked freeable\n", offset); + } + } + } + + fn create_mapping(&self, vma: &mut bindings::vm_area_struct) -> KernelResult { + let size = core::cmp::min( + (vma.vm_end - vma.vm_start) as usize, + bindings::SZ_4M as usize, + ); + let page_count = size >> bindings::PAGE_SHIFT; + + // Allocate and map all pages. + // + // N.B. If we fail halfway through mapping these pages, the kernel will unmap them. + let mut pages = Vec::new(); + pages.try_reserve_exact(page_count)?; + let mut address = vma.vm_start as usize; + for _ in 0..page_count { + let page = Pages::<0>::new()?; + page.insert_page(vma, address)?; + pages.push(page); + address += 1 << bindings::PAGE_SHIFT; + } + + // TODO: This allocates memory. + let arc = Arc::from(pages); + + // Save pages for later. + let mut inner = self.inner.lock(); + match &inner.mapping { + None => inner.mapping = Some(Mapping::new(vma.vm_start as _, size, arc)?), + Some(_) => return Err(Error::EBUSY), + } + Ok(()) + } + + fn version(&self, data: UserSlicePtr) -> KernelResult { + data.writer().write(&BinderVersion::current()) + } + + pub(crate) fn register_thread(&self) -> bool { + self.inner.lock().register_thread() + } + + fn remove_thread(&self, thread: Arc<Thread>) { + self.inner.lock().threads.remove(&thread.id); + thread.release(); + } + + fn set_max_threads(&self, max: u32) { + self.inner.lock().max_threads = max; + } + + fn get_node_debug_info(&self, data: UserSlicePtr) -> KernelResult { + let (mut reader, mut writer) = data.reader_writer(); + + // Read the starting point. + let ptr = reader.read::<BinderNodeDebugInfo>()?.ptr as usize; + let mut out = BinderNodeDebugInfo::default(); + + { + let inner = self.inner.lock(); + for (node_ptr, node) in &inner.nodes { + if *node_ptr > ptr { + node.populate_debug_info(&mut out, &inner); + break; + } + } + } + + writer.write(&out) + } + + fn get_node_info_from_ref(&self, data: UserSlicePtr) -> KernelResult { + let (mut reader, mut writer) = data.reader_writer(); + let mut out = reader.read::<BinderNodeInfoForRef>()?; + + if out.strong_count != 0 + || out.weak_count != 0 + || out.reserved1 != 0 + || out.reserved2 != 0 + || out.reserved3 != 0 + { + return Err(Error::EINVAL); + } + + // Only the context manager is allowed to use this ioctl. + if !self.inner.lock().is_manager { + return Err(Error::EPERM); + } + + let node_ref = self + .get_node_from_handle(out.handle, true) + .or(Err(Error::EINVAL))?; + + // Get the counts from the node. + { + let owner_inner = node_ref.node.owner.inner.lock(); + node_ref.node.populate_counts(&mut out, &owner_inner); + } + + // Write the result back. + writer.write(&out) + } + + pub(crate) fn needs_thread(&self) -> bool { + let mut inner = self.inner.lock(); + let ret = inner.requested_thread_count == 0 + && inner.ready_threads.is_empty() + && inner.started_thread_count < inner.max_threads; + if ret { + inner.requested_thread_count += 1 + }; + ret + } + + pub(crate) fn request_death( + &self, + reader: &mut UserSlicePtrReader, + thread: &Thread, + ) -> KernelResult { + let handle: u32 = reader.read()?; + let cookie: usize = reader.read()?; + + // TODO: First two should result in error, but not the others. + + // TODO: Do we care about the context manager dying? + + // Queue BR_ERROR if we can't allocate memory for the death notification. + let death = ArcReservation::new().map_err(|err| { + thread.push_return_work(BR_ERROR); + err + })?; + + let mut refs = self.node_refs.lock(); + let info = refs.by_handle.get_mut(&handle).ok_or(Error::EINVAL)?; + + // Nothing to do if there is already a death notification request for this handle. + if info.death.is_some() { + return Ok(()); + } + + // SAFETY: `init` is called below. + let death = death.commit(unsafe { + NodeDeath::new(info.node_ref.node.clone(), Ref::new_from(self), cookie) + }); + // SAFETY: `death` is pinned behind the `Arc` reference. + unsafe { Pin::new_unchecked(death.as_ref()) }.init(); + info.death = Some(death.clone()); + + // Register the death notification. + { + let mut owner_inner = info.node_ref.node.owner.inner.lock(); + if owner_inner.is_dead { + drop(owner_inner); + let _ = self.push_work(death); + } else { + info.node_ref.node.add_death(death, &mut owner_inner); + } + } + Ok(()) + } + + pub(crate) fn clear_death( + &self, + reader: &mut UserSlicePtrReader, + thread: &Thread, + ) -> KernelResult { + let handle: u32 = reader.read()?; + let cookie: usize = reader.read()?; + + let mut refs = self.node_refs.lock(); + let info = refs.by_handle.get_mut(&handle).ok_or(Error::EINVAL)?; + + let death = info.death.take().ok_or(Error::EINVAL)?; + if death.cookie != cookie { + info.death = Some(death); + return Err(Error::EINVAL); + } + + // Update state and determine if we need to queue a work item. We only need to do it when + // the node is not dead or if the user already completed the death notification. + if death.set_cleared(false) { + let _ = thread.push_work_if_looper(death); + } + + Ok(()) + } + + pub(crate) fn dead_binder_done(&self, cookie: usize, thread: &Thread) { + if let Some(death) = self.inner.lock().pull_delivered_death(cookie) { + death.set_notification_done(thread); + } + } +} + +impl IoctlHandler for Process { + fn write(&self, _file: &File, cmd: u32, reader: &mut UserSlicePtrReader) -> KernelResult<i32> { + let thread = self.get_thread(unsafe { rust_helper_current_pid() })?; + match cmd { + bindings::BINDER_SET_MAX_THREADS => self.set_max_threads(reader.read()?), + bindings::BINDER_SET_CONTEXT_MGR => self.set_as_manager(None, &thread)?, + bindings::BINDER_THREAD_EXIT => self.remove_thread(thread), + bindings::BINDER_SET_CONTEXT_MGR_EXT => { + self.set_as_manager(Some(reader.read()?), &thread)? + } + _ => return Err(Error::EINVAL), + } + Ok(0) + } + + fn read_write(&self, file: &File, cmd: u32, data: UserSlicePtr) -> KernelResult<i32> { + let thread = self.get_thread(unsafe { rust_helper_current_pid() })?; + match cmd { + bindings::BINDER_WRITE_READ => thread.write_read(data, file.is_blocking())?, + bindings::BINDER_GET_NODE_DEBUG_INFO => self.get_node_debug_info(data)?, + bindings::BINDER_GET_NODE_INFO_FOR_REF => self.get_node_info_from_ref(data)?, + bindings::BINDER_VERSION => self.version(data)?, + _ => return Err(Error::EINVAL), + } + Ok(0) + } +} + +unsafe impl RefCounted for Process { + fn get_count(&self) -> &RefCount { + &self.ref_count + } +} + +impl FileOpener<Arc<Context>> for Process { + fn open(ctx: &Arc<Context>) -> KernelResult<Self::Wrapper> { + let process = Self::new(ctx.clone())?; + // SAFETY: Pointer is pinned behind `Ref`. + Ok(unsafe { Pin::new_unchecked(process) }) + } +} + +impl FileOperations for Process { + type Wrapper = Pin<Ref<Self>>; + + kernel::declare_file_operations!(ioctl, compat_ioctl, mmap, poll); + + fn release(obj: Self::Wrapper, _file: &File) { + // Mark this process as dead. We'll do the same for the threads later. + obj.inner.lock().is_dead = true; + + // If this process is the manager, unset it. + if obj.inner.lock().is_manager { + obj.ctx.unset_manager_node(); + } + + // TODO: Do this in a worker? + + // Cancel all pending work items. + while let Some(work) = obj.get_work() { + work.cancel(); + } + + // Free any resources kept alive by allocated buffers. + let omapping = obj.inner.lock().mapping.take(); + if let Some(mut mapping) = omapping { + let address = mapping.address; + let pages = mapping.pages.clone(); + mapping.alloc.for_each(|offset, size, odata| { + let ptr = offset + address; + let mut alloc = Allocation::new(&obj, offset, size, ptr, pages.clone()); + if let Some(data) = odata { + alloc.set_info(data); + } + drop(alloc) + }); + } + + // Drop all references. We do this dance with `swap` to avoid destroying the references + // while holding the lock. + let mut refs = obj.node_refs.lock(); + let mut node_refs = BTreeMap::new(); + swap(&mut refs.by_handle, &mut node_refs); + drop(refs); + + // Remove all death notifications from the nodes (that belong to a different process). + for info in node_refs.values_mut() { + let death = if let Some(existing) = info.death.take() { + existing + } else { + continue; + }; + + death.set_cleared(false); + } + + // Do similar dance for the state lock. + let mut inner = obj.inner.lock(); + let mut threads = BTreeMap::new(); + let mut nodes = BTreeMap::new(); + swap(&mut inner.threads, &mut threads); + swap(&mut inner.nodes, &mut nodes); + drop(inner); + + // Release all threads. + for thread in threads.values() { + thread.release(); + } + + // Deliver death notifications. + for node in nodes.values() { + loop { + let death = { + let mut inner = obj.inner.lock(); + if let Some(death) = node.next_death(&mut inner) { + death + } else { + break; + } + }; + + death.set_dead(); + } + } + } + + fn ioctl(&self, file: &File, cmd: &mut IoctlCommand) -> KernelResult<i32> { + cmd.dispatch(self, file) + } + + fn compat_ioctl(&self, file: &File, cmd: &mut IoctlCommand) -> KernelResult<i32> { + cmd.dispatch(self, file) + } + + fn mmap(&self, _file: &File, vma: &mut bindings::vm_area_struct) -> KernelResult { + // TODO: Only group leader is allowed to create mappings. + + if vma.vm_start == 0 { + return Err(Error::EINVAL); + } + + if (vma.vm_flags & (bindings::VM_WRITE as c_types::c_ulong)) != 0 { + return Err(Error::EPERM); + } + + vma.vm_flags |= (bindings::VM_DONTCOPY | bindings::VM_MIXEDMAP) as c_types::c_ulong; + vma.vm_flags &= !(bindings::VM_MAYWRITE as c_types::c_ulong); + + // TODO: Set ops. We need to learn when the user unmaps so that we can stop using it. + self.create_mapping(vma) + } + + fn poll(&self, file: &File, table: &PollTable) -> KernelResult<u32> { + let thread = self.get_thread(unsafe { rust_helper_current_pid() })?; + let (from_proc, mut mask) = thread.poll(file, table); + if mask == 0 && from_proc && !self.inner.lock().work.is_empty() { + mask |= bindings::POLLIN; + } + Ok(mask) + } +} + +pub(crate) struct Registration<'a> { + process: &'a Process, + thread: &'a Arc<Thread>, +} + +impl<'a> Registration<'a> { + fn new( + process: &'a Process, + thread: &'a Arc<Thread>, + guard: &mut Guard<Mutex<ProcessInner>>, + ) -> Self { + guard.ready_threads.push_back(thread.clone()); + Self { process, thread } + } +} + +impl Drop for Registration<'_> { + fn drop(&mut self) { + let mut inner = self.process.inner.lock(); + unsafe { inner.ready_threads.remove(self.thread) }; + } +} diff --git a/drivers/android/range_alloc.rs b/drivers/android/range_alloc.rs new file mode 100644 index 0000000000000..0278041cdf763 --- /dev/null +++ b/drivers/android/range_alloc.rs @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: GPL-2.0 + +use alloc::boxed::Box; +use core::ptr::NonNull; +use kernel::{ + linked_list::{CursorMut, GetLinks, Links, List}, + prelude::*, + Error, +}; + +pub(crate) struct RangeAllocator<T> { + list: List<Box<Descriptor<T>>>, +} + +#[derive(Debug, PartialEq, Eq)] +enum DescriptorState { + Free, + Reserved, + Allocated, +} + +impl<T> RangeAllocator<T> { + pub(crate) fn new(size: usize) -> KernelResult<Self> { + let desc = Box::try_new(Descriptor::new(0, size))?; + let mut list = List::new(); + list.push_back(desc); + Ok(Self { list }) + } + + fn find_best_match(&self, size: usize) -> Option<NonNull<Descriptor<T>>> { + // TODO: Use a binary tree instead of list for this lookup. + let mut best = None; + let mut best_size = usize::MAX; + let mut cursor = self.list.cursor_front(); + while let Some(desc) = cursor.current() { + if desc.state == DescriptorState::Free { + if size == desc.size { + return Some(NonNull::from(desc)); + } + + if size < desc.size && desc.size < best_size { + best = Some(NonNull::from(desc)); + best_size = desc.size; + } + } + + cursor.move_next(); + } + best + } + + pub(crate) fn reserve_new(&mut self, size: usize) -> KernelResult<usize> { + let desc_ptr = match self.find_best_match(size) { + None => return Err(Error::ENOMEM), + Some(found) => found, + }; + + // SAFETY: We hold the only mutable reference to list, so it cannot have changed. + let desc = unsafe { &mut *desc_ptr.as_ptr() }; + if desc.size == size { + desc.state = DescriptorState::Reserved; + return Ok(desc.offset); + } + + // We need to break up the descriptor. + let new = Box::try_new(Descriptor::new(desc.offset + size, desc.size - size))?; + unsafe { self.list.insert_after(desc_ptr, new) }; + desc.state = DescriptorState::Reserved; + desc.size = size; + Ok(desc.offset) + } + + fn free_with_cursor(cursor: &mut CursorMut<Box<Descriptor<T>>>) -> KernelResult { + let mut size = match cursor.current() { + None => return Err(Error::EINVAL), + Some(ref mut entry) => { + match entry.state { + DescriptorState::Free => return Err(Error::EINVAL), + DescriptorState::Allocated => return Err(Error::EPERM), + DescriptorState::Reserved => {} + } + entry.state = DescriptorState::Free; + entry.size + } + }; + + // Try to merge with the next entry. + if let Some(next) = cursor.peek_next() { + if next.state == DescriptorState::Free { + next.offset -= size; + next.size += size; + size = next.size; + cursor.remove_current(); + } + } + + // Try to merge with the previous entry. + if let Some(prev) = cursor.peek_prev() { + if prev.state == DescriptorState::Free { + prev.size += size; + cursor.remove_current(); + } + } + + Ok(()) + } + + fn find_at_offset(&mut self, offset: usize) -> Option<CursorMut<Box<Descriptor<T>>>> { + let mut cursor = self.list.cursor_front_mut(); + while let Some(desc) = cursor.current() { + if desc.offset == offset { + return Some(cursor); + } + + if desc.offset > offset { + return None; + } + + cursor.move_next(); + } + None + } + + pub(crate) fn reservation_abort(&mut self, offset: usize) -> KernelResult { + // TODO: The force case is currently O(n), but could be made O(1) with unsafe. + let mut cursor = self.find_at_offset(offset).ok_or(Error::EINVAL)?; + Self::free_with_cursor(&mut cursor) + } + + pub(crate) fn reservation_commit(&mut self, offset: usize, data: Option<T>) -> KernelResult { + // TODO: This is currently O(n), make it O(1). + let mut cursor = self.find_at_offset(offset).ok_or(Error::ENOENT)?; + let desc = cursor.current().unwrap(); + desc.state = DescriptorState::Allocated; + desc.data = data; + Ok(()) + } + + /// Takes an entry at the given offset from [`DescriptorState::Allocated`] to + /// [`DescriptorState::Reserved`]. + /// + /// Returns the size of the existing entry and the data associated with it. + pub(crate) fn reserve_existing(&mut self, offset: usize) -> KernelResult<(usize, Option<T>)> { + // TODO: This is currently O(n), make it O(log n). + let mut cursor = self.find_at_offset(offset).ok_or(Error::ENOENT)?; + let desc = cursor.current().unwrap(); + if desc.state != DescriptorState::Allocated { + return Err(Error::ENOENT); + } + desc.state = DescriptorState::Reserved; + Ok((desc.size, desc.data.take())) + } + + pub(crate) fn for_each<F: Fn(usize, usize, Option<T>)>(&mut self, callback: F) { + let mut cursor = self.list.cursor_front_mut(); + while let Some(desc) = cursor.current() { + if desc.state == DescriptorState::Allocated { + callback(desc.offset, desc.size, desc.data.take()); + } + + cursor.move_next(); + } + } +} + +struct Descriptor<T> { + state: DescriptorState, + size: usize, + offset: usize, + links: Links<Descriptor<T>>, + data: Option<T>, +} + +impl<T> Descriptor<T> { + fn new(offset: usize, size: usize) -> Self { + Self { + size, + offset, + state: DescriptorState::Free, + links: Links::new(), + data: None, + } + } +} + +impl<T> GetLinks for Descriptor<T> { + type EntryType = Self; + fn get_links(desc: &Self) -> &Links<Self> { + &desc.links + } +} diff --git a/drivers/android/rust_binder.rs b/drivers/android/rust_binder.rs new file mode 100644 index 0000000000000..aaef4517d48ff --- /dev/null +++ b/drivers/android/rust_binder.rs @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Binder -- the Android IPC mechanism. +//! +//! TODO: This module is a work in progress. + +#![no_std] +#![feature(global_asm, try_reserve, allocator_api, concat_idents)] + +use alloc::{boxed::Box, sync::Arc}; +use core::pin::Pin; +use kernel::{ + cstr, + linked_list::{GetLinks, GetLinksWrapped, Links}, + miscdev::Registration, + prelude::*, + user_ptr::UserSlicePtrWriter, +}; + +mod allocation; +mod context; +mod defs; +mod node; +mod process; +mod range_alloc; +mod thread; +mod transaction; + +use {context::Context, thread::Thread}; + +module! { + type: BinderModule, + name: b"rust_binder", + author: b"Wedson Almeida Filho", + description: b"Android Binder", + license: b"GPL v2", + params: {}, +} + +enum Either<L, R> { + Left(L), + Right(R), +} + +trait DeliverToRead { + /// Performs work. Returns true if remaining work items in the queue should be processed + /// immediately, or false if it should return to caller before processing additional work + /// items. + fn do_work( + self: Arc<Self>, + thread: &Thread, + writer: &mut UserSlicePtrWriter, + ) -> KernelResult<bool>; + + /// Cancels the given work item. This is called instead of [`DeliverToRead::do_work`] when work + /// won't be delivered. + fn cancel(self: Arc<Self>) {} + + /// Returns the linked list links for the work item. + fn get_links(&self) -> &Links<dyn DeliverToRead>; +} + +struct DeliverToReadListAdapter {} + +impl GetLinks for DeliverToReadListAdapter { + type EntryType = dyn DeliverToRead; + + fn get_links(data: &Self::EntryType) -> &Links<Self::EntryType> { + data.get_links() + } +} + +impl GetLinksWrapped for DeliverToReadListAdapter { + type Wrapped = Arc<dyn DeliverToRead>; +} + +struct DeliverCode { + code: u32, + links: Links<dyn DeliverToRead>, +} + +impl DeliverCode { + fn new(code: u32) -> Self { + Self { + code, + links: Links::new(), + } + } +} + +impl DeliverToRead for DeliverCode { + fn do_work( + self: Arc<Self>, + _thread: &Thread, + writer: &mut UserSlicePtrWriter, + ) -> KernelResult<bool> { + writer.write(&self.code)?; + Ok(true) + } + + fn get_links(&self) -> &Links<dyn DeliverToRead> { + &self.links + } +} + +const fn ptr_align(value: usize) -> usize { + let size = core::mem::size_of::<usize>() - 1; + (value + size) & !size +} + +unsafe impl Sync for BinderModule {} + +struct BinderModule { + _reg: Pin<Box<Registration<Arc<Context>>>>, +} + +impl KernelModule for BinderModule { + fn init() -> KernelResult<Self> { + let pinned_ctx = Context::new()?; + let ctx = unsafe { Pin::into_inner_unchecked(pinned_ctx) }; + let reg = Registration::<Arc<Context>>::new_pinned::<process::Process>( + cstr!("rust_binder"), + None, + ctx, + )?; + Ok(Self { _reg: reg }) + } +} diff --git a/drivers/android/thread.rs b/drivers/android/thread.rs new file mode 100644 index 0000000000000..f84fe9df988bc --- /dev/null +++ b/drivers/android/thread.rs @@ -0,0 +1,821 @@ +// SPDX-License-Identifier: GPL-2.0 + +use alloc::sync::Arc; +use core::{alloc::AllocError, mem::size_of, pin::Pin}; +use kernel::{ + bindings, + file_operations::{File, PollTable}, + linked_list::{GetLinks, Links, List}, + prelude::*, + sync::{CondVar, Ref, SpinLock}, + user_ptr::{UserSlicePtr, UserSlicePtrWriter}, + Error, +}; + +use crate::{ + allocation::{Allocation, AllocationView}, + defs::*, + process::{AllocationInfo, Process}, + ptr_align, + transaction::Transaction, + DeliverCode, DeliverToRead, DeliverToReadListAdapter, Either, +}; + +pub(crate) type BinderResult<T = ()> = Result<T, BinderError>; + +pub(crate) struct BinderError { + pub(crate) reply: u32, +} + +impl BinderError { + pub(crate) fn new_failed() -> Self { + Self { + reply: BR_FAILED_REPLY, + } + } + + pub(crate) fn new_dead() -> Self { + Self { + reply: BR_DEAD_REPLY, + } + } +} + +impl From<Error> for BinderError { + fn from(_: Error) -> Self { + Self::new_failed() + } +} + +impl From<AllocError> for BinderError { + fn from(_: AllocError) -> Self { + Self::new_failed() + } +} + +const LOOPER_REGISTERED: u32 = 0x01; +const LOOPER_ENTERED: u32 = 0x02; +const LOOPER_EXITED: u32 = 0x04; +const LOOPER_INVALID: u32 = 0x08; +const LOOPER_WAITING: u32 = 0x10; +const LOOPER_POLL: u32 = 0x20; + +struct InnerThread { + /// Determines the looper state of the thread. It is a bit-wise combination of the constants + /// prefixed with `LOOPER_`. + looper_flags: u32, + + /// Determines if thread is dead. + is_dead: bool, + + /// Work item used to deliver error codes to the thread that started a transaction. When set to + /// `Some(x)`, it will hold the only reference to the object so that it can update the error + /// code to be delivered before queuing it. + reply_work: Option<Arc<ThreadError>>, + + /// Work item used to deliver error codes to the current thread. When set to `Some(x)`, it will + /// hold the only reference to the object so that it can update the error code to be delivered + /// before queuing. + return_work: Option<Arc<ThreadError>>, + + /// Determines whether the work list below should be processed. When set to false, `work_list` + /// is treated as if it were empty. + process_work_list: bool, + work_list: List<DeliverToReadListAdapter>, + current_transaction: Option<Arc<Transaction>>, +} + +impl InnerThread { + fn new() -> Self { + Self { + looper_flags: 0, + is_dead: false, + process_work_list: false, + work_list: List::new(), + current_transaction: None, + return_work: None, + reply_work: None, + } + } + + fn set_reply_work(&mut self, reply_work: Arc<ThreadError>) { + self.reply_work = Some(reply_work); + } + + fn push_reply_work(&mut self, code: u32) { + let work = self.reply_work.take(); + self.push_existing_work(work, code); + } + + fn set_return_work(&mut self, return_work: Arc<ThreadError>) { + self.return_work = Some(return_work); + } + + fn push_return_work(&mut self, code: u32) { + let work = self.return_work.take(); + self.push_existing_work(work, code); + } + + fn push_existing_work(&mut self, owork: Option<Arc<ThreadError>>, code: u32) { + // TODO: Write some warning when the following fails. It should not happen, and + // if it does, there is likely something wrong. + if let Some(mut work) = owork { + if let Some(work_mut) = Arc::get_mut(&mut work) { + work_mut.error_code = code; + self.push_work(work); + } + } + } + + fn pop_work(&mut self) -> Option<Arc<dyn DeliverToRead>> { + if !self.process_work_list { + return None; + } + + let ret = self.work_list.pop_front(); + // Once the queue is drained, we stop processing it until a non-deferred item is pushed + // again onto it. + self.process_work_list = !self.work_list.is_empty(); + ret + } + + fn push_work_deferred(&mut self, work: Arc<dyn DeliverToRead>) { + self.work_list.push_back(work); + } + + fn push_work(&mut self, work: Arc<dyn DeliverToRead>) { + self.push_work_deferred(work); + self.process_work_list = true; + } + + fn has_work(&self) -> bool { + self.process_work_list && !self.work_list.is_empty() + } + + /// Fetches the transaction the thread can reply to. If the thread has a pending transaction + /// (that it could respond to) but it has also issued a transaction, it must first wait for the + /// previously-issued transaction to complete. + fn pop_transaction_to_reply(&mut self, thread: &Thread) -> KernelResult<Arc<Transaction>> { + let transaction = self.current_transaction.take().ok_or(Error::EINVAL)?; + + if core::ptr::eq(thread, transaction.from.as_ref()) { + self.current_transaction = Some(transaction); + return Err(Error::EINVAL); + } + + // Find a new current transaction for this thread. + self.current_transaction = transaction.find_from(thread); + Ok(transaction) + } + + fn pop_transaction_replied(&mut self, transaction: &Arc<Transaction>) -> bool { + match self.current_transaction.take() { + None => false, + Some(old) => { + if !Arc::ptr_eq(transaction, &old) { + self.current_transaction = Some(old); + return false; + } + self.current_transaction = old.clone_next(); + true + } + } + } + + fn looper_enter(&mut self) { + self.looper_flags |= LOOPER_ENTERED; + if self.looper_flags & LOOPER_REGISTERED != 0 { + self.looper_flags |= LOOPER_INVALID; + } + } + + fn looper_register(&mut self, valid: bool) { + self.looper_flags |= LOOPER_REGISTERED; + if !valid || self.looper_flags & LOOPER_ENTERED != 0 { + self.looper_flags |= LOOPER_INVALID; + } + } + + fn looper_exit(&mut self) { + self.looper_flags |= LOOPER_EXITED; + } + + /// Determines whether the thread is part of a pool, i.e., if it is a looper. + fn is_looper(&self) -> bool { + self.looper_flags & (LOOPER_ENTERED | LOOPER_REGISTERED) != 0 + } + + /// Determines whether the thread should attempt to fetch work items from the process queue + /// (when its own queue is empty). This is case when the thread is not part of a transaction + /// stack and it is registered as a looper. + fn should_use_process_work_queue(&self) -> bool { + self.current_transaction.is_none() && self.is_looper() + } + + fn poll(&mut self) -> u32 { + self.looper_flags |= LOOPER_POLL; + if self.has_work() { + bindings::POLLIN + } else { + 0 + } + } +} + +pub(crate) struct Thread { + pub(crate) id: i32, + pub(crate) process: Ref<Process>, + inner: SpinLock<InnerThread>, + work_condvar: CondVar, + links: Links<Thread>, +} + +impl Thread { + pub(crate) fn new(id: i32, process: Ref<Process>) -> KernelResult<Arc<Self>> { + let return_work = Arc::try_new(ThreadError::new(InnerThread::set_return_work))?; + let reply_work = Arc::try_new(ThreadError::new(InnerThread::set_reply_work))?; + let mut arc = Arc::try_new(Self { + id, + process, + // SAFETY: `inner` is initialised in the call to `spinlock_init` below. + inner: unsafe { SpinLock::new(InnerThread::new()) }, + // SAFETY: `work_condvar` is initalised in the call to `condvar_init` below. + work_condvar: unsafe { CondVar::new() }, + links: Links::new(), + })?; + { + let mut inner = arc.inner.lock(); + inner.set_reply_work(reply_work); + inner.set_return_work(return_work); + } + let thread = Arc::get_mut(&mut arc).unwrap(); + // SAFETY: `inner` is pinned behind the `Arc` reference. + let inner = unsafe { Pin::new_unchecked(&thread.inner) }; + kernel::spinlock_init!(inner, "Thread::inner"); + kernel::condvar_init!(thread.pinned_condvar(), "Thread::work_condvar"); + Ok(arc) + } + + fn pinned_condvar(&self) -> Pin<&CondVar> { + unsafe { Pin::new_unchecked(&self.work_condvar) } + } + + pub(crate) fn set_current_transaction(&self, transaction: Arc<Transaction>) { + self.inner.lock().current_transaction = Some(transaction); + } + + /// Attempts to fetch a work item from the thread-local queue. The behaviour if the queue is + /// empty depends on `wait`: if it is true, the function waits for some work to be queued (or a + /// signal); otherwise it returns indicating that none is available. + fn get_work_local(self: &Arc<Self>, wait: bool) -> KernelResult<Arc<dyn DeliverToRead>> { + // Try once if the caller does not want to wait. + if !wait { + return self.inner.lock().pop_work().ok_or(Error::EAGAIN); + } + + // Loop waiting only on the local queue (i.e., not registering with the process queue). + let cv = self.pinned_condvar(); + let mut inner = self.inner.lock(); + loop { + if let Some(work) = inner.pop_work() { + return Ok(work); + } + + inner.looper_flags |= LOOPER_WAITING; + let signal_pending = cv.wait(&mut inner); + inner.looper_flags &= !LOOPER_WAITING; + + if signal_pending { + return Err(Error::ERESTARTSYS); + } + } + } + + /// Attempts to fetch a work item from the thread-local queue, falling back to the process-wide + /// queue if none is available locally. + /// + /// This must only be called when the thread is not participating in a transaction chain. If it + /// is, the local version (`get_work_local`) should be used instead. + fn get_work(self: &Arc<Self>, wait: bool) -> KernelResult<Arc<dyn DeliverToRead>> { + // Try to get work from the thread's work queue, using only a local lock. + { + let mut inner = self.inner.lock(); + if let Some(work) = inner.pop_work() { + return Ok(work); + } + } + + // If the caller doesn't want to wait, try to grab work from the process queue. + // + // We know nothing will have been queued directly to the thread queue because it is not in + // a transaction and it is not in the process' ready list. + if !wait { + return self.process.get_work().ok_or(Error::EAGAIN); + } + + // Get work from the process queue. If none is available, atomically register as ready. + let reg = match self.process.get_work_or_register(self) { + Either::Left(work) => return Ok(work), + Either::Right(reg) => reg, + }; + + let cv = self.pinned_condvar(); + let mut inner = self.inner.lock(); + loop { + if let Some(work) = inner.pop_work() { + return Ok(work); + } + + inner.looper_flags |= LOOPER_WAITING; + let signal_pending = cv.wait(&mut inner); + inner.looper_flags &= !LOOPER_WAITING; + + if signal_pending { + // A signal is pending. We need to pull the thread off the list, then check the + // state again after it's off the list to ensure that something was not queued in + // the meantime. If something has been queued, we just return it (instead of the + // error). + drop(inner); + drop(reg); + return self.inner.lock().pop_work().ok_or(Error::ERESTARTSYS); + } + } + } + + pub(crate) fn push_work(&self, work: Arc<dyn DeliverToRead>) -> BinderResult { + { + let mut inner = self.inner.lock(); + if inner.is_dead { + return Err(BinderError::new_dead()); + } + inner.push_work(work); + } + self.pinned_condvar().notify_one(); + Ok(()) + } + + /// Attempts to push to given work item to the thread if it's a looper thread (i.e., if it's + /// part of a thread pool) and is alive. Otherwise, push the work item to the process instead. + pub(crate) fn push_work_if_looper(&self, work: Arc<dyn DeliverToRead>) -> BinderResult { + let mut inner = self.inner.lock(); + if inner.is_looper() && !inner.is_dead { + inner.push_work(work); + Ok(()) + } else { + drop(inner); + self.process.push_work(work) + } + } + + pub(crate) fn push_work_deferred(&self, work: Arc<dyn DeliverToRead>) { + self.inner.lock().push_work_deferred(work); + } + + fn translate_object( + &self, + index_offset: usize, + alloc: &Allocation, + view: &AllocationView, + ) -> BinderResult { + let offset = alloc.read(index_offset)?; + let header = view.read::<bindings::binder_object_header>(offset)?; + // TODO: Handle other types. + match header.type_ { + bindings::BINDER_TYPE_WEAK_BINDER | bindings::BINDER_TYPE_BINDER => { + let strong = header.type_ == bindings::BINDER_TYPE_BINDER; + view.transfer_binder_object(offset, strong, |obj| { + // SAFETY: The type is `BINDER_TYPE_{WEAK_}BINDER`, so `binder` is populated. + let ptr = unsafe { obj.__bindgen_anon_1.binder } as _; + let cookie = obj.cookie as _; + Ok(self.process.get_node(ptr, cookie, strong, Some(self))?) + })?; + } + bindings::BINDER_TYPE_WEAK_HANDLE | bindings::BINDER_TYPE_HANDLE => { + let strong = header.type_ == bindings::BINDER_TYPE_HANDLE; + view.transfer_binder_object(offset, strong, |obj| { + // SAFETY: The type is `BINDER_TYPE_{WEAK_}HANDLE`, so `handle` is populated. + let handle = unsafe { obj.__bindgen_anon_1.handle } as _; + self.process.get_node_from_handle(handle, strong) + })?; + } + _ => pr_warn!("Unsupported binder object type: {:x}\n", header.type_), + } + Ok(()) + } + + fn translate_objects(&self, alloc: &mut Allocation, start: usize, end: usize) -> BinderResult { + let view = AllocationView::new(&alloc, start); + for i in (start..end).step_by(size_of::<usize>()) { + if let Err(err) = self.translate_object(i, alloc, &view) { + alloc.set_info(AllocationInfo { offsets: start..i }); + return Err(err); + } + } + alloc.set_info(AllocationInfo { + offsets: start..end, + }); + Ok(()) + } + + pub(crate) fn copy_transaction_data<'a>( + &self, + to_process: &'a Process, + tr: &BinderTransactionData, + ) -> BinderResult<Allocation<'a>> { + let data_size = tr.data_size as _; + let adata_size = ptr_align(data_size); + let offsets_size = tr.offsets_size as _; + let aoffsets_size = ptr_align(offsets_size); + + // This guarantees that at least `sizeof(usize)` bytes will be allocated. + let len = core::cmp::max( + adata_size.checked_add(aoffsets_size).ok_or(Error::ENOMEM)?, + size_of::<usize>(), + ); + let mut alloc = to_process.buffer_alloc(len)?; + + // Copy raw data. + let mut reader = unsafe { UserSlicePtr::new(tr.data.ptr.buffer as _, data_size) }.reader(); + alloc.copy_into(&mut reader, 0, data_size)?; + + // Copy offsets if there are any. + if offsets_size > 0 { + let mut reader = + unsafe { UserSlicePtr::new(tr.data.ptr.offsets as _, offsets_size) }.reader(); + alloc.copy_into(&mut reader, adata_size, offsets_size)?; + + // Traverse the objects specified. + self.translate_objects(&mut alloc, adata_size, adata_size + aoffsets_size)?; + } + + Ok(alloc) + } + + fn unwind_transaction_stack(self: &Arc<Self>) { + let mut thread = self.clone(); + while let Ok(transaction) = { + let mut inner = thread.inner.lock(); + inner.pop_transaction_to_reply(thread.as_ref()) + } { + let reply = Either::Right(BR_DEAD_REPLY); + if !transaction.from.deliver_single_reply(reply, &transaction) { + break; + } + + thread = transaction.from.clone(); + } + } + + pub(crate) fn deliver_reply( + &self, + reply: Either<Arc<Transaction>, u32>, + transaction: &Arc<Transaction>, + ) { + if self.deliver_single_reply(reply, transaction) { + transaction.from.unwind_transaction_stack(); + } + } + + /// Delivers a reply to the thread that started a transaction. The reply can either be a + /// reply-transaction or an error code to be delivered instead. + /// + /// Returns whether the thread is dead. If it is, the caller is expected to unwind the + /// transaction stack by completing transactions for threads that are dead. + fn deliver_single_reply( + &self, + reply: Either<Arc<Transaction>, u32>, + transaction: &Arc<Transaction>, + ) -> bool { + { + let mut inner = self.inner.lock(); + if !inner.pop_transaction_replied(transaction) { + return false; + } + + if inner.is_dead { + return true; + } + + match reply { + Either::Left(work) => inner.push_work(work), + Either::Right(code) => inner.push_reply_work(code), + } + } + + // Notify the thread now that we've released the inner lock. + self.pinned_condvar().notify_one(); + false + } + + /// Determines if the given transaction is the current transaction for this thread. + fn is_current_transaction(&self, transaction: &Arc<Transaction>) -> bool { + let inner = self.inner.lock(); + match &inner.current_transaction { + None => false, + Some(current) => Arc::ptr_eq(current, transaction), + } + } + + fn transaction<T>(self: &Arc<Self>, tr: &BinderTransactionData, inner: T) + where + T: FnOnce(&Arc<Self>, &BinderTransactionData) -> BinderResult, + { + if let Err(err) = inner(self, tr) { + self.inner.lock().push_return_work(err.reply); + } + } + + fn reply_inner(self: &Arc<Self>, tr: &BinderTransactionData) -> BinderResult { + let orig = self.inner.lock().pop_transaction_to_reply(self)?; + if !orig.from.is_current_transaction(&orig) { + return Err(BinderError::new_failed()); + } + + // We need to complete the transaction even if we cannot complete building the reply. + (|| -> BinderResult<_> { + let completion = Arc::try_new(DeliverCode::new(BR_TRANSACTION_COMPLETE))?; + let process = orig.from.process.clone(); + let reply = Arc::try_new(Transaction::new_reply(self, process, tr)?)?; + self.inner.lock().push_work(completion); + orig.from.deliver_reply(Either::Left(reply), &orig); + Ok(()) + })() + .map_err(|mut err| { + // At this point we only return `BR_TRANSACTION_COMPLETE` to the caller, and we must let + // the sender know that the transaction has completed (with an error in this case). + let reply = Either::Right(BR_FAILED_REPLY); + orig.from.deliver_reply(reply, &orig); + err.reply = BR_TRANSACTION_COMPLETE; + err + }) + } + + /// Determines the current top of the transaction stack. It fails if the top is in another + /// thread (i.e., this thread belongs to a stack but it has called another thread). The top is + /// [`None`] if the thread is not currently participating in a transaction stack. + fn top_of_transaction_stack(&self) -> KernelResult<Option<Arc<Transaction>>> { + let inner = self.inner.lock(); + Ok(if let Some(cur) = &inner.current_transaction { + if core::ptr::eq(self, cur.from.as_ref()) { + return Err(Error::EINVAL); + } + Some(cur.clone()) + } else { + None + }) + } + + fn oneway_transaction_inner(self: &Arc<Self>, tr: &BinderTransactionData) -> BinderResult { + let handle = unsafe { tr.target.handle }; + let node_ref = self.process.get_transaction_node(handle)?; + let completion = Arc::try_new(DeliverCode::new(BR_TRANSACTION_COMPLETE))?; + let transaction = Arc::try_new(Transaction::new(node_ref, None, self, tr)?)?; + self.inner.lock().push_work(completion); + // TODO: Remove the completion on error? + transaction.submit()?; + Ok(()) + } + + fn transaction_inner(self: &Arc<Self>, tr: &BinderTransactionData) -> BinderResult { + let handle = unsafe { tr.target.handle }; + let node_ref = self.process.get_transaction_node(handle)?; + // TODO: We need to ensure that there isn't a pending transaction in the work queue. How + // could this happen? + let top = self.top_of_transaction_stack()?; + let completion = Arc::try_new(DeliverCode::new(BR_TRANSACTION_COMPLETE))?; + let transaction = Arc::try_new(Transaction::new(node_ref, top, self, tr)?)?; + + // Check that the transaction stack hasn't changed while the lock was released, then update + // it with the new transaction. + { + let mut inner = self.inner.lock(); + if !transaction.is_stacked_on(&inner.current_transaction) { + return Err(BinderError::new_failed()); + } + inner.current_transaction = Some(transaction.clone()); + } + + // We push the completion as a deferred work so that we wait for the reply before returning + // to userland. + self.push_work_deferred(completion); + // TODO: Remove completion if submission fails? + transaction.submit()?; + Ok(()) + } + + fn write(self: &Arc<Self>, req: &mut BinderWriteRead) -> KernelResult { + let write_start = req.write_buffer.wrapping_add(req.write_consumed); + let write_len = req.write_size - req.write_consumed; + let mut reader = unsafe { UserSlicePtr::new(write_start as _, write_len as _).reader() }; + + while reader.len() >= size_of::<u32>() && self.inner.lock().return_work.is_some() { + let before = reader.len(); + match reader.read::<u32>()? { + BC_TRANSACTION => { + let tr = reader.read::<BinderTransactionData>()?; + if tr.flags & bindings::transaction_flags_TF_ONE_WAY != 0 { + self.transaction(&tr, Self::oneway_transaction_inner) + } else { + self.transaction(&tr, Self::transaction_inner) + } + } + BC_REPLY => self.transaction(&reader.read()?, Self::reply_inner), + BC_FREE_BUFFER => drop(self.process.buffer_get(reader.read()?)), + BC_INCREFS => self.process.update_ref(reader.read()?, true, false)?, + BC_ACQUIRE => self.process.update_ref(reader.read()?, true, true)?, + BC_RELEASE => self.process.update_ref(reader.read()?, false, true)?, + BC_DECREFS => self.process.update_ref(reader.read()?, false, false)?, + BC_INCREFS_DONE => self.process.inc_ref_done(&mut reader, false)?, + BC_ACQUIRE_DONE => self.process.inc_ref_done(&mut reader, true)?, + BC_REQUEST_DEATH_NOTIFICATION => self.process.request_death(&mut reader, self)?, + BC_CLEAR_DEATH_NOTIFICATION => self.process.clear_death(&mut reader, self)?, + BC_DEAD_BINDER_DONE => self.process.dead_binder_done(reader.read()?, self), + BC_REGISTER_LOOPER => { + let valid = self.process.register_thread(); + self.inner.lock().looper_register(valid); + } + BC_ENTER_LOOPER => self.inner.lock().looper_enter(), + BC_EXIT_LOOPER => self.inner.lock().looper_exit(), + + // TODO: Add support for BC_TRANSACTION_SG and BC_REPLY_SG. + // BC_ATTEMPT_ACQUIRE and BC_ACQUIRE_RESULT are no longer supported. + _ => return Err(Error::EINVAL), + } + + // Update the number of write bytes consumed. + req.write_consumed += (before - reader.len()) as u64; + } + Ok(()) + } + + fn read(self: &Arc<Self>, req: &mut BinderWriteRead, wait: bool) -> KernelResult { + let read_start = req.read_buffer.wrapping_add(req.read_consumed); + let read_len = req.read_size - req.read_consumed; + let mut writer = unsafe { UserSlicePtr::new(read_start as _, read_len as _) }.writer(); + let (in_pool, getter) = { + let inner = self.inner.lock(); + ( + inner.is_looper(), + if inner.should_use_process_work_queue() { + Self::get_work + } else { + Self::get_work_local + }, + ) + }; + + // Reserve some room at the beginning of the read buffer so that we can send a + // BR_SPAWN_LOOPER if we need to. + if req.read_consumed == 0 { + writer.write(&BR_NOOP)?; + } + + // Loop doing work while there is room in the buffer. + let initial_len = writer.len(); + while writer.len() >= size_of::<u32>() { + match getter(self, wait && initial_len == writer.len()) { + Ok(work) => { + if !work.do_work(self, &mut writer)? { + break; + } + } + Err(err) => { + // Propagate the error if we haven't written anything else. + if initial_len == writer.len() { + return Err(err); + } else { + break; + } + } + } + } + + req.read_consumed += read_len - writer.len() as u64; + + // Write BR_SPAWN_LOOPER if the process needs more threads for its pool. + if in_pool && self.process.needs_thread() { + let mut writer = + unsafe { UserSlicePtr::new(req.read_buffer as _, req.read_size as _) }.writer(); + writer.write(&BR_SPAWN_LOOPER)?; + } + + Ok(()) + } + + pub(crate) fn write_read(self: &Arc<Self>, data: UserSlicePtr, wait: bool) -> KernelResult { + let (mut reader, mut writer) = data.reader_writer(); + let mut req = reader.read::<BinderWriteRead>()?; + + // TODO: `write(&req)` happens in all exit paths from here on. Find a better way to encode + // it. + + // Go through the write buffer. + if req.write_size > 0 { + if let Err(err) = self.write(&mut req) { + req.read_consumed = 0; + writer.write(&req)?; + return Err(err); + } + } + + // Go through the work queue. + let mut ret = Ok(()); + if req.read_size > 0 { + ret = self.read(&mut req, wait); + } + + // Write the request back so that the consumed fields are visible to the caller. + writer.write(&req)?; + ret + } + + pub(crate) fn poll(&self, file: &File, table: &PollTable) -> (bool, u32) { + // SAFETY: `free_waiters` is called on release. + unsafe { table.register_wait(file, &self.work_condvar) }; + let mut inner = self.inner.lock(); + (inner.should_use_process_work_queue(), inner.poll()) + } + + pub(crate) fn notify_if_poll_ready(&self) { + // Determine if we need to notify. This requires the lock. + let inner = self.inner.lock(); + let notify = inner.looper_flags & LOOPER_POLL != 0 + && inner.should_use_process_work_queue() + && !inner.has_work(); + drop(inner); + + // Now that the lock is no longer held, notify the waiters if we have to. + if notify { + self.pinned_condvar().notify_one(); + } + } + + pub(crate) fn push_return_work(&self, code: u32) { + self.inner.lock().push_return_work(code) + } + + pub(crate) fn release(self: &Arc<Thread>) { + // Mark the thread as dead. + self.inner.lock().is_dead = true; + + // Cancel all pending work items. + while let Ok(work) = self.get_work_local(false) { + work.cancel(); + } + + // Complete the transaction stack as far as we can. + self.unwind_transaction_stack(); + + // Remove epoll items if polling was ever used on the thread. + let poller = self.inner.lock().looper_flags & LOOPER_POLL != 0; + if poller { + self.pinned_condvar().free_waiters(); + + unsafe { bindings::synchronize_rcu() }; + } + } +} + +impl GetLinks for Thread { + type EntryType = Thread; + fn get_links(data: &Thread) -> &Links<Thread> { + &data.links + } +} + +struct ThreadError { + error_code: u32, + return_fn: fn(&mut InnerThread, Arc<ThreadError>), + links: Links<dyn DeliverToRead>, +} + +impl ThreadError { + fn new(return_fn: fn(&mut InnerThread, Arc<ThreadError>)) -> Self { + Self { + error_code: BR_OK, + return_fn, + links: Links::new(), + } + } +} + +impl DeliverToRead for ThreadError { + fn do_work( + self: Arc<Self>, + thread: &Thread, + writer: &mut UserSlicePtrWriter, + ) -> KernelResult<bool> { + let code = self.error_code; + + // Return the `ThreadError` to the thread. + (self.return_fn)(&mut *thread.inner.lock(), self); + + // Deliver the error code to userspace. + writer.write(&code)?; + Ok(true) + } + + fn get_links(&self) -> &Links<dyn DeliverToRead> { + &self.links + } +} diff --git a/drivers/android/transaction.rs b/drivers/android/transaction.rs new file mode 100644 index 0000000000000..98c2c86e12f01 --- /dev/null +++ b/drivers/android/transaction.rs @@ -0,0 +1,206 @@ +// SPDX-License-Identifier: GPL-2.0 + +use alloc::sync::Arc; +use core::sync::atomic::{AtomicBool, Ordering}; +use kernel::{bindings, linked_list::Links, prelude::*, sync::Ref, user_ptr::UserSlicePtrWriter}; + +use crate::{ + defs::*, + node::NodeRef, + process::Process, + ptr_align, + thread::{BinderResult, Thread}, + DeliverToRead, Either, +}; + +pub(crate) struct Transaction { + // TODO: Node should be released when the buffer is released. + node_ref: Option<NodeRef>, + stack_next: Option<Arc<Transaction>>, + pub(crate) from: Arc<Thread>, + to: Ref<Process>, + free_allocation: AtomicBool, + code: u32, + flags: u32, + data_size: usize, + offsets_size: usize, + data_address: usize, + links: Links<dyn DeliverToRead>, +} + +impl Transaction { + pub(crate) fn new( + node_ref: NodeRef, + stack_next: Option<Arc<Transaction>>, + from: &Arc<Thread>, + tr: &BinderTransactionData, + ) -> BinderResult<Self> { + let to = node_ref.node.owner.clone(); + let alloc = from.copy_transaction_data(&to, tr)?; + let data_address = alloc.ptr; + alloc.keep_alive(); + Ok(Self { + node_ref: Some(node_ref), + stack_next, + from: from.clone(), + to, + code: tr.code, + flags: tr.flags, + data_size: tr.data_size as _, + data_address, + offsets_size: tr.offsets_size as _, + links: Links::new(), + free_allocation: AtomicBool::new(true), + }) + } + + pub(crate) fn new_reply( + from: &Arc<Thread>, + to: Ref<Process>, + tr: &BinderTransactionData, + ) -> BinderResult<Self> { + let alloc = from.copy_transaction_data(&to, tr)?; + let data_address = alloc.ptr; + alloc.keep_alive(); + Ok(Self { + node_ref: None, + stack_next: None, + from: from.clone(), + to, + code: tr.code, + flags: tr.flags, + data_size: tr.data_size as _, + data_address, + offsets_size: tr.offsets_size as _, + links: Links::new(), + free_allocation: AtomicBool::new(true), + }) + } + + /// Determines if the transaction is stacked on top of the given transaction. + pub(crate) fn is_stacked_on(&self, onext: &Option<Arc<Self>>) -> bool { + match (&self.stack_next, onext) { + (None, None) => true, + (Some(stack_next), Some(next)) => Arc::ptr_eq(stack_next, next), + _ => false, + } + } + + /// Returns a pointer to the next transaction on the transaction stack, if there is one. + pub(crate) fn clone_next(&self) -> Option<Arc<Self>> { + let next = self.stack_next.as_ref()?; + Some(next.clone()) + } + + /// Searches in the transaction stack for a thread that belongs to the target process. This is + /// useful when finding a target for a new transaction: if the node belongs to a process that + /// is already part of the transaction stack, we reuse the thread. + fn find_target_thread(&self) -> Option<Arc<Thread>> { + let process = &self.node_ref.as_ref()?.node.owner; + + let mut it = &self.stack_next; + while let Some(transaction) = it { + if Ref::ptr_eq(&transaction.from.process, process) { + return Some(transaction.from.clone()); + } + it = &transaction.stack_next; + } + None + } + + /// Searches in the transaction stack for a transaction originating at the given thread. + pub(crate) fn find_from(&self, thread: &Thread) -> Option<Arc<Transaction>> { + let mut it = &self.stack_next; + while let Some(transaction) = it { + if core::ptr::eq(thread, transaction.from.as_ref()) { + return Some(transaction.clone()); + } + + it = &transaction.stack_next; + } + None + } + + /// Submits the transaction to a work queue. Use a thread if there is one in the transaction + /// stack, otherwise use the destination process. + pub(crate) fn submit(self: Arc<Self>) -> BinderResult { + if let Some(thread) = self.find_target_thread() { + thread.push_work(self) + } else { + let process = self.to.clone(); + process.push_work(self) + } + } +} + +impl DeliverToRead for Transaction { + fn do_work( + self: Arc<Self>, + thread: &Thread, + writer: &mut UserSlicePtrWriter, + ) -> KernelResult<bool> { + /* TODO: Initialise the following fields from tr: + pub sender_pid: pid_t, + pub sender_euid: uid_t, + */ + let mut tr = BinderTransactionData::default(); + + if let Some(nref) = &self.node_ref { + let (ptr, cookie) = nref.node.get_id(); + tr.target.ptr = ptr as _; + tr.cookie = cookie as _; + }; + + tr.code = self.code; + tr.flags = self.flags; + tr.data_size = self.data_size as _; + tr.data.ptr.buffer = self.data_address as _; + tr.offsets_size = self.offsets_size as _; + if tr.offsets_size > 0 { + tr.data.ptr.offsets = (self.data_address + ptr_align(self.data_size)) as _; + } + + // When `drop` is called, we don't want the allocation to be freed because it is now the + // user's reponsibility to free it. + self.free_allocation.store(false, Ordering::Relaxed); + + let code = if self.node_ref.is_none() { + BR_REPLY + } else { + BR_TRANSACTION + }; + + // Write the transaction code and data to the user buffer. On failure we complete the + // transaction with an error. + if let Err(err) = writer.write(&code).and_then(|_| writer.write(&tr)) { + let reply = Either::Right(BR_FAILED_REPLY); + self.from.deliver_reply(reply, &self); + return Err(err); + } + + // When this is not a reply and not an async transaction, update `current_transaction`. If + // it's a reply, `current_transaction` has already been updated appropriately. + if self.node_ref.is_some() && tr.flags & bindings::transaction_flags_TF_ONE_WAY == 0 { + thread.set_current_transaction(self); + } + + Ok(false) + } + + fn cancel(self: Arc<Self>) { + let reply = Either::Right(BR_DEAD_REPLY); + self.from.deliver_reply(reply, &self); + } + + fn get_links(&self) -> &Links<dyn DeliverToRead> { + &self.links + } +} + +impl Drop for Transaction { + fn drop(&mut self) { + if self.free_allocation.load(Ordering::Relaxed) { + self.to.buffer_get(self.data_address); + } + } +} diff --git a/include/linux/kallsyms.h b/include/linux/kallsyms.h index 465060acc9816..5cdc6903abca3 100644 --- a/include/linux/kallsyms.h +++ b/include/linux/kallsyms.h @@ -14,7 +14,7 @@ #include <asm/sections.h> -#define KSYM_NAME_LEN 128 +#define KSYM_NAME_LEN 512 #define KSYM_SYMBOL_LEN (sizeof("%s+%#lx/%#lx [%s]") + (KSYM_NAME_LEN - 1) + \ 2*(BITS_PER_LONG*3/10) + (MODULE_NAME_LEN - 1) + 1) diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 79897841a2cc8..a022992725be3 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -331,12 +331,17 @@ static __always_inline raw_spinlock_t *spinlock_check(spinlock_t *lock) #ifdef CONFIG_DEBUG_SPINLOCK -# define spin_lock_init(lock) \ -do { \ - static struct lock_class_key __key; \ - \ - __raw_spin_lock_init(spinlock_check(lock), \ - #lock, &__key, LD_WAIT_CONFIG); \ +static inline void __spin_lock_init(spinlock_t *lock, const char *name, + struct lock_class_key *key) +{ + __raw_spin_lock_init(spinlock_check(lock), name, key, LD_WAIT_CONFIG); +} + +# define spin_lock_init(lock) \ +do { \ + static struct lock_class_key __key; \ + \ + __spin_lock_init(lock, #lock, &__key); \ } while (0) #else diff --git a/include/uapi/linux/android/binder.h b/include/uapi/linux/android/binder.h index 20e435fe657a1..9fca291e01324 100644 --- a/include/uapi/linux/android/binder.h +++ b/include/uapi/linux/android/binder.h @@ -229,19 +229,21 @@ struct binder_frozen_status_info { __u32 async_recv; }; -#define BINDER_WRITE_READ _IOWR('b', 1, struct binder_write_read) -#define BINDER_SET_IDLE_TIMEOUT _IOW('b', 3, __s64) -#define BINDER_SET_MAX_THREADS _IOW('b', 5, __u32) -#define BINDER_SET_IDLE_PRIORITY _IOW('b', 6, __s32) -#define BINDER_SET_CONTEXT_MGR _IOW('b', 7, __s32) -#define BINDER_THREAD_EXIT _IOW('b', 8, __s32) -#define BINDER_VERSION _IOWR('b', 9, struct binder_version) -#define BINDER_GET_NODE_DEBUG_INFO _IOWR('b', 11, struct binder_node_debug_info) -#define BINDER_GET_NODE_INFO_FOR_REF _IOWR('b', 12, struct binder_node_info_for_ref) -#define BINDER_SET_CONTEXT_MGR_EXT _IOW('b', 13, struct flat_binder_object) -#define BINDER_FREEZE _IOW('b', 14, struct binder_freeze_info) -#define BINDER_GET_FROZEN_INFO _IOWR('b', 15, struct binder_frozen_status_info) -#define BINDER_ENABLE_ONEWAY_SPAM_DETECTION _IOW('b', 16, __u32) +enum { + BINDER_WRITE_READ = _IOWR('b', 1, struct binder_write_read), + BINDER_SET_IDLE_TIMEOUT = _IOW('b', 3, __s64), + BINDER_SET_MAX_THREADS = _IOW('b', 5, __u32), + BINDER_SET_IDLE_PRIORITY = _IOW('b', 6, __s32), + BINDER_SET_CONTEXT_MGR = _IOW('b', 7, __s32), + BINDER_THREAD_EXIT = _IOW('b', 8, __s32), + BINDER_VERSION = _IOWR('b', 9, struct binder_version), + BINDER_GET_NODE_DEBUG_INFO = _IOWR('b', 11, struct binder_node_debug_info), + BINDER_GET_NODE_INFO_FOR_REF = _IOWR('b', 12, struct binder_node_info_for_ref), + BINDER_SET_CONTEXT_MGR_EXT = _IOW('b', 13, struct flat_binder_object), + BINDER_FREEZE = _IOW('b', 14, struct binder_freeze_info), + BINDER_GET_FROZEN_INFO = _IOWR('b', 15, struct binder_frozen_status_info), + BINDER_ENABLE_ONEWAY_SPAM_DETECTION = _IOW('b', 16, __u32), +}; /* * NOTE: Two special error codes you should check for when calling diff --git a/init/Kconfig b/init/Kconfig index a61c92066c2e4..b075234001a36 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -60,6 +60,15 @@ config LLD_VERSION default $(ld-version) if LD_IS_LLD default 0 +config HAS_RUST + depends on ARM64 || CPU_32v6 || CPU_32v6K || (PPC64 && CPU_LITTLE_ENDIAN) || X86_64 + def_bool $(success,$(RUSTC) --version) + +config RUSTC_VERSION + depends on HAS_RUST + int + default $(shell,$(srctree)/scripts/rust-version.sh $(RUSTC)) + config CC_CAN_LINK bool default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(m64-flag)) if 64BIT @@ -2011,6 +2020,24 @@ config PROFILING Say Y here to enable the extended profiling support mechanisms used by profilers. +config RUST + bool "Rust support" + depends on HAS_RUST + depends on !COMPILE_TEST + default n + help + Enables Rust support in the kernel. + + This allows other Rust-related options, like drivers written in Rust, + to be selected. + + It is also required to be able to load external kernel modules + written in Rust. + + See Documentation/rust/ for more information. + + If unsure, say N. + # # Place an empty function call at each tracepoint site. Can be # dynamically changed for a probe function. diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c index c851ca0ed3576..9d0c23e1993c1 100644 --- a/kernel/kallsyms.c +++ b/kernel/kallsyms.c @@ -73,6 +73,13 @@ static unsigned int kallsyms_expand_symbol(unsigned int off, */ off += len + 1; + /* If zero, it is a "big" symbol, so a two byte length follows. */ + if (len == 0) { + len = (data[0] << 8) | data[1]; + data += 2; + off += len + 2; + } + /* * For every byte on the compressed symbol data, copy the table * entry for that byte. diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index 335d988bd8111..73874e5edfda6 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -213,7 +213,7 @@ static int klp_resolve_symbols(Elf64_Shdr *sechdrs, const char *strtab, * we use the smallest/strictest upper bound possible (56, based on * the current definition of MODULE_NAME_LEN) to prevent overflows. */ - BUILD_BUG_ON(MODULE_NAME_LEN < 56 || KSYM_NAME_LEN != 128); + BUILD_BUG_ON(MODULE_NAME_LEN < 56 || KSYM_NAME_LEN != 512); relas = (Elf_Rela *) relasec->sh_addr; /* For each rela in this klp relocation section */ @@ -227,7 +227,7 @@ static int klp_resolve_symbols(Elf64_Shdr *sechdrs, const char *strtab, /* Format: .klp.sym.sym_objname.sym_name,sympos */ cnt = sscanf(strtab + sym->st_name, - ".klp.sym.%55[^.].%127[^,],%lu", + ".klp.sym.%55[^.].%511[^,],%lu", sym_objname, sym_name, &sympos); if (cnt != 3) { pr_err("symbol %s has an incorrectly formatted name\n", diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 421c35571797e..114e9963f903f 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -398,6 +398,7 @@ static struct latched_seq clear_seq = { #define CONSOLE_LOG_MAX 1024 /* the maximum size allowed to be reserved for a record */ +/* Keep in sync with rust/kernel/print.rs */ #define LOG_LINE_MAX (CONSOLE_LOG_MAX - PREFIX_MAX) #define LOG_LEVEL(v) ((v) & 0x07) diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index a6b68ee531605..35540ac7388e9 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2616,6 +2616,112 @@ config HYPERV_TESTING endmenu # "Kernel Testing and Coverage" +menu "Rust hacking" + +config RUST_DEBUG_ASSERTIONS + bool "Debug assertions" + default n + depends on RUST + help + Enables rustc's `-Cdebug-assertions` codegen option. + + This flag lets you turn `cfg(debug_assertions)` conditional + compilation on or off. This can be used to enable extra debugging + code in development but not in production. For example, it controls + the behavior of the standard library's `debug_assert!` macro. + + Note that this will apply to all Rust code, including `core`. + + If unsure, say N. + +config RUST_OVERFLOW_CHECKS + bool "Overflow checks" + default y + depends on RUST + help + Enables rustc's `-Coverflow-checks` codegen option. + + This flag allows you to control the behavior of runtime integer + overflow. When overflow-checks are enabled, a panic will occur + on overflow. + + Note that this will apply to all Rust code, including `core`. + + If unsure, say Y. + +choice + prompt "Optimization level" + default RUST_OPT_LEVEL_SIMILAR_AS_CHOSEN_FOR_C + depends on RUST + help + Controls rustc's `-Copt-level` codegen option. + + This flag controls the optimization level. + + If unsure, say "Similar as chosen for C". + +config RUST_OPT_LEVEL_SIMILAR_AS_CHOSEN_FOR_C + bool "Similar as chosen for C" + help + This choice will pick a similar optimization level as chosen in + the "Compiler optimization level" for C: + + -O2 is currently mapped to -Copt-level=2 + -O3 is currently mapped to -Copt-level=3 + -Os is currently mapped to -Copt-level=z + + The mapping may change over time to follow the intended semantics + of the choice for C as sensibly as possible. + + This is the default. + +config RUST_OPT_LEVEL_0 + bool "No optimizations (-Copt-level=0)" + help + Not recommended for most purposes. It may come in handy for debugging + suspected optimizer bugs, unexpected undefined behavior, etc. + + Note that this level will *not* enable debug assertions nor overflow + checks on its own (like it happens when interacting with rustc + directly). Use the corresponding configuration options to control + that instead, orthogonally. + + Note this level may cause excessive stack usage, which can lead to stack + overflow and subsequent crashes. + +config RUST_OPT_LEVEL_1 + bool "Basic optimizations (-Copt-level=1)" + help + Useful for debugging without getting too lost, but without + the overhead and boilerplate of no optimizations at all. + + Note this level may cause excessive stack usage, which can lead to stack + overflow and subsequent crashes. + +config RUST_OPT_LEVEL_2 + bool "Some optimizations (-Copt-level=2)" + help + The sensible choice in most cases. + +config RUST_OPT_LEVEL_3 + bool "All optimizations (-Copt-level=3)" + help + Yet more performance (hopefully). + +config RUST_OPT_LEVEL_S + bool "Optimize for size (-Copt-level=s)" + help + Smaller kernel, ideally without too much performance loss. + +config RUST_OPT_LEVEL_Z + bool "Optimize for size, no loop vectorization (-Copt-level=z)" + help + Like the previous level, but also turn off loop vectorization. + +endchoice + +endmenu # "Rust" + source "Documentation/Kconfig" endmenu # Kernel hacking diff --git a/rust/.gitignore b/rust/.gitignore new file mode 100644 index 0000000000000..8875e08ed0b17 --- /dev/null +++ b/rust/.gitignore @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 + +bindings_generated.rs +exports_*_generated.h +doc/
\ No newline at end of file diff --git a/rust/Makefile b/rust/Makefile new file mode 100644 index 0000000000000..2d1d9d7af77ca --- /dev/null +++ b/rust/Makefile @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_RUST) += core.o compiler_builtins.o helpers.o +extra-$(CONFIG_RUST) += exports_core_generated.h + +extra-$(CONFIG_RUST) += libmodule.so + +extra-$(CONFIG_RUST) += bindings_generated.rs +obj-$(CONFIG_RUST) += alloc.o kernel.o +extra-$(CONFIG_RUST) += exports_alloc_generated.h exports_kernel_generated.h + +obj-$(CONFIG_RUST) += exports.o + +RUSTDOC = rustdoc + +quiet_cmd_rustdoc = RUSTDOC $< + cmd_rustdoc = \ + RUST_BINDINGS_FILE=$(abspath $(objtree)/rust/bindings_generated.rs) \ + $(RUSTDOC) $(filter-out --emit=%, $(rustc_flags)) \ + $(rustdoc_target_flags) -L $(objtree)/rust/ \ + --output $(objtree)/rust/doc --crate-name $(subst rustdoc-,,$@) \ + -Fmissing-docs @$(objtree)/include/generated/rustc_cfg $< + +rustdoc: rustdoc-module rustdoc-compiler_builtins rustdoc-kernel + +rustdoc-module: private rustdoc_target_flags = --crate-type proc-macro \ + --extern proc_macro +rustdoc-module: $(srctree)/rust/module.rs FORCE + $(call if_changed,rustdoc) + +rustdoc-compiler_builtins: $(srctree)/rust/compiler_builtins.rs FORCE + $(call if_changed,rustdoc) + +rustdoc-kernel: private rustdoc_target_flags = --extern alloc \ + --extern module=$(objtree)/rust/libmodule.so +rustdoc-kernel: $(srctree)/rust/kernel/lib.rs rustdoc-module \ + $(objtree)/rust/libmodule.so $(objtree)/rust/bindings_generated.rs FORCE + $(call if_changed,rustdoc) + +ifdef CONFIG_CC_IS_CLANG +bindgen_c_flags = $(c_flags) +else +# bindgen relies on libclang to parse C. Ideally, bindgen would support a GCC +# plugin backend and/or the Clang driver would be perfectly compatible with GCC. +# +# For the moment, here we are tweaking the flags on the fly. Some config +# options may not work (e.g. `GCC_PLUGIN_RANDSTRUCT` if we end up using one +# of those structs). We might want to redo how Clang flags are kept track of +# in the general `Makefile` even for GCC builds, similar to what we did with +# `TENTATIVE_CLANG_FLAGS`. +bindgen_skip_c_flags := -mno-fp-ret-in-387 -mpreferred-stack-boundary=% \ + -mskip-rax-setup -mgeneral-regs-only -msign-return-address=% \ + -mindirect-branch=thunk-extern -mindirect-branch-register -mrecord-mcount \ + -mabi=lp64 -mstack-protector-guard% -fconserve-stack -falign-jumps=% \ + -falign-loops=% -fno-ipa-cp-clone -fno-partial-inlining \ + -fno-reorder-blocks -fno-allow-store-data-races -fasan-shadow-offset=% \ + -Wno-packed-not-aligned -Wno-format-truncation -Wno-format-overflow \ + -Wno-stringop-truncation -Wno-unused-but-set-variable \ + -Wno-stringop-overflow -Wno-restrict -Wno-maybe-uninitialized \ + -Werror=designated-init -Wno-zero-length-bounds \ + --param=% --param asan-% + +# PowerPC +bindgen_skip_c_flags += -mtraceback=no -mno-pointers-to-nested-functions \ + -mno-string -mno-strict-align + +bindgen_extra_c_flags = $(TENTATIVE_CLANG_FLAGS) -Wno-address-of-packed-member +bindgen_c_flags = $(filter-out $(bindgen_skip_c_flags), $(c_flags)) \ + $(bindgen_extra_c_flags) +endif + +bindgen_opaque_types := xregs_state desc_struct arch_lbr_state + +# To avoid several recompilations in PowerPC, which inserts `-D_TASK_CPU` +bindgen_c_flags_final = $(filter-out -D_TASK_CPU=%, $(bindgen_c_flags)) + +quiet_cmd_bindgen = BINDGEN $@ + cmd_bindgen = \ + $(BINDGEN) $< $(addprefix --opaque-type , $(bindgen_opaque_types)) \ + --use-core --with-derive-default --ctypes-prefix c_types \ + --size_t-is-usize -o $@ -- $(bindgen_c_flags_final) -DMODULE + +$(objtree)/rust/bindings_generated.rs: $(srctree)/rust/kernel/bindings_helper.h FORCE + $(call if_changed_dep,bindgen) + +quiet_cmd_exports = EXPORTS $@ + cmd_exports = \ + $(NM) -p --defined-only $< \ + | grep -E ' (T|R|D) ' | cut -d ' ' -f 3 | grep -E '^(__rust_|_R)' \ + | xargs -Isymbol \ + echo 'EXPORT_SYMBOL_RUST_GPL(symbol);' > $@ + +$(objtree)/rust/exports_core_generated.h: $(objtree)/rust/core.o FORCE + $(call if_changed,exports) + +$(objtree)/rust/exports_alloc_generated.h: $(objtree)/rust/alloc.o FORCE + $(call if_changed,exports) + +$(objtree)/rust/exports_kernel_generated.h: $(objtree)/rust/kernel.o FORCE + $(call if_changed,exports) + +# `-Cpanic=unwind -Cforce-unwind-tables=y` overrides `rustc_flags` in order to +# avoid the https://github.com/rust-lang/rust/issues/82320 rustc crash. +quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@ + cmd_rustc_procmacro = \ + $(RUSTC_OR_CLIPPY) $(rustc_flags) \ + --emit=dep-info,link --extern proc_macro \ + -Cpanic=unwind -Cforce-unwind-tables=y \ + --crate-type proc-macro --out-dir $(objtree)/rust/ \ + --crate-name $(patsubst lib%.so,%,$(notdir $@)) $<; \ + mv $(objtree)/rust/$(patsubst lib%.so,%,$(notdir $@)).d $(depfile); \ + sed -i '/^\#/d' $(depfile) + +$(objtree)/rust/libmodule.so: $(srctree)/rust/module.rs FORCE + $(call if_changed_dep,rustc_procmacro) + +quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L $@ + cmd_rustc_library = \ + RUST_BINDINGS_FILE=$(abspath $(objtree)/rust/bindings_generated.rs) \ + $(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \ + $(rustc_flags) $(rustc_cross_flags) $(rustc_target_flags) \ + --crate-type rlib --out-dir $(objtree)/rust/ -L $(objtree)/rust/ \ + --crate-name $(patsubst %.o,%,$(notdir $@)) $<; \ + mv $(objtree)/rust/$(patsubst %.o,%,$(notdir $@)).d $(depfile); \ + sed -i '/^\#/d' $(depfile) \ + $(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@) + +# `$(rustc_flags)` is passed in case the user added `--sysroot`. +rustc_sysroot = $(shell $(RUSTC) $(rustc_flags) --print sysroot) +rustc_src = $(rustc_sysroot)/lib/rustlib/src/rust + +.SECONDEXPANSION: +$(objtree)/rust/core.o: private skip_clippy = 1 +$(objtree)/rust/core.o: $$(rustc_src)/library/core/src/lib.rs FORCE + $(call if_changed_dep,rustc_library) + +$(objtree)/rust/compiler_builtins.o: private rustc_objcopy = -w -W '__*' +$(objtree)/rust/compiler_builtins.o: $(srctree)/rust/compiler_builtins.rs \ + $(objtree)/rust/core.o FORCE + $(call if_changed_dep,rustc_library) + +$(objtree)/rust/alloc.o: private skip_clippy = 1 +$(objtree)/rust/alloc.o: $$(rustc_src)/library/alloc/src/lib.rs \ + $(objtree)/rust/compiler_builtins.o FORCE + $(call if_changed_dep,rustc_library) + +# ICE on `--extern module`: https://github.com/rust-lang/rust/issues/56935 +$(objtree)/rust/kernel.o: private rustc_target_flags = --extern alloc \ + --extern module=$(objtree)/rust/libmodule.so +$(objtree)/rust/kernel.o: $(srctree)/rust/kernel/lib.rs $(objtree)/rust/alloc.o \ + $(objtree)/rust/libmodule.so $(objtree)/rust/bindings_generated.rs FORCE + $(call if_changed_dep,rustc_library) diff --git a/rust/compiler_builtins.rs b/rust/compiler_builtins.rs new file mode 100644 index 0000000000000..58d1a9b4d2182 --- /dev/null +++ b/rust/compiler_builtins.rs @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Our own `compiler_builtins`. +//! +//! Rust provides [`compiler_builtins`] as a port of LLVM's [`compiler-rt`]. +//! Since we do not need the vast majority of them, we avoid the dependency +//! by providing this file. +//! +//! At the moment, some builtins are required that should not be. For instance, +//! [`core`] has floating-point functionality which we should not be compiling +//! in. For the moment, we define them to [`panic!`] at runtime for simplicity. +//! These are actually a superset of the ones we actually need to define, +//! but it seems simpler to ban entire categories at once. In the future, +//! we might be able to remove all this by providing our own custom [`core`] +//! etc., or perhaps [`core`] itself might provide `cfg` options to disable +//! enough functionality to avoid requiring some of these. +//! +//! In any case, all these symbols are weakened to ensure we do not override +//! those that may be provided by the rest of the kernel. +//! +//! [`compiler_builtins`]: https://github.com/rust-lang/compiler-builtins +//! [`compiler-rt`]: https://compiler-rt.llvm.org/ + +#![feature(compiler_builtins)] +#![compiler_builtins] +#![no_builtins] +#![no_std] +#![deny(clippy::complexity)] +#![deny(clippy::correctness)] +#![deny(clippy::perf)] +#![deny(clippy::style)] + +macro_rules! define_panicking_intrinsics( + ($reason: tt, { $($ident: ident, )* }) => { + $( + #[doc(hidden)] + #[no_mangle] + pub extern "C" fn $ident() { + panic!($reason); + } + )* + } +); + +define_panicking_intrinsics!("non-inline stack probes should not be used", { + __rust_probestack, +}); + +define_panicking_intrinsics!("`f32` should not be used", { + __addsf3, + __addsf3vfp, + __aeabi_fcmpeq, + __aeabi_ul2f, + __divsf3, + __divsf3vfp, + __eqsf2, + __eqsf2vfp, + __fixsfdi, + __fixsfsi, + __fixsfti, + __fixunssfdi, + __fixunssfsi, + __fixunssfti, + __floatdisf, + __floatsisf, + __floattisf, + __floatundisf, + __floatunsisf, + __floatuntisf, + __gesf2, + __gesf2vfp, + __gtsf2, + __gtsf2vfp, + __lesf2, + __lesf2vfp, + __ltsf2, + __ltsf2vfp, + __mulsf3, + __mulsf3vfp, + __nesf2, + __nesf2vfp, + __powisf2, + __subsf3, + __subsf3vfp, + __unordsf2, +}); + +define_panicking_intrinsics!("`f64` should not be used", { + __adddf3, + __adddf3vfp, + __aeabi_dcmpeq, + __aeabi_ul2d, + __divdf3, + __divdf3vfp, + __eqdf2, + __eqdf2vfp, + __fixdfdi, + __fixdfsi, + __fixdfti, + __fixunsdfdi, + __fixunsdfsi, + __fixunsdfti, + __floatdidf, + __floatsidf, + __floattidf, + __floatundidf, + __floatunsidf, + __floatuntidf, + __gedf2, + __gedf2vfp, + __gtdf2, + __gtdf2vfp, + __ledf2, + __ledf2vfp, + __ltdf2, + __ltdf2vfp, + __muldf3, + __muldf3vfp, + __nedf2, + __nedf2vfp, + __powidf2, + __subdf3, + __subdf3vfp, + __unorddf2, +}); + +define_panicking_intrinsics!("`i128` should not be used", { + __ashrti3, + __muloti4, + __multi3, +}); + +define_panicking_intrinsics!("`u128` should not be used", { + __ashlti3, + __lshrti3, + __udivmodti4, + __udivti3, + __umodti3, +}); + +#[cfg(target_arch = "arm")] +define_panicking_intrinsics!("`u64` division/modulo should not be used", { + __aeabi_uldivmod, + __mulodi4, +}); + +extern "C" { + fn rust_helper_BUG() -> !; +} + +#[panic_handler] +fn panic(_info: &core::panic::PanicInfo) -> ! { + unsafe { + rust_helper_BUG(); + } +} diff --git a/rust/exports.c b/rust/exports.c new file mode 100644 index 0000000000000..d7dff1b3b9199 --- /dev/null +++ b/rust/exports.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0 +// +// A hack to export Rust symbols for loadable modules without having to redo +// the entire `include/linux/export.h` logic in Rust. +// +// This requires the Rust's new/future `v0` mangling scheme because the default +// one ("legacy") uses invalid characters for C identifiers (thus we cannot use +// the `EXPORT_SYMBOL_*` macros). + +#include <linux/module.h> + +#define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym); + +#include "exports_core_generated.h" +#include "exports_alloc_generated.h" +#include "exports_kernel_generated.h" diff --git a/rust/helpers.c b/rust/helpers.c new file mode 100644 index 0000000000000..aa1de0eb0cbe3 --- /dev/null +++ b/rust/helpers.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/bug.h> +#include <linux/build_bug.h> +#include <linux/uaccess.h> +#include <linux/sched/signal.h> +#include <linux/gfp.h> +#include <linux/highmem.h> + +void rust_helper_BUG(void) +{ + BUG(); +} + +unsigned long rust_helper_copy_from_user(void *to, const void __user *from, unsigned long n) +{ + return copy_from_user(to, from, n); +} + +unsigned long rust_helper_copy_to_user(void __user *to, const void *from, unsigned long n) +{ + return copy_to_user(to, from, n); +} + +void rust_helper_spin_lock_init(spinlock_t *lock, const char *name, + struct lock_class_key *key) +{ +#ifdef CONFIG_DEBUG_SPINLOCK + __spin_lock_init(lock, name, key); +#else + spin_lock_init(lock); +#endif +} +EXPORT_SYMBOL_GPL(rust_helper_spin_lock_init); + +void rust_helper_spin_lock(spinlock_t *lock) +{ + spin_lock(lock); +} +EXPORT_SYMBOL_GPL(rust_helper_spin_lock); + +void rust_helper_spin_unlock(spinlock_t *lock) +{ + spin_unlock(lock); +} +EXPORT_SYMBOL_GPL(rust_helper_spin_unlock); + +void rust_helper_init_wait(struct wait_queue_entry *wq_entry) +{ + init_wait(wq_entry); +} +EXPORT_SYMBOL_GPL(rust_helper_init_wait); + +int rust_helper_current_pid(void) +{ + return current->pid; +} +EXPORT_SYMBOL_GPL(rust_helper_current_pid); + +int rust_helper_signal_pending(void) +{ + return signal_pending(current); +} +EXPORT_SYMBOL_GPL(rust_helper_signal_pending); + +struct page *rust_helper_alloc_pages(gfp_t gfp_mask, unsigned int order) +{ + return alloc_pages(gfp_mask, order); +} +EXPORT_SYMBOL_GPL(rust_helper_alloc_pages); + +void *rust_helper_kmap(struct page *page) +{ + return kmap(page); +} +EXPORT_SYMBOL_GPL(rust_helper_kmap); + +void rust_helper_kunmap(struct page *page) +{ + return kunmap(page); +} +EXPORT_SYMBOL_GPL(rust_helper_kunmap); + +int rust_helper_cond_resched(void) +{ + return cond_resched(); +} +EXPORT_SYMBOL_GPL(rust_helper_cond_resched); + +#if !defined(CONFIG_ARM) +// See https://github.com/rust-lang/rust-bindgen/issues/1671 +static_assert(__builtin_types_compatible_p(size_t, uintptr_t), + "size_t must match uintptr_t, what architecture is this??"); +#endif diff --git a/rust/kernel/allocator.rs b/rust/kernel/allocator.rs new file mode 100644 index 0000000000000..81104f2d8ffae --- /dev/null +++ b/rust/kernel/allocator.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Allocator support. + +use core::alloc::{GlobalAlloc, Layout}; +use core::ptr; + +use crate::bindings; +use crate::c_types; + +pub struct KernelAllocator; + +unsafe impl GlobalAlloc for KernelAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + // `krealloc()` is used instead of `kmalloc()` because the latter is + // an inline function and cannot be bound to as a result. + bindings::krealloc(ptr::null(), layout.size(), bindings::GFP_KERNEL) as *mut u8 + } + + unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) { + bindings::kfree(ptr as *const c_types::c_void); + } +} + +#[alloc_error_handler] +fn oom(_layout: Layout) -> ! { + panic!("Out of memory!"); +} + +// `rustc` only generates these for some crate types. Even then, we would need +// to extract the object file that has them from the archive. For the moment, +// let's generate them ourselves instead. +#[no_mangle] +pub fn __rust_alloc(size: usize, _align: usize) -> *mut u8 { + unsafe { bindings::krealloc(core::ptr::null(), size, bindings::GFP_KERNEL) as *mut u8 } +} + +#[no_mangle] +pub fn __rust_dealloc(ptr: *mut u8, _size: usize, _align: usize) { + unsafe { bindings::kfree(ptr as *const c_types::c_void) }; +} + +#[no_mangle] +pub fn __rust_realloc(ptr: *mut u8, _old_size: usize, _align: usize, new_size: usize) -> *mut u8 { + unsafe { + bindings::krealloc( + ptr as *const c_types::c_void, + new_size, + bindings::GFP_KERNEL, + ) as *mut u8 + } +} + +#[no_mangle] +pub fn __rust_alloc_zeroed(size: usize, _align: usize) -> *mut u8 { + unsafe { + bindings::krealloc( + core::ptr::null(), + size, + bindings::GFP_KERNEL | bindings::__GFP_ZERO, + ) as *mut u8 + } +} + +#[no_mangle] +pub fn __rust_alloc_error_handler(_size: usize, _align: usize) -> ! { + panic!("Out of memory!"); +} diff --git a/rust/kernel/bindings.rs b/rust/kernel/bindings.rs new file mode 100644 index 0000000000000..6a300f52335cd --- /dev/null +++ b/rust/kernel/bindings.rs @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Bindings +//! +//! Imports the generated bindings by `bindgen`. + +#[allow( + clippy::all, + non_camel_case_types, + non_upper_case_globals, + non_snake_case, + improper_ctypes +)] +mod bindings_raw { + use crate::c_types; + include!(env!("RUST_BINDINGS_FILE")); +} +pub use bindings_raw::*; + +pub const GFP_KERNEL: gfp_t = BINDINGS_GFP_KERNEL; +pub const __GFP_ZERO: gfp_t = BINDINGS___GFP_ZERO; +pub const __GFP_HIGHMEM: gfp_t = ___GFP_HIGHMEM; diff --git a/rust/kernel/bindings_helper.h b/rust/kernel/bindings_helper.h new file mode 100644 index 0000000000000..6b08f42cfcceb --- /dev/null +++ b/rust/kernel/bindings_helper.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#include <linux/cdev.h> +#include <linux/fs.h> +#include <linux/module.h> +#include <linux/random.h> +#include <linux/slab.h> +#include <linux/sysctl.h> +#include <linux/uaccess.h> +#include <linux/version.h> +#include <linux/miscdevice.h> +#include <linux/poll.h> +#include <linux/mm.h> +#include <uapi/linux/android/binder.h> + +// `bindgen` gets confused at certain things +const gfp_t BINDINGS_GFP_KERNEL = GFP_KERNEL; +const gfp_t BINDINGS___GFP_ZERO = __GFP_ZERO; diff --git a/rust/kernel/buffer.rs b/rust/kernel/buffer.rs new file mode 100644 index 0000000000000..b2502fa968fe9 --- /dev/null +++ b/rust/kernel/buffer.rs @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Struct for writing to a pre-allocated buffer with the [`write!`] macro. + +use core::fmt; + +/// A pre-allocated buffer that implements [`core::fmt::Write`]. +/// +/// Consecutive writes will append to what has already been written. +/// Writes that don't fit in the buffer will fail. +pub struct Buffer<'a> { + slice: &'a mut [u8], + pos: usize, +} + +impl<'a> Buffer<'a> { + /// Create a new buffer from an existing array. + pub fn new(slice: &'a mut [u8]) -> Self { + Buffer { slice, pos: 0 } + } + + /// Number of bytes that have already been written to the buffer. + /// This will always be less than the length of the original array. + pub fn bytes_written(&self) -> usize { + self.pos + } +} + +impl<'a> fmt::Write for Buffer<'a> { + fn write_str(&mut self, s: &str) -> fmt::Result { + if s.len() > self.slice.len() - self.pos { + Err(fmt::Error) + } else { + self.slice[self.pos..self.pos + s.len()].copy_from_slice(s.as_bytes()); + self.pos += s.len(); + Ok(()) + } + } +} diff --git a/rust/kernel/c_types.rs b/rust/kernel/c_types.rs new file mode 100644 index 0000000000000..10486b41efa99 --- /dev/null +++ b/rust/kernel/c_types.rs @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! C types for the bindings. +//! +//! The bindings generated by `bindgen` use these types to map to the C ones. +//! +//! C's standard integer types may differ in width depending on +//! the architecture, thus we need to conditionally compile those. + +#![allow(non_camel_case_types)] + +#[cfg(any(target_arch = "arm", target_arch = "x86"))] +mod c { + /// C `void` type. + pub type c_void = core::ffi::c_void; + + /// C `char` type. + pub type c_char = i8; + + /// C `signed char` type. + pub type c_schar = i8; + + /// C `unsigned char` type. + pub type c_uchar = u8; + + /// C `short` type. + pub type c_short = i16; + + /// C `unsigned short` type. + pub type c_ushort = u16; + + /// C `int` type. + pub type c_int = i32; + + /// C `unsigned int` type. + pub type c_uint = u32; + + /// C `long` type. + pub type c_long = i32; + + /// C `unsigned long` type. + pub type c_ulong = u32; + + /// C `long long` type. + pub type c_longlong = i64; + + /// C `unsigned long long` type. + pub type c_ulonglong = u64; + + /// C `ssize_t` type (typically defined in `<sys/types.h>` by POSIX). + /// + /// For some 32-bit architectures like this one, the kernel defines it as + /// `int`, i.e. it is an [`i32`]. + pub type c_ssize_t = isize; + + /// C `size_t` type (typically defined in `<stddef.h>`). + /// + /// For some 32-bit architectures like this one, the kernel defines it as + /// `unsigned int`, i.e. it is an [`u32`]. + pub type c_size_t = usize; +} + +#[cfg(any( + target_arch = "aarch64", + target_arch = "x86_64", + target_arch = "powerpc64" +))] +mod c { + /// C `void` type. + pub type c_void = core::ffi::c_void; + + /// C `char` type. + pub type c_char = i8; + + /// C `signed char` type. + pub type c_schar = i8; + + /// C `unsigned char` type. + pub type c_uchar = u8; + + /// C `short` type. + pub type c_short = i16; + + /// C `unsigned short` type. + pub type c_ushort = u16; + + /// C `int` type. + pub type c_int = i32; + + /// C `unsigned int` type. + pub type c_uint = u32; + + /// C `long` type. + pub type c_long = i64; + + /// C `unsigned long` type. + pub type c_ulong = u64; + + /// C `long long` type. + pub type c_longlong = i64; + + /// C `unsigned long long` type. + pub type c_ulonglong = u64; + + /// C `ssize_t` type (typically defined in `<sys/types.h>` by POSIX). + /// + /// For 64-bit architectures like this one, the kernel defines it as + /// `long`, i.e. it is an [`i64`]. + pub type c_ssize_t = isize; + + /// C `size_t` type (typically defined in `<stddef.h>`). + /// + /// For 64-bit architectures like this one, the kernel defines it as + /// `unsigned long`, i.e. it is an [`u64`]. + pub type c_size_t = usize; +} + +pub use c::*; + +/// Reads string until null byte is reached and returns slice excluding the +/// terminating null. +/// +/// # Safety +/// +/// The data from the pointer until the null terminator must be valid for reads +/// and not mutated for all of `'a`. The length of the string must also be less +/// than `isize::MAX`. See the documentation on +/// [`core::slice::from_raw_parts()`] for further details on safety of +/// converting a pointer to a slice. +pub unsafe fn c_string_bytes<'a>(ptr: *const crate::c_types::c_char) -> &'a [u8] { + let length = crate::bindings::strlen(ptr) as usize; + &core::slice::from_raw_parts(ptr as *const u8, length) +} diff --git a/rust/kernel/chrdev.rs b/rust/kernel/chrdev.rs new file mode 100644 index 0000000000000..6772a3a925ccf --- /dev/null +++ b/rust/kernel/chrdev.rs @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Character devices. +//! +//! Also called "char devices", `chrdev`, `cdev`. +//! +//! C header: [`include/linux/cdev.h`](../../../../include/linux/cdev.h) +//! +//! Reference: <https://www.kernel.org/doc/html/latest/core-api/kernel-api.html#char-devices> + +use alloc::boxed::Box; +use core::convert::TryInto; +use core::marker::PhantomPinned; +use core::mem::MaybeUninit; +use core::pin::Pin; + +use crate::bindings; +use crate::c_types; +use crate::error::{Error, KernelResult}; +use crate::file_operations; +use crate::types::CStr; + +struct RegistrationInner<const N: usize> { + dev: bindings::dev_t, + used: usize, + cdevs: [MaybeUninit<bindings::cdev>; N], + _pin: PhantomPinned, +} + +/// Character device registration. +/// +/// May contain up to a fixed number (`N`) of devices. Must be pinned. +pub struct Registration<const N: usize> { + name: CStr<'static>, + minors_start: u16, + this_module: &'static crate::ThisModule, + inner: Option<RegistrationInner<N>>, +} + +impl<const N: usize> Registration<{ N }> { + /// Creates a [`Registration`] object for a character device. + /// + /// This does *not* register the device: see [`Self::register()`]. + /// + /// This associated function is intended to be used when you need to avoid + /// a memory allocation, e.g. when the [`Registration`] is a member of + /// a bigger structure inside your [`crate::KernelModule`] instance. If you + /// are going to pin the registration right away, call + /// [`Self::new_pinned()`] instead. + pub fn new( + name: CStr<'static>, + minors_start: u16, + this_module: &'static crate::ThisModule, + ) -> Self { + Registration { + name, + minors_start, + this_module, + inner: None, + } + } + + /// Creates a pinned [`Registration`] object for a character device. + /// + /// This does *not* register the device: see [`Self::register()`]. + pub fn new_pinned( + name: CStr<'static>, + minors_start: u16, + this_module: &'static crate::ThisModule, + ) -> KernelResult<Pin<Box<Self>>> { + Ok(Pin::from(Box::try_new(Self::new( + name, + minors_start, + this_module, + ))?)) + } + + /// Registers a character device. + /// + /// You may call this once per device type, up to `N` times. + pub fn register<T: file_operations::FileOpener<()>>(self: Pin<&mut Self>) -> KernelResult { + // SAFETY: We must ensure that we never move out of `this`. + let this = unsafe { self.get_unchecked_mut() }; + if this.inner.is_none() { + let mut dev: bindings::dev_t = 0; + // SAFETY: Calling unsafe function. `this.name` has `'static` + // lifetime. + let res = unsafe { + bindings::alloc_chrdev_region( + &mut dev, + this.minors_start.into(), + N.try_into()?, + this.name.as_ptr() as *const c_types::c_char, + ) + }; + if res != 0 { + return Err(Error::from_kernel_errno(res)); + } + this.inner = Some(RegistrationInner { + dev, + used: 0, + cdevs: [MaybeUninit::<bindings::cdev>::uninit(); N], + _pin: PhantomPinned, + }); + } + + let mut inner = this.inner.as_mut().unwrap(); + if inner.used == N { + return Err(Error::EINVAL); + } + let cdev = inner.cdevs[inner.used].as_mut_ptr(); + // SAFETY: Calling unsafe functions and manipulating `MaybeUninit` + // pointer. + unsafe { + bindings::cdev_init( + cdev, + // SAFETY: The adapter doesn't retrieve any state yet, so it's compatible with any + // registration. + file_operations::FileOperationsVtable::<Self, T>::build(), + ); + (*cdev).owner = this.this_module.0; + let rc = bindings::cdev_add(cdev, inner.dev + inner.used as bindings::dev_t, 1); + if rc != 0 { + return Err(Error::from_kernel_errno(rc)); + } + } + inner.used += 1; + Ok(()) + } +} + +impl<const N: usize> file_operations::FileOpenAdapter for Registration<{ N }> { + type Arg = (); + + unsafe fn convert( + _inode: *mut bindings::inode, + _file: *mut bindings::file, + ) -> *const Self::Arg { + // TODO: Update the SAFETY comment on the call to `FileOperationsVTable::build` above once + // this is updated to retrieve state. + &() + } +} + +// SAFETY: `Registration` does not expose any of its state across threads +// (it is fine for multiple threads to have a shared reference to it). +unsafe impl<const N: usize> Sync for Registration<{ N }> {} + +impl<const N: usize> Drop for Registration<{ N }> { + fn drop(&mut self) { + if let Some(inner) = self.inner.as_mut() { + // SAFETY: Calling unsafe functions, `0..inner.used` of + // `inner.cdevs` are initialized in `Registration::register`. + unsafe { + for i in 0..inner.used { + bindings::cdev_del(inner.cdevs[i].as_mut_ptr()); + } + bindings::unregister_chrdev_region(inner.dev, N.try_into().unwrap()); + } + } + } +} diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs new file mode 100644 index 0000000000000..432d866232c13 --- /dev/null +++ b/rust/kernel/error.rs @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Kernel errors. +//! +//! C header: [`include/uapi/asm-generic/errno-base.h`](../../../include/uapi/asm-generic/errno-base.h) + +use crate::{bindings, c_types}; +use alloc::{alloc::AllocError, collections::TryReserveError}; +use core::{num::TryFromIntError, str::Utf8Error}; + +/// Generic integer kernel error. +/// +/// The kernel defines a set of integer generic error codes based on C and +/// POSIX ones. These codes may have a more specific meaning in some contexts. +pub struct Error(c_types::c_int); + +impl Error { + /// Invalid argument. + pub const EINVAL: Self = Error(-(bindings::EINVAL as i32)); + + /// Out of memory. + pub const ENOMEM: Self = Error(-(bindings::ENOMEM as i32)); + + /// Bad address. + pub const EFAULT: Self = Error(-(bindings::EFAULT as i32)); + + /// Illegal seek. + pub const ESPIPE: Self = Error(-(bindings::ESPIPE as i32)); + + /// Try again. + pub const EAGAIN: Self = Error(-(bindings::EAGAIN as i32)); + + /// Device or resource busy. + pub const EBUSY: Self = Error(-(bindings::EBUSY as i32)); + + /// Restart the system call. + pub const ERESTARTSYS: Self = Error(-(bindings::ERESTARTSYS as i32)); + + /// Operation not permitted. + pub const EPERM: Self = Error(-(bindings::EPERM as i32)); + + /// No such process. + pub const ESRCH: Self = Error(-(bindings::ESRCH as i32)); + + /// No such file or directory. + pub const ENOENT: Self = Error(-(bindings::ENOENT as i32)); + + /// Interrupted system call. + pub const EINTR: Self = Error(-(bindings::EINTR as i32)); + + /// Creates an [`Error`] from a kernel error code. + pub fn from_kernel_errno(errno: c_types::c_int) -> Error { + Error(errno) + } + + /// Returns the kernel error code. + pub fn to_kernel_errno(&self) -> c_types::c_int { + self.0 + } +} + +impl From<TryFromIntError> for Error { + fn from(_: TryFromIntError) -> Error { + Error::EINVAL + } +} + +impl From<Utf8Error> for Error { + fn from(_: Utf8Error) -> Error { + Error::EINVAL + } +} + +impl From<TryReserveError> for Error { + fn from(_: TryReserveError) -> Error { + Error::ENOMEM + } +} + +/// A [`Result`] with an [`Error`] error type. +/// +/// To be used as the return type for functions that may fail. +/// +/// # Error codes in C and Rust +/// +/// In C, it is common that functions indicate success or failure through +/// their return value; modifying or returning extra data through non-`const` +/// pointer parameters. In particular, in the kernel, functions that may fail +/// typically return an `int` that represents a generic error code. We model +/// those as [`Error`]. +/// +/// In Rust, it is idiomatic to model functions that may fail as returning +/// a [`Result`]. Since in the kernel many functions return an error code, +/// [`KernelResult`] is a type alias for a [`Result`] that uses [`Error`] as +/// its error type. +/// +/// Note that even if a function does not return anything when it succeeds, +/// it should still be modeled as returning a `KernelResult` rather than +/// just an [`Error`]. +pub type KernelResult<T = ()> = Result<T, Error>; + +impl From<AllocError> for Error { + fn from(_: AllocError) -> Error { + Error::ENOMEM + } +} diff --git a/rust/kernel/file_operations.rs b/rust/kernel/file_operations.rs new file mode 100644 index 0000000000000..da13d4b6b0dbf --- /dev/null +++ b/rust/kernel/file_operations.rs @@ -0,0 +1,674 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! File operations. +//! +//! C header: [`include/linux/fs.h`](../../../../include/linux/fs.h) + +use core::convert::{TryFrom, TryInto}; +use core::{marker, mem, ops::Deref, pin::Pin, ptr}; + +use alloc::boxed::Box; +use alloc::sync::Arc; + +use crate::bindings; +use crate::c_types; +use crate::error::{Error, KernelResult}; +use crate::sync::{CondVar, Ref, RefCounted}; +use crate::user_ptr::{UserSlicePtr, UserSlicePtrReader, UserSlicePtrWriter}; + +/// Wraps the kernel's `struct file`. +/// +/// # Invariants +/// +/// The pointer [`File::ptr`] is non-null and valid. +pub struct File { + ptr: *const bindings::file, +} + +impl File { + /// Constructs a new [`struct file`] wrapper. + /// + /// # Safety + /// + /// The pointer `ptr` must be non-null and valid for the lifetime of the object. + unsafe fn from_ptr(ptr: *const bindings::file) -> File { + // INVARIANTS: the safety contract ensures the type invariant will hold. + File { ptr } + } + + /// Returns the current seek/cursor/pointer position (`struct file::f_pos`). + pub fn pos(&self) -> u64 { + // SAFETY: `File::ptr` is guaranteed to be valid by the type invariants. + unsafe { (*self.ptr).f_pos as u64 } + } + + /// Returns whether the file is in blocking mode. + pub fn is_blocking(&self) -> bool { + // SAFETY: `File::ptr` is guaranteed to be valid by the type invariants. + unsafe { (*self.ptr).f_flags & bindings::O_NONBLOCK == 0 } + } +} + +/// Wraps the kernel's `struct poll_table_struct`. +/// +/// # Invariants +/// +/// The pointer [`PollTable::ptr`] is null or valid. +pub struct PollTable { + ptr: *mut bindings::poll_table_struct, +} + +impl PollTable { + /// Constructors a new `struct poll_table_struct` wrapper. + /// + /// # Safety + /// + /// The pointer `ptr` must be either null or a valid pointer for the lifetime of the object. + unsafe fn from_ptr(ptr: *mut bindings::poll_table_struct) -> Self { + Self { ptr } + } + + /// Associates the given file and condition variable to this poll table. It means notifying the + /// condition variable will notify the poll table as well; additionally, the association + /// between the condition variable and the file will automatically be undone by the kernel when + /// the file is destructed. To unilaterally remove the association before then, one can call + /// [`CondVar::free_waiters`]. + /// + /// # Safety + /// + /// If the condition variable is destroyed before the file, then [`CondVar::free_waiters`] must + /// be called to ensure that all waiters are flushed out. + pub unsafe fn register_wait<'a>(&self, file: &'a File, cv: &'a CondVar) { + if self.ptr.is_null() { + return; + } + + // SAFETY: `PollTable::ptr` is guaranteed to be valid by the type invariants and the null + // check above. + let table = &*self.ptr; + if let Some(proc) = table._qproc { + // SAFETY: All pointers are known to be valid. + proc(file.ptr as _, cv.wait_list.get(), self.ptr) + } + } +} + +/// Equivalent to [`std::io::SeekFrom`]. +/// +/// [`std::io::SeekFrom`]: https://doc.rust-lang.org/std/io/enum.SeekFrom.html +pub enum SeekFrom { + /// Equivalent to C's `SEEK_SET`. + Start(u64), + + /// Equivalent to C's `SEEK_END`. + End(i64), + + /// Equivalent to C's `SEEK_CUR`. + Current(i64), +} + +fn from_kernel_result<T>(r: KernelResult<T>) -> T +where + T: TryFrom<c_types::c_int>, + T::Error: core::fmt::Debug, +{ + match r { + Ok(v) => v, + Err(e) => T::try_from(e.to_kernel_errno()).unwrap(), + } +} + +macro_rules! from_kernel_result { + ($($tt:tt)*) => {{ + from_kernel_result((|| { + $($tt)* + })()) + }}; +} + +unsafe extern "C" fn open_callback<A: FileOpenAdapter, T: FileOpener<A::Arg>>( + inode: *mut bindings::inode, + file: *mut bindings::file, +) -> c_types::c_int { + from_kernel_result! { + let arg = A::convert(inode, file); + let ptr = T::open(&*arg)?.into_pointer(); + (*file).private_data = ptr as *mut c_types::c_void; + Ok(0) + } +} + +unsafe extern "C" fn read_callback<T: FileOperations>( + file: *mut bindings::file, + buf: *mut c_types::c_char, + len: c_types::c_size_t, + offset: *mut bindings::loff_t, +) -> c_types::c_ssize_t { + from_kernel_result! { + let mut data = UserSlicePtr::new(buf as *mut c_types::c_void, len).writer(); + let f = &*((*file).private_data as *const T); + // No `FMODE_UNSIGNED_OFFSET` support, so `offset` must be in [0, 2^63). + // See discussion in https://github.com/fishinabarrel/linux-kernel-module-rust/pull/113 + let read = f.read(&File::from_ptr(file), &mut data, (*offset).try_into()?)?; + (*offset) += bindings::loff_t::try_from(read).unwrap(); + Ok(read as _) + } +} + +unsafe extern "C" fn write_callback<T: FileOperations>( + file: *mut bindings::file, + buf: *const c_types::c_char, + len: c_types::c_size_t, + offset: *mut bindings::loff_t, +) -> c_types::c_ssize_t { + from_kernel_result! { + let mut data = UserSlicePtr::new(buf as *mut c_types::c_void, len).reader(); + let f = &*((*file).private_data as *const T); + // No `FMODE_UNSIGNED_OFFSET` support, so `offset` must be in [0, 2^63). + // See discussion in https://github.com/fishinabarrel/linux-kernel-module-rust/pull/113 + let written = f.write(&mut data, (*offset).try_into()?)?; + (*offset) += bindings::loff_t::try_from(written).unwrap(); + Ok(written as _) + } +} + +unsafe extern "C" fn release_callback<T: FileOperations>( + _inode: *mut bindings::inode, + file: *mut bindings::file, +) -> c_types::c_int { + let ptr = mem::replace(&mut (*file).private_data, ptr::null_mut()); + T::release(T::Wrapper::from_pointer(ptr as _), &File::from_ptr(file)); + 0 +} + +unsafe extern "C" fn llseek_callback<T: FileOperations>( + file: *mut bindings::file, + offset: bindings::loff_t, + whence: c_types::c_int, +) -> bindings::loff_t { + from_kernel_result! { + let off = match whence as u32 { + bindings::SEEK_SET => SeekFrom::Start(offset.try_into()?), + bindings::SEEK_CUR => SeekFrom::Current(offset), + bindings::SEEK_END => SeekFrom::End(offset), + _ => return Err(Error::EINVAL), + }; + let f = &*((*file).private_data as *const T); + let off = f.seek(&File::from_ptr(file), off)?; + Ok(off as bindings::loff_t) + } +} + +unsafe extern "C" fn unlocked_ioctl_callback<T: FileOperations>( + file: *mut bindings::file, + cmd: c_types::c_uint, + arg: c_types::c_ulong, +) -> c_types::c_long { + from_kernel_result! { + let f = &*((*file).private_data as *const T); + // SAFETY: This function is called by the kernel, so it must set `fs` appropriately. + let mut cmd = IoctlCommand::new(cmd as _, arg as _); + let ret = f.ioctl(&File::from_ptr(file), &mut cmd)?; + Ok(ret as _) + } +} + +unsafe extern "C" fn compat_ioctl_callback<T: FileOperations>( + file: *mut bindings::file, + cmd: c_types::c_uint, + arg: c_types::c_ulong, +) -> c_types::c_long { + from_kernel_result! { + let f = &*((*file).private_data as *const T); + // SAFETY: This function is called by the kernel, so it must set `fs` appropriately. + let mut cmd = IoctlCommand::new(cmd as _, arg as _); + let ret = f.compat_ioctl(&File::from_ptr(file), &mut cmd)?; + Ok(ret as _) + } +} + +unsafe extern "C" fn mmap_callback<T: FileOperations>( + file: *mut bindings::file, + vma: *mut bindings::vm_area_struct, +) -> c_types::c_int { + from_kernel_result! { + let f = &*((*file).private_data as *const T); + f.mmap(&File::from_ptr(file), &mut *vma)?; + Ok(0) + } +} + +unsafe extern "C" fn fsync_callback<T: FileOperations>( + file: *mut bindings::file, + start: bindings::loff_t, + end: bindings::loff_t, + datasync: c_types::c_int, +) -> c_types::c_int { + from_kernel_result! { + let start = start.try_into()?; + let end = end.try_into()?; + let datasync = datasync != 0; + let f = &*((*file).private_data as *const T); + let res = f.fsync(&File::from_ptr(file), start, end, datasync)?; + Ok(res.try_into().unwrap()) + } +} + +unsafe extern "C" fn poll_callback<T: FileOperations>( + file: *mut bindings::file, + wait: *mut bindings::poll_table_struct, +) -> bindings::__poll_t { + let f = &*((*file).private_data as *const T); + match f.poll(&File::from_ptr(file), &PollTable::from_ptr(wait)) { + Ok(v) => v, + Err(_) => bindings::POLLERR, + } +} + +pub(crate) struct FileOperationsVtable<A, T>(marker::PhantomData<A>, marker::PhantomData<T>); + +impl<A: FileOpenAdapter, T: FileOpener<A::Arg>> FileOperationsVtable<A, T> { + const VTABLE: bindings::file_operations = bindings::file_operations { + open: Some(open_callback::<A, T>), + release: Some(release_callback::<T>), + read: if T::TO_USE.read { + Some(read_callback::<T>) + } else { + None + }, + write: if T::TO_USE.write { + Some(write_callback::<T>) + } else { + None + }, + llseek: if T::TO_USE.seek { + Some(llseek_callback::<T>) + } else { + None + }, + + check_flags: None, + compat_ioctl: if T::TO_USE.compat_ioctl { + Some(compat_ioctl_callback::<T>) + } else { + None + }, + copy_file_range: None, + fallocate: None, + fadvise: None, + fasync: None, + flock: None, + flush: None, + fsync: if T::TO_USE.fsync { + Some(fsync_callback::<T>) + } else { + None + }, + get_unmapped_area: None, + iterate: None, + iterate_shared: None, + iopoll: None, + lock: None, + mmap: if T::TO_USE.mmap { + Some(mmap_callback::<T>) + } else { + None + }, + mmap_supported_flags: 0, + owner: ptr::null_mut(), + poll: if T::TO_USE.poll { + Some(poll_callback::<T>) + } else { + None + }, + read_iter: None, + remap_file_range: None, + sendpage: None, + setlease: None, + show_fdinfo: None, + splice_read: None, + splice_write: None, + unlocked_ioctl: if T::TO_USE.ioctl { + Some(unlocked_ioctl_callback::<T>) + } else { + None + }, + write_iter: None, + }; + + /// Builds an instance of [`struct file_operations`]. + /// + /// # Safety + /// + /// The caller must ensure that the adapter is compatible with the way the device is registered. + pub(crate) const unsafe fn build() -> &'static bindings::file_operations { + &Self::VTABLE + } +} + +/// Represents which fields of [`struct file_operations`] should be populated with pointers. +pub struct ToUse { + /// The `read` field of [`struct file_operations`]. + pub read: bool, + + /// The `write` field of [`struct file_operations`]. + pub write: bool, + + /// The `llseek` field of [`struct file_operations`]. + pub seek: bool, + + /// The `unlocked_ioctl` field of [`struct file_operations`]. + pub ioctl: bool, + + /// The `compat_ioctl` field of [`struct file_operations`]. + pub compat_ioctl: bool, + + /// The `fsync` field of [`struct file_operations`]. + pub fsync: bool, + + /// The `mmap` field of [`struct file_operations`]. + pub mmap: bool, + + /// The `poll` field of [`struct file_operations`]. + pub poll: bool, +} + +/// A constant version where all values are to set to `false`, that is, all supported fields will +/// be set to null pointers. +pub const USE_NONE: ToUse = ToUse { + read: false, + write: false, + seek: false, + ioctl: false, + compat_ioctl: false, + fsync: false, + mmap: false, + poll: false, +}; + +/// Defines the [`FileOperations::TO_USE`] field based on a list of fields to be populated. +#[macro_export] +macro_rules! declare_file_operations { + () => { + const TO_USE: $crate::file_operations::ToUse = $crate::file_operations::USE_NONE; + }; + ($($i:ident),+) => { + const TO_USE: kernel::file_operations::ToUse = + $crate::file_operations::ToUse { + $($i: true),+ , + ..$crate::file_operations::USE_NONE + }; + }; +} + +/// Allows the handling of ioctls defined with the `_IO`, `_IOR`, `_IOW`, and `_IOWR` macros. +/// +/// For each macro, there is a handler function that takes the appropriate types as arguments. +pub trait IoctlHandler: Sync { + /// Handles ioctls defined with the `_IO` macro, that is, with no buffer as argument. + fn pure(&self, _file: &File, _cmd: u32, _arg: usize) -> KernelResult<i32> { + Err(Error::EINVAL) + } + + /// Handles ioctls defined with the `_IOR` macro, that is, with an output buffer provided as + /// argument. + fn read(&self, _file: &File, _cmd: u32, _writer: &mut UserSlicePtrWriter) -> KernelResult<i32> { + Err(Error::EINVAL) + } + + /// Handles ioctls defined with the `_IOW` macro, that is, with an input buffer provided as + /// argument. + fn write( + &self, + _file: &File, + _cmd: u32, + _reader: &mut UserSlicePtrReader, + ) -> KernelResult<i32> { + Err(Error::EINVAL) + } + + /// Handles ioctls defined with the `_IOWR` macro, that is, with a buffer for both input and + /// output provided as argument. + fn read_write(&self, _file: &File, _cmd: u32, _data: UserSlicePtr) -> KernelResult<i32> { + Err(Error::EINVAL) + } +} + +/// Represents an ioctl command. +/// +/// It can use the components of an ioctl command to dispatch ioctls using +/// [`IoctlCommand::dispatch`]. +pub struct IoctlCommand { + cmd: u32, + arg: usize, + user_slice: Option<UserSlicePtr>, +} + +impl IoctlCommand { + /// Constructs a new [`IoctlCommand`]. + fn new(cmd: u32, arg: usize) -> Self { + let size = (cmd >> bindings::_IOC_SIZESHIFT) & bindings::_IOC_SIZEMASK; + + // SAFETY: We only create one instance of the user slice per ioctl call, so TOCTOU issues + // are not possible. + let user_slice = Some(unsafe { UserSlicePtr::new(arg as _, size as _) }); + Self { + cmd, + arg, + user_slice, + } + } + + /// Dispatches the given ioctl to the appropriate handler based on the value of the command. It + /// also creates a [`UserSlicePtr`], [`UserSlicePtrReader`], or [`UserSlicePtrWriter`] + /// depending on the direction of the buffer of the command. + /// + /// It is meant to be used in implementations of [`FileOperations::ioctl`] and + /// [`FileOperations::compat_ioctl`]. + pub fn dispatch<T: IoctlHandler>(&mut self, handler: &T, file: &File) -> KernelResult<i32> { + let dir = (self.cmd >> bindings::_IOC_DIRSHIFT) & bindings::_IOC_DIRMASK; + if dir == bindings::_IOC_NONE { + return handler.pure(file, self.cmd, self.arg); + } + + let data = self.user_slice.take().ok_or(Error::EINVAL)?; + const READ_WRITE: u32 = bindings::_IOC_READ | bindings::_IOC_WRITE; + match dir { + bindings::_IOC_WRITE => handler.write(file, self.cmd, &mut data.reader()), + bindings::_IOC_READ => handler.read(file, self.cmd, &mut data.writer()), + READ_WRITE => handler.read_write(file, self.cmd, data), + _ => Err(Error::EINVAL), + } + } + + /// Returns the raw 32-bit value of the command and the ptr-sized argument. + pub fn raw(&self) -> (u32, usize) { + (self.cmd, self.arg) + } +} + +/// Trait for extracting file open arguments from kernel data structures. +/// +/// This is meant to be implemented by registration managers. +pub trait FileOpenAdapter { + /// The type of argument this adapter extracts. + type Arg; + + /// Converts untyped data stored in [`struct inode`] and [`struct file`] (when [`struct + /// file_operations::open`] is called) into the given type. For example, for `miscdev` + /// devices, a pointer to the registered [`struct miscdev`] is stored in [`struct + /// file::private_data`]. + /// + /// # Safety + /// + /// This function must be called only when [`struct file_operations::open`] is being called for + /// a file that was registered by the implementer. + unsafe fn convert(_inode: *mut bindings::inode, _file: *mut bindings::file) + -> *const Self::Arg; +} + +/// Trait for implementers of kernel files. +/// +/// In addition to the methods in [`FileOperations`], implementers must also provide +/// [`FileOpener::open`] with a customised argument. This allows a single implementation of +/// [`FileOperations`] to be used for different types of registrations, for example, `miscdev` and +/// `chrdev`. +pub trait FileOpener<T: ?Sized>: FileOperations { + /// Creates a new instance of this file. + /// + /// Corresponds to the `open` function pointer in `struct file_operations`. + fn open(context: &T) -> KernelResult<Self::Wrapper>; +} + +impl<T: FileOperations<Wrapper = Box<T>> + Default> FileOpener<()> for T { + fn open(_: &()) -> KernelResult<Self::Wrapper> { + Ok(Box::try_new(T::default())?) + } +} + +/// Corresponds to the kernel's `struct file_operations`. +/// +/// You implement this trait whenever you would create a `struct file_operations`. +/// +/// File descriptors may be used from multiple threads/processes concurrently, so your type must be +/// [`Sync`]. It must also be [`Send`] because [`FileOperations::release`] will be called from the +/// thread that decrements that associated file's refcount to zero. +pub trait FileOperations: Send + Sync + Sized { + /// The methods to use to populate [`struct file_operations`]. + const TO_USE: ToUse; + + /// The pointer type that will be used to hold ourselves. + type Wrapper: PointerWrapper<Self> = Box<Self>; + + /// Cleans up after the last reference to the file goes away. + /// + /// Note that the object is moved, so it will be freed automatically unless the implementation + /// moves it elsewhere. + /// + /// Corresponds to the `release` function pointer in `struct file_operations`. + fn release(_obj: Self::Wrapper, _file: &File) {} + + /// Reads data from this file to userspace. + /// + /// Corresponds to the `read` function pointer in `struct file_operations`. + fn read( + &self, + _file: &File, + _data: &mut UserSlicePtrWriter, + _offset: u64, + ) -> KernelResult<usize> { + Err(Error::EINVAL) + } + + /// Writes data from userspace to this file. + /// + /// Corresponds to the `write` function pointer in `struct file_operations`. + fn write(&self, _data: &mut UserSlicePtrReader, _offset: u64) -> KernelResult<usize> { + Err(Error::EINVAL) + } + + /// Changes the position of the file. + /// + /// Corresponds to the `llseek` function pointer in `struct file_operations`. + fn seek(&self, _file: &File, _offset: SeekFrom) -> KernelResult<u64> { + Err(Error::EINVAL) + } + + /// Performs IO control operations that are specific to the file. + /// + /// Corresponds to the `unlocked_ioctl` function pointer in `struct file_operations`. + fn ioctl(&self, _file: &File, _cmd: &mut IoctlCommand) -> KernelResult<i32> { + Err(Error::EINVAL) + } + + /// Performs 32-bit IO control operations on that are specific to the file on 64-bit kernels. + /// + /// Corresponds to the `compat_ioctl` function pointer in `struct file_operations`. + fn compat_ioctl(&self, _file: &File, _cmd: &mut IoctlCommand) -> KernelResult<i32> { + Err(Error::EINVAL) + } + + /// Syncs pending changes to this file. + /// + /// Corresponds to the `fsync` function pointer in `struct file_operations`. + fn fsync(&self, _file: &File, _start: u64, _end: u64, _datasync: bool) -> KernelResult<u32> { + Err(Error::EINVAL) + } + + /// Maps areas of the caller's virtual memory with device/file memory. + /// + /// Corresponds to the `mmap` function pointer in `struct file_operations`. + /// TODO: wrap `vm_area_struct` so that we don't have to expose it. + fn mmap(&self, _file: &File, _vma: &mut bindings::vm_area_struct) -> KernelResult { + Err(Error::EINVAL) + } + + /// Checks the state of the file and optionally registers for notification when the state + /// changes. + /// + /// Corresponds to the `poll` function pointer in `struct file_operations`. + fn poll(&self, _file: &File, _table: &PollTable) -> KernelResult<u32> { + Ok(bindings::POLLIN | bindings::POLLOUT | bindings::POLLRDNORM | bindings::POLLWRNORM) + } +} + +/// Used to convert an object into a raw pointer that represents it. +/// +/// It can eventually be converted back into the object. This is used to store objects as pointers +/// in kernel data structures, for example, an implementation of [`FileOperations`] in `struct +/// file::private_data`. +pub trait PointerWrapper<T> { + /// Returns the raw pointer. + fn into_pointer(self) -> *const T; + + /// Returns the instance back from the raw pointer. + /// + /// # Safety + /// + /// The passed pointer must come from a previous call to [`PointerWrapper::into_pointer()`]. + unsafe fn from_pointer(ptr: *const T) -> Self; +} + +impl<T> PointerWrapper<T> for Box<T> { + fn into_pointer(self) -> *const T { + Box::into_raw(self) + } + + unsafe fn from_pointer(ptr: *const T) -> Self { + Box::from_raw(ptr as _) + } +} + +impl<T: RefCounted> PointerWrapper<T> for Ref<T> { + fn into_pointer(self) -> *const T { + Ref::into_raw(self) + } + + unsafe fn from_pointer(ptr: *const T) -> Self { + Ref::from_raw(ptr as _) + } +} + +impl<T> PointerWrapper<T> for Arc<T> { + fn into_pointer(self) -> *const T { + Arc::into_raw(self) + } + + unsafe fn from_pointer(ptr: *const T) -> Self { + Arc::from_raw(ptr) + } +} + +impl<T, W: PointerWrapper<T> + Deref> PointerWrapper<T> for Pin<W> { + fn into_pointer(self) -> *const T { + // SAFETY: We continue to treat the pointer as pinned by returning just a pointer to it to + // the caller. + let inner = unsafe { Pin::into_inner_unchecked(self) }; + inner.into_pointer() + } + + unsafe fn from_pointer(p: *const T) -> Self { + // SAFETY: The object was originally pinned. + Pin::new_unchecked(W::from_pointer(p)) + } +} diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs new file mode 100644 index 0000000000000..2ad22dc1e6dec --- /dev/null +++ b/rust/kernel/lib.rs @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! The `kernel` crate. +//! +//! This crate contains the kernel APIs that have been ported or wrapped for +//! usage by Rust code in the kernel and is shared by all of them. +//! +//! In other words, all the rest of the Rust code in the kernel (e.g. kernel +//! modules written in Rust) depends on [`core`], [`alloc`] and this crate. +//! +//! If you need a kernel C API that is not ported or wrapped yet here, then +//! do so first instead of bypassing this crate. + +#![no_std] +#![feature( + allocator_api, + alloc_error_handler, + associated_type_defaults, + const_fn, + const_mut_refs, + const_panic, + try_reserve +)] +#![deny(clippy::complexity)] +#![deny(clippy::correctness)] +#![deny(clippy::perf)] +#![deny(clippy::style)] + +// Ensure conditional compilation based on the kernel configuration works; +// otherwise we may silently break things like initcall handling. +#[cfg(not(CONFIG_RUST))] +compile_error!("Missing kernel configuration for conditional compilation"); + +mod allocator; + +#[doc(hidden)] +pub mod bindings; + +pub mod buffer; +pub mod c_types; +pub mod chrdev; +mod error; +pub mod file_operations; +pub mod miscdev; +pub mod pages; + +pub mod linked_list; +mod raw_list; + +#[doc(hidden)] +pub mod module_param; + +pub mod prelude; +pub mod print; +pub mod random; +mod static_assert; +pub mod sync; + +#[cfg(CONFIG_SYSCTL)] +pub mod sysctl; + +mod types; +pub mod user_ptr; + +pub use crate::error::{Error, KernelResult}; +pub use crate::types::{CStr, Mode}; + +/// Page size defined in terms of the `PAGE_SHIFT` macro from C. +/// +/// [`PAGE_SHIFT`]: ../../../include/asm-generic/page.h +pub const PAGE_SIZE: usize = 1 << bindings::PAGE_SHIFT; + +/// Prefix to appear before log messages printed from within the kernel crate. +const __LOG_PREFIX: &[u8] = b"rust_kernel\0"; + +/// The top level entrypoint to implementing a kernel module. +/// +/// For any teardown or cleanup operations, your type may implement [`Drop`]. +pub trait KernelModule: Sized + Sync { + /// Called at module initialization time. + /// + /// Use this method to perform whatever setup or registration your module + /// should do. + /// + /// Equivalent to the `module_init` macro in the C API. + fn init() -> KernelResult<Self>; +} + +/// Equivalent to `THIS_MODULE` in the C API. +/// +/// C header: `include/linux/export.h` +pub struct ThisModule(*mut bindings::module); + +// SAFETY: `THIS_MODULE` may be used from all threads within a module. +unsafe impl Sync for ThisModule {} + +impl ThisModule { + /// Creates a [`ThisModule`] given the `THIS_MODULE` pointer. + /// + /// # Safety + /// + /// The pointer must be equal to the right `THIS_MODULE`. + pub const unsafe fn from_ptr(ptr: *mut bindings::module) -> ThisModule { + ThisModule(ptr) + } + + /// Locks the module parameters to access them. + /// + /// Returns a [`KParamGuard`] that will release the lock when dropped. + pub fn kernel_param_lock(&self) -> KParamGuard<'_> { + // SAFETY: `kernel_param_lock` will check if the pointer is null and + // use the built-in mutex in that case. + #[cfg(CONFIG_SYSFS)] + unsafe { + bindings::kernel_param_lock(self.0) + } + + KParamGuard { this_module: self } + } +} + +/// Scoped lock on the kernel parameters of [`ThisModule`]. +/// +/// Lock will be released when this struct is dropped. +pub struct KParamGuard<'a> { + this_module: &'a ThisModule, +} + +#[cfg(CONFIG_SYSFS)] +impl<'a> Drop for KParamGuard<'a> { + fn drop(&mut self) { + // SAFETY: `kernel_param_lock` will check if the pointer is null and + // use the built-in mutex in that case. The existance of `self` + // guarantees that the lock is held. + unsafe { bindings::kernel_param_unlock(self.this_module.0) } + } +} + +/// Calculates the offset of a field from the beginning of the struct it belongs to. +/// +/// # Example +/// +/// ``` +/// struct Test { +/// a: u64, +/// b: u32, +/// } +/// +/// fn test() { +/// // This prints `8`. +/// pr_info!("{}\n", offset_of!(Test, b)); +/// } +/// ``` +#[macro_export] +macro_rules! offset_of { + ($type:ty, $($f:tt)*) => {{ + let tmp = core::mem::MaybeUninit::<$type>::uninit(); + let outer = tmp.as_ptr(); + // To avoid warnings when nesting `unsafe` blocks. + #[allow(unused_unsafe)] + // SAFETY: The pointer is valid and aligned, just not initialised; `addr_of` ensures that + // we don't actually read from `outer` (which would be UB) nor create an intermediate + // reference. + let inner = unsafe { core::ptr::addr_of!((*outer).$($f)*) } as *const u8; + // To avoid warnings when nesting `unsafe` blocks. + #[allow(unused_unsafe)] + // SAFETY: The two pointers are within the same allocation block. + unsafe { inner.offset_from(outer as *const u8) } + }} +} + +/// Produces a pointer to an object from a pointer to one of its fields. +/// +/// # Safety +/// +/// Callers must ensure that the pointer to the field is in fact a pointer to the specified field, +/// as opposed to a pointer to another object of the same type. +/// +/// # Example +/// +/// ``` +/// struct Test { +/// a: u64, +/// b: u32, +/// } +/// +/// fn test() { +/// let test = Test { a: 10, b: 20 }; +/// let b_ptr = &test.b; +/// let test_alias = unsafe { container_of!(b_ptr, Test, b) }; +/// // This prints `true`. +/// pr_info!("{}\n", core::ptr::eq(&test, test_alias)); +/// } +/// ``` +#[macro_export] +macro_rules! container_of { + ($ptr:expr, $type:ty, $($f:tt)*) => {{ + let offset = $crate::offset_of!($type, $($f)*); + ($ptr as *const _ as *const u8).offset(-offset) as *const $type + }} +} + +#[global_allocator] +static ALLOCATOR: allocator::KernelAllocator = allocator::KernelAllocator; diff --git a/rust/kernel/linked_list.rs b/rust/kernel/linked_list.rs new file mode 100644 index 0000000000000..5b0b811eae220 --- /dev/null +++ b/rust/kernel/linked_list.rs @@ -0,0 +1,245 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Linked lists. +//! +//! TODO: This module is a work in progress. + +use alloc::{boxed::Box, sync::Arc}; +use core::ptr::NonNull; + +pub use crate::raw_list::{Cursor, GetLinks, Links}; +use crate::{raw_list, raw_list::RawList}; + +// TODO: Use the one from `kernel::file_operations::PointerWrapper` instead. +/// Wraps an object to be inserted in a linked list. +pub trait Wrapper<T: ?Sized> { + /// Converts the wrapped object into a pointer that represents it. + fn into_pointer(self) -> NonNull<T>; + + /// Converts the object back from the pointer representation. + /// + /// # Safety + /// + /// The passed pointer must come from a previous call to [`Wrapper::into_pointer()`]. + unsafe fn from_pointer(ptr: NonNull<T>) -> Self; + + /// Returns a reference to the wrapped object. + fn as_ref(&self) -> &T; +} + +impl<T: ?Sized> Wrapper<T> for Box<T> { + fn into_pointer(self) -> NonNull<T> { + NonNull::new(Box::into_raw(self)).unwrap() + } + + unsafe fn from_pointer(ptr: NonNull<T>) -> Self { + Box::from_raw(ptr.as_ptr()) + } + + fn as_ref(&self) -> &T { + AsRef::as_ref(self) + } +} + +impl<T: ?Sized> Wrapper<T> for Arc<T> { + fn into_pointer(self) -> NonNull<T> { + NonNull::new(Arc::into_raw(self) as _).unwrap() + } + + unsafe fn from_pointer(ptr: NonNull<T>) -> Self { + Arc::from_raw(ptr.as_ptr()) + } + + fn as_ref(&self) -> &T { + AsRef::as_ref(self) + } +} + +impl<T: ?Sized> Wrapper<T> for &T { + fn into_pointer(self) -> NonNull<T> { + NonNull::from(self) + } + + unsafe fn from_pointer(ptr: NonNull<T>) -> Self { + &*ptr.as_ptr() + } + + fn as_ref(&self) -> &T { + self + } +} + +/// A descriptor of wrapped list elements. +pub trait GetLinksWrapped: GetLinks { + /// Specifies which wrapper (e.g., `Box` and `Arc`) wraps the list entries. + type Wrapped: Wrapper<Self::EntryType>; +} + +impl<T: ?Sized> GetLinksWrapped for Box<T> +where + Box<T>: GetLinks, +{ + type Wrapped = Box<<Box<T> as GetLinks>::EntryType>; +} + +impl<T: GetLinks + ?Sized> GetLinks for Box<T> { + type EntryType = T::EntryType; + fn get_links(data: &Self::EntryType) -> &Links<Self::EntryType> { + <T as GetLinks>::get_links(data) + } +} + +impl<T: ?Sized> GetLinksWrapped for Arc<T> +where + Arc<T>: GetLinks, +{ + type Wrapped = Arc<<Arc<T> as GetLinks>::EntryType>; +} + +impl<T: GetLinks + ?Sized> GetLinks for Arc<T> { + type EntryType = T::EntryType; + fn get_links(data: &Self::EntryType) -> &Links<Self::EntryType> { + <T as GetLinks>::get_links(data) + } +} + +/// A linked list. +/// +/// Elements in the list are wrapped and ownership is transferred to the list while the element is +/// in the list. +pub struct List<G: GetLinksWrapped> { + list: RawList<G>, +} + +impl<G: GetLinksWrapped> List<G> { + /// Constructs a new empty linked list. + pub fn new() -> Self { + Self { + list: RawList::new(), + } + } + + /// Returns whether the list is empty. + pub fn is_empty(&self) -> bool { + self.list.is_empty() + } + + /// Adds the given object to the end (back) of the list. + /// + /// It is dropped if it's already on this (or another) list; this can happen for + /// reference-counted objects, so dropping means decrementing the reference count. + pub fn push_back(&mut self, data: G::Wrapped) { + let ptr = data.into_pointer(); + + // SAFETY: We took ownership of the entry, so it is safe to insert it. + if !unsafe { self.list.push_back(ptr.as_ref()) } { + // If insertion failed, rebuild object so that it can be freed. + // SAFETY: We just called `into_pointer` above. + unsafe { G::Wrapped::from_pointer(ptr) }; + } + } + + /// Inserts the given object after `existing`. + /// + /// It is dropped if it's already on this (or another) list; this can happen for + /// reference-counted objects, so dropping means decrementing the reference count. + /// + /// # Safety + /// + /// Callers must ensure that `existing` points to a valid entry that is on the list. + pub unsafe fn insert_after(&mut self, existing: NonNull<G::EntryType>, data: G::Wrapped) { + let ptr = data.into_pointer(); + let entry = &*existing.as_ptr(); + if !self.list.insert_after(entry, ptr.as_ref()) { + // If insertion failed, rebuild object so that it can be freed. + G::Wrapped::from_pointer(ptr); + } + } + + /// Removes the given entry. + /// + /// # Safety + /// + /// Callers must ensure that `data` is either on this list or in no list. It being on another + /// list leads to memory unsafety. + pub unsafe fn remove(&mut self, data: &G::Wrapped) -> Option<G::Wrapped> { + let entry_ref = Wrapper::as_ref(data); + if self.list.remove(entry_ref) { + Some(G::Wrapped::from_pointer(NonNull::from(entry_ref))) + } else { + None + } + } + + /// Removes the element currently at the front of the list and returns it. + /// + /// Returns `None` if the list is empty. + pub fn pop_front(&mut self) -> Option<G::Wrapped> { + let front = self.list.pop_front()?; + // SAFETY: Elements on the list were inserted after a call to `into_pointer `. + Some(unsafe { G::Wrapped::from_pointer(front) }) + } + + /// Returns a cursor starting on the first (front) element of the list. + pub fn cursor_front(&self) -> Cursor<'_, G> { + self.list.cursor_front() + } + + /// Returns a mutable cursor starting on the first (front) element of the list. + pub fn cursor_front_mut(&mut self) -> CursorMut<'_, G> { + CursorMut::new(self.list.cursor_front_mut()) + } +} + +impl<G: GetLinksWrapped> Default for List<G> { + fn default() -> Self { + Self::new() + } +} + +impl<G: GetLinksWrapped> Drop for List<G> { + fn drop(&mut self) { + while self.pop_front().is_some() {} + } +} + +/// A list cursor that allows traversing a linked list and inspecting & mutating elements. +pub struct CursorMut<'a, G: GetLinksWrapped> { + cursor: raw_list::CursorMut<'a, G>, +} + +impl<'a, G: GetLinksWrapped> CursorMut<'a, G> { + fn new(cursor: raw_list::CursorMut<'a, G>) -> Self { + Self { cursor } + } + + /// Returns the element the cursor is currently positioned on. + pub fn current(&mut self) -> Option<&mut G::EntryType> { + self.cursor.current() + } + + /// Removes the element the cursor is currently positioned on. + /// + /// After removal, it advances the cursor to the next element. + pub fn remove_current(&mut self) -> Option<G::Wrapped> { + let ptr = self.cursor.remove_current()?; + + // SAFETY: Elements on the list were inserted after a call to `into_pointer `. + Some(unsafe { G::Wrapped::from_pointer(ptr) }) + } + + /// Returns the element immediately after the one the cursor is positioned on. + pub fn peek_next(&mut self) -> Option<&mut G::EntryType> { + self.cursor.peek_next() + } + + /// Returns the element immediately before the one the cursor is positioned on. + pub fn peek_prev(&mut self) -> Option<&mut G::EntryType> { + self.cursor.peek_prev() + } + + /// Moves the cursor to the next element. + pub fn move_next(&mut self) { + self.cursor.move_next(); + } +} diff --git a/rust/kernel/miscdev.rs b/rust/kernel/miscdev.rs new file mode 100644 index 0000000000000..92c2181f30536 --- /dev/null +++ b/rust/kernel/miscdev.rs @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Miscellaneous devices. +//! +//! C header: [`include/linux/miscdevice.h`](../../../../include/linux/miscdevice.h) +//! +//! Reference: <https://www.kernel.org/doc/html/latest/driver-api/misc_devices.html> + +use crate::error::{Error, KernelResult}; +use crate::file_operations::{FileOpenAdapter, FileOpener, FileOperationsVtable}; +use crate::{bindings, c_types, CStr}; +use alloc::boxed::Box; +use core::marker::PhantomPinned; +use core::pin::Pin; + +/// A registration of a miscellaneous device. +pub struct Registration<T: Sync = ()> { + registered: bool, + mdev: bindings::miscdevice, + _pin: PhantomPinned, + + /// Context initialised on construction and made available to all file instances on + /// [`FileOpener::open`]. + pub context: T, +} + +impl<T: Sync> Registration<T> { + /// Creates a new [`Registration`] but does not register it yet. + /// + /// It is allowed to move. + pub fn new(context: T) -> Self { + Self { + registered: false, + mdev: bindings::miscdevice::default(), + _pin: PhantomPinned, + context, + } + } + + /// Registers a miscellaneous device. + /// + /// Returns a pinned heap-allocated representation of the registration. + pub fn new_pinned<F: FileOpener<T>>( + name: CStr<'static>, + minor: Option<i32>, + context: T, + ) -> KernelResult<Pin<Box<Self>>> { + let mut r = Pin::from(Box::try_new(Self::new(context))?); + r.as_mut().register::<F>(name, minor)?; + Ok(r) + } + + /// Registers a miscellaneous device with the rest of the kernel. + /// + /// It must be pinned because the memory block that represents the registration is + /// self-referential. If a minor is not given, the kernel allocates a new one if possible. + pub fn register<F: FileOpener<T>>( + self: Pin<&mut Self>, + name: CStr<'static>, + minor: Option<i32>, + ) -> KernelResult { + // SAFETY: We must ensure that we never move out of `this`. + let this = unsafe { self.get_unchecked_mut() }; + if this.registered { + // Already registered. + return Err(Error::EINVAL); + } + + // SAFETY: The adapter is compatible with `misc_register`. + this.mdev.fops = unsafe { FileOperationsVtable::<Self, F>::build() }; + this.mdev.name = name.as_ptr() as *const c_types::c_char; + this.mdev.minor = minor.unwrap_or(bindings::MISC_DYNAMIC_MINOR as i32); + + let ret = unsafe { bindings::misc_register(&mut this.mdev) }; + if ret < 0 { + return Err(Error::from_kernel_errno(ret)); + } + this.registered = true; + Ok(()) + } +} + +impl<T: Sync> FileOpenAdapter for Registration<T> { + type Arg = T; + + unsafe fn convert(_inode: *mut bindings::inode, file: *mut bindings::file) -> *const Self::Arg { + let reg = crate::container_of!((*file).private_data, Self, mdev); + &(*reg).context + } +} + +// SAFETY: The only method is `register()`, which requires a (pinned) mutable `Registration`, so it +// is safe to pass `&Registration` to multiple threads because it offers no interior mutability, +// except maybe through `Registration::context`, but it is itself `Sync`. +unsafe impl<T: Sync> Sync for Registration<T> {} + +// SAFETY: All functions work from any thread. So as long as the `Registration::context` is +// `Send`, so is `Registration<T>`. `T` needs to be `Sync` because it's a requirement of +// `Registration<T>`. +unsafe impl<T: Send + Sync> Send for Registration<T> {} + +impl<T: Sync> Drop for Registration<T> { + /// Removes the registration from the kernel if it has completed successfully before. + fn drop(&mut self) { + if self.registered { + unsafe { bindings::misc_deregister(&mut self.mdev) } + } + } +} diff --git a/rust/kernel/module_param.rs b/rust/kernel/module_param.rs new file mode 100644 index 0000000000000..e8d51fe613f5b --- /dev/null +++ b/rust/kernel/module_param.rs @@ -0,0 +1,497 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Types for module parameters. +//! +//! C header: [`include/linux/moduleparam.h`](../../../include/linux/moduleparam.h) + +use core::fmt::Write; + +/// Types that can be used for module parameters. +/// +/// Note that displaying the type in `sysfs` will fail if +/// [`alloc::string::ToString::to_string`] (as implemented through the +/// [`core::fmt::Display`] trait) writes more than [`PAGE_SIZE`] +/// bytes (including an additional null terminator). +/// +/// [`PAGE_SIZE`]: `crate::PAGE_SIZE` +pub trait ModuleParam: core::fmt::Display + core::marker::Sized { + /// The `ModuleParam` will be used by the kernel module through this type. + /// + /// This may differ from `Self` if, for example, `Self` needs to track + /// ownership without exposing it or allocate extra space for other possible + /// parameter values. See [`StringParam`] or [`ArrayParam`] for examples. + type Value: ?Sized; + + /// Whether the parameter is allowed to be set without an argument. + /// + /// Setting this to `true` allows the parameter to be passed without an + /// argument (e.g. just `module.param` instead of `module.param=foo`). + const NOARG_ALLOWED: bool; + + /// Convert a parameter argument into the parameter value. + /// + /// `None` should be returned when parsing of the argument fails. + /// `arg == None` indicates that the parameter was passed without an + /// argument. If `NOARG_ALLOWED` is set to `false` then `arg` is guaranteed + /// to always be `Some(_)`. + /// + /// Parameters passed at boot time will be set before [`kmalloc`] is + /// available (even if the module is loaded at a later time). However, in + /// this case, the argument buffer will be valid for the entire lifetime of + /// the kernel. So implementations of this method which need to allocate + /// should first check that the allocator is available (with + /// [`crate::bindings::slab_is_available`]) and when it is not available + /// provide an alternative implementation which doesn't allocate. In cases + /// where the allocator is not available it is safe to save references to + /// `arg` in `Self`, but in other cases a copy should be made. + /// + /// [`kmalloc`]: ../../../include/linux/slab.h + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option<Self>; + + /// Get the current value of the parameter for use in the kernel module. + /// + /// This function should not be used directly. Instead use the wrapper + /// `read` which will be generated by [`module::module`]. + fn value(&self) -> &Self::Value; + + /// Set the module parameter from a string. + /// + /// Used to set the parameter value when loading the module or when set + /// through `sysfs`. + /// + /// # Safety + /// + /// If `val` is non-null then it must point to a valid null-terminated + /// string. The `arg` field of `param` must be an instance of `Self`. + unsafe extern "C" fn set_param( + val: *const crate::c_types::c_char, + param: *const crate::bindings::kernel_param, + ) -> crate::c_types::c_int { + let arg = if val.is_null() { + None + } else { + Some(crate::c_types::c_string_bytes(val)) + }; + match Self::try_from_param_arg(arg) { + Some(new_value) => { + let old_value = (*param).__bindgen_anon_1.arg as *mut Self; + let _ = core::ptr::replace(old_value, new_value); + 0 + } + None => crate::error::Error::EINVAL.to_kernel_errno(), + } + } + + /// Write a string representation of the current parameter value to `buf`. + /// + /// Used for displaying the current parameter value in `sysfs`. + /// + /// # Safety + /// + /// `buf` must be a buffer of length at least `kernel::PAGE_SIZE` that is + /// writeable. The `arg` field of `param` must be an instance of `Self`. + unsafe extern "C" fn get_param( + buf: *mut crate::c_types::c_char, + param: *const crate::bindings::kernel_param, + ) -> crate::c_types::c_int { + let slice = core::slice::from_raw_parts_mut(buf as *mut u8, crate::PAGE_SIZE); + let mut buf = crate::buffer::Buffer::new(slice); + match write!(buf, "{}\0", *((*param).__bindgen_anon_1.arg as *mut Self)) { + Err(_) => crate::error::Error::EINVAL.to_kernel_errno(), + Ok(()) => buf.bytes_written() as crate::c_types::c_int, + } + } + + /// Drop the parameter. + /// + /// Called when unloading a module. + /// + /// # Safety + /// + /// The `arg` field of `param` must be an instance of `Self`. + unsafe extern "C" fn free(arg: *mut crate::c_types::c_void) { + core::ptr::drop_in_place(arg as *mut Self); + } +} + +/// Trait for parsing integers. +/// +/// Strings begining with `0x`, `0o`, or `0b` are parsed as hex, octal, or +/// binary respectively. Strings beginning with `0` otherwise are parsed as +/// octal. Anything else is parsed as decimal. A leading `+` or `-` is also +/// permitted. Any string parsed by [`kstrtol()`] or [`kstrtoul()`] will be +/// successfully parsed. +/// +/// [`kstrtol()`]: https://www.kernel.org/doc/html/latest/core-api/kernel-api.html#c.kstrtol +/// [`kstrtoul()`]: https://www.kernel.org/doc/html/latest/core-api/kernel-api.html#c.kstrtoul +trait ParseInt: Sized { + fn from_str_radix(src: &str, radix: u32) -> Result<Self, core::num::ParseIntError>; + fn checked_neg(self) -> Option<Self>; + + fn from_str_unsigned(src: &str) -> Result<Self, core::num::ParseIntError> { + let (radix, digits) = if let Some(n) = src.strip_prefix("0x") { + (16, n) + } else if let Some(n) = src.strip_prefix("0X") { + (16, n) + } else if let Some(n) = src.strip_prefix("0o") { + (8, n) + } else if let Some(n) = src.strip_prefix("0O") { + (8, n) + } else if let Some(n) = src.strip_prefix("0b") { + (2, n) + } else if let Some(n) = src.strip_prefix("0B") { + (2, n) + } else if src.starts_with('0') { + (8, src) + } else { + (10, src) + }; + Self::from_str_radix(digits, radix) + } + + fn from_str(src: &str) -> Option<Self> { + match src.bytes().next() { + None => None, + Some(b'-') => Self::from_str_unsigned(&src[1..]).ok()?.checked_neg(), + Some(b'+') => Some(Self::from_str_unsigned(&src[1..]).ok()?), + Some(_) => Some(Self::from_str_unsigned(src).ok()?), + } + } +} + +macro_rules! impl_parse_int { + ($ty:ident) => { + impl ParseInt for $ty { + fn from_str_radix(src: &str, radix: u32) -> Result<Self, core::num::ParseIntError> { + $ty::from_str_radix(src, radix) + } + + fn checked_neg(self) -> Option<Self> { + self.checked_neg() + } + } + }; +} + +impl_parse_int!(i8); +impl_parse_int!(u8); +impl_parse_int!(i16); +impl_parse_int!(u16); +impl_parse_int!(i32); +impl_parse_int!(u32); +impl_parse_int!(i64); +impl_parse_int!(u64); +impl_parse_int!(isize); +impl_parse_int!(usize); + +macro_rules! impl_module_param { + ($ty:ident) => { + impl ModuleParam for $ty { + type Value = $ty; + + const NOARG_ALLOWED: bool = false; + + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option<Self> { + let bytes = arg?; + let utf8 = core::str::from_utf8(bytes).ok()?; + <$ty as crate::module_param::ParseInt>::from_str(utf8) + } + + fn value(&self) -> &Self::Value { + self + } + } + }; +} + +#[doc(hidden)] +#[macro_export] +/// Generate a static [`kernel_param_ops`](../../../include/linux/moduleparam.h) struct. +/// +/// # Example +/// ```rust +/// make_param_ops!( +/// /// Documentation for new param ops. +/// PARAM_OPS_MYTYPE, // Name for the static. +/// MyType // A type which implements [`ModuleParam`]. +/// ); +/// ``` +macro_rules! make_param_ops { + ($ops:ident, $ty:ty) => { + $crate::make_param_ops!( + #[doc=""] + $ops, + $ty + ); + }; + ($(#[$meta:meta])* $ops:ident, $ty:ty) => { + $(#[$meta])* + /// + /// Static [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// struct generated by [`make_param_ops`]. + pub static $ops: $crate::bindings::kernel_param_ops = $crate::bindings::kernel_param_ops { + flags: if <$ty as $crate::module_param::ModuleParam>::NOARG_ALLOWED { + $crate::bindings::KERNEL_PARAM_OPS_FL_NOARG + } else { + 0 + }, + set: Some(<$ty as $crate::module_param::ModuleParam>::set_param), + get: Some(<$ty as $crate::module_param::ModuleParam>::get_param), + free: Some(<$ty as $crate::module_param::ModuleParam>::free), + }; + }; +} + +impl_module_param!(i8); +impl_module_param!(u8); +impl_module_param!(i16); +impl_module_param!(u16); +impl_module_param!(i32); +impl_module_param!(u32); +impl_module_param!(i64); +impl_module_param!(u64); +impl_module_param!(isize); +impl_module_param!(usize); + +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`i8`]. + PARAM_OPS_I8, + i8 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`u8`]. + PARAM_OPS_U8, + u8 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`i16`]. + PARAM_OPS_I16, + i16 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`u16`]. + PARAM_OPS_U16, + u16 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`i32`]. + PARAM_OPS_I32, + i32 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`u32`]. + PARAM_OPS_U32, + u32 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`i64`]. + PARAM_OPS_I64, + i64 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`u64`]. + PARAM_OPS_U64, + u64 +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`isize`]. + PARAM_OPS_ISIZE, + isize +); +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`usize`]. + PARAM_OPS_USIZE, + usize +); + +impl ModuleParam for bool { + type Value = bool; + + const NOARG_ALLOWED: bool = true; + + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option<Self> { + match arg { + None => Some(true), + Some(b"y") | Some(b"Y") | Some(b"1") | Some(b"true") => Some(true), + Some(b"n") | Some(b"N") | Some(b"0") | Some(b"false") => Some(false), + _ => None, + } + } + + fn value(&self) -> &Self::Value { + self + } +} + +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`bool`]. + PARAM_OPS_BOOL, + bool +); + +/// An array of at __most__ `N` values. +/// +/// # Invariant +/// +/// The first `self.used` elements of `self.values` are initialized. +pub struct ArrayParam<T, const N: usize> { + values: [core::mem::MaybeUninit<T>; N], + used: usize, +} + +impl<T, const N: usize> ArrayParam<T, { N }> { + fn values(&self) -> &[T] { + // SAFETY: The invariant maintained by `ArrayParam` allows us to cast + // the first `self.used` elements to `T`. + unsafe { + &*(&self.values[0..self.used] as *const [core::mem::MaybeUninit<T>] as *const [T]) + } + } +} + +impl<T: Copy, const N: usize> ArrayParam<T, { N }> { + const fn new() -> Self { + // INVARIANT: The first `self.used` elements of `self.values` are + // initialized. + ArrayParam { + values: [core::mem::MaybeUninit::uninit(); N], + used: 0, + } + } + + const fn push(&mut self, val: T) { + if self.used < N { + // INVARIANT: The first `self.used` elements of `self.values` are + // initialized. + self.values[self.used] = core::mem::MaybeUninit::new(val); + self.used += 1; + } + } + + /// Create an instance of `ArrayParam` initialized with `vals`. + /// + /// This function is only meant to be used in the [`module::module`] macro. + pub const fn create(vals: &[T]) -> Self { + let mut result = ArrayParam::new(); + let mut i = 0; + while i < vals.len() { + result.push(vals[i]); + i += 1; + } + result + } +} + +impl<T: core::fmt::Display, const N: usize> core::fmt::Display for ArrayParam<T, { N }> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + for val in self.values() { + write!(f, "{},", val)?; + } + Ok(()) + } +} + +impl<T: Copy + core::fmt::Display + ModuleParam, const N: usize> ModuleParam + for ArrayParam<T, { N }> +{ + type Value = [T]; + + const NOARG_ALLOWED: bool = false; + + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option<Self> { + arg.and_then(|args| { + let mut result = Self::new(); + for arg in args.split(|b| *b == b',') { + result.push(T::try_from_param_arg(Some(arg))?); + } + Some(result) + }) + } + + fn value(&self) -> &Self::Value { + self.values() + } +} + +/// A C-style string parameter. +/// +/// The Rust version of the [`charp`] parameter. This type is meant to be +/// used by the [`module::module`] macro, not handled directly. Instead use the +/// `read` method generated by that macro. +/// +/// [`charp`]: ../../../include/linux/moduleparam.h +pub enum StringParam { + /// A borrowed parameter value. + /// + /// Either the default value (which is static in the module) or borrowed + /// from the original argument buffer used to set the value. + Ref(&'static [u8]), + + /// A value that was allocated when the parameter was set. + /// + /// The value needs to be freed when the parameter is reset or the module is + /// unloaded. + Owned(alloc::vec::Vec<u8>), +} + +impl StringParam { + fn bytes(&self) -> &[u8] { + match self { + StringParam::Ref(bytes) => *bytes, + StringParam::Owned(vec) => &vec[..], + } + } +} + +impl core::fmt::Display for StringParam { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let bytes = self.bytes(); + match core::str::from_utf8(bytes) { + Ok(utf8) => write!(f, "{}", utf8), + Err(_) => write!(f, "{:?}", bytes), + } + } +} + +impl ModuleParam for StringParam { + type Value = [u8]; + + const NOARG_ALLOWED: bool = false; + + fn try_from_param_arg(arg: Option<&'static [u8]>) -> Option<Self> { + // SAFETY: It is always safe to call [`slab_is_available`](../../../include/linux/slab.h). + let slab_available = unsafe { crate::bindings::slab_is_available() }; + arg.and_then(|arg| { + if slab_available { + let mut vec = alloc::vec::Vec::new(); + vec.try_reserve_exact(arg.len()).ok()?; + vec.extend_from_slice(arg); + Some(StringParam::Owned(vec)) + } else { + Some(StringParam::Ref(arg)) + } + }) + } + + fn value(&self) -> &Self::Value { + self.bytes() + } +} + +make_param_ops!( + /// Rust implementation of [`kernel_param_ops`](../../../include/linux/moduleparam.h) + /// for [`StringParam`]. + PARAM_OPS_STR, + StringParam +); diff --git a/rust/kernel/pages.rs b/rust/kernel/pages.rs new file mode 100644 index 0000000000000..0426d411470da --- /dev/null +++ b/rust/kernel/pages.rs @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Kernel page allocation and management. +//! +//! TODO: This module is a work in progress. + +use crate::{bindings, c_types, user_ptr::UserSlicePtrReader, Error, KernelResult, PAGE_SIZE}; +use core::{marker::PhantomData, ptr}; + +extern "C" { + #[allow(improper_ctypes)] + fn rust_helper_alloc_pages( + gfp_mask: bindings::gfp_t, + order: c_types::c_uint, + ) -> *mut bindings::page; + + #[allow(improper_ctypes)] + fn rust_helper_kmap(page: *mut bindings::page) -> *mut c_types::c_void; + + #[allow(improper_ctypes)] + fn rust_helper_kunmap(page: *mut bindings::page); +} + +/// A set of physical pages. +/// +/// `Pages` holds a reference to a set of pages of order `ORDER`. Having the order as a generic +/// const allows the struct to have the same size as a pointer. +/// +/// # Invariants +/// +/// The pointer [`Pages::pages`] is valid and points to 2^ORDER pages. +pub struct Pages<const ORDER: u32> { + pages: *mut bindings::page, +} + +impl<const ORDER: u32> Pages<ORDER> { + /// Allocates a new set of contiguous pages. + pub fn new() -> KernelResult<Self> { + // TODO: Consider whether we want to allow callers to specify flags. + // SAFETY: This only allocates pages. We check that it succeeds in the next statement. + let pages = unsafe { + rust_helper_alloc_pages( + bindings::GFP_KERNEL | bindings::__GFP_ZERO | bindings::__GFP_HIGHMEM, + ORDER, + ) + }; + if pages.is_null() { + return Err(Error::ENOMEM); + } + // INVARIANTS: We checked that the allocation above succeeded> + Ok(Self { pages }) + } + + /// Maps a single page at the given address in the given VM area. + /// + /// This is only meant to be used by pages of order 0. + pub fn insert_page(&self, vma: &mut bindings::vm_area_struct, address: usize) -> KernelResult { + if ORDER != 0 { + return Err(Error::EINVAL); + } + + // SAFETY: We check above that the allocation is of order 0. The range of `address` is + // already checked by `vm_insert_page`. + let ret = unsafe { bindings::vm_insert_page(vma, address as _, self.pages) }; + if ret != 0 { + Err(Error::from_kernel_errno(ret)) + } else { + Ok(()) + } + } + + /// Copies data from the given [`UserSlicePtrReader`] into the pages. + pub fn copy_into_page( + &self, + reader: &mut UserSlicePtrReader, + offset: usize, + len: usize, + ) -> KernelResult { + // TODO: For now this only works on the first page. + let end = offset.checked_add(len).ok_or(Error::EINVAL)?; + if end > PAGE_SIZE { + return Err(Error::EINVAL); + } + + let mapping = self.kmap(0).ok_or(Error::EINVAL)?; + + // SAFETY: We ensured that the buffer was valid with the check above. + unsafe { reader.read_raw((mapping.ptr as usize + offset) as _, len) }?; + Ok(()) + } + + /// Maps the pages and reads from them into the given buffer. + /// + /// # Safety + /// + /// Callers must ensure that the destination buffer is valid for the given length. + /// Additionally, if the raw buffer is intended to be recast, they must ensure that the data + /// can be safely cast; [`crate::user_ptr::ReadableFromBytes`] has more details about it. + pub unsafe fn read(&self, dest: *mut u8, offset: usize, len: usize) -> KernelResult { + // TODO: For now this only works on the first page. + let end = offset.checked_add(len).ok_or(Error::EINVAL)?; + if end > PAGE_SIZE { + return Err(Error::EINVAL); + } + + let mapping = self.kmap(0).ok_or(Error::EINVAL)?; + ptr::copy((mapping.ptr as *mut u8).add(offset), dest, len); + Ok(()) + } + + /// Maps the pages and writes into them from the given bufer. + /// + /// # Safety + /// + /// Callers must ensure that the buffer is valid for the given length. Additionally, if the + /// page is (or will be) mapped by userspace, they must ensure that no kernel data is leaked + /// through padding if it was cast from another type; [`crate::user_ptr::WritableToBytes`] has + /// more details about it. + pub unsafe fn write(&self, src: *const u8, offset: usize, len: usize) -> KernelResult { + // TODO: For now this only works on the first page. + let end = offset.checked_add(len).ok_or(Error::EINVAL)?; + if end > PAGE_SIZE { + return Err(Error::EINVAL); + } + + let mapping = self.kmap(0).ok_or(Error::EINVAL)?; + ptr::copy(src, (mapping.ptr as *mut u8).add(offset), len); + Ok(()) + } + + /// Maps the page at index `index`. + fn kmap(&self, index: usize) -> Option<PageMapping> { + if index >= 1usize << ORDER { + return None; + } + + // SAFETY: We checked above that `index` is within range. + let page = unsafe { self.pages.add(index) }; + + // SAFETY: `page` is valid based on the checks above. + let ptr = unsafe { rust_helper_kmap(page) }; + if ptr.is_null() { + return None; + } + + Some(PageMapping { + page, + ptr, + _phantom: PhantomData, + }) + } +} + +impl<const ORDER: u32> Drop for Pages<ORDER> { + fn drop(&mut self) { + // SAFETY: By the type invariants, we know the pages are allocated with the given order. + unsafe { bindings::__free_pages(self.pages, ORDER) }; + } +} + +struct PageMapping<'a> { + page: *mut bindings::page, + ptr: *mut c_types::c_void, + _phantom: PhantomData<&'a i32>, +} + +impl Drop for PageMapping<'_> { + fn drop(&mut self) { + // SAFETY: An instance of `PageMapping` is created only when `kmap` succeeded for the given + // page, so it is safe to unmap it here. + unsafe { rust_helper_kunmap(self.page) }; + } +} diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs new file mode 100644 index 0000000000000..2f8215bc23525 --- /dev/null +++ b/rust/kernel/prelude.rs @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! The `kernel` prelude. +//! +//! These are most common items used by Rust code in the kernel, intended to +//! be imported by all Rust code, for convenience. +//! +//! # Examples +//! +//! ```rust,no_run +//! use kernel::prelude::*; +//! ``` + +pub use alloc::{borrow::ToOwned, string::String}; + +pub use module::{module, module_misc_device}; + +pub use super::{pr_alert, pr_cont, pr_crit, pr_emerg, pr_err, pr_info, pr_notice, pr_warn}; + +pub use super::static_assert; + +pub use super::{KernelModule, KernelResult}; diff --git a/rust/kernel/print.rs b/rust/kernel/print.rs new file mode 100644 index 0000000000000..ff9d1269cdae6 --- /dev/null +++ b/rust/kernel/print.rs @@ -0,0 +1,463 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Printing facilities. +//! +//! C header: [`include/linux/printk.h`](../../../../include/linux/printk.h) +//! +//! Reference: <https://www.kernel.org/doc/html/latest/core-api/printk-basics.html> + +use core::cmp; +use core::fmt; + +use crate::bindings; +use crate::c_types::c_int; + +/// Format strings. +/// +/// Public but hidden since it should only be used from public macros. +#[doc(hidden)] +pub mod format_strings { + use crate::bindings; + + /// The length we copy from the `KERN_*` kernel prefixes. + const LENGTH_PREFIX: usize = 2; + + /// The length of the fixed format strings. + pub const LENGTH: usize = 11; + + /// Generates a fixed format string for the kernel's [`printk`]. + /// + /// The format string is always the same for a given level, i.e. for a + /// given `prefix`, which are the kernel's `KERN_*` constants. + /// + /// [`printk`]: ../../../../include/linux/printk.h + const fn generate(is_cont: bool, prefix: &[u8; 3]) -> [u8; LENGTH] { + // Ensure the `KERN_*` macros are what we expect. + assert!(prefix[0] == b'\x01'); + if is_cont { + assert!(prefix[1] == b'c'); + } else { + assert!(prefix[1] >= b'0' && prefix[1] <= b'7'); + } + assert!(prefix[2] == b'\x00'); + + let suffix: &[u8; LENGTH - LENGTH_PREFIX] = if is_cont { + b"%.*s\0\0\0\0\0" + } else { + b"%s: %.*s\0" + }; + + [ + prefix[0], prefix[1], suffix[0], suffix[1], suffix[2], suffix[3], suffix[4], suffix[5], + suffix[6], suffix[7], suffix[8], + ] + } + + // Generate the format strings at compile-time. + // + // This avoids the compiler generating the contents on the fly in the stack. + // + // Furthermore, `static` instead of `const` is used to share the strings + // for all the kernel. + pub static EMERG: [u8; LENGTH] = generate(false, bindings::KERN_EMERG); + pub static ALERT: [u8; LENGTH] = generate(false, bindings::KERN_ALERT); + pub static CRIT: [u8; LENGTH] = generate(false, bindings::KERN_CRIT); + pub static ERR: [u8; LENGTH] = generate(false, bindings::KERN_ERR); + pub static WARNING: [u8; LENGTH] = generate(false, bindings::KERN_WARNING); + pub static NOTICE: [u8; LENGTH] = generate(false, bindings::KERN_NOTICE); + pub static INFO: [u8; LENGTH] = generate(false, bindings::KERN_INFO); + pub static DEBUG: [u8; LENGTH] = generate(false, bindings::KERN_DEBUG); + pub static CONT: [u8; LENGTH] = generate(true, bindings::KERN_CONT); +} + +/// Prints a message via the kernel's [`printk`]. +/// +/// Public but hidden since it should only be used from public macros. +/// +/// # Safety +/// +/// The format string must be one of the ones in [`format_strings`], and +/// the module name must be null-terminated. +/// +/// [`printk`]: ../../../../include/linux/printk.h +#[doc(hidden)] +pub unsafe fn call_printk( + format_string: &[u8; format_strings::LENGTH], + module_name: &[u8], + string: &[u8], +) { + // `printk` does not seem to fail in any path. + bindings::printk( + format_string.as_ptr() as _, + module_name.as_ptr(), + string.len() as c_int, + string.as_ptr(), + ); +} + +/// Prints a message via the kernel's [`printk`] for the `CONT` level. +/// +/// Public but hidden since it should only be used from public macros. +/// +/// [`printk`]: ../../../../include/linux/printk.h +#[doc(hidden)] +pub fn call_printk_cont(string: &[u8]) { + // `printk` does not seem to fail in any path. + // + // SAFETY: The format string is fixed. + unsafe { + bindings::printk( + format_strings::CONT.as_ptr() as _, + string.len() as c_int, + string.as_ptr(), + ); + } +} + +/// The maximum size of a log line in the kernel. +/// +/// From `kernel/printk/printk.c`. +const LOG_LINE_MAX: usize = 1024 - 32; + +/// The maximum size of a log line in our side. +/// +/// FIXME: We should be smarter than this, but for the moment, to reduce stack +/// usage, we only allow this much which should work for most purposes. +const LOG_LINE_SIZE: usize = 300; +crate::static_assert!(LOG_LINE_SIZE <= LOG_LINE_MAX); + +/// Public but hidden since it should only be used from public macros. +#[doc(hidden)] +pub struct LogLineWriter { + data: [u8; LOG_LINE_SIZE], + pos: usize, +} + +impl LogLineWriter { + /// Creates a new [`LogLineWriter`]. + pub fn new() -> LogLineWriter { + LogLineWriter { + data: [0u8; LOG_LINE_SIZE], + pos: 0, + } + } + + /// Returns the internal buffer as a byte slice. + pub fn as_bytes(&self) -> &[u8] { + &self.data[..self.pos] + } +} + +impl Default for LogLineWriter { + fn default() -> Self { + Self::new() + } +} + +impl fmt::Write for LogLineWriter { + fn write_str(&mut self, s: &str) -> fmt::Result { + let copy_len = cmp::min(LOG_LINE_SIZE - self.pos, s.as_bytes().len()); + self.data[self.pos..self.pos + copy_len].copy_from_slice(&s.as_bytes()[..copy_len]); + self.pos += copy_len; + Ok(()) + } +} + +/// Helper function for the [`print_macro!`] to reduce stack usage. +/// +/// Public but hidden since it should only be used from public macros. +/// +/// # Safety +/// +/// The format string must be one of the ones in [`format_strings`], and +/// the module name must be null-terminated. +#[doc(hidden)] +pub unsafe fn format_and_call<const CONT: bool>( + format_string: &[u8; format_strings::LENGTH], + module_name: &[u8], + args: fmt::Arguments, +) { + // Careful: this object takes quite a bit of stack. + let mut writer = LogLineWriter::new(); + + match fmt::write(&mut writer, args) { + Ok(_) => { + if CONT { + call_printk_cont(writer.as_bytes()); + } else { + call_printk(format_string, module_name, writer.as_bytes()); + } + } + + Err(_) => { + call_printk( + &format_strings::CRIT, + module_name, + b"Failure to format string.\n", + ); + } + }; +} + +/// Performs formatting and forwards the string to [`call_printk`]. +/// +/// Public but hidden since it should only be used from public macros. +#[doc(hidden)] +#[macro_export] +macro_rules! print_macro ( + // Without extra arguments: no need to format anything. + ($format_string:path, false, $fmt:expr) => ( + // SAFETY: This hidden macro should only be called by the documented + // printing macros which ensure the format string is one of the fixed + // ones. All `__LOG_PREFIX`s are null-terminated as they are generated + // by the `module!` proc macro or fixed values defined in a kernel + // crate. + unsafe { + $crate::print::call_printk( + &$format_string, + crate::__LOG_PREFIX, + $fmt.as_bytes(), + ); + } + ); + + // Without extra arguments: no need to format anything (`CONT` case). + ($format_string:path, true, $fmt:expr) => ( + $crate::print::call_printk_cont( + $fmt.as_bytes(), + ); + ); + + // With extra arguments: we need to perform formatting. + ($format_string:path, $cont:literal, $fmt:expr, $($arg:tt)*) => ( + // Forwarding the call to a function to perform the formatting + // is needed here to avoid stack overflows in non-optimized builds when + // invoking the printing macros a lot of times in the same function. + // Without it, the compiler reserves one `LogLineWriter` per macro + // invocation, which is a huge type. + // + // We could use an immediately-invoked closure for this, which + // seems to lower even more the stack usage at `opt-level=0` because + // `fmt::Arguments` objects do not pile up. However, that breaks + // the `?` operator if used in one of the arguments. + // + // At `opt-level=2`, the generated code is basically the same for + // all alternatives. + // + // SAFETY: This hidden macro should only be called by the documented + // printing macros which ensure the format string is one of the fixed + // ones. All `__LOG_PREFIX`s are null-terminated as they are generated + // by the `module!` proc macro or fixed values defined in a kernel + // crate. + unsafe { + $crate::print::format_and_call::<$cont>( + &$format_string, + crate::__LOG_PREFIX, + format_args!($fmt, $($arg)*), + ); + } + ); +); + +// We could use a macro to generate these macros. However, doing so ends +// up being a bit ugly: it requires the dollar token trick to escape `$` as +// well as playing with the `doc` attribute. Furthermore, they cannot be easily +// imported in the prelude due to [1]. So, for the moment, we just write them +// manually, like in the C side; while keeping most of the logic in another +// macro, i.e. [`print_macro`]. +// +// [1]: https://github.com/rust-lang/rust/issues/52234 + +/// Prints an emergency-level message (level 0). +/// +/// Use this level if the system is unusable. +/// +/// Equivalent to the kernel's [`pr_emerg`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`alloc::format!`] for information about the formatting syntax. +/// +/// [`pr_emerg`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_emerg +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// pr_emerg!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_emerg ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::EMERG, false, $($arg)*) + ) +); + +/// Prints an alert-level message (level 1). +/// +/// Use this level if action must be taken immediately. +/// +/// Equivalent to the kernel's [`pr_alert`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`alloc::format!`] for information about the formatting syntax. +/// +/// [`pr_alert`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_alert +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// pr_alert!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_alert ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::ALERT, false, $($arg)*) + ) +); + +/// Prints a critical-level message (level 2). +/// +/// Use this level for critical conditions. +/// +/// Equivalent to the kernel's [`pr_crit`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`alloc::format!`] for information about the formatting syntax. +/// +/// [`pr_crit`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_crit +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// pr_crit!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_crit ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::CRIT, false, $($arg)*) + ) +); + +/// Prints an error-level message (level 3). +/// +/// Use this level for error conditions. +/// +/// Equivalent to the kernel's [`pr_err`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`alloc::format!`] for information about the formatting syntax. +/// +/// [`pr_err`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_err +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// pr_err!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_err ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::ERR, false, $($arg)*) + ) +); + +/// Prints a warning-level message (level 4). +/// +/// Use this level for warning conditions. +/// +/// Equivalent to the kernel's [`pr_warn`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`alloc::format!`] for information about the formatting syntax. +/// +/// [`pr_warn`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_warn +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// pr_warn!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_warn ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::WARNING, false, $($arg)*) + ) +); + +/// Prints a notice-level message (level 5). +/// +/// Use this level for normal but significant conditions. +/// +/// Equivalent to the kernel's [`pr_notice`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`alloc::format!`] for information about the formatting syntax. +/// +/// [`pr_notice`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_notice +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// pr_notice!("hello {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_notice ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::NOTICE, false, $($arg)*) + ) +); + +/// Prints an info-level message (level 6). +/// +/// Use this level for informational messages. +/// +/// Equivalent to the kernel's [`pr_info`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`alloc::format!`] for information about the formatting syntax. +/// +/// [`pr_info`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_info +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// pr_info!("hello {}\n", "there"); +/// ``` +#[macro_export] +#[doc(alias = "print")] +macro_rules! pr_info ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::INFO, false, $($arg)*) + ) +); + +/// Continues a previous log message in the same line. +/// +/// Use only when continuing a previous `pr_*!` macro (e.g. [`pr_info!`]). +/// +/// Equivalent to the kernel's [`pr_cont`] macro. +/// +/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and +/// [`alloc::format!`] for information about the formatting syntax. +/// +/// [`pr_cont`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_cont +/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html +/// +/// # Examples +/// +/// ``` +/// pr_info!("hello"); +/// pr_cont!(" {}\n", "there"); +/// ``` +#[macro_export] +macro_rules! pr_cont ( + ($($arg:tt)*) => ( + $crate::print_macro!($crate::print::format_strings::CONT, true, $($arg)*) + ) +); diff --git a/rust/kernel/random.rs b/rust/kernel/random.rs new file mode 100644 index 0000000000000..a7df79c1f7bf7 --- /dev/null +++ b/rust/kernel/random.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Random numbers. +//! +//! C header: [`include/linux/random.h`](../../../../include/linux/random.h) + +use core::convert::TryInto; + +use crate::{bindings, c_types, error}; + +/// Fills a byte slice with random bytes generated from the kernel's CSPRNG. +/// +/// Ensures that the CSPRNG has been seeded before generating any random bytes, +/// and will block until it is ready. +pub fn getrandom(dest: &mut [u8]) -> error::KernelResult { + let res = unsafe { bindings::wait_for_random_bytes() }; + if res != 0 { + return Err(error::Error::from_kernel_errno(res)); + } + + unsafe { + bindings::get_random_bytes( + dest.as_mut_ptr() as *mut c_types::c_void, + dest.len().try_into()?, + ); + } + Ok(()) +} + +/// Fills a byte slice with random bytes generated from the kernel's CSPRNG. +/// +/// If the CSPRNG is not yet seeded, returns an `Err(EAGAIN)` immediately. +pub fn getrandom_nonblock(dest: &mut [u8]) -> error::KernelResult { + if !unsafe { bindings::rng_is_initialized() } { + return Err(error::Error::EAGAIN); + } + getrandom(dest) +} + +/// Contributes the contents of a byte slice to the kernel's entropy pool. +/// +/// Does *not* credit the kernel entropy counter though. +pub fn add_randomness(data: &[u8]) { + unsafe { + bindings::add_device_randomness( + data.as_ptr() as *const c_types::c_void, + data.len().try_into().unwrap(), + ); + } +} diff --git a/rust/kernel/raw_list.rs b/rust/kernel/raw_list.rs new file mode 100644 index 0000000000000..0202b44d560ac --- /dev/null +++ b/rust/kernel/raw_list.rs @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Raw lists. +//! +//! TODO: This module is a work in progress. + +use core::{ + cell::UnsafeCell, + ptr, + ptr::NonNull, + sync::atomic::{AtomicBool, Ordering}, +}; + +/// A descriptor of list elements. +/// +/// It describes the type of list elements and provides a function to determine how to get the +/// links to be used on a list. +/// +/// A type that may be in multiple lists simultaneously neneds to implement one of these for each +/// simultaneous list. +pub trait GetLinks { + /// The type of the entries in the list. + type EntryType: ?Sized; + + /// Returns the links to be used when linking an entry within a list. + fn get_links(data: &Self::EntryType) -> &Links<Self::EntryType>; +} + +/// The links used to link an object on a linked list. +/// +/// Instances of this type are usually embedded in structures and returned in calls to +/// [`GetLinks::get_links`]. +pub struct Links<T: ?Sized> { + inserted: AtomicBool, + entry: UnsafeCell<ListEntry<T>>, +} + +impl<T: ?Sized> Links<T> { + /// Constructs a new [`Links`] instance that isn't inserted on any lists yet. + pub fn new() -> Self { + Self { + inserted: AtomicBool::new(false), + entry: UnsafeCell::new(ListEntry::new()), + } + } + + fn acquire_for_insertion(&self) -> bool { + self.inserted + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + } + + fn release_after_removal(&self) { + self.inserted.store(false, Ordering::Release); + } +} + +impl<T: ?Sized> Default for Links<T> { + fn default() -> Self { + Self::new() + } +} + +struct ListEntry<T: ?Sized> { + next: Option<NonNull<T>>, + prev: Option<NonNull<T>>, +} + +impl<T: ?Sized> ListEntry<T> { + fn new() -> Self { + Self { + next: None, + prev: None, + } + } +} + +/// A linked list. +/// +/// # Invariants +/// +/// The links of objects added to a list are owned by the list. +pub(crate) struct RawList<G: GetLinks> { + head: Option<NonNull<G::EntryType>>, +} + +impl<G: GetLinks> RawList<G> { + pub(crate) fn new() -> Self { + Self { head: None } + } + + pub(crate) fn is_empty(&self) -> bool { + self.head.is_none() + } + + fn insert_after_priv( + &mut self, + existing: &G::EntryType, + new_entry: &mut ListEntry<G::EntryType>, + new_ptr: Option<NonNull<G::EntryType>>, + ) { + { + // SAFETY: It's safe to get the previous entry of `existing` because the list cannot + // change. + let existing_links = unsafe { &mut *G::get_links(existing).entry.get() }; + new_entry.next = existing_links.next; + existing_links.next = new_ptr; + } + + new_entry.prev = Some(NonNull::from(existing)); + + // SAFETY: It's safe to get the next entry of `existing` because the list cannot change. + let next_links = + unsafe { &mut *G::get_links(new_entry.next.unwrap().as_ref()).entry.get() }; + next_links.prev = new_ptr; + } + + /// Inserts the given object after `existing`. + /// + /// # Safety + /// + /// Callers must ensure that `existing` points to a valid entry that is on the list. + pub(crate) unsafe fn insert_after( + &mut self, + existing: &G::EntryType, + new: &G::EntryType, + ) -> bool { + let links = G::get_links(new); + if !links.acquire_for_insertion() { + // Nothing to do if already inserted. + return false; + } + + // SAFETY: The links are now owned by the list, so it is safe to get a mutable reference. + let new_entry = &mut *links.entry.get(); + self.insert_after_priv(existing, new_entry, Some(NonNull::from(new))); + true + } + + fn push_back_internal(&mut self, new: &G::EntryType) -> bool { + let links = G::get_links(new); + if !links.acquire_for_insertion() { + // Nothing to do if already inserted. + return false; + } + + // SAFETY: The links are now owned by the list, so it is safe to get a mutable reference. + let new_entry = unsafe { &mut *links.entry.get() }; + let new_ptr = Some(NonNull::from(new)); + match self.back() { + // SAFETY: `back` is valid as the list cannot change. + Some(back) => self.insert_after_priv(unsafe { back.as_ref() }, new_entry, new_ptr), + None => { + self.head = new_ptr; + new_entry.next = new_ptr; + new_entry.prev = new_ptr; + } + } + true + } + + pub(crate) unsafe fn push_back(&mut self, new: &G::EntryType) -> bool { + self.push_back_internal(new) + } + + fn remove_internal(&mut self, data: &G::EntryType) -> bool { + let links = G::get_links(data); + + // SAFETY: The links are now owned by the list, so it is safe to get a mutable reference. + let entry = unsafe { &mut *links.entry.get() }; + let next = if let Some(next) = entry.next { + next + } else { + // Nothing to do if the entry is not on the list. + return false; + }; + + if ptr::eq(data, next.as_ptr()) { + // We're removing the only element. + self.head = None + } else { + // Update the head if we're removing it. + if let Some(raw_head) = self.head { + if ptr::eq(data, raw_head.as_ptr()) { + self.head = Some(next); + } + } + + // SAFETY: It's safe to get the previous entry because the list cannot change. + unsafe { &mut *G::get_links(entry.prev.unwrap().as_ref()).entry.get() }.next = + entry.next; + + // SAFETY: It's safe to get the next entry because the list cannot change. + unsafe { &mut *G::get_links(next.as_ref()).entry.get() }.prev = entry.prev; + } + + // Reset the links of the element we're removing so that we know it's not on any list. + entry.next = None; + entry.prev = None; + links.release_after_removal(); + true + } + + /// Removes the given entry. + /// + /// # Safety + /// + /// Callers must ensure that `data` is either on this list or in no list. It being on another + /// list leads to memory unsafety. + pub(crate) unsafe fn remove(&mut self, data: &G::EntryType) -> bool { + self.remove_internal(data) + } + + fn pop_front_internal(&mut self) -> Option<NonNull<G::EntryType>> { + let head = self.head?; + // SAFETY: The head is on the list as we just got it from there and it cannot change. + unsafe { self.remove(head.as_ref()) }; + Some(head) + } + + pub(crate) fn pop_front(&mut self) -> Option<NonNull<G::EntryType>> { + self.pop_front_internal() + } + + pub(crate) fn front(&self) -> Option<NonNull<G::EntryType>> { + self.head + } + + pub(crate) fn back(&self) -> Option<NonNull<G::EntryType>> { + // SAFETY: The links of head are owned by the list, so it is safe to get a reference. + unsafe { &*G::get_links(self.head?.as_ref()).entry.get() }.prev + } + + pub(crate) fn cursor_front(&self) -> Cursor<'_, G> { + Cursor::new(self, self.front()) + } + + pub(crate) fn cursor_front_mut(&mut self) -> CursorMut<'_, G> { + CursorMut::new(self, self.front()) + } +} + +struct CommonCursor<G: GetLinks> { + cur: Option<NonNull<G::EntryType>>, +} + +impl<G: GetLinks> CommonCursor<G> { + fn new(cur: Option<NonNull<G::EntryType>>) -> Self { + Self { cur } + } + + fn move_next(&mut self, list: &RawList<G>) { + match self.cur.take() { + None => self.cur = list.head, + Some(cur) => { + if let Some(head) = list.head { + // SAFETY: We have a shared ref to the linked list, so the links can't change. + let links = unsafe { &*G::get_links(cur.as_ref()).entry.get() }; + if links.next.unwrap() != head { + self.cur = links.next; + } + } + } + } + } + + fn move_prev(&mut self, list: &RawList<G>) { + match list.head { + None => self.cur = None, + Some(head) => { + let next = match self.cur.take() { + None => head, + Some(cur) => { + if cur == head { + return; + } + cur + } + }; + // SAFETY: There's a shared ref to the list, so the links can't change. + let links = unsafe { &*G::get_links(next.as_ref()).entry.get() }; + self.cur = links.prev; + } + } + } +} + +/// A list cursor that allows traversing a linked list and inspecting elements. +pub struct Cursor<'a, G: GetLinks> { + cursor: CommonCursor<G>, + list: &'a RawList<G>, +} + +impl<'a, G: GetLinks> Cursor<'a, G> { + fn new(list: &'a RawList<G>, cur: Option<NonNull<G::EntryType>>) -> Self { + Self { + list, + cursor: CommonCursor::new(cur), + } + } + + /// Returns the element the cursor is currently positioned on. + pub fn current(&self) -> Option<&'a G::EntryType> { + let cur = self.cursor.cur?; + // SAFETY: Objects must be kept alive while on the list. + Some(unsafe { &*cur.as_ptr() }) + } + + /// Moves the cursor to the next element. + pub fn move_next(&mut self) { + self.cursor.move_next(self.list); + } +} + +pub(crate) struct CursorMut<'a, G: GetLinks> { + cursor: CommonCursor<G>, + list: &'a mut RawList<G>, +} + +impl<'a, G: GetLinks> CursorMut<'a, G> { + fn new(list: &'a mut RawList<G>, cur: Option<NonNull<G::EntryType>>) -> Self { + Self { + list, + cursor: CommonCursor::new(cur), + } + } + + pub(crate) fn current(&mut self) -> Option<&mut G::EntryType> { + let cur = self.cursor.cur?; + // SAFETY: Objects must be kept alive while on the list. + Some(unsafe { &mut *cur.as_ptr() }) + } + + /// Removes the entry the cursor is pointing to and advances the cursor to the next entry. It + /// returns a raw pointer to the removed element (if one is removed). + pub(crate) fn remove_current(&mut self) -> Option<NonNull<G::EntryType>> { + let entry = self.cursor.cur?; + self.cursor.move_next(self.list); + // SAFETY: The entry is on the list as we just got it from there and it cannot change. + unsafe { self.list.remove(entry.as_ref()) }; + Some(entry) + } + + pub(crate) fn peek_next(&mut self) -> Option<&mut G::EntryType> { + let mut new = CommonCursor::new(self.cursor.cur); + new.move_next(self.list); + // SAFETY: Objects must be kept alive while on the list. + Some(unsafe { &mut *new.cur?.as_ptr() }) + } + + pub(crate) fn peek_prev(&mut self) -> Option<&mut G::EntryType> { + let mut new = CommonCursor::new(self.cursor.cur); + new.move_prev(self.list); + // SAFETY: Objects must be kept alive while on the list. + Some(unsafe { &mut *new.cur?.as_ptr() }) + } + + pub(crate) fn move_next(&mut self) { + self.cursor.move_next(self.list); + } +} diff --git a/rust/kernel/static_assert.rs b/rust/kernel/static_assert.rs new file mode 100644 index 0000000000000..1d8f137155c68 --- /dev/null +++ b/rust/kernel/static_assert.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Static assert. + +/// Static assert (i.e. compile-time assert). +/// +/// Similar to C11 [`_Static_assert`] and C++11 [`static_assert`]. +/// +/// The feature may be added to Rust in the future: see [RFC 2790]. +/// +/// [`_Static_assert`]: https://en.cppreference.com/w/c/language/_Static_assert +/// [`static_assert`]: https://en.cppreference.com/w/cpp/language/static_assert +/// [RFC 2790]: https://github.com/rust-lang/rfcs/issues/2790 +/// +/// # Examples +/// +/// ``` +/// static_assert!(42 > 24); +/// static_assert!(core::mem::size_of::<u8>() == 1); +/// +/// const X: &[u8] = b"bar"; +/// static_assert!(X[1] == 'a' as u8); +/// +/// const fn f(x: i32) -> i32 { +/// x + 2 +/// } +/// static_assert!(f(40) == 42); +/// ``` +#[macro_export] +macro_rules! static_assert { + ($condition:expr) => { + // Based on the latest one in `rustc`'s one before it was [removed]. + // + // [removed]: https://github.com/rust-lang/rust/commit/c2dad1c6b9f9636198d7c561b47a2974f5103f6d + #[allow(dead_code)] + const _: () = [()][!($condition) as usize]; + }; +} diff --git a/rust/kernel/sync/arc.rs b/rust/kernel/sync/arc.rs new file mode 100644 index 0000000000000..9b3ae49514436 --- /dev/null +++ b/rust/kernel/sync/arc.rs @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A reference-counted pointer. +//! +//! This module implements a way for users to create reference-counted objects and pointers to +//! them. Such a pointer automatically increments and decrements the count, and drops the +//! underlying object when it reaches zero. It is also safe to use concurrently from multiple +//! threads. +//! +//! It is different from the standard library's [`Arc`] in two ways: it does not support weak +//! references, which allows it to be smaller -- a single pointer-sized integer; it allows users to +//! safely increment the reference count from a single reference to the underlying object. +//! +//! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html + +use crate::KernelResult; +use alloc::boxed::Box; +use core::{ + mem::ManuallyDrop, + ops::Deref, + ptr::NonNull, + sync::atomic::{fence, AtomicUsize, Ordering}, +}; + +/// A reference-counted pointer to an instance of `T`. +/// +/// The reference count is incremented when new instances of [`Ref`] are created, and decremented +/// when they are dropped. When the count reaches zero, the underlying `T` is also dropped. +/// +/// # Invariants +/// +/// The value stored in [`RefCounted::get_count`] corresponds to the number of instances of [`Ref`] +/// that point to that instance of `T`. +pub struct Ref<T: RefCounted + ?Sized> { + ptr: NonNull<T>, +} + +// SAFETY: It is safe to send `Ref<T>` to another thread when the underlying `T` is `Sync` because +// it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs +// `T` to be `Send` because any thread that has a `Ref<T>` may ultimately access `T` directly, for +// example, when the reference count reaches zero and `T` is dropped. +unsafe impl<T: RefCounted + ?Sized + Sync + Send> Send for Ref<T> {} + +// SAFETY: It is safe to send `&Ref<T>` to another thread when the underlying `T` is `Sync` for +// the same reason as above. `T` needs to be `Send` as well because a thread can clone a `&Ref<T>` +// into a `Ref<T>`, which may lead to `T` being accessed by the same reasoning as above. +unsafe impl<T: RefCounted + ?Sized + Sync + Send> Sync for Ref<T> {} + +impl<T: RefCounted> Ref<T> { + /// Constructs a new reference counted instance of `T`. + pub fn try_new(contents: T) -> KernelResult<Self> { + let boxed = Box::try_new(contents)?; + boxed.get_count().count.store(1, Ordering::Relaxed); + let ptr = NonNull::from(Box::leak(boxed)); + Ok(Ref { ptr }) + } +} + +impl<T: RefCounted + ?Sized> Ref<T> { + /// Creates a new reference-counted pointer to the given instance of `T`. + /// + /// It works by incrementing the current reference count as part of constructing the new + /// pointer. + pub fn new_from(obj: &T) -> Self { + let ref_count = obj.get_count(); + let cur = ref_count.count.fetch_add(1, Ordering::Relaxed); + if cur == usize::MAX { + panic!("Reference count overflowed"); + } + Self { + ptr: NonNull::from(obj), + } + } + + /// Returns a mutable reference to `T` iff the reference count is one. Otherwise returns + /// [`None`]. + pub fn get_mut(&mut self) -> Option<&mut T> { + // Synchronises with the decrement in `drop`. + if self.get_count().count.load(Ordering::Acquire) != 1 { + return None; + } + // SAFETY: Since there is only one reference, we know it isn't possible for another thread + // to concurrently call this. + Some(unsafe { self.ptr.as_mut() }) + } + + /// Determines if two reference-counted pointers point to the same underlying instance of `T`. + pub fn ptr_eq(a: &Self, b: &Self) -> bool { + core::ptr::eq(a.ptr.as_ptr(), b.ptr.as_ptr()) + } + + /// Deconstructs a [`Ref`] object into a raw pointer. + /// + /// It can be reconstructed once via [`Ref::from_raw`]. + pub fn into_raw(obj: Self) -> *const T { + let no_drop = ManuallyDrop::new(obj); + no_drop.ptr.as_ptr() + } + + /// Recreates a [`Ref`] instance previously deconstructed via [`Ref::into_raw`]. + /// + /// # Safety + /// + /// `ptr` must have been returned by a previous call to [`Ref::into_raw`]. Additionally, it + /// can only be called once for each previous call to [``Ref::into_raw`]. + pub unsafe fn from_raw(ptr: *const T) -> Self { + Ref { + ptr: NonNull::new(ptr as _).unwrap(), + } + } +} + +impl<T: RefCounted + ?Sized> Deref for Ref<T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is + // safe to dereference it. + unsafe { self.ptr.as_ref() } + } +} + +impl<T: RefCounted + ?Sized> Clone for Ref<T> { + fn clone(&self) -> Self { + Self::new_from(self) + } +} + +impl<T: RefCounted + ?Sized> Drop for Ref<T> { + fn drop(&mut self) { + { + // SAFETY: By the type invariant, there is necessarily a reference to the object. + let obj = unsafe { self.ptr.as_ref() }; + + // Synchronises with the acquire below or with the acquire in `get_mut`. + if obj.get_count().count.fetch_sub(1, Ordering::Release) != 1 { + return; + } + } + + // Synchronises with the release when decrementing above. This ensures that modifications + // from all previous threads/CPUs are visible to the underlying object's `drop`. + fence(Ordering::Acquire); + + // The count reached zero, we must free the memory. + // + // SAFETY: The pointer was initialised from the result of `Box::into_raw`. + unsafe { Box::from_raw(self.ptr.as_ptr()) }; + } +} + +/// Trait for reference counted objects. +/// +/// # Safety +/// +/// Implementers of [`RefCounted`] must ensure that all of their constructors call +/// [`Ref::try_new`]. +pub unsafe trait RefCounted { + /// Returns a pointer to the object field holds the reference count. + fn get_count(&self) -> &RefCount; +} + +/// Holds the reference count of an object. +/// +/// It is meant to be embedded in objects to be reference-counted, with [`RefCounted::get_count`] +/// returning a reference to it. +pub struct RefCount { + count: AtomicUsize, +} + +impl RefCount { + /// Constructs a new instance of [`RefCount`]. + pub fn new() -> Self { + Self { + count: AtomicUsize::new(1), + } + } +} + +impl Default for RefCount { + fn default() -> Self { + Self::new() + } +} diff --git a/rust/kernel/sync/condvar.rs b/rust/kernel/sync/condvar.rs new file mode 100644 index 0000000000000..be6012794777c --- /dev/null +++ b/rust/kernel/sync/condvar.rs @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A condition variable. +//! +//! This module allows Rust code to use the kernel's [`struct wait_queue_head`] as a condition +//! variable. + +use super::{Guard, Lock, NeedsLockClass}; +use crate::{bindings, CStr}; +use core::{cell::UnsafeCell, marker::PhantomPinned, mem::MaybeUninit, pin::Pin}; + +extern "C" { + fn rust_helper_init_wait(wq: *mut bindings::wait_queue_entry); +} + +/// Safely initialises a [`CondVar`] with the given name, generating a new lock class. +#[macro_export] +macro_rules! condvar_init { + ($condvar:expr, $name:literal) => { + $crate::init_with_lockdep!($condvar, $name) + }; +} + +// TODO: `bindgen` is not generating this constant. Figure out why. +const POLLFREE: u32 = 0x4000; + +/// Exposes the kernel's [`struct wait_queue_head`] as a condition variable. It allows the caller to +/// atomically release the given lock and go to sleep. It reacquires the lock when it wakes up. And +/// it wakes up when notified by another thread (via [`CondVar::notify_one`] or +/// [`CondVar::notify_all`]) or because the thread received a signal. +/// +/// [`struct wait_queue_head`]: ../../../include/linux/wait.h +pub struct CondVar { + pub(crate) wait_list: UnsafeCell<bindings::wait_queue_head>, + + /// A condvar needs to be pinned because it contains a [`struct list_head`] that is + /// self-referential, so it cannot be safely moved once it is initialised. + _pin: PhantomPinned, +} + +// SAFETY: `CondVar` only uses a `struct wait_queue_head`, which is safe to use on any thread. +unsafe impl Send for CondVar {} + +// SAFETY: `CondVar` only uses a `struct wait_queue_head`, which is safe to use on multiple threads +// concurrently. +unsafe impl Sync for CondVar {} + +impl CondVar { + /// Constructs a new conditional variable. + /// + /// # Safety + /// + /// The caller must call `CondVar::init` before using the conditional variable. + pub unsafe fn new() -> Self { + Self { + wait_list: UnsafeCell::new(bindings::wait_queue_head::default()), + _pin: PhantomPinned, + } + } + + /// Atomically releases the given lock (whose ownership is proven by the guard) and puts the + /// thread to sleep. It wakes up when notified by [`CondVar::notify_one`] or + /// [`CondVar::notify_all`], or when the thread receives a signal. + /// + /// Returns whether there is a signal pending. + #[must_use = "wait returns if a signal is pending, so the caller must check the return value"] + pub fn wait<L: Lock>(&self, guard: &mut Guard<L>) -> bool { + let lock = guard.lock; + let mut wait = MaybeUninit::<bindings::wait_queue_entry>::uninit(); + + // SAFETY: `wait` points to valid memory. + unsafe { rust_helper_init_wait(wait.as_mut_ptr()) }; + + // SAFETY: Both `wait` and `wait_list` point to valid memory. + unsafe { + bindings::prepare_to_wait_exclusive( + self.wait_list.get(), + wait.as_mut_ptr(), + bindings::TASK_INTERRUPTIBLE as _, + ); + } + + // SAFETY: The guard is evidence that the caller owns the lock. + unsafe { lock.unlock() }; + + // SAFETY: No arguments, switches to another thread. + unsafe { bindings::schedule() }; + + lock.lock_noguard(); + + // SAFETY: Both `wait` and `wait_list` point to valid memory. + unsafe { bindings::finish_wait(self.wait_list.get(), wait.as_mut_ptr()) }; + + super::signal_pending() + } + + /// Calls the kernel function to notify the appropriate number of threads with the given flags. + fn notify(&self, count: i32, flags: u32) { + // SAFETY: `wait_list` points to valid memory. + unsafe { + bindings::__wake_up( + self.wait_list.get(), + bindings::TASK_NORMAL, + count, + flags as _, + ) + }; + } + + /// Wakes a single waiter up, if any. This is not 'sticky' in the sense that if no thread is + /// waiting, the notification is lost completely (as opposed to automatically waking up the + /// next waiter). + pub fn notify_one(&self) { + self.notify(1, 0); + } + + /// Wakes all waiters up, if any. This is not 'sticky' in the sense that if no thread is + /// waiting, the notification is lost completely (as opposed to automatically waking up the + /// next waiter). + pub fn notify_all(&self) { + self.notify(0, 0); + } + + /// Wakes all waiters up. If they were added by `epoll`, they are also removed from the list of + /// waiters. This is useful when cleaning up a condition variable that may be waited on by + /// threads that use `epoll`. + pub fn free_waiters(&self) { + self.notify(1, bindings::POLLHUP | POLLFREE); + } +} + +impl NeedsLockClass for CondVar { + unsafe fn init(self: Pin<&Self>, name: CStr<'static>, key: *mut bindings::lock_class_key) { + bindings::__init_waitqueue_head(self.wait_list.get(), name.as_ptr() as _, key); + } +} diff --git a/rust/kernel/sync/guard.rs b/rust/kernel/sync/guard.rs new file mode 100644 index 0000000000000..84e5d319a5fd8 --- /dev/null +++ b/rust/kernel/sync/guard.rs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A generic lock guard and trait. +//! +//! This module contains a lock guard that can be used with any locking primitive that implements +//! the ([`Lock`]) trait. It also contains the definition of the trait, which can be leveraged by +//! other constructs to work on generic locking primitives. + +/// Allows mutual exclusion primitives that implement the [`Lock`] trait to automatically unlock +/// when a guard goes out of scope. It also provides a safe and convenient way to access the data +/// protected by the lock. +#[must_use = "the lock unlocks immediately when the guard is unused"] +pub struct Guard<'a, L: Lock + ?Sized> { + pub(crate) lock: &'a L, +} + +// SAFETY: `Guard` is sync when the data protected by the lock is also sync. This is more +// conservative than the default compiler implementation; more details can be found on +// https://github.com/rust-lang/rust/issues/41622 -- it refers to `MutexGuard` from the standard +// library. +unsafe impl<L> Sync for Guard<'_, L> +where + L: Lock + ?Sized, + L::Inner: Sync, +{ +} + +impl<L: Lock + ?Sized> core::ops::Deref for Guard<'_, L> { + type Target = L::Inner; + + fn deref(&self) -> &Self::Target { + // SAFETY: The caller owns the lock, so it is safe to deref the protected data. + unsafe { &*self.lock.locked_data().get() } + } +} + +impl<L: Lock + ?Sized> core::ops::DerefMut for Guard<'_, L> { + fn deref_mut(&mut self) -> &mut L::Inner { + // SAFETY: The caller owns the lock, so it is safe to deref the protected data. + unsafe { &mut *self.lock.locked_data().get() } + } +} + +impl<L: Lock + ?Sized> Drop for Guard<'_, L> { + fn drop(&mut self) { + // SAFETY: The caller owns the lock, so it is safe to unlock it. + unsafe { self.lock.unlock() }; + } +} + +impl<'a, L: Lock + ?Sized> Guard<'a, L> { + /// Constructs a new lock guard. + /// + /// # Safety + /// + /// The caller must ensure that it owns the lock. + pub(crate) unsafe fn new(lock: &'a L) -> Self { + Self { lock } + } +} + +/// A generic mutual exclusion primitive. +/// +/// [`Guard`] is written such that any mutual exclusion primitive that can implement this trait can +/// also benefit from having an automatic way to unlock itself. +pub trait Lock { + /// The type of the data protected by the lock. + type Inner: ?Sized; + + /// Acquires the lock, making the caller its owner. + fn lock_noguard(&self); + + /// Releases the lock, giving up ownership of the lock. + /// + /// # Safety + /// + /// It must only be called by the current owner of the lock. + unsafe fn unlock(&self); + + /// Returns the data protected by the lock. + fn locked_data(&self) -> &core::cell::UnsafeCell<Self::Inner>; +} diff --git a/rust/kernel/sync/locked_by.rs b/rust/kernel/sync/locked_by.rs new file mode 100644 index 0000000000000..fc540b35c53ae --- /dev/null +++ b/rust/kernel/sync/locked_by.rs @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A wrapper for data protected by a lock that does not wrap it. + +use super::{Guard, Lock}; +use core::{cell::UnsafeCell, ops::Deref, ptr}; + +/// Allows access to some data to be serialised by a lock that does not wrap it. +/// +/// In most cases, data protected by a lock is wrapped by the appropriate lock type, e.g., +/// [`super::Mutex`] or [`super::SpinLock`]. [`LockedBy`] is meant for cases when this is not +/// possible. For example, if a container has a lock and some data in the contained elements needs +/// to be protected by the same lock. +/// +/// [`LockedBy`] wraps the data in lieu of another locking primitive, and only allows access to it +/// when the caller shows evidence that 'external' lock is locked. +/// +/// # Example +/// +/// The following is an example for illustrative purposes: `InnerDirectory::bytes_used` is an +/// aggregate of all `InnerFile::bytes_used` and must be kept consistent; so we wrap `InnerFile` in +/// a `LockedBy` so that it shares a lock with `InnerDirectory`. This allows us to enforce at +/// compile-time that access to `InnerFile` is only granted when an `InnerDirectory` is also +/// locked; we enforce at run time that the right `InnerDirectory` is locked. +/// +/// ``` +/// use super::Mutex; +/// use alloc::{string::String, vec::Vec}; +/// +/// struct InnerFile { +/// bytes_used: u64, +/// } +/// +/// struct File { +/// name: String, +/// inner: LockedBy<InnerFile, Mutex<InnerDirectory>>, +/// } +/// +/// struct InnerDirectory { +/// /// The sum of the bytes used by all files. +/// bytes_used: u64, +/// files: Vec<File>, +/// } +/// +/// struct Directory { +/// name: String, +/// inner: Mutex<InnerDirectory>, +/// } +/// ``` +pub struct LockedBy<T: ?Sized, L: Lock + ?Sized> { + owner: *const L::Inner, + data: UnsafeCell<T>, +} + +// SAFETY: `LockedBy` can be transferred across thread boundaries iff the data it protects can. +unsafe impl<T: ?Sized + Send, L: Lock + ?Sized> Send for LockedBy<T, L> {} + +// SAFETY: `LockedBy` serialises the interior mutability it provides, so it is `Sync` as long as the +// data it protects is `Send`. +unsafe impl<T: ?Sized + Send, L: Lock + ?Sized> Sync for LockedBy<T, L> {} + +impl<T, L: Lock + ?Sized> LockedBy<T, L> { + /// Constructs a new instance of [`LockedBy`]. + /// + /// It stores a raw pointer to the owner that is never dereferenced. It is only used to ensure + /// that the right owner is being used to access the protected data. If the owner is freed, the + /// data becomes inaccessible; if another instance of the owner is allocated *on the same + /// memory location*, the data becomes accessible again: none of this affects memory safety + /// because in any case at most one thread (or CPU) can access the protected data at a time. + pub fn new(owner: &L, data: T) -> Self { + Self { + owner: owner.locked_data().get(), + data: UnsafeCell::new(data), + } + } +} + +impl<T: ?Sized, L: Lock + ?Sized> LockedBy<T, L> { + /// Returns a reference to the protected data when the caller provides evidence (via a + /// [`Guard`]) that the owner is locked. + pub fn access<'a>(&'a self, guard: &'a Guard<L>) -> &'a T { + if !ptr::eq(guard.deref(), self.owner) { + panic!("guard does not match owner"); + } + + // SAFETY: `guard` is evidence that the owner is locked. + unsafe { &mut *self.data.get() } + } + + /// Returns a mutable reference to the protected data when the caller provides evidence (via a + /// mutable [`Guard`]) that the owner is locked mutably. + pub fn access_mut<'a>(&'a self, guard: &'a mut Guard<L>) -> &'a mut T { + if !ptr::eq(guard.deref().deref(), self.owner) { + panic!("guard does not match owner"); + } + + // SAFETY: `guard` is evidence that the owner is locked. + unsafe { &mut *self.data.get() } + } + + /// Returns a mutable reference to the protected data when the caller provides evidence (via a + /// mutable owner) that the owner is locked mutably. Showing a mutable reference to the owner + /// is sufficient because we know no other references can exist to it. + pub fn access_from_mut<'a>(&'a self, owner: &'a mut L::Inner) -> &'a mut T { + if !ptr::eq(owner, self.owner) { + panic!("mismatched owners"); + } + + // SAFETY: `owner` is evidence that there is only one reference to the owner. + unsafe { &mut *self.data.get() } + } +} diff --git a/rust/kernel/sync/mod.rs b/rust/kernel/sync/mod.rs new file mode 100644 index 0000000000000..0a8cec021e5c5 --- /dev/null +++ b/rust/kernel/sync/mod.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Synchronisation primitives. +//! +//! This module contains the kernel APIs related to synchronisation that have been ported or +//! wrapped for usage by Rust code in the kernel and is shared by all of them. +//! +//! # Example +//! +//! ``` +//! fn test() { +//! // SAFETY: `init` is called below. +//! let data = alloc::sync::Arc::pin(unsafe { Mutex::new(0) }); +//! mutex_init!(data.as_ref(), "test::data"); +//! *data.lock() = 10; +//! pr_info!("{}\n", *data.lock()); +//! } +//! ``` + +use crate::{bindings, c_types, CStr}; +use core::pin::Pin; + +mod arc; +mod condvar; +mod guard; +mod locked_by; +mod mutex; +mod spinlock; + +pub use arc::{Ref, RefCount, RefCounted}; +pub use condvar::CondVar; +pub use guard::{Guard, Lock}; +pub use locked_by::LockedBy; +pub use mutex::Mutex; +pub use spinlock::SpinLock; + +extern "C" { + fn rust_helper_signal_pending() -> c_types::c_int; + fn rust_helper_cond_resched() -> c_types::c_int; +} + +/// Safely initialises an object that has an `init` function that takes a name and a lock class as +/// arguments, examples of these are [`Mutex`] and [`SpinLock`]. Each of them also provides a more +/// specialised name that uses this macro. +#[doc(hidden)] +#[macro_export] +macro_rules! init_with_lockdep { + ($obj:expr, $name:literal) => {{ + static mut CLASS: core::mem::MaybeUninit<$crate::bindings::lock_class_key> = + core::mem::MaybeUninit::uninit(); + // SAFETY: `CLASS` is never used by Rust code directly; the kernel may change it though. + #[allow(unused_unsafe)] + unsafe { + $crate::sync::NeedsLockClass::init($obj, $crate::cstr!($name), CLASS.as_mut_ptr()) + }; + }}; +} + +/// A trait for types that need a lock class during initialisation. +/// +/// Implementers of this trait benefit from the [`init_with_lockdep`] macro that generates a new +/// class for each initialisation call site. +pub trait NeedsLockClass { + /// Initialises the type instance so that it can be safely used. + /// + /// Callers are encouraged to use the [`init_with_lockdep`] macro as it automatically creates a + /// new lock class on each usage. + /// + /// # Safety + /// + /// `key` must point to a valid memory location as it will be used by the kernel. + unsafe fn init(self: Pin<&Self>, name: CStr<'static>, key: *mut bindings::lock_class_key); +} + +/// Determines if a signal is pending on the current process. +pub fn signal_pending() -> bool { + // SAFETY: No arguments, just checks `current` for pending signals. + unsafe { rust_helper_signal_pending() != 0 } +} + +/// Reschedules the caller's task if needed. +pub fn cond_resched() -> bool { + // SAFETY: No arguments, reschedules `current` if needed. + unsafe { rust_helper_cond_resched() != 0 } +} diff --git a/rust/kernel/sync/mutex.rs b/rust/kernel/sync/mutex.rs new file mode 100644 index 0000000000000..e528228d16c10 --- /dev/null +++ b/rust/kernel/sync/mutex.rs @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A kernel mutex. +//! +//! This module allows Rust code to use the kernel's [`struct mutex`]. + +use super::{Guard, Lock, NeedsLockClass}; +use crate::{bindings, CStr}; +use core::{cell::UnsafeCell, marker::PhantomPinned, pin::Pin}; + +/// Safely initialises a [`Mutex`] with the given name, generating a new lock class. +#[macro_export] +macro_rules! mutex_init { + ($mutex:expr, $name:literal) => { + $crate::init_with_lockdep!($mutex, $name) + }; +} + +/// Exposes the kernel's [`struct mutex`]. When multiple threads attempt to lock the same mutex, +/// only one at a time is allowed to progress, the others will block (sleep) until the mutex is +/// unlocked, at which point another thread will be allowed to wake up and make progress. +/// +/// A [`Mutex`] must first be initialised with a call to [`Mutex::init`] before it can be used. The +/// [`mutex_init`] macro is provided to automatically assign a new lock class to a mutex instance. +/// +/// Since it may block, [`Mutex`] needs to be used with care in atomic contexts. +/// +/// [`struct mutex`]: ../../../include/linux/mutex.h +pub struct Mutex<T: ?Sized> { + /// The kernel `struct mutex` object. + mutex: UnsafeCell<bindings::mutex>, + + /// A mutex needs to be pinned because it contains a [`struct list_head`] that is + /// self-referential, so it cannot be safely moved once it is initialised. + _pin: PhantomPinned, + + /// The data protected by the mutex. + data: UnsafeCell<T>, +} + +// SAFETY: `Mutex` can be transferred across thread boundaries iff the data it protects can. +unsafe impl<T: ?Sized + Send> Send for Mutex<T> {} + +// SAFETY: `Mutex` serialises the interior mutability it provides, so it is `Sync` as long as the +// data it protects is `Send`. +unsafe impl<T: ?Sized + Send> Sync for Mutex<T> {} + +impl<T> Mutex<T> { + /// Constructs a new mutex. + /// + /// # Safety + /// + /// The caller must call [`Mutex::init`] before using the mutex. + pub unsafe fn new(t: T) -> Self { + Self { + mutex: UnsafeCell::new(bindings::mutex::default()), + data: UnsafeCell::new(t), + _pin: PhantomPinned, + } + } +} + +impl<T: ?Sized> Mutex<T> { + /// Locks the mutex and gives the caller access to the data protected by it. Only one thread at + /// a time is allowed to access the protected data. + pub fn lock(&self) -> Guard<Self> { + self.lock_noguard(); + // SAFETY: The mutex was just acquired. + unsafe { Guard::new(self) } + } +} + +impl<T: ?Sized> NeedsLockClass for Mutex<T> { + unsafe fn init(self: Pin<&Self>, name: CStr<'static>, key: *mut bindings::lock_class_key) { + bindings::__mutex_init(self.mutex.get(), name.as_ptr() as _, key); + } +} + +impl<T: ?Sized> Lock for Mutex<T> { + type Inner = T; + + #[cfg(not(CONFIG_DEBUG_LOCK_ALLOC))] + fn lock_noguard(&self) { + // SAFETY: `mutex` points to valid memory. + unsafe { bindings::mutex_lock(self.mutex.get()) }; + } + + #[cfg(CONFIG_DEBUG_LOCK_ALLOC)] + fn lock_noguard(&self) { + // SAFETY: `mutex` points to valid memory. + unsafe { bindings::mutex_lock_nested(self.mutex.get(), 0) }; + } + + unsafe fn unlock(&self) { + bindings::mutex_unlock(self.mutex.get()); + } + + fn locked_data(&self) -> &UnsafeCell<T> { + &self.data + } +} diff --git a/rust/kernel/sync/spinlock.rs b/rust/kernel/sync/spinlock.rs new file mode 100644 index 0000000000000..49a7d5fd837b2 --- /dev/null +++ b/rust/kernel/sync/spinlock.rs @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! A kernel spinlock. +//! +//! This module allows Rust code to use the kernel's [`struct spinlock`]. +//! +//! See <https://www.kernel.org/doc/Documentation/locking/spinlocks.txt>. + +use super::{Guard, Lock, NeedsLockClass}; +use crate::{bindings, c_types, CStr}; +use core::{cell::UnsafeCell, marker::PhantomPinned, pin::Pin}; + +extern "C" { + #[allow(improper_ctypes)] + fn rust_helper_spin_lock_init( + lock: *mut bindings::spinlock_t, + name: *const c_types::c_char, + key: *mut bindings::lock_class_key, + ); + fn rust_helper_spin_lock(lock: *mut bindings::spinlock); + fn rust_helper_spin_unlock(lock: *mut bindings::spinlock); +} + +/// Safely initialises a [`SpinLock`] with the given name, generating a new lock class. +#[macro_export] +macro_rules! spinlock_init { + ($spinlock:expr, $name:literal) => { + $crate::init_with_lockdep!($spinlock, $name) + }; +} + +/// Exposes the kernel's [`spinlock_t`]. When multiple CPUs attempt to lock the same spinlock, only +/// one at a time is allowed to progress, the others will block (spinning) until the spinlock is +/// unlocked, at which point another CPU will be allowed to make progress. +/// +/// A [`SpinLock`] must first be initialised with a call to [`SpinLock::init`] before it can be +/// used. The [`spinlock_init`] macro is provided to automatically assign a new lock class to a +/// spinlock instance. +/// +/// [`SpinLock`] does not manage the interrupt state, so it can be used in only two cases: (a) when +/// the caller knows that interrupts are disabled, or (b) when callers never use it in interrupt +/// handlers (in which case it is ok for interrupts to be enabled). +/// +/// [`spinlock_t`]: ../../../include/linux/spinlock.h +pub struct SpinLock<T: ?Sized> { + spin_lock: UnsafeCell<bindings::spinlock>, + + /// Spinlocks are architecture-defined. So we conservatively require them to be pinned in case + /// some architecture uses self-references now or in the future. + _pin: PhantomPinned, + + data: UnsafeCell<T>, +} + +// SAFETY: `SpinLock` can be transferred across thread boundaries iff the data it protects can. +unsafe impl<T: ?Sized + Send> Send for SpinLock<T> {} + +// SAFETY: `SpinLock` serialises the interior mutability it provides, so it is `Sync` as long as the +// data it protects is `Send`. +unsafe impl<T: ?Sized + Send> Sync for SpinLock<T> {} + +impl<T> SpinLock<T> { + /// Constructs a new spinlock. + /// + /// # Safety + /// + /// The caller must call [`SpinLock::init`] before using the spinlock. + pub unsafe fn new(t: T) -> Self { + Self { + spin_lock: UnsafeCell::new(bindings::spinlock::default()), + data: UnsafeCell::new(t), + _pin: PhantomPinned, + } + } +} + +impl<T: ?Sized> SpinLock<T> { + /// Locks the spinlock and gives the caller access to the data protected by it. Only one thread + /// at a time is allowed to access the protected data. + pub fn lock(&self) -> Guard<Self> { + self.lock_noguard(); + // SAFETY: The spinlock was just acquired. + unsafe { Guard::new(self) } + } +} + +impl<T: ?Sized> NeedsLockClass for SpinLock<T> { + unsafe fn init(self: Pin<&Self>, name: CStr<'static>, key: *mut bindings::lock_class_key) { + rust_helper_spin_lock_init(self.spin_lock.get(), name.as_ptr() as _, key); + } +} + +impl<T: ?Sized> Lock for SpinLock<T> { + type Inner = T; + + fn lock_noguard(&self) { + // SAFETY: `spin_lock` points to valid memory. + unsafe { rust_helper_spin_lock(self.spin_lock.get()) }; + } + + unsafe fn unlock(&self) { + rust_helper_spin_unlock(self.spin_lock.get()); + } + + fn locked_data(&self) -> &UnsafeCell<T> { + &self.data + } +} diff --git a/rust/kernel/sysctl.rs b/rust/kernel/sysctl.rs new file mode 100644 index 0000000000000..a1928a8523db8 --- /dev/null +++ b/rust/kernel/sysctl.rs @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! System control. +//! +//! C header: [`include/linux/sysctl.h`](../../../../include/linux/sysctl.h) +//! +//! Reference: <https://www.kernel.org/doc/Documentation/sysctl/README> + +use alloc::boxed::Box; +use alloc::vec; +use core::mem; +use core::ptr; +use core::sync::atomic; + +use crate::bindings; +use crate::c_types; +use crate::error; +use crate::types; +use crate::user_ptr::{UserSlicePtr, UserSlicePtrWriter}; + +/// Sysctl storage. +pub trait SysctlStorage: Sync { + /// Writes a byte slice. + fn store_value(&self, data: &[u8]) -> (usize, error::KernelResult); + + /// Reads via a [`UserSlicePtrWriter`]. + fn read_value(&self, data: &mut UserSlicePtrWriter) -> (usize, error::KernelResult); +} + +fn trim_whitespace(mut data: &[u8]) -> &[u8] { + while !data.is_empty() && (data[0] == b' ' || data[0] == b'\t' || data[0] == b'\n') { + data = &data[1..]; + } + while !data.is_empty() + && (data[data.len() - 1] == b' ' + || data[data.len() - 1] == b'\t' + || data[data.len() - 1] == b'\n') + { + data = &data[..data.len() - 1]; + } + data +} + +impl<T> SysctlStorage for &T +where + T: SysctlStorage, +{ + fn store_value(&self, data: &[u8]) -> (usize, error::KernelResult) { + (*self).store_value(data) + } + + fn read_value(&self, data: &mut UserSlicePtrWriter) -> (usize, error::KernelResult) { + (*self).read_value(data) + } +} + +impl SysctlStorage for atomic::AtomicBool { + fn store_value(&self, data: &[u8]) -> (usize, error::KernelResult) { + let result = match trim_whitespace(data) { + b"0" => { + self.store(false, atomic::Ordering::Relaxed); + Ok(()) + } + b"1" => { + self.store(true, atomic::Ordering::Relaxed); + Ok(()) + } + _ => Err(error::Error::EINVAL), + }; + (data.len(), result) + } + + fn read_value(&self, data: &mut UserSlicePtrWriter) -> (usize, error::KernelResult) { + let value = if self.load(atomic::Ordering::Relaxed) { + b"1\n" + } else { + b"0\n" + }; + (value.len(), data.write_slice(value)) + } +} + +/// Holds a single `sysctl` entry (and its table). +pub struct Sysctl<T: SysctlStorage> { + inner: Box<T>, + // Responsible for keeping the `ctl_table` alive. + _table: Box<[bindings::ctl_table]>, + header: *mut bindings::ctl_table_header, +} + +// SAFETY: The only public method we have is `get()`, which returns `&T`, and +// `T: Sync`. Any new methods must adhere to this requirement. +unsafe impl<T: SysctlStorage> Sync for Sysctl<T> {} + +unsafe extern "C" fn proc_handler<T: SysctlStorage>( + ctl: *mut bindings::ctl_table, + write: c_types::c_int, + buffer: *mut c_types::c_void, + len: *mut usize, + ppos: *mut bindings::loff_t, +) -> c_types::c_int { + // If we are reading from some offset other than the beginning of the file, + // return an empty read to signal EOF. + if *ppos != 0 && write == 0 { + *len = 0; + return 0; + } + + let data = UserSlicePtr::new(buffer, *len); + let storage = &*((*ctl).data as *const T); + let (bytes_processed, result) = if write != 0 { + let data = match data.read_all() { + Ok(r) => r, + Err(e) => return e.to_kernel_errno(), + }; + storage.store_value(&data) + } else { + let mut writer = data.writer(); + storage.read_value(&mut writer) + }; + *len = bytes_processed; + *ppos += *len as bindings::loff_t; + match result { + Ok(()) => 0, + Err(e) => e.to_kernel_errno(), + } +} + +impl<T: SysctlStorage> Sysctl<T> { + /// Registers a single entry in `sysctl`. + pub fn register( + path: types::CStr<'static>, + name: types::CStr<'static>, + storage: T, + mode: types::Mode, + ) -> error::KernelResult<Sysctl<T>> { + if name.contains('/') { + return Err(error::Error::EINVAL); + } + + let storage = Box::try_new(storage)?; + let mut table = vec![ + bindings::ctl_table { + procname: name.as_ptr() as *const i8, + mode: mode.as_int(), + data: &*storage as *const T as *mut c_types::c_void, + proc_handler: Some(proc_handler::<T>), + + maxlen: 0, + child: ptr::null_mut(), + poll: ptr::null_mut(), + extra1: ptr::null_mut(), + extra2: ptr::null_mut(), + }, + unsafe { mem::zeroed() }, + ] + .into_boxed_slice(); + + let result = + unsafe { bindings::register_sysctl(path.as_ptr() as *const i8, table.as_mut_ptr()) }; + if result.is_null() { + return Err(error::Error::ENOMEM); + } + + Ok(Sysctl { + inner: storage, + _table: table, + header: result, + }) + } + + /// Gets the storage. + pub fn get(&self) -> &T { + &self.inner + } +} + +impl<T: SysctlStorage> Drop for Sysctl<T> { + fn drop(&mut self) { + unsafe { + bindings::unregister_sysctl_table(self.header); + } + self.header = ptr::null_mut(); + } +} diff --git a/rust/kernel/types.rs b/rust/kernel/types.rs new file mode 100644 index 0000000000000..6207670c32904 --- /dev/null +++ b/rust/kernel/types.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Kernel types. +//! +//! C header: [`include/linux/types.h`](../../../../include/linux/types.h) + +use core::ops::Deref; + +use crate::bindings; + +/// Permissions. +/// +/// C header: [`include/uapi/linux/stat.h`](../../../../include/uapi/linux/stat.h) +/// +/// C header: [`include/linux/stat.h`](../../../../include/linux/stat.h) +pub struct Mode(bindings::umode_t); + +impl Mode { + /// Creates a [`Mode`] from an integer. + pub fn from_int(m: u16) -> Mode { + Mode(m) + } + + /// Returns the mode as an integer. + pub fn as_int(&self) -> u16 { + self.0 + } +} + +/// A string that is guaranteed to have exactly one `NUL` byte, which is at the +/// end. +/// +/// Used for interoperability with kernel APIs that take C strings. +#[repr(transparent)] +pub struct CStr<'a>(&'a str); + +impl CStr<'_> { + /// Creates a [`CStr`] from a [`str`] without performing any additional + /// checks. + /// + /// # Safety + /// + /// `data` *must* end with a `NUL` byte, and should only have only a single + /// `NUL` byte (or the string will be truncated). + pub const unsafe fn new_unchecked(data: &str) -> CStr { + CStr(data) + } +} + +impl Deref for CStr<'_> { + type Target = str; + + fn deref(&self) -> &str { + self.0 + } +} + +/// Creates a new `CStr` from a string literal. +/// +/// The string literal should not contain any `NUL` bytes. +/// +/// # Examples +/// +/// ```rust,no_run +/// const MY_CSTR: CStr<'static> = cstr!("My awesome CStr!"); +/// ``` +#[macro_export] +macro_rules! cstr { + ($str:expr) => {{ + let s = concat!($str, "\x00"); + unsafe { $crate::CStr::new_unchecked(s) } + }}; +} diff --git a/rust/kernel/user_ptr.rs b/rust/kernel/user_ptr.rs new file mode 100644 index 0000000000000..d9304d269d06d --- /dev/null +++ b/rust/kernel/user_ptr.rs @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! User pointers. +//! +//! C header: [`include/linux/uaccess.h`](../../../../include/linux/uaccess.h) + +use crate::{c_types, error::Error, KernelResult}; +use alloc::vec::Vec; +use core::mem::{size_of, MaybeUninit}; + +extern "C" { + fn rust_helper_copy_from_user( + to: *mut c_types::c_void, + from: *const c_types::c_void, + n: c_types::c_ulong, + ) -> c_types::c_ulong; + + fn rust_helper_copy_to_user( + to: *mut c_types::c_void, + from: *const c_types::c_void, + n: c_types::c_ulong, + ) -> c_types::c_ulong; +} + +/// Specifies that a type is safely readable from byte slices. +/// +/// Not all types can be safely read from byte slices; examples from +/// <https://doc.rust-lang.org/reference/behavior-considered-undefined.html> include `bool` +/// that must be either `0` or `1`, and `char` that cannot be a surrogate or above `char::MAX`. +/// +/// # Safety +/// +/// Implementers must ensure that the type is made up only of types that can be safely read from +/// arbitrary byte sequences (e.g., `u32`, `u64`, etc.). +pub unsafe trait ReadableFromBytes {} + +// SAFETY: All bit patterns are acceptable values of the types below. +unsafe impl ReadableFromBytes for u8 {} +unsafe impl ReadableFromBytes for u16 {} +unsafe impl ReadableFromBytes for u32 {} +unsafe impl ReadableFromBytes for u64 {} +unsafe impl ReadableFromBytes for usize {} +unsafe impl ReadableFromBytes for i8 {} +unsafe impl ReadableFromBytes for i16 {} +unsafe impl ReadableFromBytes for i32 {} +unsafe impl ReadableFromBytes for i64 {} +unsafe impl ReadableFromBytes for isize {} + +/// Specifies that a type is safely writable to byte slices. +/// +/// This means that we don't read undefined values (which leads to UB) in preparation for writing +/// to the byte slice. It also ensures that no potentially sensitive information is leaked into the +/// byte slices. +/// +/// # Safety +/// +/// A type must not include padding bytes and must be fully initialised to safely implement +/// [`WritableToBytes`] (i.e., it doesn't contain [`MaybeUninit`] fields). A composition of +/// writable types in a structure is not necessarily writable because it may result in padding +/// bytes. +pub unsafe trait WritableToBytes {} + +// SAFETY: Initialised instances of the following types have no uninitialised portions. +unsafe impl WritableToBytes for u8 {} +unsafe impl WritableToBytes for u16 {} +unsafe impl WritableToBytes for u32 {} +unsafe impl WritableToBytes for u64 {} +unsafe impl WritableToBytes for usize {} +unsafe impl WritableToBytes for i8 {} +unsafe impl WritableToBytes for i16 {} +unsafe impl WritableToBytes for i32 {} +unsafe impl WritableToBytes for i64 {} +unsafe impl WritableToBytes for isize {} + +/// A reference to an area in userspace memory, which can be either +/// read-only or read-write. +/// +/// All methods on this struct are safe: invalid pointers return +/// `EFAULT`. Concurrent access, *including data races to/from userspace +/// memory*, is permitted, because fundamentally another userspace +/// thread/process could always be modifying memory at the same time +/// (in the same way that userspace Rust's [`std::io`] permits data races +/// with the contents of files on disk). In the presence of a race, the +/// exact byte values read/written are unspecified but the operation is +/// well-defined. Kernelspace code should validate its copy of data +/// after completing a read, and not expect that multiple reads of the +/// same address will return the same value. +/// +/// All APIs enforce the invariant that a given byte of memory from userspace +/// may only be read once. By preventing double-fetches we avoid TOCTOU +/// vulnerabilities. This is accomplished by taking `self` by value to prevent +/// obtaining multiple readers on a given [`UserSlicePtr`], and the readers +/// only permitting forward reads. +/// +/// Constructing a [`UserSlicePtr`] performs no checks on the provided +/// address and length, it can safely be constructed inside a kernel thread +/// with no current userspace process. Reads and writes wrap the kernel APIs +/// `copy_from_user` and `copy_to_user`, which check the memory map of the +/// current process and enforce that the address range is within the user +/// range (no additional calls to `access_ok` are needed). +/// +/// [`std::io`]: https://doc.rust-lang.org/std/io/index.html +pub struct UserSlicePtr(*mut c_types::c_void, usize); + +impl UserSlicePtr { + /// Constructs a user slice from a raw pointer and a length in bytes. + /// + /// # Safety + /// + /// Callers must be careful to avoid time-of-check-time-of-use + /// (TOCTOU) issues. The simplest way is to create a single instance of + /// [`UserSlicePtr`] per user memory block as it reads each byte at + /// most once. + pub unsafe fn new(ptr: *mut c_types::c_void, length: usize) -> Self { + UserSlicePtr(ptr, length) + } + + /// Reads the entirety of the user slice. + /// + /// Returns `EFAULT` if the address does not currently point to + /// mapped, readable memory. + pub fn read_all(self) -> KernelResult<Vec<u8>> { + self.reader().read_all() + } + + /// Constructs a [`UserSlicePtrReader`]. + pub fn reader(self) -> UserSlicePtrReader { + UserSlicePtrReader(self.0, self.1) + } + + /// Writes the provided slice into the user slice. + /// + /// Returns `EFAULT` if the address does not currently point to + /// mapped, writable memory (in which case some data from before the + /// fault may be written), or `data` is larger than the user slice + /// (in which case no data is written). + pub fn write_all(self, data: &[u8]) -> KernelResult { + self.writer().write_slice(data) + } + + /// Constructs a [`UserSlicePtrWriter`]. + pub fn writer(self) -> UserSlicePtrWriter { + UserSlicePtrWriter(self.0, self.1) + } + + /// Constructs both a [`UserSlicePtrReader`] and a [`UserSlicePtrWriter`]. + pub fn reader_writer(self) -> (UserSlicePtrReader, UserSlicePtrWriter) { + ( + UserSlicePtrReader(self.0, self.1), + UserSlicePtrWriter(self.0, self.1), + ) + } +} + +/// A reader for [`UserSlicePtr`]. +/// +/// Used to incrementally read from the user slice. +pub struct UserSlicePtrReader(*mut c_types::c_void, usize); + +impl UserSlicePtrReader { + /// Returns the number of bytes left to be read from this. + /// + /// Note that even reading less than this number of bytes may fail. + pub fn len(&self) -> usize { + self.1 + } + + /// Returns `true` if `self.len()` is 0. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Reads all data remaining in the user slice. + /// + /// Returns `EFAULT` if the address does not currently point to + /// mapped, readable memory. + pub fn read_all(&mut self) -> KernelResult<Vec<u8>> { + let mut data = Vec::<u8>::new(); + data.try_reserve_exact(self.1)?; + data.resize(self.1, 0); + // SAFETY: The output buffer is valid as we just allocated it. + unsafe { self.read_raw(data.as_mut_ptr(), data.len())? }; + Ok(data) + } + + /// Reads a byte slice from the user slice. + /// + /// Returns `EFAULT` if the byte slice is bigger than the remaining size + /// of the user slice or if the address does not currently point to mapped, + /// readable memory. + pub fn read_slice(&mut self, data: &mut [u8]) -> KernelResult { + // SAFETY: The output buffer is valid as it's coming from a live reference. + unsafe { self.read_raw(data.as_mut_ptr(), data.len()) } + } + + /// Reads raw data from the user slice into a raw kernel buffer. + /// + /// # Safety + /// + /// The output buffer must be valid. + pub unsafe fn read_raw(&mut self, out: *mut u8, len: usize) -> KernelResult { + if len > self.1 || len > u32::MAX as usize { + return Err(Error::EFAULT); + } + let res = rust_helper_copy_from_user(out as _, self.0, len as _); + if res != 0 { + return Err(Error::EFAULT); + } + // Since this is not a pointer to a valid object in our program, + // we cannot use `add`, which has C-style rules for defined + // behavior. + self.0 = self.0.wrapping_add(len); + self.1 -= len; + Ok(()) + } + + /// Reads the contents of a plain old data (POD) type from the user slice. + pub fn read<T: ReadableFromBytes>(&mut self) -> KernelResult<T> { + let mut out = MaybeUninit::<T>::uninit(); + // SAFETY: The buffer is valid as it was just allocated. + unsafe { self.read_raw(out.as_mut_ptr() as _, size_of::<T>()) }?; + // SAFETY: We just initialised the data. + Ok(unsafe { out.assume_init() }) + } +} + +/// A writer for [`UserSlicePtr`]. +/// +/// Used to incrementally write into the user slice. +pub struct UserSlicePtrWriter(*mut c_types::c_void, usize); + +impl UserSlicePtrWriter { + /// Returns the number of bytes left to be written from this. + /// + /// Note that even writing less than this number of bytes may fail. + pub fn len(&self) -> usize { + self.1 + } + + /// Returns `true` if `self.len()` is 0. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Writes a byte slice to the user slice. + /// + /// Returns `EFAULT` if the byte slice is bigger than the remaining size + /// of the user slice or if the address does not currently point to mapped, + /// writable memory. + pub fn write_slice(&mut self, data: &[u8]) -> KernelResult { + // SAFETY: The input buffer is valid as it's coming from a live reference. + unsafe { self.write_raw(data.as_ptr(), data.len()) } + } + + /// Writes raw data to the user slice from a raw kernel buffer. + /// + /// # Safety + /// + /// The input buffer must be valid. + unsafe fn write_raw(&mut self, data: *const u8, len: usize) -> KernelResult { + if len > self.1 || len > u32::MAX as usize { + return Err(Error::EFAULT); + } + let res = rust_helper_copy_to_user(self.0, data as _, len as _); + if res != 0 { + return Err(Error::EFAULT); + } + // Since this is not a pointer to a valid object in our program, + // we cannot use `add`, which has C-style rules for defined + // behavior. + self.0 = self.0.wrapping_add(len); + self.1 -= len; + Ok(()) + } + + /// Writes the contents of the given data into the user slice. + pub fn write<T: WritableToBytes>(&mut self, data: &T) -> KernelResult<()> { + // SAFETY: The input buffer is valid as it's coming from a live + // reference to a type that implements `WritableToBytes`. + unsafe { self.write_raw(data as *const T as _, size_of::<T>()) } + } +} diff --git a/rust/module.rs b/rust/module.rs new file mode 100644 index 0000000000000..bddb90d4822d4 --- /dev/null +++ b/rust/module.rs @@ -0,0 +1,764 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Proc macro crate implementing the [`module!`] magic. +//! +//! C header: [`include/linux/moduleparam.h`](../../../include/linux/moduleparam.h) + +#![deny(clippy::complexity)] +#![deny(clippy::correctness)] +#![deny(clippy::perf)] +#![deny(clippy::style)] + +use proc_macro::{token_stream, Delimiter, Group, TokenStream, TokenTree}; + +fn try_ident(it: &mut token_stream::IntoIter) -> Option<String> { + if let Some(TokenTree::Ident(ident)) = it.next() { + Some(ident.to_string()) + } else { + None + } +} + +fn try_literal(it: &mut token_stream::IntoIter) -> Option<String> { + if let Some(TokenTree::Literal(literal)) = it.next() { + Some(literal.to_string()) + } else { + None + } +} + +fn try_byte_string(it: &mut token_stream::IntoIter) -> Option<String> { + try_literal(it).and_then(|byte_string| { + if byte_string.starts_with("b\"") && byte_string.ends_with('\"') { + Some(byte_string[2..byte_string.len() - 1].to_string()) + } else { + None + } + }) +} + +fn expect_ident(it: &mut token_stream::IntoIter) -> String { + try_ident(it).expect("Expected Ident") +} + +fn expect_punct(it: &mut token_stream::IntoIter) -> char { + if let TokenTree::Punct(punct) = it.next().expect("Reached end of token stream for Punct") { + punct.as_char() + } else { + panic!("Expected Punct"); + } +} + +fn expect_literal(it: &mut token_stream::IntoIter) -> String { + try_literal(it).expect("Expected Literal") +} + +fn expect_group(it: &mut token_stream::IntoIter) -> Group { + if let TokenTree::Group(group) = it.next().expect("Reached end of token stream for Group") { + group + } else { + panic!("Expected Group"); + } +} + +fn expect_byte_string(it: &mut token_stream::IntoIter) -> String { + try_byte_string(it).expect("Expected byte string") +} + +#[derive(Clone, PartialEq)] +enum ParamType { + Ident(String), + Array { vals: String, max_length: usize }, +} + +fn expect_array_fields(it: &mut token_stream::IntoIter) -> ParamType { + assert_eq!(expect_punct(it), '<'); + let vals = expect_ident(it); + assert_eq!(expect_punct(it), ','); + let max_length_str = expect_literal(it); + let max_length = max_length_str + .parse::<usize>() + .expect("Expected usize length"); + assert_eq!(expect_punct(it), '>'); + ParamType::Array { vals, max_length } +} + +fn expect_type(it: &mut token_stream::IntoIter) -> ParamType { + if let TokenTree::Ident(ident) = it + .next() + .expect("Reached end of token stream for param type") + { + match ident.to_string().as_ref() { + "ArrayParam" => expect_array_fields(it), + _ => ParamType::Ident(ident.to_string()), + } + } else { + panic!("Expected Param Type") + } +} + +fn expect_end(it: &mut token_stream::IntoIter) { + if it.next().is_some() { + panic!("Expected end"); + } +} + +fn get_ident(it: &mut token_stream::IntoIter, expected_name: &str) -> String { + assert_eq!(expect_ident(it), expected_name); + assert_eq!(expect_punct(it), ':'); + let ident = expect_ident(it); + assert_eq!(expect_punct(it), ','); + ident +} + +fn get_literal(it: &mut token_stream::IntoIter, expected_name: &str) -> String { + assert_eq!(expect_ident(it), expected_name); + assert_eq!(expect_punct(it), ':'); + let literal = expect_literal(it); + assert_eq!(expect_punct(it), ','); + literal +} + +fn get_group(it: &mut token_stream::IntoIter, expected_name: &str) -> Group { + assert_eq!(expect_ident(it), expected_name); + assert_eq!(expect_punct(it), ':'); + let group = expect_group(it); + assert_eq!(expect_punct(it), ','); + group +} + +fn get_byte_string(it: &mut token_stream::IntoIter, expected_name: &str) -> String { + assert_eq!(expect_ident(it), expected_name); + assert_eq!(expect_punct(it), ':'); + let byte_string = expect_byte_string(it); + assert_eq!(expect_punct(it), ','); + byte_string +} + +fn __build_modinfo_string_base( + module: &str, + field: &str, + content: &str, + variable: &str, + builtin: bool, +) -> String { + let string = if builtin { + // Built-in modules prefix their modinfo strings by `module.`. + format!( + "{module}.{field}={content}", + module = module, + field = field, + content = content + ) + } else { + // Loadable modules' modinfo strings go as-is. + format!("{field}={content}", field = field, content = content) + }; + + format!( + " + {cfg} + #[link_section = \".modinfo\"] + #[used] + pub static {variable}: [u8; {length}] = *b\"{string}\\0\"; + ", + cfg = if builtin { + "#[cfg(not(MODULE))]" + } else { + "#[cfg(MODULE)]" + }, + variable = variable, + length = string.len() + 1, + string = string, + ) +} + +fn __build_modinfo_string_variable(module: &str, field: &str) -> String { + format!("__{module}_{field}", module = module, field = field) +} + +fn build_modinfo_string_only_builtin(module: &str, field: &str, content: &str) -> String { + __build_modinfo_string_base( + module, + field, + content, + &__build_modinfo_string_variable(module, field), + true, + ) +} + +fn build_modinfo_string_only_loadable(module: &str, field: &str, content: &str) -> String { + __build_modinfo_string_base( + module, + field, + content, + &__build_modinfo_string_variable(module, field), + false, + ) +} + +fn build_modinfo_string(module: &str, field: &str, content: &str) -> String { + build_modinfo_string_only_builtin(module, field, content) + + &build_modinfo_string_only_loadable(module, field, content) +} + +fn build_modinfo_string_param(module: &str, field: &str, param: &str, content: &str) -> String { + let variable = format!( + "__{module}_{field}_{param}", + module = module, + field = field, + param = param + ); + let content = format!("{param}:{content}", param = param, content = content); + __build_modinfo_string_base(module, field, &content, &variable, true) + + &__build_modinfo_string_base(module, field, &content, &variable, false) +} + +fn permissions_are_readonly(perms: &str) -> bool { + let (radix, digits) = if let Some(n) = perms.strip_prefix("0x") { + (16, n) + } else if let Some(n) = perms.strip_prefix("0o") { + (8, n) + } else if let Some(n) = perms.strip_prefix("0b") { + (2, n) + } else { + (10, perms) + }; + match u32::from_str_radix(digits, radix) { + Ok(perms) => perms & 0o222 == 0, + Err(_) => false, + } +} + +fn param_ops_path(param_type: &str) -> &'static str { + match param_type { + "bool" => "kernel::module_param::PARAM_OPS_BOOL", + "i8" => "kernel::module_param::PARAM_OPS_I8", + "u8" => "kernel::module_param::PARAM_OPS_U8", + "i16" => "kernel::module_param::PARAM_OPS_I16", + "u16" => "kernel::module_param::PARAM_OPS_U16", + "i32" => "kernel::module_param::PARAM_OPS_I32", + "u32" => "kernel::module_param::PARAM_OPS_U32", + "i64" => "kernel::module_param::PARAM_OPS_I64", + "u64" => "kernel::module_param::PARAM_OPS_U64", + "isize" => "kernel::module_param::PARAM_OPS_ISIZE", + "usize" => "kernel::module_param::PARAM_OPS_USIZE", + "str" => "kernel::module_param::PARAM_OPS_STR", + t => panic!("Unrecognized type {}", t), + } +} + +fn try_simple_param_val( + param_type: &str, +) -> Box<dyn Fn(&mut token_stream::IntoIter) -> Option<String>> { + match param_type { + "bool" => Box::new(|param_it| try_ident(param_it)), + "str" => Box::new(|param_it| { + try_byte_string(param_it) + .map(|s| format!("kernel::module_param::StringParam::Ref(b\"{}\")", s)) + }), + _ => Box::new(|param_it| try_literal(param_it)), + } +} + +fn get_default(param_type: &ParamType, param_it: &mut token_stream::IntoIter) -> String { + let try_param_val = match param_type { + ParamType::Ident(ref param_type) + | ParamType::Array { + vals: ref param_type, + max_length: _, + } => try_simple_param_val(param_type), + }; + assert_eq!(expect_ident(param_it), "default"); + assert_eq!(expect_punct(param_it), ':'); + let default = match param_type { + ParamType::Ident(_) => try_param_val(param_it).expect("Expected default param value"), + ParamType::Array { + vals: _, + max_length: _, + } => { + let group = expect_group(param_it); + assert_eq!(group.delimiter(), Delimiter::Bracket); + let mut default_vals = Vec::new(); + let mut it = group.stream().into_iter(); + + while let Some(default_val) = try_param_val(&mut it) { + default_vals.push(default_val); + match it.next() { + Some(TokenTree::Punct(punct)) => assert_eq!(punct.as_char(), ','), + None => break, + _ => panic!("Expected ',' or end of array default values"), + } + } + + let mut default_array = "kernel::module_param::ArrayParam::create(&[".to_string(); + default_array.push_str( + &default_vals + .iter() + .map(|val| val.to_string()) + .collect::<Vec<String>>() + .join(","), + ); + default_array.push_str("])"); + default_array + } + }; + assert_eq!(expect_punct(param_it), ','); + default +} + +fn generated_array_ops_name(vals: &str, max_length: usize) -> String { + format!( + "__generated_array_ops_{vals}_{max_length}", + vals = vals, + max_length = max_length + ) +} + +/// Declares a kernel module. +/// +/// The `type` argument should be a type which implements the [`KernelModule`] +/// trait. Also accepts various forms of kernel metadata. +/// +/// [`KernelModule`]: ../kernel/trait.KernelModule.html +/// +/// # Examples +/// +/// ```rust,no_run +/// use kernel::prelude::*; +/// +/// module!{ +/// type: MyKernelModule, +/// name: b"my_kernel_module", +/// author: b"Rust for Linux Contributors", +/// description: b"My very own kernel module!", +/// license: b"GPL v2", +/// params: { +/// my_i32: i32 { +/// default: 42, +/// permissions: 0o000, +/// description: b"Example of i32", +/// }, +/// writeable_i32: i32 { +/// default: 42, +/// permissions: 0o644, +/// description: b"Example of i32", +/// }, +/// }, +/// } +/// +/// struct MyKernelModule; +/// +/// impl KernelModule for MyKernelModule { +/// fn init() -> KernelResult<Self> { +/// // If the parameter is writeable, then the kparam lock must be +/// // taken to read the parameter: +/// { +/// let lock = THIS_MODULE.kernel_param_lock(); +/// pr_info!("i32 param is: {}\n", writeable_i32.read(&lock)); +/// } +/// // If the parameter is read only, it can be read without locking +/// // the kernel parameters: +/// pr_info!("i32 param is: {}\n", my_i32.read()); +/// Ok(MyKernelModule) +/// } +/// } +/// ``` +/// +/// # Supported parameter types +/// +/// - `bool`: Corresponds to C `bool` param type. +/// - `i8`: No equivalent C param type. +/// - `u8`: Corresponds to C `char` param type. +/// - `i16`: Corresponds to C `short` param type. +/// - `u16`: Corresponds to C `ushort` param type. +/// - `i32`: Corresponds to C `int` param type. +/// - `u32`: Corresponds to C `uint` param type. +/// - `i64`: No equivalent C param type. +/// - `u64`: Corresponds to C `ullong` param type. +/// - `isize`: No equivalent C param type. +/// - `usize`: No equivalent C param type. +/// - `str`: Corresponds to C `charp` param type. Reading returns a byte slice. +/// - `ArrayParam<T,N>`: Corresponds to C parameters created using `module_param_array`. An array +/// of `T`'s of length at **most** `N`. +/// +/// `invbool` is unsupported: it was only ever used in a few modules. +/// Consider using a `bool` and inverting the logic instead. +#[proc_macro] +pub fn module(ts: TokenStream) -> TokenStream { + let mut it = ts.into_iter(); + + let type_ = get_ident(&mut it, "type"); + let name = get_byte_string(&mut it, "name"); + let author = get_byte_string(&mut it, "author"); + let description = get_byte_string(&mut it, "description"); + let license = get_byte_string(&mut it, "license"); + let params = get_group(&mut it, "params"); + + expect_end(&mut it); + + assert_eq!(params.delimiter(), Delimiter::Brace); + + let mut it = params.stream().into_iter(); + + let mut params_modinfo = String::new(); + + let mut array_types_to_generate = Vec::new(); + + loop { + let param_name = match it.next() { + Some(TokenTree::Ident(ident)) => ident.to_string(), + Some(_) => panic!("Expected Ident or end"), + None => break, + }; + + assert_eq!(expect_punct(&mut it), ':'); + let param_type = expect_type(&mut it); + let group = expect_group(&mut it); + assert_eq!(expect_punct(&mut it), ','); + + assert_eq!(group.delimiter(), Delimiter::Brace); + + let mut param_it = group.stream().into_iter(); + let param_default = get_default(¶m_type, &mut param_it); + let param_permissions = get_literal(&mut param_it, "permissions"); + let param_description = get_byte_string(&mut param_it, "description"); + expect_end(&mut param_it); + + // TODO: more primitive types + // TODO: other kinds: unsafes, etc. + let (param_kernel_type, ops): (String, _) = match param_type { + ParamType::Ident(ref param_type) => ( + param_type.to_string(), + param_ops_path(¶m_type).to_string(), + ), + ParamType::Array { + ref vals, + max_length, + } => { + array_types_to_generate.push((vals.clone(), max_length)); + ( + format!("__rust_array_param_{}_{}", vals, max_length), + generated_array_ops_name(vals, max_length), + ) + } + }; + + params_modinfo.push_str(&build_modinfo_string_param( + &name, + "parmtype", + ¶m_name, + ¶m_kernel_type, + )); + params_modinfo.push_str(&build_modinfo_string_param( + &name, + "parm", + ¶m_name, + ¶m_description, + )); + let param_type_internal = match param_type { + ParamType::Ident(ref param_type) => match param_type.as_ref() { + "str" => "kernel::module_param::StringParam".to_string(), + other => other.to_string(), + }, + ParamType::Array { + ref vals, + max_length, + } => format!( + "kernel::module_param::ArrayParam<{vals}, {max_length}>", + vals = vals, + max_length = max_length + ), + }; + let read_func = if permissions_are_readonly(¶m_permissions) { + format!( + " + fn read(&self) -> &<{param_type_internal} as kernel::module_param::ModuleParam>::Value {{ + // SAFETY: Parameters do not need to be locked because they are read only or sysfs is not enabled. + unsafe {{ <{param_type_internal} as kernel::module_param::ModuleParam>::value(&__{name}_{param_name}_value) }} + }} + ", + name = name, + param_name = param_name, + param_type_internal = param_type_internal, + ) + } else { + format!( + " + fn read<'lck>(&self, lock: &'lck kernel::KParamGuard) -> &'lck <{param_type_internal} as kernel::module_param::ModuleParam>::Value {{ + // SAFETY: Parameters are locked by `KParamGuard`. + unsafe {{ <{param_type_internal} as kernel::module_param::ModuleParam>::value(&__{name}_{param_name}_value) }} + }} + ", + name = name, + param_name = param_name, + param_type_internal = param_type_internal, + ) + }; + let kparam = format!( + " + kernel::bindings::kernel_param__bindgen_ty_1 {{ + arg: unsafe {{ &__{name}_{param_name}_value }} as *const _ as *mut kernel::c_types::c_void, + }}, + ", + name = name, + param_name = param_name, + ); + params_modinfo.push_str( + &format!( + " + static mut __{name}_{param_name}_value: {param_type_internal} = {param_default}; + + struct __{name}_{param_name}; + + impl __{name}_{param_name} {{ {read_func} }} + + const {param_name}: __{name}_{param_name} = __{name}_{param_name}; + + // Note: the C macro that generates the static structs for the `__param` section + // asks for them to be `aligned(sizeof(void *))`. However, that was put in place + // in 2003 in commit 38d5b085d2 (\"[PATCH] Fix over-alignment problem on x86-64\") + // to undo GCC over-alignment of static structs of >32 bytes. It seems that is + // not the case anymore, so we simplify to a transparent representation here + // in the expectation that it is not needed anymore. + // TODO: revisit this to confirm the above comment and remove it if it happened + #[repr(transparent)] + struct __{name}_{param_name}_RacyKernelParam(kernel::bindings::kernel_param); + + unsafe impl Sync for __{name}_{param_name}_RacyKernelParam {{ + }} + + #[cfg(not(MODULE))] + const __{name}_{param_name}_name: *const kernel::c_types::c_char = b\"{name}.{param_name}\\0\" as *const _ as *const kernel::c_types::c_char; + + #[cfg(MODULE)] + const __{name}_{param_name}_name: *const kernel::c_types::c_char = b\"{param_name}\\0\" as *const _ as *const kernel::c_types::c_char; + + #[link_section = \"__param\"] + #[used] + static __{name}_{param_name}_struct: __{name}_{param_name}_RacyKernelParam = __{name}_{param_name}_RacyKernelParam(kernel::bindings::kernel_param {{ + name: __{name}_{param_name}_name, + // SAFETY: `__this_module` is constructed by the kernel at load time and will not be freed until the module is unloaded. + #[cfg(MODULE)] + mod_: unsafe {{ &kernel::bindings::__this_module as *const _ as *mut _ }}, + #[cfg(not(MODULE))] + mod_: core::ptr::null_mut(), + ops: unsafe {{ &{ops} }} as *const kernel::bindings::kernel_param_ops, + perm: {permissions}, + level: -1, + flags: 0, + __bindgen_anon_1: {kparam} + }}); + ", + name = name, + param_type_internal = param_type_internal, + read_func = read_func, + param_default = param_default, + param_name = param_name, + ops = ops, + permissions = param_permissions, + kparam = kparam, + ) + ); + } + + let mut generated_array_types = String::new(); + + for (vals, max_length) in array_types_to_generate { + let ops_name = generated_array_ops_name(&vals, max_length); + generated_array_types.push_str(&format!( + " + kernel::make_param_ops!( + {ops_name}, + kernel::module_param::ArrayParam<{vals}, {{ {max_length} }}> + ); + ", + ops_name = ops_name, + vals = vals, + max_length = max_length, + )); + } + + let file = + std::env::var("RUST_MODFILE").expect("Unable to fetch RUST_MODFILE environmental variable"); + + format!( + " + /// The module name. + /// + /// Used by the printing macros, e.g. [`info!`]. + const __LOG_PREFIX: &[u8] = b\"{name}\\0\"; + + static mut __MOD: Option<{type_}> = None; + + // SAFETY: `__this_module` is constructed by the kernel at load time and will not be freed until the module is unloaded. + #[cfg(MODULE)] + static THIS_MODULE: kernel::ThisModule = unsafe {{ kernel::ThisModule::from_ptr(&kernel::bindings::__this_module as *const _ as *mut _) }}; + #[cfg(not(MODULE))] + static THIS_MODULE: kernel::ThisModule = unsafe {{ kernel::ThisModule::from_ptr(core::ptr::null_mut()) }}; + + // Loadable modules need to export the `{{init,cleanup}}_module` identifiers + #[cfg(MODULE)] + #[no_mangle] + pub extern \"C\" fn init_module() -> kernel::c_types::c_int {{ + __init() + }} + + #[cfg(MODULE)] + #[no_mangle] + pub extern \"C\" fn cleanup_module() {{ + __exit() + }} + + // Built-in modules are initialized through an initcall pointer + // and the identifiers need to be unique + #[cfg(not(MODULE))] + #[cfg(not(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS))] + #[link_section = \"{initcall_section}\"] + #[used] + pub static __{name}_initcall: extern \"C\" fn() -> kernel::c_types::c_int = __{name}_init; + + #[cfg(not(MODULE))] + #[cfg(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS)] + global_asm!( + r#\".section \"{initcall_section}\", \"a\" + __{name}_initcall: + .long __{name}_init - . + .previous + \"# + ); + + #[cfg(not(MODULE))] + #[no_mangle] + pub extern \"C\" fn __{name}_init() -> kernel::c_types::c_int {{ + __init() + }} + + #[cfg(not(MODULE))] + #[no_mangle] + pub extern \"C\" fn __{name}_exit() {{ + __exit() + }} + + fn __init() -> kernel::c_types::c_int {{ + match <{type_} as kernel::KernelModule>::init() {{ + Ok(m) => {{ + unsafe {{ + __MOD = Some(m); + }} + return 0; + }} + Err(e) => {{ + return e.to_kernel_errno(); + }} + }} + }} + + fn __exit() {{ + unsafe {{ + // Invokes `drop()` on `__MOD`, which should be used for cleanup. + __MOD = None; + }} + }} + + {author} + {description} + {license} + + // Built-in modules also export the `file` modinfo string + {file} + + {params_modinfo} + + {generated_array_types} + ", + type_ = type_, + name = name, + author = &build_modinfo_string(&name, "author", &author), + description = &build_modinfo_string(&name, "description", &description), + license = &build_modinfo_string(&name, "license", &license), + file = &build_modinfo_string_only_builtin(&name, "file", &file), + params_modinfo = params_modinfo, + generated_array_types = generated_array_types, + initcall_section = ".initcall6.init" + ).parse().expect("Error parsing formatted string into token stream.") +} + +/// Declares a kernel module that exposes a single misc device. +/// +/// The `type` argument should be a type which implements the [`FileOpener`] trait. Also accepts +/// various forms of kernel metadata. +/// +/// [`FileOpener`]: ../kernel/file_operations/trait.FileOpener.html +/// +/// # Examples +/// +/// ```rust,no_run +/// use kernel::prelude::*; +/// +/// module_misc_device! { +/// type: MyFile, +/// name: b"my_miscdev_kernel_module", +/// author: b"Rust for Linux Contributors", +/// description: b"My very own misc device kernel module!", +/// license: b"GPL v2", +/// } +/// +/// #[derive(Default)] +/// struct MyFile; +/// +/// impl kernel::file_operations::FileOperations for MyFile { +/// kernel::declare_file_operations!(); +/// } +/// ``` +#[proc_macro] +pub fn module_misc_device(ts: TokenStream) -> TokenStream { + let mut it = ts.into_iter(); + + let type_ = get_ident(&mut it, "type"); + let name = get_byte_string(&mut it, "name"); + let author = get_byte_string(&mut it, "author"); + let description = get_byte_string(&mut it, "description"); + let license = get_byte_string(&mut it, "license"); + expect_end(&mut it); + + let module = format!("__internal_ModuleFor{}", type_); + + format!( + " + #[doc(hidden)] + struct {module} {{ + _dev: core::pin::Pin<alloc::boxed::Box<kernel::miscdev::Registration>>, + }} + + impl kernel::KernelModule for {module} {{ + fn init() -> kernel::KernelResult<Self> {{ + Ok(Self {{ + _dev: kernel::miscdev::Registration::new_pinned::<{type_}>( + kernel::cstr!(\"{name}\"), + None, + (), + )?, + }}) + }} + }} + + kernel::prelude::module! {{ + type: {module}, + name: b\"{name}\", + author: b\"{author}\", + description: b\"{description}\", + license: b\"{license}\", + params: {{}}, + }} + ", + module = module, + type_ = type_, + name = name, + author = author, + description = description, + license = license + ) + .parse() + .expect("Error parsing formatted string into token stream.") +} diff --git a/samples/Kconfig b/samples/Kconfig index b5a1a7aa7e23a..eaa06c05d37f1 100644 --- a/samples/Kconfig +++ b/samples/Kconfig @@ -223,4 +223,6 @@ config SAMPLE_WATCH_QUEUE Build example userspace program to use the new mount_notify(), sb_notify() syscalls and the KEYCTL_WATCH_KEY keyctl() function. +source "samples/rust/Kconfig" + endif # SAMPLES diff --git a/samples/Makefile b/samples/Makefile index 087e0988ccc56..291663e56a3cd 100644 --- a/samples/Makefile +++ b/samples/Makefile @@ -30,3 +30,4 @@ obj-$(CONFIG_SAMPLE_INTEL_MEI) += mei/ subdir-$(CONFIG_SAMPLE_WATCHDOG) += watchdog subdir-$(CONFIG_SAMPLE_WATCH_QUEUE) += watch_queue obj-$(CONFIG_DEBUG_KMEMLEAK_TEST) += kmemleak/ +obj-$(CONFIG_SAMPLES_RUST) += rust/ diff --git a/samples/rust/Kconfig b/samples/rust/Kconfig new file mode 100644 index 0000000000000..183a3c4dc80cd --- /dev/null +++ b/samples/rust/Kconfig @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: GPL-2.0 + +menuconfig SAMPLES_RUST + bool "Rust samples" + depends on RUST + help + You can build sample Rust kernel code here. + + If unsure, say N. + +if SAMPLES_RUST + +config SAMPLE_RUST_MINIMAL + tristate "Minimal" + help + This option builds the Rust minimal module sample. + + To compile this as a module, choose M here: + the module will be called rust_minimal. + + If unsure, say N. + +config SAMPLE_RUST_PRINT + tristate "Printing macros" + help + This option builds the Rust printing macros sample. + + To compile this as a module, choose M here: + the module will be called rust_print. + + If unsure, say N. + +config SAMPLE_RUST_MODULE_PARAMETERS + tristate "Module parameters" + help + This option builds the Rust module parameters sample. + + To compile this as a module, choose M here: + the module will be called rust_module_parameters. + + If unsure, say N. + +config SAMPLE_RUST_SYNC + tristate "Synchronisation primitives" + help + This option builds the Rust synchronisation primitives sample. + + To compile this as a module, choose M here: + the module will be called rust_sync. + + If unsure, say N. + +config SAMPLE_RUST_CHRDEV + tristate "Character device" + help + This option builds the Rust character device sample. + + To compile this as a module, choose M here: + the module will be called rust_chrdev. + + If unsure, say N. + +config SAMPLE_RUST_MISCDEV + tristate "Miscellaneous device" + help + This option builds the Rust miscellaneous device sample. + + To compile this as a module, choose M here: + the module will be called rust_miscdev. + + If unsure, say N. + +config SAMPLE_RUST_STACK_PROBING + tristate "Stack probing" + help + This option builds the Rust stack probing sample. + + To compile this as a module, choose M here: + the module will be called rust_stack_probing. + + If unsure, say N. + +config SAMPLE_RUST_SEMAPHORE + tristate "Semaphore" + help + This option builds the Rust semaphore sample. + + To compile this as a module, choose M here: + the module will be called rust_semaphore. + + If unsure, say N. + +config SAMPLE_RUST_SEMAPHORE_C + tristate "Semaphore (in C, for comparison)" + help + This option builds the Rust semaphore sample (in C, for comparison). + + To compile this as a module, choose M here: + the module will be called rust_semaphore_c. + + If unsure, say N. + +config SAMPLE_RUST_RANDOM + tristate "Random" + help + This option builds the Rust random sample. + + To compile this as a module, choose M here: + the module will be called rust_random. + + If unsure, say N. + +endif # SAMPLES_RUST diff --git a/samples/rust/Makefile b/samples/rust/Makefile new file mode 100644 index 0000000000000..48bc871ea1f8f --- /dev/null +++ b/samples/rust/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_SAMPLE_RUST_MINIMAL) += rust_minimal.o +obj-$(CONFIG_SAMPLE_RUST_PRINT) += rust_print.o +obj-$(CONFIG_SAMPLE_RUST_MODULE_PARAMETERS) += rust_module_parameters.o +obj-$(CONFIG_SAMPLE_RUST_SYNC) += rust_sync.o +obj-$(CONFIG_SAMPLE_RUST_CHRDEV) += rust_chrdev.o +obj-$(CONFIG_SAMPLE_RUST_MISCDEV) += rust_miscdev.o +obj-$(CONFIG_SAMPLE_RUST_STACK_PROBING) += rust_stack_probing.o +obj-$(CONFIG_SAMPLE_RUST_SEMAPHORE) += rust_semaphore.o +obj-$(CONFIG_SAMPLE_RUST_SEMAPHORE_C) += rust_semaphore_c.o +obj-$(CONFIG_SAMPLE_RUST_RANDOM) += rust_random.o diff --git a/samples/rust/rust_chrdev.rs b/samples/rust/rust_chrdev.rs new file mode 100644 index 0000000000000..78423b1e3d116 --- /dev/null +++ b/samples/rust/rust_chrdev.rs @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust character device sample + +#![no_std] +#![feature(allocator_api, global_asm)] + +use alloc::boxed::Box; +use core::pin::Pin; +use kernel::prelude::*; +use kernel::{chrdev, cstr, file_operations::FileOperations}; + +module! { + type: RustChrdev, + name: b"rust_chrdev", + author: b"Rust for Linux Contributors", + description: b"Rust character device sample", + license: b"GPL v2", + params: { + }, +} + +#[derive(Default)] +struct RustFile; + +impl FileOperations for RustFile { + kernel::declare_file_operations!(); +} + +struct RustChrdev { + _dev: Pin<Box<chrdev::Registration<2>>>, +} + +impl KernelModule for RustChrdev { + fn init() -> KernelResult<Self> { + pr_info!("Rust character device sample (init)\n"); + + let mut chrdev_reg = + chrdev::Registration::new_pinned(cstr!("rust_chrdev"), 0, &THIS_MODULE)?; + + // Register the same kind of device twice, we're just demonstrating + // that you can use multiple minors. There are two minors in this case + // because its type is `chrdev::Registration<2>` + chrdev_reg.as_mut().register::<RustFile>()?; + chrdev_reg.as_mut().register::<RustFile>()?; + + Ok(RustChrdev { _dev: chrdev_reg }) + } +} + +impl Drop for RustChrdev { + fn drop(&mut self) { + pr_info!("Rust character device sample (exit)\n"); + } +} diff --git a/samples/rust/rust_minimal.rs b/samples/rust/rust_minimal.rs new file mode 100644 index 0000000000000..21627ce5656e2 --- /dev/null +++ b/samples/rust/rust_minimal.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust minimal sample + +#![no_std] +#![feature(allocator_api, global_asm)] + +use kernel::prelude::*; + +module! { + type: RustMinimal, + name: b"rust_minimal", + author: b"Rust for Linux Contributors", + description: b"Rust minimal sample", + license: b"GPL v2", + params: { + }, +} + +struct RustMinimal { + message: String, +} + +impl KernelModule for RustMinimal { + fn init() -> KernelResult<Self> { + pr_info!("Rust minimal sample (init)\n"); + pr_info!("Am I built-in? {}\n", !cfg!(MODULE)); + + Ok(RustMinimal { + message: "on the heap!".to_owned(), + }) + } +} + +impl Drop for RustMinimal { + fn drop(&mut self) { + pr_info!("My message is {}\n", self.message); + pr_info!("Rust minimal sample (exit)\n"); + } +} diff --git a/samples/rust/rust_miscdev.rs b/samples/rust/rust_miscdev.rs new file mode 100644 index 0000000000000..3d48f1e2fad3a --- /dev/null +++ b/samples/rust/rust_miscdev.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust miscellaneous device sample + +#![no_std] +#![feature(allocator_api, global_asm)] + +use alloc::{boxed::Box, sync::Arc}; +use core::pin::Pin; +use kernel::prelude::*; +use kernel::{ + cstr, + file_operations::{File, FileOpener, FileOperations}, + miscdev, + sync::{CondVar, Mutex}, + user_ptr::{UserSlicePtrReader, UserSlicePtrWriter}, + Error, +}; + +module! { + type: RustMiscdev, + name: b"rust_miscdev", + author: b"Rust for Linux Contributors", + description: b"Rust miscellaneous device sample", + license: b"GPL v2", + params: { + }, +} + +const MAX_TOKENS: usize = 3; + +struct SharedStateInner { + token_count: usize, +} + +struct SharedState { + state_changed: CondVar, + inner: Mutex<SharedStateInner>, +} + +impl SharedState { + fn try_new() -> KernelResult<Arc<Self>> { + let state = Arc::try_new(Self { + // SAFETY: `condvar_init!` is called below. + state_changed: unsafe { CondVar::new() }, + // SAFETY: `mutex_init!` is called below. + inner: unsafe { Mutex::new(SharedStateInner { token_count: 0 }) }, + })?; + // SAFETY: `state_changed` is pinned behind `Arc`. + let state_changed = unsafe { Pin::new_unchecked(&state.state_changed) }; + kernel::condvar_init!(state_changed, "SharedState::state_changed"); + // SAFETY: `inner` is pinned behind `Arc`. + let inner = unsafe { Pin::new_unchecked(&state.inner) }; + kernel::mutex_init!(inner, "SharedState::inner"); + Ok(state) + } +} + +struct Token { + shared: Arc<SharedState>, +} + +impl FileOpener<Arc<SharedState>> for Token { + fn open(shared: &Arc<SharedState>) -> KernelResult<Self::Wrapper> { + Ok(Box::try_new(Self { + shared: shared.clone(), + })?) + } +} + +impl FileOperations for Token { + type Wrapper = Box<Self>; + + kernel::declare_file_operations!(read, write); + + fn read(&self, _: &File, data: &mut UserSlicePtrWriter, offset: u64) -> KernelResult<usize> { + // Succeed if the caller doesn't provide a buffer or if not at the start. + if data.is_empty() || offset != 0 { + return Ok(0); + } + + { + let mut inner = self.shared.inner.lock(); + + // Wait until we are allowed to decrement the token count or a signal arrives. + while inner.token_count == 0 { + if self.shared.state_changed.wait(&mut inner) { + return Err(Error::EINTR); + } + } + + // Consume a token. + inner.token_count -= 1; + } + + // Notify a possible writer waiting. + self.shared.state_changed.notify_all(); + + // Write a one-byte 1 to the reader. + data.write_slice(&[1u8; 1])?; + Ok(1) + } + + fn write(&self, data: &mut UserSlicePtrReader, _offset: u64) -> KernelResult<usize> { + { + let mut inner = self.shared.inner.lock(); + + // Wait until we are allowed to increment the token count or a signal arrives. + while inner.token_count == MAX_TOKENS { + if self.shared.state_changed.wait(&mut inner) { + return Err(Error::EINTR); + } + } + + // Increment the number of token so that a reader can be released. + inner.token_count += 1; + } + + // Notify a possible reader waiting. + self.shared.state_changed.notify_all(); + Ok(data.len()) + } +} + +struct RustMiscdev { + _dev: Pin<Box<miscdev::Registration<Arc<SharedState>>>>, +} + +impl KernelModule for RustMiscdev { + fn init() -> KernelResult<Self> { + pr_info!("Rust miscellaneous device sample (init)\n"); + + let state = SharedState::try_new()?; + + Ok(RustMiscdev { + _dev: miscdev::Registration::new_pinned::<Token>(cstr!("rust_miscdev"), None, state)?, + }) + } +} + +impl Drop for RustMiscdev { + fn drop(&mut self) { + pr_info!("Rust miscellaneous device sample (exit)\n"); + } +} diff --git a/samples/rust/rust_module_parameters.rs b/samples/rust/rust_module_parameters.rs new file mode 100644 index 0000000000000..d9b6de695384f --- /dev/null +++ b/samples/rust/rust_module_parameters.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust module parameters sample + +#![no_std] +#![feature(allocator_api, global_asm)] + +use kernel::prelude::*; + +module! { + type: RustModuleParameters, + name: b"rust_module_parameters", + author: b"Rust for Linux Contributors", + description: b"Rust module parameters sample", + license: b"GPL v2", + params: { + my_bool: bool { + default: true, + permissions: 0, + description: b"Example of bool", + }, + my_i32: i32 { + default: 42, + permissions: 0o644, + description: b"Example of i32", + }, + my_str: str { + default: b"default str val", + permissions: 0o644, + description: b"Example of a string param", + }, + my_usize: usize { + default: 42, + permissions: 0o644, + description: b"Example of usize", + }, + my_array: ArrayParam<i32, 3> { + default: [0, 1], + permissions: 0, + description: b"Example of array", + }, + }, +} + +struct RustModuleParameters; + +impl KernelModule for RustModuleParameters { + fn init() -> KernelResult<Self> { + pr_info!("Rust module parameters sample (init)\n"); + + { + let lock = THIS_MODULE.kernel_param_lock(); + pr_info!("Parameters:\n"); + pr_info!(" my_bool: {}\n", my_bool.read()); + pr_info!(" my_i32: {}\n", my_i32.read(&lock)); + pr_info!( + " my_str: {}\n", + core::str::from_utf8(my_str.read(&lock))? + ); + pr_info!(" my_usize: {}\n", my_usize.read(&lock)); + pr_info!(" my_array: {:?}\n", my_array.read()); + } + + Ok(RustModuleParameters) + } +} + +impl Drop for RustModuleParameters { + fn drop(&mut self) { + pr_info!("Rust module parameters sample (exit)\n"); + } +} diff --git a/samples/rust/rust_print.rs b/samples/rust/rust_print.rs new file mode 100644 index 0000000000000..ddfac800f425d --- /dev/null +++ b/samples/rust/rust_print.rs @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust printing macros sample + +#![no_std] +#![feature(allocator_api, global_asm)] + +use kernel::prelude::*; + +module! { + type: RustPrint, + name: b"rust_print", + author: b"Rust for Linux Contributors", + description: b"Rust printing macros sample", + license: b"GPL v2", + params: { + }, +} + +struct RustPrint; + +impl KernelModule for RustPrint { + fn init() -> KernelResult<Self> { + pr_info!("Rust printing macros sample (init)\n"); + + pr_emerg!("Emergency message (level 0) without args\n"); + pr_alert!("Alert message (level 1) without args\n"); + pr_crit!("Critical message (level 2) without args\n"); + pr_err!("Error message (level 3) without args\n"); + pr_warn!("Warning message (level 4) without args\n"); + pr_notice!("Notice message (level 5) without args\n"); + pr_info!("Info message (level 6) without args\n"); + + pr_info!("A line that"); + pr_cont!(" is continued"); + pr_cont!(" without args\n"); + + pr_emerg!("{} message (level {}) with args\n", "Emergency", 0); + pr_alert!("{} message (level {}) with args\n", "Alert", 1); + pr_crit!("{} message (level {}) with args\n", "Critical", 2); + pr_err!("{} message (level {}) with args\n", "Error", 3); + pr_warn!("{} message (level {}) with args\n", "Warning", 4); + pr_notice!("{} message (level {}) with args\n", "Notice", 5); + pr_info!("{} message (level {}) with args\n", "Info", 6); + + pr_info!("A {} that", "line"); + pr_cont!(" is {}", "continued"); + pr_cont!(" with {}\n", "args"); + + Ok(RustPrint) + } +} + +impl Drop for RustPrint { + fn drop(&mut self) { + pr_info!("Rust printing macros sample (exit)\n"); + } +} diff --git a/samples/rust/rust_random.rs b/samples/rust/rust_random.rs new file mode 100644 index 0000000000000..9f17b94404661 --- /dev/null +++ b/samples/rust/rust_random.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust random device +//! +//! Adapted from Alex Gaynor's original available at +//! <https://github.com/alex/just-use/blob/master/src/lib.rs>. + +#![no_std] +#![feature(allocator_api, global_asm)] + +use kernel::{ + file_operations::{File, FileOperations}, + prelude::*, + user_ptr::{UserSlicePtrReader, UserSlicePtrWriter}, +}; + +#[derive(Default)] +struct RandomFile; + +impl FileOperations for RandomFile { + kernel::declare_file_operations!(read, write); + + fn read(&self, file: &File, buf: &mut UserSlicePtrWriter, _offset: u64) -> KernelResult<usize> { + let total_len = buf.len(); + let mut chunkbuf = [0; 256]; + + while !buf.is_empty() { + let len = chunkbuf.len().min(buf.len()); + let chunk = &mut chunkbuf[0..len]; + + if file.is_blocking() { + kernel::random::getrandom(chunk)?; + } else { + kernel::random::getrandom_nonblock(chunk)?; + } + buf.write_slice(chunk)?; + } + Ok(total_len) + } + + fn write(&self, buf: &mut UserSlicePtrReader, _offset: u64) -> KernelResult<usize> { + let total_len = buf.len(); + let mut chunkbuf = [0; 256]; + while !buf.is_empty() { + let len = chunkbuf.len().min(buf.len()); + let chunk = &mut chunkbuf[0..len]; + buf.read_slice(chunk)?; + kernel::random::add_randomness(chunk); + } + Ok(total_len) + } +} + +module_misc_device! { + type: RandomFile, + name: b"rust_random", + author: b"Rust for Linux Contributors", + description: b"Just use /dev/urandom: Now with early-boot safety", + license: b"GPL v2", +} diff --git a/samples/rust/rust_semaphore.rs b/samples/rust/rust_semaphore.rs new file mode 100644 index 0000000000000..eb1dcc202e719 --- /dev/null +++ b/samples/rust/rust_semaphore.rs @@ -0,0 +1,176 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust semaphore sample +//! +//! A counting semaphore that can be used by userspace. +//! +//! The count is incremented by writes to the device. A write of `n` bytes results in an increment +//! of `n`. It is decremented by reads; each read results in the count being decremented by 1. If +//! the count is already zero, a read will block until another write increments it. +//! +//! This can be used in user space from the shell for example as follows (assuming a node called +//! `semaphore`): `cat semaphore` decrements the count by 1 (waiting for it to become non-zero +//! before decrementing); `echo -n 123 > semaphore` increments the semaphore by 3, potentially +//! unblocking up to 3 blocked readers. + +#![no_std] +#![feature(allocator_api, global_asm)] + +use alloc::{boxed::Box, sync::Arc}; +use core::{ + pin::Pin, + sync::atomic::{AtomicU64, Ordering}, +}; +use kernel::{ + condvar_init, cstr, declare_file_operations, + file_operations::{File, FileOpener, FileOperations, IoctlCommand, IoctlHandler}, + miscdev::Registration, + mutex_init, + prelude::*, + sync::{CondVar, Mutex}, + user_ptr::{UserSlicePtrReader, UserSlicePtrWriter}, + Error, +}; + +module! { + type: RustSemaphore, + name: b"rust_semaphore", + author: b"Rust for Linux Contributors", + description: b"Rust semaphore sample", + license: b"GPL v2", + params: {}, +} + +struct SemaphoreInner { + count: usize, + max_seen: usize, +} + +struct Semaphore { + changed: CondVar, + inner: Mutex<SemaphoreInner>, +} + +struct FileState { + read_count: AtomicU64, + shared: Arc<Semaphore>, +} + +impl FileState { + fn consume(&self) -> KernelResult { + let mut inner = self.shared.inner.lock(); + while inner.count == 0 { + if self.shared.changed.wait(&mut inner) { + return Err(Error::EINTR); + } + } + inner.count -= 1; + Ok(()) + } +} + +impl FileOpener<Arc<Semaphore>> for FileState { + fn open(shared: &Arc<Semaphore>) -> KernelResult<Box<Self>> { + Ok(Box::try_new(Self { + read_count: AtomicU64::new(0), + shared: shared.clone(), + })?) + } +} + +impl FileOperations for FileState { + type Wrapper = Box<Self>; + + declare_file_operations!(read, write, ioctl); + + fn read(&self, _: &File, data: &mut UserSlicePtrWriter, offset: u64) -> KernelResult<usize> { + if data.is_empty() || offset > 0 { + return Ok(0); + } + self.consume()?; + data.write_slice(&[0u8; 1])?; + self.read_count.fetch_add(1, Ordering::Relaxed); + Ok(1) + } + + fn write(&self, data: &mut UserSlicePtrReader, _offset: u64) -> KernelResult<usize> { + { + let mut inner = self.shared.inner.lock(); + inner.count = inner.count.saturating_add(data.len()); + if inner.count > inner.max_seen { + inner.max_seen = inner.count; + } + } + + self.shared.changed.notify_all(); + Ok(data.len()) + } + + fn ioctl(&self, file: &File, cmd: &mut IoctlCommand) -> KernelResult<i32> { + cmd.dispatch(self, file) + } +} + +struct RustSemaphore { + _dev: Pin<Box<Registration<Arc<Semaphore>>>>, +} + +impl KernelModule for RustSemaphore { + fn init() -> KernelResult<Self> { + pr_info!("Rust semaphore sample (init)\n"); + + let sema = Arc::try_new(Semaphore { + // SAFETY: `condvar_init!` is called below. + changed: unsafe { CondVar::new() }, + + // SAFETY: `mutex_init!` is called below. + inner: unsafe { + Mutex::new(SemaphoreInner { + count: 0, + max_seen: 0, + }) + }, + })?; + + // SAFETY: `changed` is pinned behind `Arc`. + condvar_init!(Pin::new_unchecked(&sema.changed), "Semaphore::changed"); + + // SAFETY: `inner` is pinned behind `Arc`. + mutex_init!(Pin::new_unchecked(&sema.inner), "Semaphore::inner"); + + Ok(Self { + _dev: Registration::new_pinned::<FileState>(cstr!("rust_semaphore"), None, sema)?, + }) + } +} + +impl Drop for RustSemaphore { + fn drop(&mut self) { + pr_info!("Rust semaphore sample (exit)\n"); + } +} + +const IOCTL_GET_READ_COUNT: u32 = 0x80086301; +const IOCTL_SET_READ_COUNT: u32 = 0x40086301; + +impl IoctlHandler for FileState { + fn read(&self, _: &File, cmd: u32, writer: &mut UserSlicePtrWriter) -> KernelResult<i32> { + match cmd { + IOCTL_GET_READ_COUNT => { + writer.write(&self.read_count.load(Ordering::Relaxed))?; + Ok(0) + } + _ => Err(Error::EINVAL), + } + } + + fn write(&self, _: &File, cmd: u32, reader: &mut UserSlicePtrReader) -> KernelResult<i32> { + match cmd { + IOCTL_SET_READ_COUNT => { + self.read_count.store(reader.read()?, Ordering::Relaxed); + Ok(0) + } + _ => Err(Error::EINVAL), + } + } +} diff --git a/samples/rust/rust_semaphore_c.c b/samples/rust/rust_semaphore_c.c new file mode 100644 index 0000000000000..cdc121d4030d0 --- /dev/null +++ b/samples/rust/rust_semaphore_c.c @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Rust semaphore sample (in C, for comparison) + * + * This is a C implementation of `rust_semaphore.rs`. Refer to the description + * in that file for details on the device. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/miscdevice.h> +#include <linux/module.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/refcount.h> +#include <linux/wait.h> + +#define IOCTL_GET_READ_COUNT _IOR('c', 1, u64) +#define IOCTL_SET_READ_COUNT _IOW('c', 1, u64) + +struct semaphore_state { + struct kref ref; + struct miscdevice miscdev; + wait_queue_head_t changed; + struct mutex mutex; + size_t count; + size_t max_seen; +}; + +struct file_state { + atomic64_t read_count; + struct semaphore_state *shared; +}; + +static int semaphore_consume(struct semaphore_state *state) +{ + DEFINE_WAIT(wait); + + mutex_lock(&state->mutex); + while (state->count == 0) { + prepare_to_wait(&state->changed, &wait, TASK_INTERRUPTIBLE); + mutex_unlock(&state->mutex); + schedule(); + finish_wait(&state->changed, &wait); + if (signal_pending(current)) + return -EINTR; + mutex_lock(&state->mutex); + } + + state->count--; + mutex_unlock(&state->mutex); + + return 0; +} + +static int semaphore_open(struct inode *nodp, struct file *filp) +{ + struct semaphore_state *shared = + container_of(filp->private_data, struct semaphore_state, miscdev); + struct file_state *state; + + state = kzalloc(sizeof(*state), GFP_KERNEL); + if (!state) + return -ENOMEM; + + kref_get(&shared->ref); + state->shared = shared; + atomic64_set(&state->read_count, 0); + + filp->private_data = state; + + return 0; +} + +static ssize_t semaphore_write(struct file *filp, const char __user *buffer, size_t count, + loff_t *ppos) +{ + struct file_state *state = filp->private_data; + struct semaphore_state *shared = state->shared; + + mutex_lock(&shared->mutex); + + shared->count += count; + if (shared->count < count) + shared->count = SIZE_MAX; + + if (shared->count > shared->max_seen) + shared->max_seen = shared->count; + + mutex_unlock(&shared->mutex); + + wake_up_all(&shared->changed); + + return count; +} + +static ssize_t semaphore_read(struct file *filp, char __user *buffer, + size_t count, loff_t *ppos) +{ + struct file_state *state = filp->private_data; + char c = 0; + int ret; + + if (count == 0 || *ppos > 0) + return 0; + + ret = semaphore_consume(state->shared); + if (ret) + return ret; + + if (copy_to_user(buffer, &c, sizeof(c))) + return -EFAULT; + + atomic64_add(1, &state->read_count); + *ppos += 1; + return 1; +} + +static long semaphore_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + struct file_state *state = filp->private_data; + void __user *buffer = (void __user *)arg; + u64 value; + + switch (cmd) { + case IOCTL_GET_READ_COUNT: + value = atomic64_read(&state->read_count); + if (copy_to_user(buffer, &value, sizeof(value))) + return -EFAULT; + return 0; + case IOCTL_SET_READ_COUNT: + if (copy_from_user(&value, buffer, sizeof(value))) + return -EFAULT; + atomic64_set(&state->read_count, value); + return 0; + default: + return -EINVAL; + } +} + +static void semaphore_free(struct kref *kref) +{ + struct semaphore_state *device; + + device = container_of(kref, struct semaphore_state, ref); + kfree(device); +} + +static int semaphore_release(struct inode *nodp, struct file *filp) +{ + struct file_state *state = filp->private_data; + + kref_put(&state->shared->ref, semaphore_free); + kfree(state); + return 0; +} + +static const struct file_operations semaphore_fops = { + .owner = THIS_MODULE, + .open = semaphore_open, + .read = semaphore_read, + .write = semaphore_write, + .compat_ioctl = semaphore_ioctl, + .release = semaphore_release, +}; + +static struct semaphore_state *device; + +static int __init semaphore_init(void) +{ + int ret; + struct semaphore_state *state; + + pr_info("Rust semaphore sample (in C, for comparison) (init)\n"); + + state = kzalloc(sizeof(*state), GFP_KERNEL); + if (!state) + return -ENOMEM; + + mutex_init(&state->mutex); + kref_init(&state->ref); + init_waitqueue_head(&state->changed); + + state->miscdev.fops = &semaphore_fops; + state->miscdev.minor = MISC_DYNAMIC_MINOR; + state->miscdev.name = "semaphore"; + + ret = misc_register(&state->miscdev); + if (ret < 0) { + kfree(state); + return ret; + } + + device = state; + + return 0; +} + +static void __exit semaphore_exit(void) +{ + pr_info("Rust semaphore sample (in C, for comparison) (exit)\n"); + + misc_deregister(&device->miscdev); + kref_put(&device->ref, semaphore_free); +} + +module_init(semaphore_init); +module_exit(semaphore_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Rust for Linux Contributors"); +MODULE_DESCRIPTION("Rust semaphore sample (in C, for comparison)"); diff --git a/samples/rust/rust_stack_probing.rs b/samples/rust/rust_stack_probing.rs new file mode 100644 index 0000000000000..f992773545658 --- /dev/null +++ b/samples/rust/rust_stack_probing.rs @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust stack probing sample + +#![no_std] +#![feature(allocator_api, global_asm)] +#![feature(test)] + +use kernel::prelude::*; + +module! { + type: RustStackProbing, + name: b"rust_stack_probing", + author: b"Rust for Linux Contributors", + description: b"Rust stack probing sample", + license: b"GPL v2", + params: { + }, +} + +struct RustStackProbing; + +impl KernelModule for RustStackProbing { + fn init() -> KernelResult<Self> { + pr_info!("Rust stack probing sample (init)\n"); + + // Including this large variable on the stack will trigger + // stack probing on the supported archs. + // This will verify that stack probing does not lead to + // any errors if we need to link `__rust_probestack`. + let x: [u64; 514] = core::hint::black_box([5; 514]); + pr_info!("Large array has length: {}\n", x.len()); + + Ok(RustStackProbing) + } +} + +impl Drop for RustStackProbing { + fn drop(&mut self) { + pr_info!("Rust stack probing sample (exit)\n"); + } +} diff --git a/samples/rust/rust_sync.rs b/samples/rust/rust_sync.rs new file mode 100644 index 0000000000000..a921bfd7d55fb --- /dev/null +++ b/samples/rust/rust_sync.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0 + +//! Rust synchronisation primitives sample + +#![no_std] +#![feature(allocator_api, global_asm)] + +use alloc::boxed::Box; +use core::pin::Pin; +use kernel::prelude::*; +use kernel::{ + condvar_init, mutex_init, spinlock_init, + sync::{CondVar, Mutex, SpinLock}, +}; + +module! { + type: RustSync, + name: b"rust_sync", + author: b"Rust for Linux Contributors", + description: b"Rust synchronisation primitives sample", + license: b"GPL v2", + params: { + }, +} + +struct RustSync; + +impl KernelModule for RustSync { + fn init() -> KernelResult<Self> { + pr_info!("Rust synchronisation primitives sample (init)\n"); + + // Test mutexes. + { + // SAFETY: `init` is called below. + let data = Pin::from(Box::try_new(unsafe { Mutex::new(0) })?); + mutex_init!(data.as_ref(), "RustSync::init::data1"); + *data.lock() = 10; + pr_info!("Value: {}\n", *data.lock()); + + // SAFETY: `init` is called below. + let cv = Pin::from(Box::try_new(unsafe { CondVar::new() })?); + condvar_init!(cv.as_ref(), "RustSync::init::cv1"); + { + let mut guard = data.lock(); + while *guard != 10 { + let _ = cv.wait(&mut guard); + } + } + cv.notify_one(); + cv.notify_all(); + cv.free_waiters(); + } + + // Test spinlocks. + { + // SAFETY: `init` is called below. + let data = Pin::from(Box::try_new(unsafe { SpinLock::new(0) })?); + spinlock_init!(data.as_ref(), "RustSync::init::data2"); + *data.lock() = 10; + pr_info!("Value: {}\n", *data.lock()); + + // SAFETY: `init` is called below. + let cv = Pin::from(Box::try_new(unsafe { CondVar::new() })?); + condvar_init!(cv.as_ref(), "RustSync::init::cv2"); + { + let mut guard = data.lock(); + while *guard != 10 { + let _ = cv.wait(&mut guard); + } + } + cv.notify_one(); + cv.notify_all(); + cv.free_waiters(); + } + + Ok(RustSync) + } +} + +impl Drop for RustSync { + fn drop(&mut self) { + pr_info!("Rust synchronisation primitives sample (exit)\n"); + } +} diff --git a/scripts/Makefile.build b/scripts/Makefile.build index 10b2f2380d6fb..d766d3daf75b1 100644 --- a/scripts/Makefile.build +++ b/scripts/Makefile.build @@ -26,6 +26,7 @@ EXTRA_CPPFLAGS := EXTRA_LDFLAGS := asflags-y := ccflags-y := +rustcflags-y := cppflags-y := ldflags-y := @@ -287,6 +288,24 @@ quiet_cmd_cc_lst_c = MKLST $@ $(obj)/%.lst: $(src)/%.c FORCE $(call if_changed_dep,cc_lst_c) +# Compile Rust sources (.rs) +# --------------------------------------------------------------------------- + +rustc_cross_flags := --target=$(srctree)/arch/$(SRCARCH)/rust/target.json + +quiet_cmd_rustc_o_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@ + cmd_rustc_o_rs = \ + RUST_MODFILE=$(modfile) \ + $(RUSTC_OR_CLIPPY) $(rustc_flags) $(rustc_cross_flags) \ + --extern alloc --extern kernel \ + --crate-type rlib --out-dir $(obj) -L $(objtree)/rust/ \ + --crate-name $(patsubst %.o,%,$(notdir $@)) $<; \ + mv $(obj)/$(subst .o,,$(notdir $@)).d $(depfile); \ + sed -i '/^\#/d' $(depfile) + +$(obj)/%.o: $(src)/%.rs FORCE + $(call if_changed_dep,rustc_o_rs) + # Compile assembler sources (.S) # --------------------------------------------------------------------------- diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 10950559b223b..b307b40870392 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -8,6 +8,7 @@ ldflags-y += $(EXTRA_LDFLAGS) # flags that take effect in current and sub directories KBUILD_AFLAGS += $(subdir-asflags-y) KBUILD_CFLAGS += $(subdir-ccflags-y) +KBUILD_RUSTCFLAGS += $(subdir-rustcflags-y) # Figure out what we need to build from the various variables # =========================================================================== @@ -133,6 +134,10 @@ _c_flags = $(filter-out $(CFLAGS_REMOVE_$(target-stem).o), \ $(filter-out $(ccflags-remove-y), \ $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS) $(ccflags-y)) \ $(CFLAGS_$(target-stem).o)) +_rustc_flags = $(filter-out $(RUSTCFLAGS_REMOVE_$(target-stem).o), \ + $(filter-out $(rustcflags-remove-y), \ + $(KBUILD_RUSTCFLAGS) $(rustcflags-y)) \ + $(RUSTCFLAGS_$(target-stem).o)) _a_flags = $(filter-out $(AFLAGS_REMOVE_$(target-stem).o), \ $(filter-out $(asflags-remove-y), \ $(KBUILD_CPPFLAGS) $(KBUILD_AFLAGS) $(asflags-y)) \ @@ -202,6 +207,11 @@ modkern_cflags = \ $(KBUILD_CFLAGS_MODULE) $(CFLAGS_MODULE), \ $(KBUILD_CFLAGS_KERNEL) $(CFLAGS_KERNEL) $(modfile_flags)) +modkern_rustcflags = \ + $(if $(part-of-module), \ + $(KBUILD_RUSTCFLAGS_MODULE) $(RUSTCFLAGS_MODULE), \ + $(KBUILD_RUSTCFLAGS_KERNEL) $(RUSTCFLAGS_KERNEL)) + modkern_aflags = $(if $(part-of-module), \ $(KBUILD_AFLAGS_MODULE) $(AFLAGS_MODULE), \ $(KBUILD_AFLAGS_KERNEL) $(AFLAGS_KERNEL)) @@ -211,6 +221,8 @@ c_flags = -Wp,-MMD,$(depfile) $(NOSTDINC_FLAGS) $(LINUXINCLUDE) \ $(_c_flags) $(modkern_cflags) \ $(basename_flags) $(modname_flags) +rustc_flags = $(_rustc_flags) $(modkern_rustcflags) @$(objtree)/include/generated/rustc_cfg + a_flags = -Wp,-MMD,$(depfile) $(NOSTDINC_FLAGS) $(LINUXINCLUDE) \ $(_a_flags) $(modkern_aflags) diff --git a/scripts/dummy-tools/elfedit b/scripts/dummy-tools/elfedit new file mode 120000 index 0000000000000..c0648b38dd424 --- /dev/null +++ b/scripts/dummy-tools/elfedit @@ -0,0 +1 @@ +ld
\ No newline at end of file diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c index 54ad86d137849..9bab5f55ade3d 100644 --- a/scripts/kallsyms.c +++ b/scripts/kallsyms.c @@ -27,7 +27,7 @@ #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof(arr[0])) -#define KSYM_NAME_LEN 128 +#define KSYM_NAME_LEN 512 struct sym_entry { unsigned long long addr; @@ -470,12 +470,37 @@ static void write_src(void) if ((i & 0xFF) == 0) markers[i >> 8] = off; - printf("\t.byte 0x%02x", table[i]->len); + /* + * There cannot be any symbol of length zero -- we use that + * to mark a "big" symbol (and it doesn't make sense anyway). + */ + if (table[i]->len == 0) { + fprintf(stderr, "kallsyms failure: " + "unexpected zero symbol length\n"); + exit(EXIT_FAILURE); + } + + /* Only lengths that fit in up to two bytes are supported. */ + if (table[i]->len > 0xFFFF) { + fprintf(stderr, "kallsyms failure: " + "unexpected huge symbol length\n"); + exit(EXIT_FAILURE); + } + + if (table[i]->len <= 0xFF) { + /* Most symbols use a single byte for the length. */ + printf("\t.byte 0x%02x", table[i]->len); + off += table[i]->len + 1; + } else { + /* "Big" symbols use a zero and then two bytes. */ + printf("\t.byte 0x00, 0x%02x, 0x%02x", + (table[i]->len >> 8) & 0xFF, + table[i]->len & 0xFF); + off += table[i]->len + 3; + } for (k = 0; k < table[i]->len; k++) printf(", 0x%02x", table[i]->sym[k]); printf("\n"); - - off += table[i]->len + 1; } printf("\n"); diff --git a/scripts/kconfig/confdata.c b/scripts/kconfig/confdata.c index cf72680cd7692..d9fc638dfa86a 100644 --- a/scripts/kconfig/confdata.c +++ b/scripts/kconfig/confdata.c @@ -652,6 +652,56 @@ static struct conf_printer kconfig_printer_cb = }; /* + * rustc cfg printer + * + * This printer is used when generating the resulting rustc configuration + * after kconfig invocation and `defconfig` files. + */ +static void rustc_cfg_print_symbol(FILE *fp, struct symbol *sym, const char *value, void *arg) +{ + const char *str; + + switch (sym->type) { + case S_INT: + case S_HEX: + case S_BOOLEAN: + case S_TRISTATE: + str = sym_escape_string_value(value); + + /* + * We don't care about disabled ones, i.e. no need for + * what otherwise are "comments" in other printers. + */ + if (*value == 'n') + return; + + /* + * To have similar functionality to the C macro `IS_ENABLED()` + * we provide an empty `--cfg CONFIG_X` here in both `y` + * and `m` cases. + * + * Then, the common `fprintf()` below will also give us + * a `--cfg CONFIG_X="y"` or `--cfg CONFIG_X="m"`, which can + * be used as the equivalent of `IS_BUILTIN()`/`IS_MODULE()`. + */ + if (*value == 'y' || *value == 'm') + fprintf(fp, "--cfg=%s%s\n", CONFIG_, sym->name); + + break; + default: + str = value; + break; + } + + fprintf(fp, "--cfg=%s%s=%s\n", CONFIG_, sym->name, str); +} + +static struct conf_printer rustc_cfg_printer_cb = +{ + .print_symbol = rustc_cfg_print_symbol, +}; + +/* * Header printer * * This printer is used when generating the `include/generated/autoconf.h' file. @@ -1058,7 +1108,7 @@ int conf_write_autoconf(int overwrite) struct symbol *sym; const char *name; const char *autoconf_name = conf_get_autoconfig_name(); - FILE *out, *out_h; + FILE *out, *out_h, *out_rustc_cfg; int i; if (!overwrite && is_present(autoconf_name)) @@ -1079,6 +1129,13 @@ int conf_write_autoconf(int overwrite) return 1; } + out_rustc_cfg = fopen(".tmp_rustc_cfg", "w"); + if (!out_rustc_cfg) { + fclose(out); + fclose(out_h); + return 1; + } + conf_write_heading(out, &kconfig_printer_cb, NULL); conf_write_heading(out_h, &header_printer_cb, NULL); @@ -1090,9 +1147,11 @@ int conf_write_autoconf(int overwrite) /* write symbols to auto.conf and autoconf.h */ conf_write_symbol(out, sym, &kconfig_printer_cb, (void *)1); conf_write_symbol(out_h, sym, &header_printer_cb, NULL); + conf_write_symbol(out_rustc_cfg, sym, &rustc_cfg_printer_cb, NULL); } fclose(out); fclose(out_h); + fclose(out_rustc_cfg); name = getenv("KCONFIG_AUTOHEADER"); if (!name) @@ -1111,6 +1170,12 @@ int conf_write_autoconf(int overwrite) if (rename(".tmpconfig", autoconf_name)) return 1; + name = "include/generated/rustc_cfg"; + if (make_parent_dir(name)) + return 1; + if (rename(".tmp_rustc_cfg", name)) + return 1; + return 0; } diff --git a/scripts/rust-version.sh b/scripts/rust-version.sh new file mode 100755 index 0000000000000..67b6d31688e24 --- /dev/null +++ b/scripts/rust-version.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +# +# rust-version rust-command +# +# Print the compiler version of `rust-command' in a 5 or 6-digit form +# such as `14502' for rustc-1.45.2 etc. +# +# Returns 0 if not found (so that Kconfig does not complain) +compiler="$*" + +if [ ${#compiler} -eq 0 ]; then + echo "Error: No compiler specified." >&2 + printf "Usage:\n\t$0 <rust-command>\n" >&2 + exit 1 +fi + +if ! command -v $compiler >/dev/null 2>&1; then + echo 0 + exit 0 +fi + +VERSION=$($compiler --version | cut -f2 -d' ') + +# Cut suffix if any (e.g. `-dev`) +VERSION=$(echo $VERSION | cut -f1 -d'-') + +MAJOR=$(echo $VERSION | cut -f1 -d'.') +MINOR=$(echo $VERSION | cut -f2 -d'.') +PATCHLEVEL=$(echo $VERSION | cut -f3 -d'.') +printf "%d%02d%02d\\n" $MAJOR $MINOR $PATCHLEVEL diff --git a/tools/include/linux/kallsyms.h b/tools/include/linux/kallsyms.h index efb6c3f5f2a9a..5a37ccbec54fb 100644 --- a/tools/include/linux/kallsyms.h +++ b/tools/include/linux/kallsyms.h @@ -6,7 +6,7 @@ #include <stdio.h> #include <unistd.h> -#define KSYM_NAME_LEN 128 +#define KSYM_NAME_LEN 512 struct module; diff --git a/tools/include/linux/lockdep.h b/tools/include/linux/lockdep.h index e56997288f2b0..d9c163f3ab242 100644 --- a/tools/include/linux/lockdep.h +++ b/tools/include/linux/lockdep.h @@ -47,7 +47,7 @@ static inline int debug_locks_off(void) #define task_pid_nr(tsk) ((tsk)->pid) -#define KSYM_NAME_LEN 128 +#define KSYM_NAME_LEN 512 #define printk(...) dprintf(STDOUT_FILENO, __VA_ARGS__) #define pr_err(format, ...) fprintf (stderr, format, ## __VA_ARGS__) #define pr_warn pr_err diff --git a/tools/lib/perf/include/perf/event.h b/tools/lib/perf/include/perf/event.h index 4d0c02ba3f7d3..095d60144a70c 100644 --- a/tools/lib/perf/include/perf/event.h +++ b/tools/lib/perf/include/perf/event.h @@ -95,7 +95,7 @@ struct perf_record_throttle { }; #ifndef KSYM_NAME_LEN -#define KSYM_NAME_LEN 256 +#define KSYM_NAME_LEN 512 #endif struct perf_record_ksymbol { diff --git a/tools/lib/symbol/kallsyms.h b/tools/lib/symbol/kallsyms.h index 72ab9870454ba..542f9b059c3bd 100644 --- a/tools/lib/symbol/kallsyms.h +++ b/tools/lib/symbol/kallsyms.h @@ -7,7 +7,7 @@ #include <linux/types.h> #ifndef KSYM_NAME_LEN -#define KSYM_NAME_LEN 256 +#define KSYM_NAME_LEN 512 #endif static inline u8 kallsyms2elf_binding(char type) |
