Dockerfile Parts (#428)

* De-duplicate Dockerfiles by using a bash file to concatenate common parts

Resolves #375.

Dockerfiles are still committed to the repo to avoid a dependency on bash.

* Add a CI job to confirm the committed dockerfiles are the currently generated ones

* Create dedicated Dockerfiles per processor network

Ensures the compromising of network-specific dependencies doesn't lead to a
compromise of the build process for all processors.

* Dockerfile corrections

* Correct call to build processor Docker image in tests/processor
This commit is contained in:
Luke Parker 2023-11-12 23:55:15 -05:00 committed by GitHub
parent c328e5ea68
commit 351436a258
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
32 changed files with 511 additions and 154 deletions

View file

@ -66,3 +66,11 @@ jobs:
- name: Run rustfmt
run: cargo +${{ steps.nightly.outputs.version }} fmt -- --check
dockerfiles:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
- name: Verify Dockerfiles are up to date
# Runs the file which generates them and checks the diff has no lines
run: cd orchestration && ./dockerfiles.sh && git diff | wc -l | grep -x "0"

View file

@ -5,7 +5,7 @@
##### Ubuntu
```
sudo apt-get install -y build-essential cmake clang-11 git curl python3-pip protobuf-compiler libssl-dev pkg-config
sudo apt-get install -y build-essential clang-11 pkg-config cmake git curl protobuf-compiler
```
### Install rustup

View file

@ -0,0 +1,6 @@
FROM alpine:latest as image
COPY --from=mimalloc libmimalloc.so /usr/lib
ENV LD_PRELOAD=libmimalloc.so
RUN apk update && apk upgrade

View file

@ -0,0 +1,6 @@
FROM debian:bookworm-slim as image
COPY --from=mimalloc libmimalloc.so /usr/lib
RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload
RUN apt update && apt upgrade -y && apt autoremove -y && apt clean

View file

@ -0,0 +1,38 @@
FROM rust:1.73-slim-bookworm as builder
COPY --from=mimalloc libmimalloc.so /usr/lib
RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload
RUN apt update && apt upgrade -y && apt autoremove -y && apt clean
# Add dev dependencies
RUN apt install -y pkg-config clang
# Dependencies for the Serai node
RUN apt install -y make protobuf-compiler
# Add files for build
ADD common /serai/common
ADD crypto /serai/crypto
ADD coins /serai/coins
ADD message-queue /serai/message-queue
ADD processor /serai/processor
ADD coordinator /serai/coordinator
ADD substrate /serai/substrate
ADD mini /serai/mini
ADD tests /serai/tests
ADD Cargo.toml /serai
ADD Cargo.lock /serai
ADD AGPL-3.0 /serai
WORKDIR /serai
# Add the wasm toolchain
RUN rustup target add wasm32-unknown-unknown
# Mount the caches and build
RUN --mount=type=cache,target=/root/.cargo \
--mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
--mount=type=cache,target=/serai/target \
mkdir /serai/bin && \

View file

@ -0,0 +1,10 @@
FROM alpine:latest as mimalloc
RUN apk update && apk upgrade && apk --no-cache add gcc g++ libc-dev make cmake git
RUN git clone https://github.com/microsoft/mimalloc && \
cd mimalloc && \
mkdir -p out/secure && \
cd out/secure && \
cmake -DMI_SECURE=ON ../.. && \
make && \
cp ./libmimalloc-secure.so ../../../libmimalloc.so

View file

@ -0,0 +1,10 @@
FROM debian:bookworm-slim as mimalloc
RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git
RUN git clone https://github.com/microsoft/mimalloc && \
cd mimalloc && \
mkdir -p out/secure && \
cd out/secure && \
cmake -DMI_SECURE=ON ../.. && \
make && \
cp ./libmimalloc-secure.so ../../../libmimalloc.so

View file

@ -20,6 +20,7 @@ All commands are assumed to be ran from `/deploy`, not the root folder.
* `message-queue` - The message queue service.
* `processor` - Serai processor for one external network.
* `coordinator` - Serai coordinator for the entire Serai stack.
* `serai` - Serai node
* `cluster-sm` - "Alice", "Bob", "Charlie", and "Dave" Serai nodes, all as

View file

@ -1,10 +1,17 @@
# Configure Environment
FROM alpine:latest as builder
FROM debian:bookworm-slim as mimalloc
RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git
RUN git clone https://github.com/microsoft/mimalloc && \
cd mimalloc && \
mkdir -p out/secure && \
cd out/secure && \
cmake -DMI_SECURE=ON ../.. && \
make && \
cp ./libmimalloc-secure.so ../../../libmimalloc.so
FROM alpine:latest as bitcoin
ENV BITCOIN_VERSION=25.1
WORKDIR /home/bitcoin
RUN apk --no-cache add git gnupg
# Download Bitcoin
@ -23,34 +30,18 @@ RUN grep bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz SHA256SUMS | sha256s
# Prepare Image
RUN tar xzvf bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz
RUN mv bitcoin-${BITCOIN_VERSION}/bin/bitcoind .
# Also build mimalloc
FROM debian:bookworm-slim as mimalloc
RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git
RUN git clone https://github.com/microsoft/mimalloc && \
cd mimalloc && \
mkdir -p out/secure && \
cd out/secure && \
cmake -DMI_SECURE=ON ../.. && \
make && \
cp ./libmimalloc-secure.so ../../../libmimalloc.so
# Build the actual image
FROM debian:bookworm-slim as image
COPY --from=mimalloc libmimalloc.so /usr/lib
RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload
# Upgrade packages
RUN apt update && apt upgrade -y && apt autoremove -y && apt clean
# Switch to a non-root user
RUN useradd --system --create-home --shell /sbin/nologin bitcoin
USER bitcoin
WORKDIR /home/bitcoin
COPY --from=builder --chown=bitcoin /home/bitcoin/bitcoind /bin
COPY --from=bitcoin --chown=bitcoin bitcoind /bin
COPY ./scripts /scripts
EXPOSE 8332 8333 18332 18333 18443 18444

View file

@ -0,0 +1,22 @@
FROM alpine:latest as bitcoin
ENV BITCOIN_VERSION=25.1
RUN apk --no-cache add git gnupg
# Download Bitcoin
RUN wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz \
&& wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/SHA256SUMS \
&& wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/SHA256SUMS.asc
# Verify all sigs and check for a valid signature from laanwj -- 71A3
RUN git clone https://github.com/bitcoin-core/guix.sigs && \
cd guix.sigs/builder-keys && \
find . -iname '*.gpg' -exec gpg --import {} \; && \
gpg --verify --status-fd 1 --verify ../../SHA256SUMS.asc ../../SHA256SUMS | grep "^\[GNUPG:\] VALIDSIG.*71A3B16735405025D447E8F274810B012346C9A6"
RUN grep bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz SHA256SUMS | sha256sum -c
# Prepare Image
RUN tar xzvf bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz
RUN mv bitcoin-${BITCOIN_VERSION}/bin/bitcoind .

View file

@ -0,0 +1,10 @@
# Switch to a non-root user
RUN useradd --system --create-home --shell /sbin/nologin bitcoin
USER bitcoin
WORKDIR /home/bitcoin
COPY --from=bitcoin --chown=bitcoin bitcoind /bin
COPY ./scripts /scripts
EXPOSE 8332 8333 18332 18333 18443 18444
# VOLUME ["/home/bitcoin/.bitcoin"]

View file

@ -1,4 +1,14 @@
FROM alpine:latest as builder
FROM alpine:latest as mimalloc
RUN apk update && apk upgrade && apk --no-cache add gcc g++ libc-dev make cmake git
RUN git clone https://github.com/microsoft/mimalloc && \
cd mimalloc && \
mkdir -p out/secure && \
cd out/secure && \
cmake -DMI_SECURE=ON ../.. && \
make && \
cp ./libmimalloc-secure.so ../../../libmimalloc.so
FROM alpine:latest as monero
# https://downloads.getmonero.org/cli/monero-linux-x64-v0.18.3.1.tar.bz2
# Verification will fail if MONERO_VERSION doesn't match the latest
@ -8,17 +18,6 @@ FROM alpine:latest as builder
# Most publish a asc file for each release / build architecture ¯\_(ツ)_/¯
ENV MONERO_VERSION=0.18.3.1
WORKDIR /home/monero
RUN apk update && apk --no-cache add gcc g++ libc-dev make cmake git
RUN git clone https://github.com/microsoft/mimalloc && \
cd mimalloc && \
mkdir -p out/secure && \
cd out/secure && \
cmake -DMI_SECURE=ON ../.. && \
make && \
cp ./libmimalloc-secure.so ../../../libmimalloc.so
RUN apk --no-cache add gnupg
# Download Monero
@ -30,17 +29,15 @@ RUN gpg --keyserver hkp://keyserver.ubuntu.com:80 --keyserver-options no-self-si
gpg --verify hashes-v${MONERO_VERSION}.txt && \
grep "$(sha256sum monero-linux-x64-v${MONERO_VERSION}.tar.bz2 | cut -c 1-64)" hashes-v${MONERO_VERSION}.txt
# Cleanup
# Extract it
RUN tar -xvjf monero-linux-x64-v${MONERO_VERSION}.tar.bz2 --strip-components=1
# Build the actual image
FROM alpine:latest as image
COPY --from=builder /home/monero/libmimalloc.so /usr/lib
COPY --from=mimalloc libmimalloc.so /usr/lib
ENV LD_PRELOAD=libmimalloc.so
# Upgrade packages
RUN apk update && apk upgrade && apk --no-cache add gcompat
RUN apk update && apk upgrade
RUN apk --no-cache add gcompat
# Switch to a non-root user
# System user (not a human), shell of nologin, no password assigned
@ -48,7 +45,7 @@ RUN adduser -S -s /sbin/nologin -D monero
USER monero
WORKDIR /home/monero
COPY --from=builder --chown=monero /home/monero/monerod /bin
COPY --from=monero --chown=monero monerod /bin
ADD scripts /scripts
EXPOSE 18080 18081

View file

@ -0,0 +1,23 @@
FROM alpine:latest as monero
# https://downloads.getmonero.org/cli/monero-linux-x64-v0.18.3.1.tar.bz2
# Verification will fail if MONERO_VERSION doesn't match the latest
# due to the way monero publishes releases. They overwrite a single hashes.txt
# file with each release, meaning we can only grab the SHA256 of the latest
# release.
# Most publish a asc file for each release / build architecture ¯\_(ツ)_/¯
ENV MONERO_VERSION=0.18.3.1
RUN apk --no-cache add gnupg
# Download Monero
RUN wget https://downloads.getmonero.org/cli/monero-linux-x64-v${MONERO_VERSION}.tar.bz2
# Verify Binary -- fingerprint from https://github.com/monero-project/monero-site/issues/1949
ADD ./temp/hashes-v${MONERO_VERSION}.txt .
RUN gpg --keyserver hkp://keyserver.ubuntu.com:80 --keyserver-options no-self-sigs-only --receive-keys 81AC591FE9C4B65C5806AFC3F0AF4D462A0BDF92 && \
gpg --verify hashes-v${MONERO_VERSION}.txt && \
grep "$(sha256sum monero-linux-x64-v${MONERO_VERSION}.tar.bz2 | cut -c 1-64)" hashes-v${MONERO_VERSION}.txt
# Extract it
RUN tar -xvjf monero-linux-x64-v${MONERO_VERSION}.tar.bz2 --strip-components=1

View file

@ -0,0 +1,13 @@
RUN apk --no-cache add gcompat
# Switch to a non-root user
# System user (not a human), shell of nologin, no password assigned
RUN adduser -S -s /sbin/nologin -D monero
USER monero
WORKDIR /home/monero
COPY --from=monero --chown=monero monerod /bin
ADD scripts /scripts
EXPOSE 18080 18081
# VOLUME /home/monero/.bitmonero

View file

@ -1,11 +1,25 @@
FROM debian:bookworm-slim as mimalloc
RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git
RUN git clone https://github.com/microsoft/mimalloc && \
cd mimalloc && \
mkdir -p out/secure && \
cd out/secure && \
cmake -DMI_SECURE=ON ../.. && \
make && \
cp ./libmimalloc-secure.so ../../../libmimalloc.so
FROM rust:1.73-slim-bookworm as builder
LABEL description="STAGE 1: Build"
# Upgrade and add dev dependencies
RUN apt update && apt upgrade -y && apt install -y pkg-config clang && apt autoremove -y && apt clean
COPY --from=mimalloc libmimalloc.so /usr/lib
RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload
# Add the wasm toolchain
RUN rustup target add wasm32-unknown-unknown
RUN apt update && apt upgrade -y && apt autoremove -y && apt clean
# Add dev dependencies
RUN apt install -y pkg-config clang
# Dependencies for the Serai node
RUN apt install -y make protobuf-compiler
# Add files for build
ADD common /serai/common
@ -23,36 +37,25 @@ ADD AGPL-3.0 /serai
WORKDIR /serai
# Add the wasm toolchain
RUN rustup target add wasm32-unknown-unknown
# Mount the caches and build
RUN --mount=type=cache,target=/root/.cargo \
--mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
--mount=type=cache,target=/serai/target \
cd coordinator && \
cargo build --release --all-features && \
mkdir /serai/bin && \
cargo build -p serai-coordinator --release --all-features && \
mv /serai/target/release/serai-coordinator /serai/bin
# Also build mimalloc
FROM debian:bookworm-slim as mimalloc
RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git
RUN git clone https://github.com/microsoft/mimalloc && \
cd mimalloc && \
mkdir -p out/secure && \
cd out/secure && \
cmake -DMI_SECURE=ON ../.. && \
make && \
cp ./libmimalloc-secure.so ../../../libmimalloc.so
# Build the actual image
FROM debian:bookworm-slim as image
COPY --from=mimalloc libmimalloc.so /usr/lib
RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload
# Upgrade packages and install ca-certificates
RUN apt update && apt upgrade -y && apt install -y ca-certificates && apt autoremove && apt clean
RUN apt update && apt upgrade -y && apt autoremove -y && apt clean
# Install ca-certificates
RUN apt install -y ca-certificates
# Switch to a non-root user
RUN useradd --system --create-home --shell /sbin/nologin coordinator
@ -60,7 +63,7 @@ USER coordinator
WORKDIR /home/coordinator
# Copy necessary files to run node
# Copy the Coordinator binary and relevant license
COPY --from=builder --chown=processsor /serai/bin/serai-coordinator /bin/
COPY --from=builder --chown=processsor /serai/AGPL-3.0 .

View file

@ -0,0 +1,2 @@
cargo build -p serai-coordinator --release --all-features && \
mv /serai/target/release/serai-coordinator /serai/bin

View file

@ -0,0 +1,15 @@
# Install ca-certificates
RUN apt install -y ca-certificates
# Switch to a non-root user
RUN useradd --system --create-home --shell /sbin/nologin coordinator
USER coordinator
WORKDIR /home/coordinator
# Copy the Coordinator binary and relevant license
COPY --from=builder --chown=processsor /serai/bin/serai-coordinator /bin/
COPY --from=builder --chown=processsor /serai/AGPL-3.0 .
# Run coordinator
CMD ["serai-coordinator"]

View file

@ -67,12 +67,23 @@ services:
expose:
- "2287"
processor:
bitcoin-processor:
profiles:
- processor
- bitcoin-processor
build:
context: ../
dockerfile: ./orchestration/processor/Dockerfile
dockerfile: ./orchestration/processor/bitcoin/Dockerfile
restart: unless-stopped
volumes:
- "./processor/scripts:/scripts"
entrypoint: /scripts/entry-dev.sh
monero-processor:
profiles:
- monero-processor
build:
context: ../
dockerfile: ./orchestration/processor/monero/Dockerfile
restart: unless-stopped
volumes:
- "./processor/scripts:/scripts"

60
orchestration/dockerfiles.sh Executable file
View file

@ -0,0 +1,60 @@
# Bitcoin
rm ./coins/bitcoin/Dockerfile
cat \
./Dockerfile.parts/mimalloc/Dockerfile.debian \
./coins/bitcoin/Dockerfile.bitcoin \
./Dockerfile.parts/Dockerfile.debian.start \
./coins/bitcoin/Dockerfile.bitcoin.end >> ./coins/bitcoin/Dockerfile
# Monero
rm ./coins/monero/Dockerfile
cat \
./Dockerfile.parts/mimalloc/Dockerfile.alpine \
./coins/monero/Dockerfile.monero \
./Dockerfile.parts/Dockerfile.alpine.start \
./coins/monero/Dockerfile.monero.end >> ./coins/monero/Dockerfile
# Message Queue
rm ./message-queue/Dockerfile
cat \
./Dockerfile.parts/mimalloc/Dockerfile.debian \
./Dockerfile.parts/Dockerfile.serai.build \
./message-queue/Dockerfile.message-queue \
./Dockerfile.parts/Dockerfile.debian.start \
./message-queue/Dockerfile.message-queue.end >> ./message-queue/Dockerfile
# Bitcoin Processor
rm ./processor/bitcoin/Dockerfile
cat \
./Dockerfile.parts/mimalloc/Dockerfile.debian \
./Dockerfile.parts/Dockerfile.serai.build \
./processor/bitcoin/Dockerfile.processor.bitcoin \
./Dockerfile.parts/Dockerfile.debian.start \
./processor/Dockerfile.processor.end >> ./processor/bitcoin/Dockerfile
# Monero Processor
rm ./processor/monero/Dockerfile
cat \
./Dockerfile.parts/mimalloc/Dockerfile.debian \
./Dockerfile.parts/Dockerfile.serai.build \
./processor/monero/Dockerfile.processor.monero \
./Dockerfile.parts/Dockerfile.debian.start \
./processor/Dockerfile.processor.end >> ./processor/monero/Dockerfile
# Coordinator
rm ./coordinator/Dockerfile
cat \
./Dockerfile.parts/mimalloc/Dockerfile.debian \
./Dockerfile.parts/Dockerfile.serai.build \
./coordinator/Dockerfile.coordinator \
./Dockerfile.parts/Dockerfile.debian.start \
./coordinator/Dockerfile.coordinator.end >> ./coordinator/Dockerfile
# Node
rm ./serai/Dockerfile
cat \
./Dockerfile.parts/mimalloc/Dockerfile.debian \
./Dockerfile.parts/Dockerfile.serai.build \
./serai/Dockerfile.serai \
./Dockerfile.parts/Dockerfile.debian.start \
./serai/Dockerfile.serai.end >> ./serai/Dockerfile

View file

@ -1,8 +1,25 @@
FROM rust:1.73-slim-bookworm as builder
LABEL description="STAGE 1: Build"
FROM debian:bookworm-slim as mimalloc
# Upgrade and add dev dependencies
RUN apt update && apt upgrade -y && apt install -y pkg-config clang libssl-dev && apt autoremove -y && apt clean
RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git
RUN git clone https://github.com/microsoft/mimalloc && \
cd mimalloc && \
mkdir -p out/secure && \
cd out/secure && \
cmake -DMI_SECURE=ON ../.. && \
make && \
cp ./libmimalloc-secure.so ../../../libmimalloc.so
FROM rust:1.73-slim-bookworm as builder
COPY --from=mimalloc libmimalloc.so /usr/lib
RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload
RUN apt update && apt upgrade -y && apt autoremove -y && apt clean
# Add dev dependencies
RUN apt install -y pkg-config clang
# Dependencies for the Serai node
RUN apt install -y make protobuf-compiler
# Add files for build
ADD common /serai/common
@ -20,37 +37,23 @@ ADD AGPL-3.0 /serai
WORKDIR /serai
# Add the wasm toolchain
RUN rustup target add wasm32-unknown-unknown
# Mount the caches and build
RUN --mount=type=cache,target=/root/.cargo \
--mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
--mount=type=cache,target=/serai/target \
cd message-queue && \
cargo build --release --all-features && \
mkdir /serai/bin && \
cargo build --release --all-features -p serai-message-queue && \
mv /serai/target/release/serai-message-queue /serai/bin
# Also build mimalloc
FROM debian:bookworm-slim as mimalloc
RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git
RUN git clone https://github.com/microsoft/mimalloc && \
cd mimalloc && \
mkdir -p out/secure && \
cd out/secure && \
cmake -DMI_SECURE=ON ../.. && \
make && \
cp ./libmimalloc-secure.so ../../../libmimalloc.so
# Build the actual image
FROM debian:bookworm-slim as image
COPY --from=mimalloc libmimalloc.so /usr/lib
RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload
# Upgrade packages
RUN apt update && apt upgrade -y
RUN apt update && apt upgrade -y && apt autoremove -y && apt clean
# Switch to a non-root user
RUN useradd --system --home /home/message-queue --create-home --shell /sbin/nologin messagequeue
USER messagequeue

View file

@ -0,0 +1,2 @@
cargo build --release --all-features -p serai-message-queue && \
mv /serai/target/release/serai-message-queue /serai/bin

View file

@ -0,0 +1,13 @@
# Switch to a non-root user
RUN useradd --system --home /home/message-queue --create-home --shell /sbin/nologin messagequeue
USER messagequeue
WORKDIR /home/message-queue
# Copy the Message Queue binary and relevant license
COPY --from=builder --chown=messagequeue /serai/bin/serai-message-queue /bin
COPY --from=builder --chown=messagequeue /serai/AGPL-3.0 .
# Run message-queue
EXPOSE 2287
CMD ["serai-message-queue"]

View file

@ -0,0 +1,15 @@
# Install ca-certificates
RUN apt install -y ca-certificates
# Switch to a non-root user
RUN useradd --system --create-home --shell /sbin/nologin processor
USER processor
WORKDIR /home/processor
# Copy the Processor binary and relevant license
COPY --from=builder --chown=processsor /serai/bin/serai-processor /bin/
COPY --from=builder --chown=processsor /serai/AGPL-3.0 .
# Run processor
CMD ["serai-processor"]

View file

@ -0,0 +1,71 @@
FROM debian:bookworm-slim as mimalloc
RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git
RUN git clone https://github.com/microsoft/mimalloc && \
cd mimalloc && \
mkdir -p out/secure && \
cd out/secure && \
cmake -DMI_SECURE=ON ../.. && \
make && \
cp ./libmimalloc-secure.so ../../../libmimalloc.so
FROM rust:1.73-slim-bookworm as builder
COPY --from=mimalloc libmimalloc.so /usr/lib
RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload
RUN apt update && apt upgrade -y && apt autoremove -y && apt clean
# Add dev dependencies
RUN apt install -y pkg-config clang
# Dependencies for the Serai node
RUN apt install -y make protobuf-compiler
# Add files for build
ADD common /serai/common
ADD crypto /serai/crypto
ADD coins /serai/coins
ADD message-queue /serai/message-queue
ADD processor /serai/processor
ADD coordinator /serai/coordinator
ADD substrate /serai/substrate
ADD mini /serai/mini
ADD tests /serai/tests
ADD Cargo.toml /serai
ADD Cargo.lock /serai
ADD AGPL-3.0 /serai
WORKDIR /serai
# Add the wasm toolchain
RUN rustup target add wasm32-unknown-unknown
# Mount the caches and build
RUN --mount=type=cache,target=/root/.cargo \
--mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
--mount=type=cache,target=/serai/target \
mkdir /serai/bin && \
cargo build --release --features bitcoin -p serai-processor && \
mv /serai/target/release/serai-processor /serai/bin
FROM debian:bookworm-slim as image
COPY --from=mimalloc libmimalloc.so /usr/lib
RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload
RUN apt update && apt upgrade -y && apt autoremove -y && apt clean
# Install ca-certificates
RUN apt install -y ca-certificates
# Switch to a non-root user
RUN useradd --system --create-home --shell /sbin/nologin processor
USER processor
WORKDIR /home/processor
# Copy the Processor binary and relevant license
COPY --from=builder --chown=processsor /serai/bin/serai-processor /bin/
COPY --from=builder --chown=processsor /serai/AGPL-3.0 .
# Run processor
CMD ["serai-processor"]

View file

@ -0,0 +1,2 @@
cargo build --release --features bitcoin -p serai-processor && \
mv /serai/target/release/serai-processor /serai/bin

View file

@ -1,8 +1,25 @@
FROM rust:1.73-slim-bookworm as builder
LABEL description="STAGE 1: Build"
FROM debian:bookworm-slim as mimalloc
# Upgrade and add dev dependencies
RUN apt update && apt upgrade -y && apt install -y pkg-config clang && apt autoremove -y && apt clean
RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git
RUN git clone https://github.com/microsoft/mimalloc && \
cd mimalloc && \
mkdir -p out/secure && \
cd out/secure && \
cmake -DMI_SECURE=ON ../.. && \
make && \
cp ./libmimalloc-secure.so ../../../libmimalloc.so
FROM rust:1.73-slim-bookworm as builder
COPY --from=mimalloc libmimalloc.so /usr/lib
RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload
RUN apt update && apt upgrade -y && apt autoremove -y && apt clean
# Add dev dependencies
RUN apt install -y pkg-config clang
# Dependencies for the Serai node
RUN apt install -y make protobuf-compiler
# Add files for build
ADD common /serai/common
@ -28,31 +45,17 @@ RUN --mount=type=cache,target=/root/.cargo \
--mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
--mount=type=cache,target=/serai/target \
cd processor && \
cargo build --release --all-features && \
mkdir /serai/bin && \
cargo build --release --features monero -p serai-processor && \
mv /serai/target/release/serai-processor /serai/bin
# Also build mimalloc
FROM debian:bookworm-slim as mimalloc
RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git
RUN git clone https://github.com/microsoft/mimalloc && \
cd mimalloc && \
mkdir -p out/secure && \
cd out/secure && \
cmake -DMI_SECURE=ON ../.. && \
make && \
cp ./libmimalloc-secure.so ../../../libmimalloc.so
# Build the actual image
FROM debian:bookworm-slim as image
COPY --from=mimalloc libmimalloc.so /usr/lib
RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload
# Upgrade packages and install ca-certificates
RUN apt update && apt upgrade -y && apt install -y ca-certificates
RUN apt update && apt upgrade -y && apt autoremove -y && apt clean
# Install ca-certificates
RUN apt install -y ca-certificates
# Switch to a non-root user
RUN useradd --system --create-home --shell /sbin/nologin processor
@ -60,7 +63,7 @@ USER processor
WORKDIR /home/processor
# Copy necessary files to run node
# Copy the Processor binary and relevant license
COPY --from=builder --chown=processsor /serai/bin/serai-processor /bin/
COPY --from=builder --chown=processsor /serai/AGPL-3.0 .

View file

@ -0,0 +1,2 @@
cargo build --release --features monero -p serai-processor && \
mv /serai/target/release/serai-processor /serai/bin

View file

@ -1,11 +1,25 @@
FROM debian:bookworm-slim as mimalloc
RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git
RUN git clone https://github.com/microsoft/mimalloc && \
cd mimalloc && \
mkdir -p out/secure && \
cd out/secure && \
cmake -DMI_SECURE=ON ../.. && \
make && \
cp ./libmimalloc-secure.so ../../../libmimalloc.so
FROM rust:1.73-slim-bookworm as builder
LABEL description="STAGE 1: Build"
# Upgrade and add dev dependencies
RUN apt update && apt upgrade -y && apt install -y git pkg-config make clang libssl-dev protobuf-compiler && apt autoremove -y && apt clean
COPY --from=mimalloc libmimalloc.so /usr/lib
RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload
# Add the wasm toolchain
RUN rustup target add wasm32-unknown-unknown
RUN apt update && apt upgrade -y && apt autoremove -y && apt clean
# Add dev dependencies
RUN apt install -y pkg-config clang
# Dependencies for the Serai node
RUN apt install -y make protobuf-compiler
# Add files for build
ADD common /serai/common
@ -23,44 +37,30 @@ ADD AGPL-3.0 /serai
WORKDIR /serai
# Add the wasm toolchain
RUN rustup target add wasm32-unknown-unknown
# Mount the caches and build
RUN --mount=type=cache,target=/root/.cargo \
--mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
--mount=type=cache,target=/serai/target \
cd substrate/node && \
cargo build --release && \
mkdir /serai/bin && \
cargo build --release -p serai-node && \
mv /serai/target/release/serai-node /serai/bin
# Also build mimalloc
FROM debian:bookworm-slim as mimalloc
RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git
RUN git clone https://github.com/microsoft/mimalloc && \
cd mimalloc && \
mkdir -p out/secure && \
cd out/secure && \
cmake -DMI_SECURE=ON ../.. && \
make && \
cp ./libmimalloc-secure.so ../../../libmimalloc.so
# Build the actual image
FROM debian:bookworm-slim as image
COPY --from=mimalloc libmimalloc.so /usr/lib
RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload
# Upgrade packages
RUN apt update && apt upgrade -y
RUN apt update && apt upgrade -y && apt autoremove -y && apt clean
# Switch to a non-root user
RUN useradd --system --home /home/serai --shell /sbin/nologin serai
USER serai
WORKDIR /home/serai
# Copy necessary files to run node
# Copy the Serai binary and relevant license
COPY --from=builder --chown=serai /serai/bin/serai-node /bin/
COPY --from=builder --chown=serai /serai/AGPL-3.0 .

View file

@ -0,0 +1,2 @@
cargo build --release -p serai-node && \
mv /serai/target/release/serai-node /serai/bin

View file

@ -0,0 +1,13 @@
# Switch to a non-root user
RUN useradd --system --home /home/serai --shell /sbin/nologin serai
USER serai
WORKDIR /home/serai
# Copy the Serai binary and relevant license
COPY --from=builder --chown=serai /serai/bin/serai-node /bin/
COPY --from=builder --chown=serai /serai/AGPL-3.0 .
# Run node
EXPOSE 30333 9615 9933 9944
CMD ["serai-node"]

View file

@ -54,7 +54,14 @@ pub fn build(name: String) {
if HashSet::from(["bitcoin", "ethereum", "monero"]).contains(name.as_str()) {
dockerfile_path = dockerfile_path.join("coins");
}
if name.contains("-processor") {
dockerfile_path = dockerfile_path
.join("processor")
.join(name.split('-').next().unwrap())
.join("Dockerfile");
} else {
dockerfile_path = dockerfile_path.join(&name).join("Dockerfile");
}
// For all services, if the Dockerfile was edited after the image was built we should rebuild
let mut last_modified =
@ -71,7 +78,7 @@ pub fn build(name: String) {
meta(repo_path.join("substrate").join("primitives")),
meta(repo_path.join("message-queue")),
],
"processor" => vec![
"bitcoin-processor" | "ethereum-processor" | "monero-processor" => vec![
meta(repo_path.join("common")),
meta(repo_path.join("crypto")),
meta(repo_path.join("coins")),

View file

@ -29,28 +29,26 @@ pub fn processor_instance(
port: u32,
message_queue_key: <Ristretto as Ciphersuite>::F,
) -> TestBodySpecification {
serai_docker_tests::build("processor".to_string());
let mut entropy = [0; 32];
OsRng.fill_bytes(&mut entropy);
let network_str = match network {
NetworkId::Serai => panic!("starting a processor for Serai"),
NetworkId::Bitcoin => "bitcoin",
NetworkId::Ethereum => "ethereum",
NetworkId::Monero => "monero",
};
let image = format!("{network_str}-processor");
serai_docker_tests::build(image.clone());
TestBodySpecification::with_image(
Image::with_repository("serai-dev-processor").pull_policy(PullPolicy::Never),
Image::with_repository(format!("serai-dev-{image}")).pull_policy(PullPolicy::Never),
)
.replace_env(
[
("MESSAGE_QUEUE_KEY".to_string(), hex::encode(message_queue_key.to_repr())),
("ENTROPY".to_string(), hex::encode(entropy)),
(
"NETWORK".to_string(),
(match network {
NetworkId::Serai => panic!("starting a processor for Serai"),
NetworkId::Bitcoin => "bitcoin",
NetworkId::Ethereum => "ethereum",
NetworkId::Monero => "monero",
})
.to_string(),
),
("NETWORK".to_string(), network_str.to_string()),
("NETWORK_RPC_LOGIN".to_string(), format!("{RPC_USER}:{RPC_PASS}")),
("NETWORK_RPC_PORT".to_string(), port.to_string()),
("DB_PATH".to_string(), "./processor-db".to_string()),