diff --git a/books/architecture/README.md b/books/architecture/README.md index e487843..88e86eb 100644 --- a/books/architecture/README.md +++ b/books/architecture/README.md @@ -1,4 +1,4 @@ -## Cuprate's architecture (implementation) book +## Cuprate's architecture book This book documents Cuprate's architecture and implementation. See: diff --git a/books/architecture/book.toml b/books/architecture/book.toml index 76724aa..996f7fe 100644 --- a/books/architecture/book.toml +++ b/books/architecture/book.toml @@ -1,19 +1,17 @@ [book] -authors = ["hinto-janai"] +authors = ["Cuprate Contributors"] language = "en" multilingual = false src = "src" title = "Cuprate Architecture" git-repository-url = "https://github.com/Cuprate/architecture-book" -# TODO: fix after importing real files. -# -# [preprocessor.last-changed] -# command = "mdbook-last-changed" -# renderer = ["html"] -# -# [output.html] -# default-theme = "ayu" -# preferred-dark-theme = "ayu" -# git-repository-url = "https://github.com/hinto-janai/cuprate-architecture" -# additional-css = ["last-changed.css"] +[preprocessor.last-changed] +command = "mdbook-last-changed" +renderer = ["html"] + +[output.html] +default-theme = "ayu" +preferred-dark-theme = "ayu" +git-repository-url = "https://github.com/Cuprate/architecture-book" +additional-css = ["last-changed.css"] diff --git a/books/architecture/last-changed.css b/books/architecture/last-changed.css new file mode 100644 index 0000000..a9abae5 --- /dev/null +++ b/books/architecture/last-changed.css @@ -0,0 +1,7 @@ +footer { + font-size: 0.8em; + text-align: center; + border-top: 1px solid; + margin-top: 4%; + padding: 5px 0; +} \ No newline at end of file diff --git a/books/architecture/src/SUMMARY.md b/books/architecture/src/SUMMARY.md index 2b8615c..74ecda6 100644 --- a/books/architecture/src/SUMMARY.md +++ b/books/architecture/src/SUMMARY.md @@ -1,3 +1,124 @@ # Summary -- [TODO](todo.md) +[Cuprate Architecture](cuprate-architecture.md) +[🟡 Foreword](foreword.md) + +--- + +- [🟠 Intro](intro/intro.md) + - [🟡 Who this book is for](intro/who-this-book-is-for.md) + - [🔴 Required knowledge](intro/required-knowledge.md) + - [🔴 How to use this book](intro/how-to-use-this-book.md) + +--- + +- [⚪️ Bird's eye view](birds-eye-view/intro.md) + - [⚪️ Map](birds-eye-view/map.md) + - [⚪️ Components](birds-eye-view/components.md) + +--- + +- [⚪️ Formats, protocols, types](formats-protocols-types/intro.md) + - [⚪️ monero_serai](formats-protocols-types/monero-serai.md) + - [⚪️ cuprate_types](formats-protocols-types/cuprate-types.md) + - [⚪️ cuprate_helper](formats-protocols-types/cuprate-helper.md) + - [⚪️ Epee](formats-protocols-types/epee.md) + - [⚪️ Levin](formats-protocols-types/levin.md) + +--- + +- [⚪️ Storage](storage/intro.md) + - [⚪️ Database abstraction](storage/database-abstraction.md) + - [⚪️ Blockchain](storage/blockchain.md) + - [⚪️ Transaction pool](storage/transaction-pool.md) + - [⚪️ Pruning](storage/pruning.md) + +--- + +- [🔴 RPC](rpc/intro.md) + - [⚪️ Types](rpc/types/intro.md) + - [⚪️ JSON](rpc/types/json.md) + - [⚪️ Binary](rpc/types/binary.md) + - [⚪️ Other](rpc/types/other.md) + - [⚪️ Interface](rpc/interface.md) + - [⚪️ Router](rpc/router.md) + - [⚪️ Handler](rpc/handler.md) + - [⚪️ Methods](rpc/methods/intro.md) + +--- + +- [⚪️ ZMQ](zmq/intro.md) + - [⚪️ TODO](zmq/todo.md) + +--- + +- [⚪️ Consensus](consensus/intro.md) + - [⚪️ Verifier](consensus/verifier.md) + - [⚪️ TODO](consensus/todo.md) + +--- + +- [⚪️ Networking](networking/intro.md) + - [⚪️ P2P](networking/p2p.md) + - [⚪️ Dandelion++](networking/dandelion.md) + - [⚪️ Proxy](networking/proxy.md) + - [⚪️ Tor](networking/tor.md) + - [⚪️ i2p](networking/i2p.md) + - [⚪️ IPv4/IPv6](networking/ipv4-ipv6.md) + +--- + +- [🔴 Instrumentation](instrumentation/intro.md) + - [⚪️ Logging](instrumentation/logging.md) + - [⚪️ Data collection](instrumentation/data-collection.md) + +--- + +- [⚪️ Binary](binary/intro.md) + - [⚪️ CLI](binary/cli.md) + - [⚪️ Config](binary/config.md) + - [⚪️ Logging](binary/logging.md) + +--- + +- [⚪️ Resource model](resource-model/intro.md) + - [⚪️ File system](resource-model/file-system.md) + - [⚪️ Sockets](resource-model/sockets.md) + - [⚪️ Memory](resource-model/memory.md) + - [🟡 Concurrency and parallelism](resource-model/concurrency-and-parallelism/intro.md) + - [⚪️ Map](resource-model/concurrency-and-parallelism/map.md) + - [⚪️ The RPC server](resource-model/concurrency-and-parallelism/the-rpc-server.md) + - [⚪️ The database](resource-model/concurrency-and-parallelism/the-database.md) + - [⚪️ The block downloader](resource-model/concurrency-and-parallelism/the-block-downloader.md) + - [⚪️ The verifier](resource-model/concurrency-and-parallelism/the-verifier.md) + - [⚪️ Thread exit](resource-model/concurrency-and-parallelism/thread-exit.md) + +--- + +- [⚪️ External Monero libraries](external-monero-libraries/intro.md) + - [⚪️ Cryptonight](external-monero-libraries/cryptonight.md) + - [🔴 RandomX](external-monero-libraries/randomx.md) + - [🔴 monero_serai](external-monero-libraries/monero_serai.md) + +--- + +- [⚪️ Benchmarking](benchmarking/intro.md) + - [⚪️ Criterion](benchmarking/criterion.md) + - [⚪️ Harness](benchmarking/harness.md) +- [⚪️ Testing](testing/intro.md) + - [⚪️ Monero data](testing/monero-data.md) + - [⚪️ RPC client](testing/rpc-client.md) + - [⚪️ Spawning `monerod`](testing/spawning-monerod.md) +- [⚪️ Known issues and tradeoffs](known-issues-and-tradeoffs/intro.md) + - [⚪️ Networking](known-issues-and-tradeoffs/networking.md) + - [⚪️ RPC](known-issues-and-tradeoffs/rpc.md) + - [⚪️ Storage](known-issues-and-tradeoffs/storage.md) + +--- + +- [⚪️ Appendix](appendix/intro.md) + - [🔴 Contributing](appendix/contributing.md) + - [🔴 Crate documentation](appendix/crate-documentation.md) + - [🔴 Build targets](appendix/build-targets.md) + - [🔴 Protocol book](appendix/protocol-book.md) + - [⚪️ User book](appendix/user-book.md) \ No newline at end of file diff --git a/books/architecture/src/appendix/build-targets.md b/books/architecture/src/appendix/build-targets.md new file mode 100644 index 0000000..495a3d6 --- /dev/null +++ b/books/architecture/src/appendix/build-targets.md @@ -0,0 +1,7 @@ +# Build targets +- x86 +- ARM64 +- Windows +- Linux +- macOS +- FreeBSD(?) diff --git a/books/architecture/src/appendix/contributing.md b/books/architecture/src/appendix/contributing.md new file mode 100644 index 0000000..675937a --- /dev/null +++ b/books/architecture/src/appendix/contributing.md @@ -0,0 +1,2 @@ +# Contributing + \ No newline at end of file diff --git a/books/architecture/src/appendix/crate-documentation.md b/books/architecture/src/appendix/crate-documentation.md new file mode 100644 index 0000000..0f4d96d --- /dev/null +++ b/books/architecture/src/appendix/crate-documentation.md @@ -0,0 +1,4 @@ +# Crate documentation +```bash +cargo doc --package $CUPRATE_CRATE +``` \ No newline at end of file diff --git a/books/architecture/src/appendix/intro.md b/books/architecture/src/appendix/intro.md new file mode 100644 index 0000000..fad5ae4 --- /dev/null +++ b/books/architecture/src/appendix/intro.md @@ -0,0 +1 @@ +# Appendix diff --git a/books/architecture/src/appendix/protocol-book.md b/books/architecture/src/appendix/protocol-book.md new file mode 100644 index 0000000..a855b73 --- /dev/null +++ b/books/architecture/src/appendix/protocol-book.md @@ -0,0 +1,2 @@ +# Protocol book + \ No newline at end of file diff --git a/books/architecture/src/appendix/user-book.md b/books/architecture/src/appendix/user-book.md new file mode 100644 index 0000000..0f12476 --- /dev/null +++ b/books/architecture/src/appendix/user-book.md @@ -0,0 +1 @@ +# ⚪️ User book diff --git a/books/architecture/src/benchmarking/criterion.md b/books/architecture/src/benchmarking/criterion.md new file mode 100644 index 0000000..e9d61e6 --- /dev/null +++ b/books/architecture/src/benchmarking/criterion.md @@ -0,0 +1 @@ +# ⚪️ Criterion diff --git a/books/architecture/src/benchmarking/harness.md b/books/architecture/src/benchmarking/harness.md new file mode 100644 index 0000000..6f82b52 --- /dev/null +++ b/books/architecture/src/benchmarking/harness.md @@ -0,0 +1 @@ +# ⚪️ Harness diff --git a/books/architecture/src/benchmarking/intro.md b/books/architecture/src/benchmarking/intro.md new file mode 100644 index 0000000..f043a0b --- /dev/null +++ b/books/architecture/src/benchmarking/intro.md @@ -0,0 +1 @@ +# ⚪️ Benchmarking diff --git a/books/architecture/src/binary/cli.md b/books/architecture/src/binary/cli.md new file mode 100644 index 0000000..1c515f4 --- /dev/null +++ b/books/architecture/src/binary/cli.md @@ -0,0 +1 @@ +# ⚪️ CLI diff --git a/books/architecture/src/binary/config.md b/books/architecture/src/binary/config.md new file mode 100644 index 0000000..c9582d0 --- /dev/null +++ b/books/architecture/src/binary/config.md @@ -0,0 +1 @@ +# ⚪️ Config diff --git a/books/architecture/src/binary/intro.md b/books/architecture/src/binary/intro.md new file mode 100644 index 0000000..dea12fa --- /dev/null +++ b/books/architecture/src/binary/intro.md @@ -0,0 +1 @@ +# ⚪️ Binary diff --git a/books/architecture/src/binary/logging.md b/books/architecture/src/binary/logging.md new file mode 100644 index 0000000..c7c88a3 --- /dev/null +++ b/books/architecture/src/binary/logging.md @@ -0,0 +1 @@ +# ⚪️ Logging diff --git a/books/architecture/src/birds-eye-view/components.md b/books/architecture/src/birds-eye-view/components.md new file mode 100644 index 0000000..19a17e2 --- /dev/null +++ b/books/architecture/src/birds-eye-view/components.md @@ -0,0 +1 @@ +# ⚪️ Components diff --git a/books/architecture/src/birds-eye-view/intro.md b/books/architecture/src/birds-eye-view/intro.md new file mode 100644 index 0000000..5ee2eb3 --- /dev/null +++ b/books/architecture/src/birds-eye-view/intro.md @@ -0,0 +1 @@ +# ⚪️ Bird's eye view diff --git a/books/architecture/src/birds-eye-view/map.md b/books/architecture/src/birds-eye-view/map.md new file mode 100644 index 0000000..1bde994 --- /dev/null +++ b/books/architecture/src/birds-eye-view/map.md @@ -0,0 +1 @@ +# ⚪️ Map diff --git a/books/architecture/src/consensus/intro.md b/books/architecture/src/consensus/intro.md new file mode 100644 index 0000000..32013b6 --- /dev/null +++ b/books/architecture/src/consensus/intro.md @@ -0,0 +1 @@ +# ⚪️ Consensus diff --git a/books/architecture/src/consensus/todo.md b/books/architecture/src/consensus/todo.md new file mode 100644 index 0000000..460d445 --- /dev/null +++ b/books/architecture/src/consensus/todo.md @@ -0,0 +1 @@ +# ⚪️ TODO diff --git a/books/architecture/src/consensus/verifier.md b/books/architecture/src/consensus/verifier.md new file mode 100644 index 0000000..128a3b0 --- /dev/null +++ b/books/architecture/src/consensus/verifier.md @@ -0,0 +1 @@ +# ⚪️ Verifier diff --git a/books/architecture/src/cuprate-architecture.md b/books/architecture/src/cuprate-architecture.md new file mode 100644 index 0000000..3c6c073 --- /dev/null +++ b/books/architecture/src/cuprate-architecture.md @@ -0,0 +1,22 @@ +# Cuprate Architecture +WIP + +[Cuprate](https://github.com/Cuprate/cuprate)'s architecture book. + +Sections are notated with colors indicating how complete they are: + +| Color | Meaning | +|-------|---------| +| ⚪️ | Empty +| 🔴 | Severely lacking information +| 🟠 | Lacking some information +| 🟡 | Almost ready +| 🟢 | OK + +--- + +Continue to the next chapter by clicking the right `>` button, or by selecting it on the left side. + +All chapters are viewable by clicking the top-left `☰` button. + +The entire book can searched by clicking the top-left 🔍 button. \ No newline at end of file diff --git a/books/architecture/src/external-monero-libraries/cryptonight.md b/books/architecture/src/external-monero-libraries/cryptonight.md new file mode 100644 index 0000000..80647b0 --- /dev/null +++ b/books/architecture/src/external-monero-libraries/cryptonight.md @@ -0,0 +1 @@ +# ⚪️ Cryptonight diff --git a/books/architecture/src/external-monero-libraries/intro.md b/books/architecture/src/external-monero-libraries/intro.md new file mode 100644 index 0000000..440a344 --- /dev/null +++ b/books/architecture/src/external-monero-libraries/intro.md @@ -0,0 +1 @@ +# ⚪️ External Monero libraries diff --git a/books/architecture/src/external-monero-libraries/monero_serai.md b/books/architecture/src/external-monero-libraries/monero_serai.md new file mode 100644 index 0000000..f1567b1 --- /dev/null +++ b/books/architecture/src/external-monero-libraries/monero_serai.md @@ -0,0 +1,2 @@ +# monero_serai + diff --git a/books/architecture/src/external-monero-libraries/randomx.md b/books/architecture/src/external-monero-libraries/randomx.md new file mode 100644 index 0000000..7705151 --- /dev/null +++ b/books/architecture/src/external-monero-libraries/randomx.md @@ -0,0 +1,2 @@ +# RandomX + \ No newline at end of file diff --git a/books/architecture/src/foreword.md b/books/architecture/src/foreword.md new file mode 100644 index 0000000..c85f18b --- /dev/null +++ b/books/architecture/src/foreword.md @@ -0,0 +1,36 @@ +# Foreword +Monero[^1] is a large software project, coming in at 329k lines of C++, C, headers, and make files.[^2] It is directly responsible for 2.6 billion dollars worth of value.[^3] It has had over 400 contributors, more if counting unnamed contributions.[^4] It has over 10,000 node operators and a large active userbase.[^5] + +The project wasn't always this big, but somewhere in the midst of contributors coming and going, various features being added, bugs being fixed, and celebrated cryptography being implemented - there was an aspect that was lost by the project that it could not easily gain again: **maintainability**. + +Within large and complicated software projects, there is an important transfer of knowledge that must occur for long-term survival. Much like an organism that must eventually pass the torch onto the next generation, projects must do the same for future contributors. + +However, newcomers often lack experience, past contributors might not be around, and current maintainers may be too busy. For whatever reason, this transfer of knowledge is not always smooth. + +There is a solution to this problem: **documentation**. + +The activity of writing the what, where, why, and how of the solutions to technical problems can be done in an author's lonesome. + +The activity of reading these ideas can be done by future readers at any time without permission. + +These readers may be new prospective contributors, it may be the current maintainers, it may be researchers, it may be users of various scale. Whoever it may be, documentation acts as the link between the past and present; a bottle of wisdom thrown into the river of time for future participants to open. + +This book is the manifestation of this will, for Cuprate[^6], an alternative Monero node. It documents Cuprate's implementation from head-to-toe such that in the case of a contributor's untimely disappearance, the project can continue. + +People come and go, documentation is forever. + +— hinto-janai + +--- + +[^1]: [`monero-project/monero`](https://github.com/monero-project/monero) + +[^2]: `git ls-files | grep "\.cpp$\|\.h$\|\.c$\|CMake" | xargs cat | wc -l` on [`cc73fe7`](https://github.com/monero-project/monero/tree/cc73fe71162d564ffda8e549b79a350bca53c454) + +[^3]: 2024-05-24: $143.55 USD * 18,151,608 XMR = $2,605,663,258 + +[^4]: `git log --all --pretty="%an" | sort -u | wc -l` on [`cc73fe7`](https://github.com/monero-project/monero/tree/cc73fe71162d564ffda8e549b79a350bca53c454) + +[^5]: + +[^6]: \ No newline at end of file diff --git a/books/architecture/src/formats-protocols-types/cuprate-helper.md b/books/architecture/src/formats-protocols-types/cuprate-helper.md new file mode 100644 index 0000000..6227829 --- /dev/null +++ b/books/architecture/src/formats-protocols-types/cuprate-helper.md @@ -0,0 +1 @@ +# ⚪️ cuprate_helper diff --git a/books/architecture/src/formats-protocols-types/cuprate-types.md b/books/architecture/src/formats-protocols-types/cuprate-types.md new file mode 100644 index 0000000..1069ce5 --- /dev/null +++ b/books/architecture/src/formats-protocols-types/cuprate-types.md @@ -0,0 +1 @@ +# ⚪️ cuprate_types diff --git a/books/architecture/src/formats-protocols-types/epee.md b/books/architecture/src/formats-protocols-types/epee.md new file mode 100644 index 0000000..4c0b17e --- /dev/null +++ b/books/architecture/src/formats-protocols-types/epee.md @@ -0,0 +1 @@ +# ⚪️ Epee diff --git a/books/architecture/src/formats-protocols-types/intro.md b/books/architecture/src/formats-protocols-types/intro.md new file mode 100644 index 0000000..77052fd --- /dev/null +++ b/books/architecture/src/formats-protocols-types/intro.md @@ -0,0 +1 @@ +# ⚪️ Formats, protocols, types diff --git a/books/architecture/src/formats-protocols-types/levin.md b/books/architecture/src/formats-protocols-types/levin.md new file mode 100644 index 0000000..72a88cc --- /dev/null +++ b/books/architecture/src/formats-protocols-types/levin.md @@ -0,0 +1 @@ +# ⚪️ Levin diff --git a/books/architecture/src/formats-protocols-types/monero-serai.md b/books/architecture/src/formats-protocols-types/monero-serai.md new file mode 100644 index 0000000..139af8e --- /dev/null +++ b/books/architecture/src/formats-protocols-types/monero-serai.md @@ -0,0 +1 @@ +# ⚪️ monero_serai diff --git a/books/architecture/src/instrumentation/data-collection.md b/books/architecture/src/instrumentation/data-collection.md new file mode 100644 index 0000000..7ea3d9f --- /dev/null +++ b/books/architecture/src/instrumentation/data-collection.md @@ -0,0 +1 @@ +# ⚪️ Data collection diff --git a/books/architecture/src/instrumentation/intro.md b/books/architecture/src/instrumentation/intro.md new file mode 100644 index 0000000..33640dd --- /dev/null +++ b/books/architecture/src/instrumentation/intro.md @@ -0,0 +1,2 @@ +# Instrumentation +Cuprate is built with [instrumentation](https://en.wikipedia.org/wiki/Instrumentation) in mind. \ No newline at end of file diff --git a/books/architecture/src/instrumentation/logging.md b/books/architecture/src/instrumentation/logging.md new file mode 100644 index 0000000..c7c88a3 --- /dev/null +++ b/books/architecture/src/instrumentation/logging.md @@ -0,0 +1 @@ +# ⚪️ Logging diff --git a/books/architecture/src/intro.md b/books/architecture/src/intro.md new file mode 100644 index 0000000..c708d61 --- /dev/null +++ b/books/architecture/src/intro.md @@ -0,0 +1,4 @@ +# Systems +Cuprate is made up of multiple distinct internal systems that work together. + +This section provides informal specifications and implementation details about each. \ No newline at end of file diff --git a/books/architecture/src/intro/how-to-use-this-book.md b/books/architecture/src/intro/how-to-use-this-book.md new file mode 100644 index 0000000..7664e04 --- /dev/null +++ b/books/architecture/src/intro/how-to-use-this-book.md @@ -0,0 +1,5 @@ +# How to use this book + +## Maintainers +## Contributors +## Researchers \ No newline at end of file diff --git a/books/architecture/src/intro/intro.md b/books/architecture/src/intro/intro.md new file mode 100644 index 0000000..db2603c --- /dev/null +++ b/books/architecture/src/intro/intro.md @@ -0,0 +1,15 @@ +# Intro +[Cuprate](https://github.com/Cuprate/cuprate) is an alternative [Monero](https://getmonero.org) node implementation. + +This book describes Cuprate's architecture, ranging from small things like database pruning to larger meta-components like the networking stack. + +A brief overview of some aspects covered within this book: +- Component designs +- Implementation details +- File location and purpose +- Design decisions and tradeoffs +- Things in relation to `monerod` +- Dependency usage + +## Source code +The source files for this book can be found on at: . \ No newline at end of file diff --git a/books/architecture/src/intro/required-knowledge.md b/books/architecture/src/intro/required-knowledge.md new file mode 100644 index 0000000..3262f05 --- /dev/null +++ b/books/architecture/src/intro/required-knowledge.md @@ -0,0 +1,28 @@ +# Required knowledge + +## General +- Rust +- Monero +- System design + +## Components +### Storage +- Embedded databases +- LMDB +- redb + +### RPC +- `axum` +- `tower` +- `async` +- JSON-RPC 2.0 +- Epee + +### Networking +- `tower` +- `tokio` +- `async` +- Levin + +### Instrumentation +- `tracing` diff --git a/books/architecture/src/intro/who-this-book-is-for.md b/books/architecture/src/intro/who-this-book-is-for.md new file mode 100644 index 0000000..4b5be2b --- /dev/null +++ b/books/architecture/src/intro/who-this-book-is-for.md @@ -0,0 +1,31 @@ +# Who this book is for + +## Maintainers +As mentioned in [`Foreword`](../foreword.md), the group of people that benefit from this book's value the most by far are the current and future Cuprate maintainers. + +Cuprate's system design is documented in this book such that if you were ever to build it again from scratch, you would have an excellent guide on how to do such, and also where improvements could be made. + +Practically, what that means for maintainers is that it acts as _the_ reference. During maintenance, it is quite valuable to have a book that contains condensed knowledge on the behavior of components, or how certain code works, or why it was built a certain way. + +## Contributors +Contributors also have access to the inner-workings of Cuprate via this book, which helps when making larger contributions. + +Design decisions and implementation details notated in this book helps answer questions such as: +- Why is it done this way? +- Why can it _not_ be done this way? +- Were other methods attempted? + +Cuprate's testing and benchmarking suites, unknown to new contributors, are also documented within this book. + +## Researchers +This book contains the why, where, and how of the _implementation_ of formal research. + +Although it is an informal specification, this book still acts as a more accessible overview of Cuprate compared to examining the codebase itself. + +## Operators & users +This book is not a practical guide for using Cuprate itself. + +For configuration, data collection (also important for researchers), and other practical usage, see [Cuprate's user book](https://user.cuprate.org). + +## Observers +Anyone curious enough is free to learn the inner-workings of Cuprate via this book, and maybe even contribute someday. \ No newline at end of file diff --git a/books/architecture/src/known-issues-and-tradeoffs/intro.md b/books/architecture/src/known-issues-and-tradeoffs/intro.md new file mode 100644 index 0000000..20ab7b5 --- /dev/null +++ b/books/architecture/src/known-issues-and-tradeoffs/intro.md @@ -0,0 +1 @@ +# ⚪️ Known issues and tradeoffs diff --git a/books/architecture/src/known-issues-and-tradeoffs/networking.md b/books/architecture/src/known-issues-and-tradeoffs/networking.md new file mode 100644 index 0000000..20487cb --- /dev/null +++ b/books/architecture/src/known-issues-and-tradeoffs/networking.md @@ -0,0 +1 @@ +# ⚪️ Networking diff --git a/books/architecture/src/known-issues-and-tradeoffs/rpc.md b/books/architecture/src/known-issues-and-tradeoffs/rpc.md new file mode 100644 index 0000000..3337f37 --- /dev/null +++ b/books/architecture/src/known-issues-and-tradeoffs/rpc.md @@ -0,0 +1 @@ +# ⚪️ RPC diff --git a/books/architecture/src/known-issues-and-tradeoffs/storage.md b/books/architecture/src/known-issues-and-tradeoffs/storage.md new file mode 100644 index 0000000..214cf15 --- /dev/null +++ b/books/architecture/src/known-issues-and-tradeoffs/storage.md @@ -0,0 +1 @@ +# ⚪️ Storage diff --git a/books/architecture/src/networking/dandelion.md b/books/architecture/src/networking/dandelion.md new file mode 100644 index 0000000..30916b7 --- /dev/null +++ b/books/architecture/src/networking/dandelion.md @@ -0,0 +1 @@ +# ⚪️ Dandelion++ diff --git a/books/architecture/src/networking/i2p.md b/books/architecture/src/networking/i2p.md new file mode 100644 index 0000000..986ab2a --- /dev/null +++ b/books/architecture/src/networking/i2p.md @@ -0,0 +1 @@ +# ⚪️ i2p diff --git a/books/architecture/src/networking/intro.md b/books/architecture/src/networking/intro.md new file mode 100644 index 0000000..20487cb --- /dev/null +++ b/books/architecture/src/networking/intro.md @@ -0,0 +1 @@ +# ⚪️ Networking diff --git a/books/architecture/src/networking/ipv4-ipv6.md b/books/architecture/src/networking/ipv4-ipv6.md new file mode 100644 index 0000000..07339b4 --- /dev/null +++ b/books/architecture/src/networking/ipv4-ipv6.md @@ -0,0 +1 @@ +# ⚪️ IPv4/IPv6 diff --git a/books/architecture/src/networking/p2p.md b/books/architecture/src/networking/p2p.md new file mode 100644 index 0000000..11b2015 --- /dev/null +++ b/books/architecture/src/networking/p2p.md @@ -0,0 +1 @@ +# ⚪️ P2P diff --git a/books/architecture/src/networking/proxy.md b/books/architecture/src/networking/proxy.md new file mode 100644 index 0000000..bd9e5f7 --- /dev/null +++ b/books/architecture/src/networking/proxy.md @@ -0,0 +1 @@ +# ⚪️ Proxy diff --git a/books/architecture/src/networking/tor.md b/books/architecture/src/networking/tor.md new file mode 100644 index 0000000..cc0a809 --- /dev/null +++ b/books/architecture/src/networking/tor.md @@ -0,0 +1 @@ +# ⚪️ Tor diff --git a/books/architecture/src/resource-model/concurrency-and-parallelism/intro.md b/books/architecture/src/resource-model/concurrency-and-parallelism/intro.md new file mode 100644 index 0000000..2cca180 --- /dev/null +++ b/books/architecture/src/resource-model/concurrency-and-parallelism/intro.md @@ -0,0 +1,32 @@ +# Concurrency and parallelism +It is incumbent upon software like Cuprate to take advantage of today's highly parallel hardware as much as practically possible. + +With that said, programs must setup guardrails when operating in a concurrent and parallel manner, [for correctness and safety](https://en.wikipedia.org/wiki/Concurrency_(computer_science)). + +There are "synchronization primitives" that help with this, common ones being: +- [Locks](https://en.wikipedia.org/wiki/Lock_(computer_science)) +- [Channels](https://en.wikipedia.org/wiki/Channel_(programming)) +- [Atomics](https://en.wikipedia.org/wiki/Linearizability#Primitive_atomic_instructions) + +These tools are relatively easy to use in isolation, but trickier to do so when considering the entire system. It is not uncommon for _the_ bottleneck to be the [poor orchastration](https://en.wikipedia.org/wiki/Starvation_(computer_science)) of these primitives. + +## Analogy +A common analogy for a parallel system is an intersection. + +Like a parallel computer system, an intersection contains: +1. **Parallelism:** multiple individual units that want to move around (cars, pedestrians, etc) +1. **Synchronization primitives:** traffic lights, car lights, walk signals + +In theory, the amount of "work" the units can do is only limited by the speed of the units themselves, but in practice, the slow cascading reaction speeds between all units, the frequent hiccups that can occur, and the synchronization primitives themselves become bottlenecks far before the maximum speed of any unit is reached. + +A car that hogs the middle of the intersection on the wrong light is akin to a system thread holding onto a lock longer than it should be - it degrades total system output. + +Unlike humans however, computer systems at least have the potential to move at lightning speeds, but only if the above synchronization primitives are used correctly. + +## Goal +To aid the long-term maintenance of highly concurrent and parallel code, this section documents: +1. All system threads spawned and maintained +1. All major sections where synchronization primitives are used +1. The asynchronous behavior of some components + +and how these compose together efficiently in Cuprate. \ No newline at end of file diff --git a/books/architecture/src/resource-model/concurrency-and-parallelism/map.md b/books/architecture/src/resource-model/concurrency-and-parallelism/map.md new file mode 100644 index 0000000..1bde994 --- /dev/null +++ b/books/architecture/src/resource-model/concurrency-and-parallelism/map.md @@ -0,0 +1 @@ +# ⚪️ Map diff --git a/books/architecture/src/resource-model/concurrency-and-parallelism/the-block-downloader.md b/books/architecture/src/resource-model/concurrency-and-parallelism/the-block-downloader.md new file mode 100644 index 0000000..c0dccba --- /dev/null +++ b/books/architecture/src/resource-model/concurrency-and-parallelism/the-block-downloader.md @@ -0,0 +1 @@ +# ⚪️ The block downloader diff --git a/books/architecture/src/resource-model/concurrency-and-parallelism/the-database.md b/books/architecture/src/resource-model/concurrency-and-parallelism/the-database.md new file mode 100644 index 0000000..32e4b62 --- /dev/null +++ b/books/architecture/src/resource-model/concurrency-and-parallelism/the-database.md @@ -0,0 +1 @@ +# ⚪️ The database diff --git a/books/architecture/src/resource-model/concurrency-and-parallelism/the-rpc-server.md b/books/architecture/src/resource-model/concurrency-and-parallelism/the-rpc-server.md new file mode 100644 index 0000000..cd654cf --- /dev/null +++ b/books/architecture/src/resource-model/concurrency-and-parallelism/the-rpc-server.md @@ -0,0 +1 @@ +# ⚪️ The RPC server diff --git a/books/architecture/src/resource-model/concurrency-and-parallelism/the-verifier.md b/books/architecture/src/resource-model/concurrency-and-parallelism/the-verifier.md new file mode 100644 index 0000000..eeaedc6 --- /dev/null +++ b/books/architecture/src/resource-model/concurrency-and-parallelism/the-verifier.md @@ -0,0 +1 @@ +# ⚪️ The verifier diff --git a/books/architecture/src/resource-model/concurrency-and-parallelism/thread-exit.md b/books/architecture/src/resource-model/concurrency-and-parallelism/thread-exit.md new file mode 100644 index 0000000..3925975 --- /dev/null +++ b/books/architecture/src/resource-model/concurrency-and-parallelism/thread-exit.md @@ -0,0 +1 @@ +# ⚪️ Thread exit diff --git a/books/architecture/src/resource-model/file-system.md b/books/architecture/src/resource-model/file-system.md new file mode 100644 index 0000000..b67ca07 --- /dev/null +++ b/books/architecture/src/resource-model/file-system.md @@ -0,0 +1 @@ +# ⚪️ File system diff --git a/books/architecture/src/resource-model/intro.md b/books/architecture/src/resource-model/intro.md new file mode 100644 index 0000000..28d1dd6 --- /dev/null +++ b/books/architecture/src/resource-model/intro.md @@ -0,0 +1 @@ +# ⚪️ Resource model diff --git a/books/architecture/src/resource-model/memory.md b/books/architecture/src/resource-model/memory.md new file mode 100644 index 0000000..e3624b5 --- /dev/null +++ b/books/architecture/src/resource-model/memory.md @@ -0,0 +1 @@ +# ⚪️ Memory diff --git a/books/architecture/src/resource-model/sockets.md b/books/architecture/src/resource-model/sockets.md new file mode 100644 index 0000000..0d590ca --- /dev/null +++ b/books/architecture/src/resource-model/sockets.md @@ -0,0 +1 @@ +# ⚪️ Sockets diff --git a/books/architecture/src/rpc/handler.md b/books/architecture/src/rpc/handler.md new file mode 100644 index 0000000..fffa45f --- /dev/null +++ b/books/architecture/src/rpc/handler.md @@ -0,0 +1 @@ +# ⚪️ Handler diff --git a/books/architecture/src/rpc/interface.md b/books/architecture/src/rpc/interface.md new file mode 100644 index 0000000..541b744 --- /dev/null +++ b/books/architecture/src/rpc/interface.md @@ -0,0 +1 @@ +# ⚪️ Interface diff --git a/books/architecture/src/rpc/intro.md b/books/architecture/src/rpc/intro.md new file mode 100644 index 0000000..dcfc82b --- /dev/null +++ b/books/architecture/src/rpc/intro.md @@ -0,0 +1,3 @@ +# RPC +- +- \ No newline at end of file diff --git a/books/architecture/src/rpc/methods/intro.md b/books/architecture/src/rpc/methods/intro.md new file mode 100644 index 0000000..d4a3a15 --- /dev/null +++ b/books/architecture/src/rpc/methods/intro.md @@ -0,0 +1 @@ +# ⚪️ Methods diff --git a/books/architecture/src/rpc/router.md b/books/architecture/src/rpc/router.md new file mode 100644 index 0000000..1827dd3 --- /dev/null +++ b/books/architecture/src/rpc/router.md @@ -0,0 +1 @@ +# ⚪️ Router diff --git a/books/architecture/src/rpc/types/binary.md b/books/architecture/src/rpc/types/binary.md new file mode 100644 index 0000000..dea12fa --- /dev/null +++ b/books/architecture/src/rpc/types/binary.md @@ -0,0 +1 @@ +# ⚪️ Binary diff --git a/books/architecture/src/rpc/types/intro.md b/books/architecture/src/rpc/types/intro.md new file mode 100644 index 0000000..22e430c --- /dev/null +++ b/books/architecture/src/rpc/types/intro.md @@ -0,0 +1 @@ +# ⚪️ Types diff --git a/books/architecture/src/rpc/types/json.md b/books/architecture/src/rpc/types/json.md new file mode 100644 index 0000000..0bf9351 --- /dev/null +++ b/books/architecture/src/rpc/types/json.md @@ -0,0 +1 @@ +# ⚪️ JSON diff --git a/books/architecture/src/rpc/types/other.md b/books/architecture/src/rpc/types/other.md new file mode 100644 index 0000000..49a36cc --- /dev/null +++ b/books/architecture/src/rpc/types/other.md @@ -0,0 +1 @@ +# ⚪️ Other diff --git a/books/architecture/src/storage/blockchain.md b/books/architecture/src/storage/blockchain.md new file mode 100644 index 0000000..6046687 --- /dev/null +++ b/books/architecture/src/storage/blockchain.md @@ -0,0 +1 @@ +# ⚪️ Blockchain diff --git a/books/architecture/src/storage/database-abstraction.md b/books/architecture/src/storage/database-abstraction.md new file mode 100644 index 0000000..b21a192 --- /dev/null +++ b/books/architecture/src/storage/database-abstraction.md @@ -0,0 +1 @@ +# ⚪️ Database abstraction diff --git a/books/architecture/src/storage/intro.md b/books/architecture/src/storage/intro.md new file mode 100644 index 0000000..214cf15 --- /dev/null +++ b/books/architecture/src/storage/intro.md @@ -0,0 +1 @@ +# ⚪️ Storage diff --git a/books/architecture/src/storage/pruning.md b/books/architecture/src/storage/pruning.md new file mode 100644 index 0000000..cfeee69 --- /dev/null +++ b/books/architecture/src/storage/pruning.md @@ -0,0 +1 @@ +# ⚪️ Pruning diff --git a/books/architecture/src/storage/transaction-pool.md b/books/architecture/src/storage/transaction-pool.md new file mode 100644 index 0000000..4eb139b --- /dev/null +++ b/books/architecture/src/storage/transaction-pool.md @@ -0,0 +1 @@ +# ⚪️ Transaction pool diff --git a/books/architecture/src/testing/intro.md b/books/architecture/src/testing/intro.md new file mode 100644 index 0000000..397ae90 --- /dev/null +++ b/books/architecture/src/testing/intro.md @@ -0,0 +1 @@ +# ⚪️ Testing diff --git a/books/architecture/src/testing/monero-data.md b/books/architecture/src/testing/monero-data.md new file mode 100644 index 0000000..915af28 --- /dev/null +++ b/books/architecture/src/testing/monero-data.md @@ -0,0 +1 @@ +# ⚪️ Monero data diff --git a/books/architecture/src/testing/rpc-client.md b/books/architecture/src/testing/rpc-client.md new file mode 100644 index 0000000..5a373c2 --- /dev/null +++ b/books/architecture/src/testing/rpc-client.md @@ -0,0 +1 @@ +# ⚪️ RPC client diff --git a/books/architecture/src/testing/spawning-monerod.md b/books/architecture/src/testing/spawning-monerod.md new file mode 100644 index 0000000..1552266 --- /dev/null +++ b/books/architecture/src/testing/spawning-monerod.md @@ -0,0 +1 @@ +# ⚪️ Spawning monerod diff --git a/books/architecture/src/zmq/intro.md b/books/architecture/src/zmq/intro.md new file mode 100644 index 0000000..0b668b3 --- /dev/null +++ b/books/architecture/src/zmq/intro.md @@ -0,0 +1 @@ +# ⚪️ ZMQ diff --git a/books/architecture/src/todo.md b/books/architecture/src/zmq/todo.md similarity index 100% rename from books/architecture/src/todo.md rename to books/architecture/src/zmq/todo.md diff --git a/books/protocol/src/SUMMARY.md b/books/protocol/src/SUMMARY.md index 1a4b1f0..682e0e7 100644 --- a/books/protocol/src/SUMMARY.md +++ b/books/protocol/src/SUMMARY.md @@ -23,5 +23,14 @@ - [Bulletproofs+](./consensus_rules/transactions/ring_ct/bulletproofs+.md) - [P2P Network](./p2p_network.md) - [Levin Protocol](./p2p_network/levin.md) - - [P2P Messages](./p2p_network/messages.md) + - [Admin Messages](./p2p_network/levin/admin.md) + - [Protocol Messages](./p2p_network/levin/protocol.md) + - [Common Types](./p2p_network/common_types.md) + - [Message Flows](./p2p_network/message_flows.md) + - [Handshake](./p2p_network/message_flows/handshake.md) + - [Timed Sync](./p2p_network/message_flows/timed_sync.md) + - [New Block](./p2p_network/message_flows/new_block.md) + - [New Transactions](./p2p_network/message_flows/new_transactions.md) + - [Chain Sync](./p2p_network/message_flows/chain_sync.md) + - [Get Blocks](./p2p_network/message_flows/get_blocks.md) - [Pruning](./pruning.md) diff --git a/books/protocol/src/p2p_network.md b/books/protocol/src/p2p_network.md index e0d9a79..89bd1be 100644 --- a/books/protocol/src/p2p_network.md +++ b/books/protocol/src/p2p_network.md @@ -1,3 +1,3 @@ # P2P Network -This chapter contains descriptions of Monero's peer to peer network, including messages, flows, expected responses, etc. +This chapter contains descriptions of Monero's peer to peer network, including messages, flows, etc. diff --git a/books/protocol/src/p2p_network/common_types.md b/books/protocol/src/p2p_network/common_types.md new file mode 100644 index 0000000..0bf29cb --- /dev/null +++ b/books/protocol/src/p2p_network/common_types.md @@ -0,0 +1,116 @@ +# Common P2P Types + +This chapter contains definitions of types used in multiple P2P messages. + +### Support Flags + +Support flags specify any protocol extensions the peer supports, currently only the first bit is used: + +`FLUFFY_BLOCKS = 1` - for if the peer supports receiving fluffy blocks. + +### Basic Node Data [^b-n-d] { #basic-node-data } + +| Fields | Type | Description | +|------------------------|---------------------------------------|-------------------------------------------------------------------------------------------| +| `network_id` | A UUID (epee string) | A fixed constant value for a specific network (mainnet,testnet,stagenet) | +| `my_port` | u32 | The peer's inbound port, if the peer does not want inbound connections this should be `0` | +| `rpc_port` | u16 | The peer's RPC port, if the peer does not want inbound connections this should be `0` | +| `rpc_credits_per_hash` | u32 | States how much it costs to use this node in credits per hashes, `0` being free | +| `peer_id` | u64 | A fixed ID for the node, set to 1 for anonymity networks | +| `support_flags` | [support flags](#support-flags) (u32) | Specifies any protocol extensions the peer supports | + +### Core Sync Data [^c-s-d] { #core-sync-data } + +| Fields | Type | Description | +|-------------------------------|------------------------|---------------------------------------------------------------| +| `current_height` | u64 | The current chain height | +| `cumulative_difficulty` | u64 | The low 64 bits of the cumulative difficulty | +| `cumulative_difficulty_top64` | u64 | The high 64 bits of the cumulative difficulty | +| `top_id` | [u8; 32] (epee string) | The hash of the top block | +| `top_version` | u8 | The hardfork version of the top block | +| `pruning_seed` | u32 | THe pruning seed of the node, `0` if the node does no pruning | + +### Network Address [^network-addr] { #network-address } + +Network addresses are serialized differently than other types, the fields needed depend on the `type` field: + +| Fields | Type | Description | +| ------ | --------------------------------------- | ---------------- | +| `type` | u8 | The address type | +| `addr` | An object whose fields depend on `type` | The address | + +#### IPv4 + +`type = 1` + +| Fields | Type | Description | +| -------- | ---- | ---------------- | +| `m_ip` | u32 | The IPv4 address | +| `m_port` | u16 | The port | + + +#### IPv6 + +`type = 2` + +| Fields | Type | Description | +| -------- | ---------------------- | ---------------- | +| `addr` | [u8; 16] (epee string) | The IPv6 address | +| `m_port` | u16 | The port | + +#### Tor + +TODO: + +#### I2p + +TODO: + +### Peer List Entry Base [^pl-entry-base] { #peer-list-entry-base } + +| Fields | Type | Description | +|------------------------|-------------------------------------|-------------------------------------------------------------------------------------------------------| +| `adr` | [Network Address](#network-address) | The address of the peer | +| `id` | u64 | The random, self assigned, ID of this node | +| `last_seen` | i64 | A field marking when this peer was last seen, although this is zeroed before sending over the network | +| `pruning_seed` | u32 | This peer's pruning seed, `0` if the peer does no pruning | +| `rpc_port` | u16 | This node's RPC port, `0` if this peer has no public RPC port. | +| `rpc_credits_per_hash` | u32 | States how much it costs to use this node in credits per hashes, `0` being free | + +### Tx Blob Entry [^tb-entry] { #tx-blob-entry } + +| Fields | Type | Description | +| --------------- | ---------------------- | --------------------------------------- | +| `blob` | bytes (epee string) | The pruned tx blob | +| `prunable_hash` | [u8; 32] (epee string) | The hash of the prunable part of the tx | + +### Block Complete Entry [^bc-entry] { #block-complete-entry } + +| Fields | Type | Description | +|----------------|---------------------|-----------------------------------------------------------| +| `pruned` | bool | True if the block is pruned, false otherwise | +| `block` | bytes (epee string) | The block blob | +| `block_weight` | u64 | The block's weight | +| `txs` | depends on `pruned` | The transaction blobs, the exact type depends on `pruned` | + +If `pruned` is true: + +`txs` is a vector of [Tx Blob Entry](#tx-blob-entry) + +If `pruned` is false: + +`txs` is a vector of bytes. + +--- + +[^b-n-d]: + +[^c-s-d]: + +[^network-addr]: + +[^pl-entry-base]: + +[^tb-entry]: + +[^bc-entry]: diff --git a/books/protocol/src/p2p_network/epee.md b/books/protocol/src/p2p_network/epee.md deleted file mode 100644 index 2f8161d..0000000 --- a/books/protocol/src/p2p_network/epee.md +++ /dev/null @@ -1,3 +0,0 @@ -# Epee Binary Format - -The epee binary format is described here: TODO diff --git a/books/protocol/src/p2p_network/levin.md b/books/protocol/src/p2p_network/levin.md index de74660..ec92f7d 100644 --- a/books/protocol/src/p2p_network/levin.md +++ b/books/protocol/src/p2p_network/levin.md @@ -10,16 +10,16 @@ of buckets that will be combined into a single message. ### Bucket Format | Field | Type | Size (bytes) | -| ------ | ----------------------------- | ------------ | +|--------|-------------------------------|--------------| | Header | [BucketHeader](#bucketheader) | 33 | | Body | bytes | dynamic | ### BucketHeader -Format: +Format[^header-format]: | Field | Type | Size (bytes) | -| ---------------- | ------ | ------------ | +|------------------|--------|--------------| | Signature | LE u64 | 8 | | Size | LE u64 | 8 | | Expect Response | bool | 1 | @@ -32,7 +32,7 @@ Format: The signature field is fixed for every bucket and is used to tell apart peers running different protocols. -Its value should be `0x0101010101012101` +Its value should be `0x0101010101012101` [^signature] #### Size @@ -53,7 +53,7 @@ responses should be `1`. #### Flags -This is a bit-flag field that determines what type of bucket this is: +This is a bit-flag field that determines what type of bucket this is[^flags]: | Type | Bits set | | -------------- | ----------- | @@ -66,3 +66,17 @@ This is a bit-flag field that determines what type of bucket this is: #### Protocol Version This is a fixed value of 1. + +## Bucket Body + +All bucket bodies are serialized in the epee binary format which is described here: https://github.com/monero-project/monero/blob/cc73fe71162d564ffda8e549b79a350bca53c454/docs/PORTABLE_STORAGE.md + +Exact message types are described in the next chapters. + +--- + +[^header-format]: + +[^signature]: + +[^flags]: diff --git a/books/protocol/src/p2p_network/levin/admin.md b/books/protocol/src/p2p_network/levin/admin.md new file mode 100644 index 0000000..6f2b716 --- /dev/null +++ b/books/protocol/src/p2p_network/levin/admin.md @@ -0,0 +1,102 @@ +# Admin Messages + +This chapter describes admin messages, and documents the current admin messages. Admin messages are a subset of messages that handle connection +creation, making sure connections are still alive, and sharing peer lists. + +## Levin + +All admin messages are in the request/response levin format. This means requests will set the [expect response bit](./levin.md#expect-response) and +responses will set the return code to [`1`](./levin.md#return-code). + +## Messages + +### Handshake + +ID: `1001`[^handshake-id] + +#### Request [^handshake-req] { #handshake-request } + +| Fields | Type | Description | +|----------------|-------------------------------------------------------|--------------------------------------| +| `node_data` | [basic node data](../common_types.md#basic-node-data) | Static information about our node | +| `payload_data` | [core sync data](../common_types.md#core-sync-data) | Information on the node's sync state | + +#### Response [^handshake-res] { #handshake-response } + +| Fields | Type | Description | +|----------------------|--------------------------------------------------------------------------|-----------------------------------------| +| `node_data` | [basic node data](../common_types.md#basic-node-data) | Static information about our node | +| `payload_data` | [core sync data](../common_types.md#core-sync-data) | Information on the node's sync state | +| `local_peerlist_new` | A Vec of [peer list entry base](../common_types.md#peer-list-entry-base) | A list of peers in the node's peer list | + +### Timed Sync + +ID: `1002`[^timed-sync-id] + +#### Request [^timed-sync-req] { #timed-sync-request } + +| Fields | Type | Description | +| -------------- | --------------------------------------------------- | ------------------------------------ | +| `payload_data` | [core sync data](../common_types.md#core-sync-data) | Information on the node's sync state | + +#### Response [^timed-sync-res] { #timed-sync-response } + +| Fields | Type | Description | +|----------------------|--------------------------------------------------------------------------|-----------------------------------------| +| `payload_data` | [core sync data](../common_types.md#core-sync-data) | Information on the node's sync state | +| `local_peerlist_new` | A Vec of [peer list entry base](../common_types.md#peer-list-entry-base) | A list of peers in the node's peer list | + +### Ping + +ID: `1003`[^ping-id] + +#### Request [^ping-req] { #ping-request } + +No data is serialized for a ping request. + +#### Response [^ping-res] { #ping-response } + +| Fields | Type | Description | +| --------- | ------ | --------------------------------- | +| `status` | string | Will be `OK` for successful pings | +| `peer_id` | u64 | The self assigned id of the peer | + +### Request Support Flags + +ID: `1007`[^support-flags] + +#### Request [^sf-req] { #support-flags-request } + +No data is serialized for a ping request. + +#### Response [^sf-res] { #support-flags-response } + +| Fields | Type | Description | +| --------------- | ---- | ------------------------------------------------------------ | +| `support_flags` | u32 | The peer's [support flags](../common_types.md#support-flags) | + +--- + +[^handshake-id]: + +[^handshake-req]: + +[^handshake-res]: + +[^timed-sync-id]: + +[^timed-sync-req]: + +[^timed-sync-res]: + +[^ping-id]: + +[^ping-req]: + +[^ping-res]: + +[^support-flags]: + +[^sf-req]: + +[^sf-res]: diff --git a/books/protocol/src/p2p_network/levin/protocol.md b/books/protocol/src/p2p_network/levin/protocol.md new file mode 100644 index 0000000..a52ca1d --- /dev/null +++ b/books/protocol/src/p2p_network/levin/protocol.md @@ -0,0 +1,121 @@ +# Protocol Messages + +This chapter describes protocol messages, and documents the current protocol messages. Protocol messages are used to share protocol data +like blocks and transactions. + +## Levin + +All protocol messages are in the notification levin format. Although there are some messages that fall under requests/responses, levin will treat them as notifications. + +All admin messages are in the request/response levin format. This means requests will set the [expect response bit](../levin.md#expect-response) and +responses will set the return code to [`1`](../levin.md#return-code). + +## Messages + +### Notify New Block + +ID: `2001`[^notify-new-block-id] + +| Fields | Type | Description | +| --------------------------- | --------------------------------------------------------------- | ------------------------ | +| `b` | [Block Complete Entry](../common_types.md#block-complete-entry) | The full block | +| `current_blockchain_height` | u64 | The current chain height | + +### Notify New Transactions + +ID: `2002`[^notify-new-transactions-id] + +| Fields | Type | Description | +| ------------------- | ----------------- | ------------------------------------------------------ | +| `txs` | A vector of bytes | The txs | +| `_` | Bytes | Padding to prevent traffic volume analysis | +| `dandelionpp_fluff` | bool | True if this message contains fluff txs, false if stem | + +### Notify Request Get Objects + +ID: `2003`[^notify-request-get-objects-id] + +| Fields | Type | Description | +|----------|----------------------------------------------------|------------------------------------------------------------| +| `blocks` | A vector of [u8; 32] serialized as a single string | The block IDs requested | +| `prune` | bool | True if we want the blocks in pruned form, false otherwise | + +### Notify Response Get Objects + +ID: `2004`[^notify-response-get-objects-id] + +| Fields | Type | Description | +| --------------------------- | --------------------------------------------------------------------------- | ------------------------------ | +| `blocks` | A vector of [Block Complete Entry](../common_types.md#block-complete-entry) | The blocks that were requested | +| `missed_ids` | A vector of [u8; 32] serialized as a single string | IDs of any missed blocks | +| `current_blockchain_height` | u64 | The current blockchain height | + +### Notify Request Chain + +ID: `2006`[^notify-request-chain-id] + +| Fields | Type | Description | +|-------------|----------------------------------------------------|-------------------------------------------------------------------------------------------------------| +| `block_ids` | A vector of [u8; 32] serialized as a single string | A list of block IDs in reverse chronological order, the top and genesis block will always be included | +| `prune` | bool | True if we want the response to contain pruned blocks, false otherwise | + +### Notify Response Chain Entry + +ID: `2007`[^notify-response-chain-entry-id] + +| Fields | Type | Description | +|-------------------------------|----------------------------------------------------|------------------------------------------------| +| `start_height` | u64 | The start height of the entry | +| `total_height` | u64 | The height of the peer's blockchain | +| `cumulative_difficulty` | u64 | The low 64 bits of the cumulative difficulty | +| `cumulative_difficulty_top64` | u64 | The high 64 bits of the cumulative difficulty | +| `m_block_ids` | A vector of [u8; 32] serialized as a single string | The block IDs in this entry | +| `m_block_weights` | A vector of u64 serialized as a single string | The block weights | +| `first_block` | bytes (epee string) | The header of the first block in `m_block_ids` | + +### Notify New Fluffy Block + +ID: `2008`[^notify-new-fluffy-block-id] + +| Fields | Type | Description | +| --------------------------- | --------------------------------------------------------------- | ------------------------------------- | +| `b` | [Block Complete Entry](../common_types.md#block-complete-entry) | The block, may or may not contain txs | +| `current_blockchain_height` | u64 | The current chain height | + +### Notify Request Fluffy Missing Tx + +ID: `2009`[^notify-request-fluffy-missing-tx-id] + +| Fields | Type | Description | +|-----------------------------|-----------------------------------------------|--------------------------------------------| +| `block_hash` | [u8; 32] serialized as a string | The block hash txs are needed from | +| `current_blockchain_height` | u64 | The current chain height | +| `missing_tx_indices` | A vector of u64 serialized as a single string | The indices of the needed txs in the block | + +### Notify Get Txpool Compliment + +ID: `2010`[^notify-get-txpool-compliment-id] + +| Fields | Type | Description | +| -------- | ------------------------------------------- | ---------------------- | +| `hashes` | A vector of [u8; 32] serialized as a string | The current txpool txs | + +--- + +[^notify-new-block-id]: + +[^notify-new-transactions-id]: + +[^notify-request-get-objects-id]: + +[^notify-response-get-objects-id]: + +[^notify-request-chain-id]: + +[^notify-response-chain-entry-id]: + +[^notify-new-fluffy-block-id]: + +[^notify-request-fluffy-missing-tx-id]: + +[^notify-get-txpool-compliment-id]: diff --git a/books/protocol/src/p2p_network/message_flows.md b/books/protocol/src/p2p_network/message_flows.md new file mode 100644 index 0000000..8f1004c --- /dev/null +++ b/books/protocol/src/p2p_network/message_flows.md @@ -0,0 +1,19 @@ +# Message Flows + +Message flows are sets of messages sent between peers, that achieve an identifiable goal, like a handshake. +Some message flows are complex, involving many message types, whereas others are simple, requiring only 1. + +The message flows here are not every possible request/response. + +When documenting checks on the messages, not all checks are documented, only the ones notable. This should help +to reduce the maintenance burden. + +## Different Flows + +- [Handshakes](./message_flows/handshake.md) +- [Timed Sync](./message_flows/timed_sync.md) +- [New Block](./message_flows/new_block.md) +- [New Transactions](./message_flows/new_transactions.md) +- [Chain Sync](./message_flows/chain_sync.md) +- [Get Blocks](./message_flows/get_blocks.md) + diff --git a/books/protocol/src/p2p_network/message_flows/chain_sync.md b/books/protocol/src/p2p_network/message_flows/chain_sync.md new file mode 100644 index 0000000..1b66132 --- /dev/null +++ b/books/protocol/src/p2p_network/message_flows/chain_sync.md @@ -0,0 +1,28 @@ +# Chain Sync + +Chain sync is the first step in syncing a peer's blockchain, it allows a peers to find the split point in their chains and for the peer +to learn about the missing block IDs. + +## Flow + +The first step is for the initiating peer is to get its compact chain history. The compact chain history must be in reverse chronological +order, with the first block being the top block and the last the genesis, if the only block is the genesis then that only needs to be included +once. The blocks in the middle are not enforced to be at certain locations, however `monerod` will use the top 11 blocks and will then go power +of 2 offsets from then on, i.e. `{13, 17, 25, ...}` + +Then, with the compact history, the initiating peer will send a [request chain](../levin/protocol.md#notify-request-chain) message, the receiving +peer will then find the split point and return a [response chain entry](../levin/protocol.md#notify-response-chain-entry) message. + +The `response chain entry` will contain a list of block IDs with the first being a common ancestor and the rest being the next blocks that come after +that block in the peer's chain. + +### Response Checks + +- There must be an overlapping block.[^res-overlapping-block] +- The amount of returned block IDs must be less than `25,000`.[^res-max-blocks] + +--- + +[^res-overlapping-block]: + +[^res-max-blocks]: \ No newline at end of file diff --git a/books/protocol/src/p2p_network/message_flows/get_blocks.md b/books/protocol/src/p2p_network/message_flows/get_blocks.md new file mode 100644 index 0000000..eacca7f --- /dev/null +++ b/books/protocol/src/p2p_network/message_flows/get_blocks.md @@ -0,0 +1,19 @@ +# Get Blocks + +The get block flow is used to download batches of blocks from a peer. + +## Flow + +The initiating peer needs a list of block IDs that the receiving peer has, this can be done with +the [chain sync flow](./chain_sync.md). + +With a list a block IDs the initiating peer will send a [get objects request](../levin/protocol.md#notify-request-get-objects) message, the receiving +peer will then respond with [get objects response](../levin/protocol.md#notify-response-get-objects). + +### Request Checks + +- The amount of blocks must be less than `100`.[^max-block-requests] + +--- + +[^max-block-requests]: diff --git a/books/protocol/src/p2p_network/message_flows/handshake.md b/books/protocol/src/p2p_network/message_flows/handshake.md new file mode 100644 index 0000000..2a4abe1 --- /dev/null +++ b/books/protocol/src/p2p_network/message_flows/handshake.md @@ -0,0 +1,51 @@ +# Handshakes + +Handshakes are used to establish connections to peers. + +## Flow + +The default handshake flow is made up of the connecting peer sending a [handshake request](../levin/admin.md#handshake-request) and the +receiving peer responding with a [handshake response](../levin/admin.md#handshake-response). + +It should be noted that not all other messages are banned during handshakes, for example, support flag requests and even some protocol +requests can be sent. + +### Handshake Request Checks + +The receiving peer will check: + +- The `network_id` is network ID expected.[^network-id] +- The connection is an incoming connection.[^req-incoming-only] +- The peer hasn't already completed a handshake.[^double-handshake] +- If the network zone is public, then the `peer_id` must not be the same as ours.[^same-peer-id] +- The core sync data is not malformed.[^core-sync-data-checks] + +### Handshake Response Checks + +The initiating peer will check: + +- The `network_id` is network ID expected.[^res-network-id] +- The number of peers in the peer list is less than `250`.[^max-peer-list-res] +- All peers in the peer list are in the same zone.[^peers-all-in-same-zone] +- The core sync data is not malformed.[^core-sync-data-checks] +- If the network zone is public, then the `peer_id` must not be the same as ours.[^same-peer-id-res] + +--- + +[^network-id]: + +[^req-incoming-only]: + +[^double-handshake]: + +[^same-peer-id]: + +[^core-sync-data-checks]: + +[^res-network-id]: + +[^max-peer-list-res]: + +[^peers-all-in-same-zone]: + +[^same-peer-id-res]: diff --git a/books/protocol/src/p2p_network/message_flows/new_block.md b/books/protocol/src/p2p_network/message_flows/new_block.md new file mode 100644 index 0000000..452aa44 --- /dev/null +++ b/books/protocol/src/p2p_network/message_flows/new_block.md @@ -0,0 +1,29 @@ +# New Block + +This is used whenever a new block is to be sent to peers. Only the fluffy block flow is described here, as the other method is deprecated. + +## Flow + +First the peer with the new block will send a [new fluffy block](../levin/protocol.md#notify-new-fluffy-block) notification, if the receiving +peer has all the txs in the block then the flow is complete. Otherwise the peer sends a [fluffy missing transactions request](../levin/protocol.md#notify-request-fluffy-missing-tx) +to the first peer, the first peer will then respond with again a [new fluffy block](../levin/protocol.md#notify-new-fluffy-block) notification but +with the transactions requested. + +```bob + + ,-----------. ,----------. + | Initiator | | Receiver | + `-----+-----' `-----+----' + | New Fluffy Block | + |-------------------->| + | | + | Missing Txs Request | + |<- - - - - - - - - - | + | | + | New Fluffy Block | + | - - - - - - - - - ->| + | | + | | + V v +``` + diff --git a/books/protocol/src/p2p_network/message_flows/new_transactions.md b/books/protocol/src/p2p_network/message_flows/new_transactions.md new file mode 100644 index 0000000..2a90a3f --- /dev/null +++ b/books/protocol/src/p2p_network/message_flows/new_transactions.md @@ -0,0 +1,16 @@ +# New Transactions + +Monero uses the dandelion++ protocol to pass transactions around the network, this flow just describes the actual tx passing between nodes part. + +## Flow + +This flow is pretty simple, the txs are put into a [new transactions](../levin/protocol.md#notify-new-transactions) notification and sent to +peers. + +Hopefully in the future [this is changed](https://github.com/monero-project/monero/issues/9334). + +There must be no duplicate txs in the notification.[^duplicate-txs] + +--- + +[^duplicate-txs]: \ No newline at end of file diff --git a/books/protocol/src/p2p_network/message_flows/timed_sync.md b/books/protocol/src/p2p_network/message_flows/timed_sync.md new file mode 100644 index 0000000..4d258d7 --- /dev/null +++ b/books/protocol/src/p2p_network/message_flows/timed_sync.md @@ -0,0 +1,28 @@ +# Timed Syncs + +A timed sync request is sent every 60 seconds to make sure the connection is still live. + +## Flow + +First the timed sync initiator will send a [timed sync request](../levin/admin.md#timed-sync-request), the receiver will then +respond with a [timed sync response](../levin/admin.md#timed-sync-response) + +### Timed Sync Request Checks + +- The core sync data is not malformed.[^core-sync-data-checks] + + +### Timed Sync Response Checks + +- The core sync data is not malformed.[^core-sync-data-checks] +- The number of peers in the peer list is less than `250`.[^max-peer-list-res] +- All peers in the peer list are in the same zone.[^peers-all-in-same-zone] + +--- + +[^core-sync-data-checks]: + +[^max-peer-list-res]: + +[^peers-all-in-same-zone]: + diff --git a/books/protocol/src/p2p_network/messages.md b/books/protocol/src/p2p_network/messages.md deleted file mode 100644 index c3f1828..0000000 --- a/books/protocol/src/p2p_network/messages.md +++ /dev/null @@ -1,37 +0,0 @@ -# P2P Messages - -This chapter contains every P2P message. - -## Index - -## Types - -Types used in multiple P2P messages. - -### Support Flags - -Support flags specify any protocol extensions the peer supports, currently only the first bit is used: - -`FLUFFY_BLOCKS = 1` - for if the peer supports receiving fluffy blocks. - -### Basic Node Data - -| Fields | Type (Epee Type) | Description | -| ---------------------- | ------------------------------------- | ---------------------------------------------------------------------------------------- | -| `network_id` | A UUID (String) | A fixed constant value for a specific network (mainnet,testnet,stagenet) | -| `my_port` | u32 (u32) | The peer's inbound port, if the peer does not want inbound connections this should be `0` | -| `rpc_port` | u16 (u16) | The peer's RPC port, if the peer does not want inbound connections this should be `0` | -| `rpc_credits_per_hash` | u32 (u32) | TODO | -| `peer_id` | u64 (u64) | A fixed ID for the node, set to 1 for anonymity networks | -| `support_flags` | [support flags](#support-flags) (u32) | Specifies any protocol extensions the peer supports | - -## Messages - -### Handshake Requests - -levin command: 1001 - -| Fields | Type (Epee Type) | Description | -| ----------- | -------------------------------------------- | ----------- | -| `node_data` | [basic node data](#basic-node-data) (Object) | | -| | | |