add more configurations; add statistics webserver

.dockerignore

@@ -1 +1,2 @@
 **/build
+**/docker-compose

docker-compose/README.md

@@ -34,9 +34,9 @@ docker compose up
 
 #### Optional
 * Open ports 18080 (Monero p2p port) and 37889 (P2Pool p2p port) or 37888 (P2Pool-mini p2p port) in your firewall to ensure better connectivity. If you're mining from a computer behind NAT (like a router) you could consider forwarding the ports to your local machine
-* An XMRig CPU miner is included by default, but you can connect additional miners to this same p2pool node using port 3333
+* An XMRig CPU miner is included by default, but you can connect additional miners to this same p2pool node using port 3333 (or alternate if configured) when you set it as "exposed" in the configuration
 * Configure your kernel for maximum mining performance: [XMRig RandomX Optimization Guide](https://xmrig.com/docs/miner/randomx-optimization-guide)
-* Small miners can mine on p2pool-mini by editing the .env file
+* Many optional configurations and customizations are available by running './configure'
 
 
 #### Other usefull commands
@@ -45,6 +45,7 @@ docker compose up
 * You can see logs when running in the background for with the "docker logs" command:  ```docker logs -f p2pool-xmrig``` or ```docker logs -f p2pool-p2pool``` or ```docker logs -f p2pool-monero```
 * You can pause mining with: ```docker compose pause xmrig``` and resume mining with: ```docker compose unpause xmrig```
 * You can disable mining with: ```docker compose stop xmrig``` and re-enable mining with: ```docker compose start xmrig```
+* You can view your Server Statistics using a web browser if you enabled that feature in the configuration at: http://localhost:3380 (or alternate port as configured)
 
 
 #### Uninstall

docker-compose/cfg/current_config.jinja2

@@ -3,14 +3,26 @@
     "configure_monero": {{ configure_monero | tojson(indent=2) }},
     "configure_xmrig": {{ configure_xmrig | tojson(indent=2) }},
     "sidechain": {{ sidechain | tojson(indent=2) }},
+    "enable_statistics": {{ enable_statistics | tojson(indent=2) }},
+    "statistics_port": {{ statistics_port | tojson(indent=2) }},
     "expose_stratum_port": {{ expose_stratum_port | tojson(indent=2) }},
     "stratum_port": {{ stratum_port | tojson(indent=2) }},
     "p2pool_log_level": {{ p2pool_log_level | int | tojson(indent=2) }},
     "enable_autodiff": {{ enable_autodiff | tojson(indent=2) }},
+    "light_mode": {{ light_mode | tojson(indent=2) }},
+    "no_cache": {{ no_cache | tojson(indent=2) }},
     "p2pool_options": {{ p2pool_options | tojson(indent=2) }},
     "monero_version": {{ monero_version | tojson(indent=2) }},
     "prune_blockchain": {{ prune_blockchain | tojson(indent=2) }},
     "monero_log_level": {{ monero_log_level | int | tojson(indent=2) }},
+    "expose_rpc_port": {{ expose_rpc_port | tojson(indent=2) }},
+    "rpc_port": {{ rpc_port | tojson(indent=2) }},
+    "rpc_login": {{ rpc_login | tojson(indent=2) }},
+    "limit_data_rates": {{ limit_data_rates | tojson(indent=2) }},
+    "rate_limit_up": {{ rate_limit_up | tojson(indent=2) }},
+    "rate_limit_down": {{ rate_limit_down | tojson(indent=2) }},
+    "sync_pruned_blocks": {{ sync_pruned_blocks | tojson(indent=2) }},
+    "fast_sync": {{ fast_sync | tojson(indent=2) }},
     "monero_options": {{ monero_options | tojson(indent=2) }},
     "public_monero_node": {{ public_monero_node | tojson(indent=2) }},
     "monero_node_login": {{ monero_node_login | tojson(indent=2) }},
@@ -18,5 +30,6 @@
     "use_fixed_difficulty": {{ use_fixed_difficulty | tojson(indent=2) }},
     "fixed_difficulty": {{ fixed_difficulty | int | tojson(indent=2) }},
     "cpu_percent": {{ cpu_percent | int | tojson(indent=2) }},
+    "cpu_priority": {{ cpu_priority | int | tojson(indent=2) }},
     "xmrig_options": {{ xmrig_options | tojson(indent=2) }}
 }

docker-compose/cfg/defaults

@@ -3,14 +3,26 @@
     "configure_monero": true,
     "configure_xmrig": true,
     "sidechain": [0],
+    "enable_statistics": true,
+    "statistics_port": "3380",
     "expose_stratum_port": false,
     "stratum_port": "3333",
     "p2pool_log_level": 3,
     "enable_autodiff": true,
+    "light_mode": false,
+    "no_cache": false,
     "p2pool_options": "",
     "monero_version": "latest",
     "prune_blockchain": true,
     "monero_log_level": 0,
+    "expose_rpc_port": false,
+    "rpc_port": "18081",
+    "rpc_login": "",
+    "limit_data_rates": false,
+    "rate_limit_up": "2048",
+    "rate_limit_down": "8192",
+    "sync_pruned_blocks": false,
+    "fast_sync": false,
     "monero_options": "",
     "public_monero_node": "",
     "monero_node_login": "",
@@ -18,5 +30,6 @@
     "use_fixed_difficulty": true,
     "fixed_difficulty": 500000,
     "cpu_percent": 100,
+    "cpu_priority": 2,
     "xmrig_options": ""
 }

docker-compose/cfg/docker-compose.jinja2

@@ -20,6 +20,7 @@ services:
     container_name: p2pool-p2pool
     networks:
       - p2pool
+    privileged: true
     ports:
 {% if sidechain[0] == 0 %}
       - 37888:37888/tcp
@@ -27,7 +28,7 @@ services:
       - 37889:37889/tcp
 {% endif %}
 {% if expose_stratum_port == True %}
-      - {{ stratum_port | int }}:3333/tcp
+      - {{ stratum_port }}:3333/tcp
 {% endif %}
     volumes:
       - p2pool:/home/p2pool/.p2pool:rw
@@ -53,11 +54,42 @@ services:
 {% if enable_autodiff == False %}
       --no-autodiff
 {% endif %}
+{% if enable_statistics == True %}
+      --local-api
+      --data-api /home/p2pool/.p2pool
+{% endif %}
+{% if light_mode == True %}
+      --light-mode
+{% endif %}
+{% if no_cache == True %}
+      --no-cache
+{% endif %}
+{% if rpc_login != "" %}
+      --rpc-login {{ rpc_login }}
+{% endif %}
 {% if p2pool_options != "" %}
       {{ p2pool_options }}
 {% endif %}
 
 
+{% if enable_statistics == True %}
+  statistics:
+    image: statistics:latest
+    build:
+      context: statistics
+    container_name: p2pool-statistics
+    networks:
+      - p2pool
+    ports:
+      - {{ statistics_port }}:80/tcp
+    volumes:
+      - p2pool:/data:ro
+    depends_on:
+      - p2pool
+    restart: unless-stopped
+{% endif %}
+
+
 {% if configure_monero == True %}
   monero:
     image: monero:latest
@@ -70,6 +102,9 @@ services:
       - p2pool
     ports:
       - 18080:18080/tcp
+{% if expose_rpc_port == True %}
+      - {{ rpc_port }}:18081/tcp
+{% endif %}
     volumes:
       - monero:/home/monero/.bitmonero:rw
       - /dev/null:/home/monero/.bitmonero/bitmonero.log:rw
@@ -84,10 +119,26 @@ services:
       --p2p-bind-port=18080
       --rpc-bind-ip=0.0.0.0
       --rpc-bind-port=18081
+      --restricted-rpc
       --confirm-external-bind
       --log-level={{ monero_log_level | int }}
 {% if prune_blockchain == True %}
       --prune-blockchain
+{% if sync_pruned_blocks == True %}
+      --sync-pruned-blocks
+{% endif %}
+{% endif %}
+{% if rpc_login != "" %}
+      --rpc-login {{ rpc_login }}
+{% endif %}
+{% if limit_data_rates == True %}
+      --limit-rate-up {{ rate_limit_up }}
+      --limit-rate-down {{ rate_limit_down }}
+{% endif %}
+{% if fast_sync == True %}
+      --fast-block-sync=1
+{% else %}
+      --fast-block-sync=0
 {% endif %}
 {% if monero_options != "" %}
       {{ monero_options }}
@@ -119,6 +170,7 @@ services:
       -u {{ xmrig_username }}
 {% endif %}
       --cpu-max-threads-hint={{ cpu_percent | int }}
+      --cpu-priority={{ cpu_priority | int }}
 {% if xmrig_options != "" %}
       {{ xmrig_options }}
 {% endif %}

docker-compose/configure

@@ -1,6 +1,45 @@
 #!/bin/bash
 echo ""
 echo ""
+echo "Verifying Requirements:"
+DOCKER_VER=$(docker version -f "{{.Server.Version}}" 2> /dev/null)
+if [ -z "$DOCKER_VER" ]; then
+    echo "Docker not found; install it: https://docs.docker.com/engine/install/"
+    exit 1
+fi
+if [ "$(echo "$DOCKER_VER"| cut -d'.' -f 1)" -ge 19 ] && \
+   [ "$(echo "$DOCKER_VER"| cut -d'.' -f 2)" -ge 0 ]  && \
+   [ "$(echo "$DOCKER_VER"| cut -d'.' -f 3)" -ge 3 ]; then
+    echo "Docker Found; OK"
+else
+    echo "Docker version less than 19.0.3; upgrade it: https://docs.docker.com/engine/install/"
+    exit 1
+fi
+docker compose version 2>&1 > /dev/null
+COMPOSE_PLUGIN_RC=$?
+docker-compose --version 2>&1 > /dev/null
+COMPOSE_CLI_RC=$?
+if [ "$COMPOSE_PLUGIN_RC" -eq 0 ] || [ "$COMPOSE_CLI_RC" -eq 0 ]; then
+    echo "Docker Compose found; OK"
+    if [ "$COMPOSE_PLUGIN_RC" -eq 0 ]; then
+        COMPOSE_COMMAND="docker compose"
+    else
+	COMPOSE_COMMAND="docker-compose"
+    fi
+else
+    echo "Docker Compose not found; install it: https://docs.docker.com/compose/install/compose-plugin/"
+    exit 1
+fi
+echo ""
+echo ""
 echo "Building and Running P2Pool docker-compose Configuration"
 docker build -t p2pool_config:latest cfg
 docker run -it --rm -v $PWD:/docker-compose --user $(id -u):$(id -g) p2pool_config:latest
+CONFIGURE_RC=$?
+echo ""
+echo ""
+if [ "$CONFIGURE_RC" -eq 0 ]; then
+    echo "P2Pool is configured.  Start the project with: $COMPOSE_COMMAND up --build -d"
+else
+    exit 1
+fi

docker-compose/current_config

@@ -5,14 +5,26 @@
     "sidechain": [
   0
 ],
+    "enable_statistics": true,
+    "statistics_port": "3380",
     "expose_stratum_port": false,
     "stratum_port": "3333",
     "p2pool_log_level": 3,
     "enable_autodiff": true,
+    "light_mode": false,
+    "no_cache": false,
     "p2pool_options": "",
     "monero_version": "latest",
     "prune_blockchain": true,
     "monero_log_level": 0,
+    "expose_rpc_port": false,
+    "rpc_port": "18081",
+    "rpc_login": "",
+    "limit_data_rates": false,
+    "rate_limit_up": "2048",
+    "rate_limit_down": "8192",
+    "sync_pruned_blocks": false,
+    "fast_sync": false,
     "monero_options": "",
     "public_monero_node": "",
     "monero_node_login": "",
@@ -20,5 +32,6 @@
     "use_fixed_difficulty": true,
     "fixed_difficulty": 500000,
     "cpu_percent": 100,
+    "cpu_priority": 2,
     "xmrig_options": ""
 }

docker-compose/docker-compose.yml

@@ -20,6 +20,7 @@ services:
     container_name: p2pool-p2pool
     networks:
       - p2pool
+    privileged: true
     ports:
 
       - 37888:37888/tcp
@@ -31,7 +32,8 @@ services:
       - /dev/hugepages:/dev/hugepages:rw
 
     depends_on:
-      - monero
+      monero:
+        condition: service_healthy
 
     restart: unless-stopped
     command: >-
@@ -43,6 +45,31 @@ services:
 
 
 
+      --local-api
+      --data-api /home/p2pool/.p2pool
+
+
+
+
+
+
+
+
+  statistics:
+    image: statistics:latest
+    build:
+      context: statistics
+    container_name: p2pool-statistics
+    networks:
+      - p2pool
+    ports:
+      - 3380:80/tcp
+    volumes:
+      - p2pool:/data:ro
+    depends_on:
+      - p2pool
+    restart: unless-stopped
+
 
 
 
@@ -57,11 +84,17 @@ services:
       - p2pool
     ports:
       - 18080:18080/tcp
+
     volumes:
       - monero:/home/monero/.bitmonero:rw
       - /dev/null:/home/monero/.bitmonero/bitmonero.log:rw
       - /dev/hugepages:/dev/hugepages:rw
     restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "nc", "-z", "localhost", "18081"]
+      interval: 2s
+      timeout: 1s
+      start_period: 10s
     command: >-
       --zmq-pub tcp://0.0.0.0:18083
       --disable-dns-checkpoints
@@ -71,6 +104,7 @@ services:
       --p2p-bind-port=18080
       --rpc-bind-ip=0.0.0.0
       --rpc-bind-port=18081
+      --restricted-rpc
       --confirm-external-bind
       --log-level=0
 
@@ -80,6 +114,12 @@ services:
 
 
 
+      --fast-block-sync=0
+
+
+
+
+
 
   xmrig:
     image: xmrig:latest
@@ -102,4 +142,5 @@ services:
       -u p2pool
 
       --cpu-max-threads-hint=100
+      --cpu-priority=2
 

docker-compose/monero/Dockerfile

@@ -62,6 +62,7 @@ RUN set -e && \
     apt-get update -q -y --no-install-recommends && \
     DEBIAN_FRONTEND="noninteractive" apt-get install -q -y --no-install-recommends \
     libgssapi-krb5-2 \
+    netcat \
       && \
     apt-get clean && \
     rm -rf /var/lib/apt

docker-compose/statistics/Dockerfile

@@ -0,0 +1,8 @@
+FROM python:slim
+
+COPY app /app
+
+WORKDIR /app
+RUN pip install -r requirements.txt
+
+CMD ["/app/p2pool_statistics.py"]

docker-compose/statistics/app/p2pool_statistics.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+import json
+from datetime import datetime
+from prefixed import Float
+import humanfriendly
+from flask import Flask, render_template
+
+app = Flask(__name__)
+
+
+##
+# Add some custom jinja filters
+def timeago(value):
+    """Format a date time to human friendly time ago"""
+    if value is None:
+        return ""
+    if type(value) is int:
+        dt = datetime.fromtimestamp(value).replace(microsecond=0)
+    now = datetime.now().replace(microsecond=0)
+    return humanfriendly.format_timespan(now - dt)
+
+
+app.jinja_env.filters["timeago"] = timeago
+
+
+def human_numbers(value):
+    """Format a number in human readable format"""
+    if value is None:
+        return ""
+    return "{:!.3h}".format(Float(value))
+
+
+app.jinja_env.filters["humanize"] = human_numbers
+
+
+##
+# Get Pool Instance Birth Date
+def birthdate():
+    try:
+        with open("/data/p2pool.blocks") as reader:
+            first_block = reader.readline().rstrip()
+        bday_ts = int(first_block.split(" ")[0])
+        bday = timeago(bday_ts)
+        return bday
+    except Exception as e:
+        return "unknown time"
+
+
+##
+# The App Routes
+@app.route("/")
+def render():
+    try:
+        my_bday = birthdate()
+        with open("/data/stats_mod", "r") as reader:
+            stats_mod = json.loads(reader.read())
+        with open("/data/pool/stats", "r") as reader:
+            pool_stats = json.loads(reader.read())
+        with open("/data/network/stats", "r") as reader:
+            network_stats = json.loads(reader.read())
+        with open("/data/local/stats", "r") as reader:
+            local_stats = json.loads(reader.read())
+        return render_template(
+            "index.html",
+            my_bday=my_bday,
+            stats_mod=stats_mod,
+            pool_stats=pool_stats,
+            network_stats=network_stats,
+            local_stats=local_stats,
+        )
+    except Exception as e:
+        return render_template("oops.html", error=str(e))
+
+
+##
+# main()
+if __name__ == "__main__":
+    app.run(debug=False, host="0.0.0.0", port=80)

docker-compose/statistics/app/requirements.txt

@@ -0,0 +1,3 @@
+flask
+prefixed
+humanfriendly

docker-compose/statistics/app/templates/index.html

@@ -0,0 +1,158 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <link rel="stylesheet" href="{{url_for('static', filename='bootstrap.min.css')}}">
+    <title>Monero P2Pool Server Statistics</title>
+  </head>
+  <body style="font-size:85%;">
+    <script src="{{url_for('static', filename='jquery-3.2.1.slim.min.js')}}"></script>
+    <script src="{{url_for('static', filename='popper.min.js')}}"></script>
+    <script src="{{url_for('static', filename='bootstrap.min.js')}}"></script>
+    <div style="text-align:center;">
+      <img src="{{url_for('static', filename='monero-symbol-480.png')}}" width="75px" height="75px" alt="Monero"/>
+      <h1 style="color: #FFA500;">P2Pool Server Statistics</h1>
+    </div>
+ 
+    <div class="card-group">
+      <div class="col-sm-4 grid-margin stretch-card">
+        <div class="card">
+          <div class="card-header text-black mb-1 pb-1" style="background-color: #FFA500;"><h2>Local Pool</h2></div>
+          <div class="card-body mb-0 pb-0">
+            <h6 class="card-subtitle text-muted" style="font-size:80%;">(note: stats reset on restart)</h6>
+            <div class="table-responsive table-hover table-condensed table-striped">
+              <table class="table">
+                <tbody>
+                  <tr>
+                    <td>Hashrate 15 Minutes</td>
+                    <td>{{ local_stats["hashrate_15m"]|humanize }}H/s</td>
+                  </tr>
+                  <tr>
+                    <td>Hashrate 1 Hour</td>
+                    <td>{{ local_stats["hashrate_1h"]|humanize }}H/s</td>
+                  </tr>
+                  <tr>
+                    <td>Hashrate 24 Hours</td>
+                    <td>{{ local_stats["hashrate_24h"]|humanize }}H/s</td>
+                  </tr>
+                  <tr>
+                    <td>Shares Found</td>
+                    <td>{{ local_stats["shares_found"] }}</td>
+                  </tr>
+                  <tr>
+                    <td>Current Effort</td>
+                    <td>{{ local_stats["current_effort"] }}%</td>
+                  </tr>
+                  <tr>
+                    <td>Average Effort</td>
+                    <td>{{ local_stats["average_effort"] }}%</td>
+                  </tr>
+                  <tr>
+                    <td>Total Hashes</td>
+                    <td>{{ local_stats["total_hashes"]|humanize }}</td>
+                  </tr>
+                  <tr>
+                    <td>Miner Connections</td>
+                    <td>{{ local_stats["incoming_connections"] }}<td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+          </div>
+        </div>
+      </div>
+  
+      <div class="col-sm-4 grid-margin stretch-card">
+        <div class="card">
+          <div class="card-header text-black mb-1 pb-1" style="background-color: #FFA500;"><h2>Global Pool</h2></div>
+          <div class="card-body mb-0 pb-0">
+            <h6 class="card-subtitle text-muted" style="font-size:80%;">&nbsp;</h6>
+            <div class="table-responsive table-hover table-condensed table-striped">
+              <table class="table">
+                <tbody>
+                  <tr>
+                    <td>Hash Rate</td>
+                    <td>{{ pool_stats["pool_statistics"]["hashRate"]|humanize }}H/s</td>
+                  </tr>
+                  <tr>
+                    <td>Round Hashes</td>
+                    <td>{{ stats_mod["pool"]["roundHashes"]|humanize }}</td>
+                  </tr>
+                  <tr>
+                    <td>Last Block Found</td>
+                    <td>{{ pool_stats["pool_statistics"]["lastBlockFound"] }} <br>
+                        {{ pool_stats["pool_statistics"]["lastBlockFoundTime"]|timeago }} ago</td>
+                  </tr>
+                  <tr>
+                    <td>Payout Method</td>
+                    <td>{{ pool_stats["pool_list"]|join(',') }}<br>2160 block window (~6 hours)</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+            <h6 class="card-subtitle text-muted" style="font-size:80%;">(since instance birth: {{ my_bday }} ago)</h6>
+            <div class="table-responsive table-hover table-condensed table-striped">
+              <table class="table">
+                <tbody>
+                  <tr>
+                    <td>Total Hashes</td>
+                    <td>{{ pool_stats["pool_statistics"]["totalHashes"]|humanize }}</td>
+                  </tr>
+                  <tr>
+                    <td>Blocks Found</td>
+                    <td>{{ pool_stats["pool_statistics"]["totalBlocksFound"] }}</td>
+                  </tr>
+                  <tr>
+                    <td>Known Miners</td>
+                    <td>{{ pool_stats["pool_statistics"]["miners"] }}</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+          </div>
+        </div>
+      </div>
+  
+      <div class="col-sm-4 grid-margin stretch-card">
+        <div class="card">
+          <div class="card-header text-black mb-1 pb-1" style="background-color: #FFA500;"><h2>Monero Network</h2></div>
+          <div class="card-body mb-0 pb-0">
+            <h6 class="card-subtitle text-muted" style="font-size:80%;">&nbsp;</h6>
+            <div class="table-responsive table-hover table-condensed table-striped">
+              <table class="table">
+                <tbody>
+                  <tr>
+                    <td>Height</td>
+                    <td>{{ network_stats["height"] }}<br>
+                        {{ network_stats["timestamp"]|timeago }} ago</td>
+                  </tr>
+                  <tr>
+                    <td>Difficulty</td>
+                    <td>{{ network_stats["difficulty"]|humanize }}</td>
+                  </tr>
+                  <tr>
+                    <td>Reward</td>
+                    <td>0.{{ network_stats["reward"] }} Monero</td>
+                  </tr>
+                  <tr>
+                    <td>Head&nbsp;Hash</td>
+                    <td>{{ network_stats["hash"] }}</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
+
+    <script>
+      window.setInterval('refresh()', 30000); 	// Call refresh function every 30000 milliseconds (30 seconds).
+      function refresh() {
+          window .location.reload();
+      }
+    </script>
+  
+  </body>
+</html>

docker-compose/statistics/app/templates/oops.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title>Oops</title>
+  </head>
+  <body>
+    <h2>Ooops, something went wrong:</h2>
+    {{ error }}
+    (Maybe you need to wait a few minutes for p2pool to start and sync for the first time?)
+  </body>
+</html>