Snark Worker Opt Protocol Update - dev

.gitignore

@@ -13,7 +13,6 @@ result
 *.aux
 src/config.mlh
 src/config/config.mlh
-src/external/ocaml-rocksdb/rocksdb
 .stack-work/
 /wallet-keys/
 test_output

Makefile

@@ -53,6 +53,32 @@ LIBP2P_HELPER_SIG := $(shell cd src/app/libp2p_helper ; find . -type f -print0
 help: ## Display this help information
 	@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
+
+########################################
+.PHONY: add-o1labs-opam-repo
+add-o1labs-opam-repo:
+	opam repository add --yes --all --set-default o1-labs https://github.com/o1-labs/opam-repository.git
+
+.PHONY: prepare
+prepare: add-o1labs-opam-repo
+	@echo "Preparing the environment and installing dependencies..."
+	@# Check Go installation and version
+	@command -v go >/dev/null 2>&1 || { echo >&2 "Error: Go is not installed. Please install Go before continuing. You can use gvm to install the appropriate Go environment."; exit 1; }
+	@GO_VERSION=$$(go version | awk '{print $$3}' | sed 's/go//'); \
+	GO_MOD_PATH="src/app/libp2p_helper/src/go.mod"; \
+	REQUIRED_GO_VERSION=$$(grep -E "^go [0-9]+\.[0-9]" $$GO_MOD_PATH | awk '{print $$2}'); \
+	if ! printf '%s\n%s\n' "$$REQUIRED_GO_VERSION" "$$GO_VERSION" | sort -V | head -n1 | grep -q "^$$REQUIRED_GO_VERSION$$"; then \
+		echo "Error: Go version $$GO_VERSION is not compatible. Required version is $$REQUIRED_GO_VERSION or newer (only minor)."; \
+		exit 1; \
+	fi; \
+	echo "Go version $$GO_VERSION detected (requirement: $$REQUIRED_GO_VERSION or newer (only minor))"
+	opam switch import --switch mina --yes opam.export
+	eval $(opam env --switch=mina --set-switch)
+	chmod +x scripts/pin-external-packages.sh
+	./scripts/pin-external-packages.sh
+	@echo "Environment prepared. You can now run 'make build' to build the project."
+
+
 ########################################
 ## Code
 
@@ -553,6 +579,14 @@ debian-build-rosetta-devnet: ## Build the Debian Rosetta package for devnet
 debian-build-rosetta-mainnet: ## Build the Debian Rosetta package for mainnet
 	$(call build_debian_package,rosetta_mainnet)
 
+.PHONY: debian-build-daemon-devnet-hardfork
+debian-build-daemon-devnet-hardfork: ## Build the Debian daemon package for devnet hardfork
+	$(call build_debian_package,daemon_devnet_hardfork)
+
+.PHONY: debian-download-create-legacy-hardfork
+debian-download-create-legacy-hardfork: ## Download and create legacy hardfork Debian packages
+	$(info 📦 Downloading legacy hardfork Debian packages for debian $(CODENAME))
+	@./buildkite/scripts/release/manager.sh pull --artifacts mina-create-legacy-genesis  --from-special-folder legacy/debians/$(CODENAME)  --backend hetzner --target _build
 ########################################
 # Docker images
 
@@ -649,6 +683,45 @@ docker-build-rosetta-mainnet: SHELL := /bin/bash
 docker-build-rosetta-mainnet: start-local-debian-repo ## Build the Rosetta Docker image for mainnet
 	$(call build_docker_image,mina-rosetta,mainnet)
 
+########################################
+# Generate hardfork packages
+
+.PHONY: hardfork-debian
+hardfork-debian: SHELL := /bin/bash
+hardfork-debian: ocaml_checks ## Generate hardfork packages
+	$(info 📦 Generating hardfork packages for network $(NETWORK_NAME))
+
+	@BUILD_DIR=./_build \
+	MINA_DEB_CODENAME=$(CODENAME) \
+	BRANCH_NAME=$(BRANCH_NAME) \
+	KEEP_MY_TAGS_INTACT=true \
+	./scripts/hardfork/build-packages.sh daemon_devnet_hardfork
+
+
+.PHONY: hardfork-docker
+hardfork-docker: SHELL := /bin/bash
+hardfork-docker: ocaml_checks ## Generate hardfork packages
+	$(info 📦 Generating hardfork docker for network $(NETWORK_NAME))
+
+	$(MAKE) hardfork-debian
+	$(MAKE) start-local-debian-repo
+
+	@BUILD_DIR=./_build \
+	MINA_DEB_CODENAME=$(CODENAME) \
+	KEEP_MY_TAGS_INTACT=true \
+	. ./scripts/export-git-env-vars.sh \
+	&& ./scripts/docker/build.sh \
+		--deb-codename $(CODENAME) \
+		--service mina-daemon \
+		--version "$$MINA_DEB_VERSION" \
+		--branch $(BRANCH_NAME) \
+		--network $(NETWORK_NAME) \
+		--deb-suffix hardfork \
+		--no-cache
+
+	$(info 📦 stopping local Debian repository)
+	@./scripts/debian/aptly.sh stop
+
 ########################################
 # Generate odoc documentation
 

buildkite/scripts/build-release.sh

@@ -4,9 +4,6 @@ set -eo pipefail
 
 ([ -z ${DUNE_PROFILE+x} ] || [ -z ${MINA_DEB_CODENAME+x} ]) && echo "required env vars were not provided" && exit 1
 
-# shellcheck disable=SC1090
-source ~/.profile
-
 ./buildkite/scripts/build-artifact.sh
 
 echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}"

buildkite/scripts/entrypoints/run-hardfork-package-gen.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Generate Hardfork Package Build Script
+#
+# This script generates Dhall configuration for creating hardfork packages in the Mina protocol
+# build system. It converts input parameters to Dhall format and outputs YAML configuration
+# for the buildkite pipeline.
+#
+# USAGE:
+#   ./run-hardfork-package-gen.sh
+#
+# REQUIRED ENVIRONMENT VARIABLES:
+#   CODENAMES           - Comma-separated list of Debian codenames (e.g., "Bullseye,Focal")
+#   NETWORK             - Target network name (e.g., "Devnet", "Mainnet")
+#   GENESIS_TIMESTAMP   - Genesis timestamp in ISO format (e.g., "2024-04-07T11:45:00Z")
+#   CONFIG_JSON_GZ_URL  - URL to the gzipped genesis config JSON file
+#   VERSION             - Version string for the hardfork package
+#
+# EXAMPLE:
+#   export CODENAMES="Bullseye,Focal"
+#   export NETWORK="Devnet"
+#   export GENESIS_TIMESTAMP="2024-04-07T11:45:00Z"
+#   export CONFIG_JSON_GZ_URL="https://example.com/config.json.gz"
+#   export VERSION="3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e"
+#   ./run-hardfork-package-gen.sh
+#
+# OUTPUT:
+#   YAML configuration for buildkite pipeline generation
+#
+# DEPENDENCIES:
+#   - dhall-to-yaml command-line tool
+#   - Dhall configuration files in buildkite/src/
+#
+
+
+DEBIAN_VERSION_DHALL_DEF="(./buildkite/src/Constants/DebianVersions.dhall)"
+NETWORK_DHALL_DEF="(./buildkite/src/Constants/Network.dhall)"
+GENERATE_HARDFORK_PACKAGE_DHALL_DEF="(./buildkite/src/Entrypoints/GenerateHardforkPackage.dhall)"
+
+function usage() {
+  if [[ -n "$1" ]]; then
+    echo -e "${RED}☞  $1${CLEAR}\n";
+  fi
+  cat << EOF
+  CODENAMES                   The Debian codenames (Bullseye, Focal etc.)
+  NETWORK                     The Docker and Debian network (Devnet, Mainnet)
+  GENESIS_TIMESTAMP           The Genesis timestamp in ISO format (e.g. 2024-04-07T11:45:00Z)
+  CONFIG_JSON_GZ_URL          The URL to the gzipped genesis config JSON file
+  VERSION                     The version of the hardfork package to generate (e.g. 3.0.0devnet-tooling-dkijania-hardfork-package-gen-in-nightly-b37f50e)
+
+EOF
+  exit 1
+}
+
+function to_dhall_list() {
+  local input_str="$1"
+  local dhall_type="$2"
+  local arr=("${input_str//,/ }")
+  local dhall_list=""
+
+  if [[ ${#arr[@]} -eq 0 || -z "${arr[0]}" ]]; then
+    dhall_list="([] : List $dhall_type)"
+  elif [[ ${#arr[@]} -eq 1 ]]; then
+    dhall_list="[$dhall_type.${arr[0]}]"
+  else
+    for i in "${arr[@]}"; do
+      dhall_list="${dhall_list}, $dhall_type.${i}"
+    done
+    dhall_list="[${dhall_list:1}]"
+  fi
+
+  echo "$dhall_list"
+}
+
+DHALL_CODENAMES=$(to_dhall_list "$CODENAMES" "$DEBIAN_VERSION_DHALL_DEF.DebVersion")
+
+echo $GENERATE_HARDFORK_PACKAGE_DHALL_DEF'.generate_hardfork_package '"$DHALL_CODENAMES"' '$NETWORK_DHALL_DEF'.Type.'"${NETWORK}"' (None Text) "'"${CONFIG_JSON_GZ_URL}"'" "'""'" ' | dhall-to-yaml --quoted

buildkite/scripts/generate-genesis-config.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+([ -z ${CONFIG_JSON_GZ_URL+x} ]) && echo "required env vars were not provided" && exit 1
+
+echo "--- Download and extract previous network config"
+curl -o config.json.gz $CONFIG_JSON_GZ_URL
+gunzip config.json.gz
+
+# Patch against a bug in 1.4 which is fixed by PR #15462
+mv config.json config_orig.json
+jq 'del(.ledger.num_accounts) | del(.ledger.name)' config_orig.json > config.json 
+
+echo "--- Migrate accounts to new network format"
+# TODO: At this stage, we need to migrate the json accounts into the new network's format.
+#       For now, this is hard-coded to the mainnet -> berkeley migration, but we need to select
+#       a migration to perform in the future.
+# NB: we use sed here instead of jq, because jq is extremely slow at processing this file
+sed -i -e 's/"set_verification_key": "signature"/"set_verification_key": {"auth": "signature", "txn_version": "2"}/' config.json
+
+echo "--- Generate hardfork ledger tarballs"
+mkdir hardfork_ledgers
+mina-create-genesis --config-file config.json --genesis-dir hardfork_ledgers/ --hash-output-file hardfork_ledger_hashes.json | tee runtime_genesis_ledger.log | mina-logproc
+
+echo "--- Create hardfork config"
+FORK_CONFIG_JSON=config.json LEDGER_HASHES_JSON=hardfork_ledger_hashes.json scripts/hardfork/create_runtime_config.sh > new_config.json
+
+echo "--- New genesis config"
+jq 'del(.ledger.s3_data_hash, .epoch_data.staking.s3_data_hash, .epoch_data.next.s3_data_hash)' new_config.json

buildkite/scripts/hardfork/build-packages.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# Sourcing profile as this is where the environment variables are set in toolchain container
+# Thus, this script should be ran only in buildkite context using toolchain docker
+# shellcheck disable=SC1090
+source ~/.profile
+
+export BRANCH_NAME=$BUILDKITE_BRANCH
+
+./scripts/hardfork/build-packages.sh "$@"
+
+echo "--- Git diff after build is complete:"
+git diff --exit-code -- .

buildkite/scripts/release/manager.sh

@@ -1708,6 +1708,7 @@ function pull(){
     local __target="."
     local __codenames="$DEFAULT_CODENAMES"
     local __networks="$DEFAULT_NETWORKS"
+    local __from_special_folder
 
     while [ ${#} -gt 0 ]; do
         error_message="Error: a value is needed for '$1'";
@@ -1731,6 +1732,10 @@ function pull(){
                 __buildkite_build_id=${2:?$error_message}
                 shift 2;
             ;;
+            --from-special-folder )
+                __from_special_folder=${2:?$error_message}
+                shift 2;
+            ;;
             --target )
                 __target=${2:?$error_message}
                 shift 2;
@@ -1747,20 +1752,27 @@ function pull(){
         esac
     done
 
+    if [[ -z ${__buildkite_build_id+x} && -z ${__from_special_folder+x} ]]; then
+        echo -e "❌ ${RED} !! Buildkite build id (--buildkite-build-id) is required${CLEAR}\n";
+        pull_help; exit 1;
+    fi
+
+
     echo ""
     echo " ℹ️  Pulling mina artifacts with following parameters:"
     echo " - Backend: $__backend"
     echo " - Artifacts: $__artifacts"
-    echo " - Buildkite build id: $__buildkite_build_id"
     echo " - Target: $__target"
     echo " - Codenames: $__codenames"
     echo " - Networks: $__networks"
-
-    if [[ -z ${__buildkite_build_id+x} ]]; then
-        echo -e "❌ ${RED} !! Buildkite build id (--buildkite-build-id) is required${CLEAR}\n";
-        pull_help; exit 1;
+    if [[ -n ${__from_special_folder+x} ]]; then
+        echo " - From special folder: $__from_special_folder"
     fi
-
+    if [[ -n ${__buildkite_build_id+x} ]]; then
+        echo " - Buildkite build id: $__buildkite_build_id"
+    fi
+
+
     IFS=', '
     read -r -a __artifacts_arr <<< "$__artifacts"
     read -r -a __codenames_arr <<< "$__codenames"
@@ -1771,8 +1783,16 @@ function pull(){
             for network in "${__networks_arr[@]}"; do
                 echo "  📥  Pulling $__artifact for $__codename codename and $network network"
                 local __artifact_full_name
+                local __source_path
                 __artifact_full_name=$(get_artifact_with_suffix $__artifact $network)
-                storage_download "$__backend" "$(storage_root "$__backend")/$__buildkite_build_id/debians/$__codename/${__artifact_full_name}_*" "$__target"
+
+                if [[ -n ${__from_special_folder+x} ]]; then
+                    __source_path="$(storage_root "$__backend")/$__from_special_folder/${__artifact_full_name}_*"
+                else
+                    __source_path="$(storage_root "$__backend")/$__buildkite_build_id/debians/$__codename/${__artifact_full_name}_*"
+                fi
+
+                storage_download "$__backend" "$__source_path" "$__target"
             done
         done
     done

buildkite/src/Command/DockerImage.dhall

@@ -44,6 +44,7 @@ let ReleaseSpec =
           , deb_release : Text
           , deb_version : Text
           , deb_legacy_version : Text
+          , deb_suffix : Optional Text
           , deb_profile : Profiles.Type
           , deb_repo : DebianRepo.Type
           , build_flags : BuildFlags.Type
@@ -71,6 +72,7 @@ let ReleaseSpec =
           , no_debian = False
           , step_key_suffix = "-docker-image"
           , verify = False
+          , deb_suffix = None Text
           , if = None B/If
           }
       }
@@ -128,8 +130,14 @@ let generateStep =
                           "${BuildFlags.toSuffixLowercase spec.build_flags}"
                       }
                       spec.build_flags
+                  , spec.deb_suffix
                   ]
 
+          let debSuffix =
+                merge
+                  { None = "", Some = \(s : Text) -> " --deb-suffix " ++ s }
+                  spec.deb_suffix
+
           let maybeVerify =
                       if     spec.verify
                          &&  DockerPublish.shouldPublish
@@ -167,6 +175,7 @@ let generateStep =
                 ++  " --deb-build-flags ${BuildFlags.lowerName
                                             spec.build_flags}"
                 ++  " --deb-legacy-version ${spec.deb_legacy_version}"
+                ++  debSuffix
                 ++  " --repo ${spec.repo}"
 
           let releaseDockerCmd =