feat: sentry for arm

Signed-off-by: 小草林(田梓萱) <xcl@xuegao-tzx.top>

install/_lib.sh

@@ -0,0 +1,55 @@
+set -euo pipefail
+test "${DEBUG:-}" && set -x
+
+# Override any user-supplied umask that could cause problems, see #1222
+umask 002
+
+# Thanks to https://unix.stackexchange.com/a/145654/108960
+log_file=sentry_install_log-$(date +'%Y-%m-%d_%H-%M-%S').txt
+exec &> >(tee -a "$log_file")
+
+# Allow `.env` overrides using the `.env.custom` file.
+# We pass this to docker compose in a couple places.
+if [[ -f .env.custom ]]; then
+  _ENV=.env.custom
+else
+  _ENV=.env
+fi
+
+# Read .env for default values with a tip o' the hat to https://stackoverflow.com/a/59831605/90297
+t=$(mktemp) && export -p >"$t" && set -a && . $_ENV && set +a && . "$t" && rm "$t" && unset t
+
+if [ "${GITHUB_ACTIONS:-}" = "true" ]; then
+  _group="::group::"
+  _endgroup="::endgroup::"
+else
+  _group="▶ "
+  _endgroup=""
+fi
+
+# A couple of the config files are referenced from other subscripts, so they
+# get vars, while multiple subscripts call ensure_file_from_example.
+function ensure_file_from_example {
+  target="$1"
+  if [[ -f "$target" ]]; then
+    echo "$target already exists, skipped creation."
+  else
+    # sed from https://stackoverflow.com/a/25123013/90297
+    example="$(echo "$target" | sed 's/\.[^.]*$/.example&/')"
+    if [[ ! -f "$example" ]]; then
+      echo "Oops! Where did $example go? 🤨 We need it in order to create $target."
+      exit
+    fi
+    echo "Creating $target ..."
+    cp -n "$example" "$target"
+  fi
+}
+
+SENTRY_CONFIG_PY=sentry/sentry.conf.py
+SENTRY_CONFIG_YML=sentry/config.yml
+
+# Increase the default 10 second SIGTERM timeout
+# to ensure celery queues are properly drained
+# between upgrades as task signatures may change across
+# versions
+STOP_TIMEOUT=60 # seconds

install/_min-requirements.sh

@@ -0,0 +1,9 @@
+# Don't forget to update the README and other docs when you change these!
+MIN_DOCKER_VERSION='19.03.6'
+MIN_COMPOSE_VERSION='2.19.0'
+
+# 16 GB minimum host RAM, but there'll be some overhead outside of what
+# can be allotted to docker
+MIN_RAM_HARD=14000 # MB
+
+MIN_CPU_HARD=4

install/bootstrap-snuba.sh

@@ -0,0 +1,6 @@
+echo "${_group}Bootstrapping and migrating Snuba ..."
+
+$dcr snuba-api bootstrap --no-migrate --force
+$dcr snuba-api migrations migrate --force
+
+echo "${_endgroup}"

install/build-docker-images.sh

@@ -0,0 +1,14 @@
+echo "${_group}Building and tagging Docker images ..."
+
+echo ""
+# Build any service that provides the image sentry-self-hosted-local first,
+# as it is used as the base image for sentry-cleanup-self-hosted-local.
+$dcb --force-rm web
+# Build each other service individually to localize potential failures better.
+for service in $($dc config --services); do
+  $dcb --force-rm "$service"
+done
+echo ""
+echo "Docker images built."
+
+echo "${_endgroup}"

install/check-latest-commit.sh

@@ -0,0 +1,15 @@
+echo "${_group}Checking for latest commit ... "
+
+# Checks if we are on latest commit from github if it is running from master branch
+if [[ -d "../.git" && "${SKIP_COMMIT_CHECK:-0}" != 1 ]]; then
+  if [[ $(git branch --show-current) == "master" ]]; then
+    if [[ $(git rev-parse HEAD) != $(git ls-remote $(git rev-parse --abbrev-ref @{u} | sed 's/\// /g') | cut -f1) ]]; then
+      echo "Seems like you are not using the latest commit from the self-hosted repository. Please pull the latest changes and try again, or suppress this check with --skip-commit-check."
+      exit 1
+    fi
+  fi
+else
+  echo "skipped"
+fi
+
+echo "${_endgroup}"

install/check-memcached-backend.sh

@@ -0,0 +1,16 @@
+echo "${_group}Checking memcached backend ..."
+
+if grep -q "\.PyMemcacheCache" "$SENTRY_CONFIG_PY"; then
+  echo "PyMemcacheCache found in $SENTRY_CONFIG_PY, gonna assume you're good."
+else
+  if grep -q "\.MemcachedCache" "$SENTRY_CONFIG_PY"; then
+    echo "MemcachedCache found in $SENTRY_CONFIG_PY, you should switch to PyMemcacheCache."
+    echo "See:"
+    echo "  https://develop.sentry.dev/self-hosted/releases/#breaking-changes"
+    exit 1
+  else
+    echo 'Your setup looks weird. Good luck.'
+  fi
+fi
+
+echo "${_endgroup}"

install/check-minimum-requirements.sh

@@ -0,0 +1,59 @@
+echo "${_group}Checking minimum requirements ..."
+
+source install/_min-requirements.sh
+
+# Check the version of $1 is greater than or equal to $2 using sort. Note: versions must be stripped of "v"
+function vergte() {
+  printf "%s\n%s" $1 $2 | sort --version-sort --check=quiet --reverse
+  echo $?
+}
+
+DOCKER_VERSION=$(docker version --format '{{.Server.Version}}' || echo '')
+if [[ -z "$DOCKER_VERSION" ]]; then
+  echo "FAIL: Unable to get docker version, is the docker daemon running?"
+  exit 1
+fi
+
+if [[ "$(vergte ${DOCKER_VERSION//v/} $MIN_DOCKER_VERSION)" -eq 1 ]]; then
+  echo "FAIL: Expected minimum docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION"
+  exit 1
+fi
+echo "Found Docker version $DOCKER_VERSION"
+
+COMPOSE_VERSION=$($dc_base version --short || echo '')
+if [[ -z "$COMPOSE_VERSION" ]]; then
+  echo "FAIL: Docker compose is required to run self-hosted"
+  exit 1
+fi
+
+if [[ "$(vergte ${COMPOSE_VERSION//v/} $MIN_COMPOSE_VERSION)" -eq 1 ]]; then
+  echo "FAIL: Expected minimum $dc_base version to be $MIN_COMPOSE_VERSION but found $COMPOSE_VERSION"
+  exit 1
+fi
+echo "Found Docker Compose version $COMPOSE_VERSION"
+
+CPU_AVAILABLE_IN_DOCKER=$(docker run --rm busybox nproc --all)
+if [[ "$CPU_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_HARD" ]]; then
+  echo "FAIL: Required minimum CPU cores available to Docker is $MIN_CPU_HARD, found $CPU_AVAILABLE_IN_DOCKER"
+  exit 1
+fi
+
+RAM_AVAILABLE_IN_DOCKER=$(docker run --rm busybox free -m 2>/dev/null | awk '/Mem/ {print $2}')
+if [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM_HARD" ]]; then
+  echo "FAIL: Required minimum RAM available to Docker is $MIN_RAM_HARD MB, found $RAM_AVAILABLE_IN_DOCKER MB"
+  exit 1
+fi
+
+#SSE4.2 required by Clickhouse (https://clickhouse.yandex/docs/en/operations/requirements/)
+# On KVM, cpuinfo could falsely not report SSE 4.2 support, so skip the check. https://github.com/ClickHouse/ClickHouse/issues/20#issuecomment-226849297
+# This may also happen on other virtualization software such as on VMWare ESXi hosts.
+IS_KVM=$(docker run --rm busybox grep -c 'Common KVM processor' /proc/cpuinfo || :)
+if [[ ! "$SKIP_SSE42_REQUIREMENTS" -eq 1 && "$IS_KVM" -eq 0 && "$DOCKER_ARCH" = "x86_64" ]]; then
+  SUPPORTS_SSE42=$(docker run --rm busybox grep -c sse4_2 /proc/cpuinfo || :)
+  if [[ "$SUPPORTS_SSE42" -eq 0 ]]; then
+    echo "FAIL: The CPU your machine is running on does not support the SSE 4.2 instruction set, which is required for one of the services Sentry uses (Clickhouse). See https://github.com/getsentry/self-hosted/issues/340 for more info."
+    exit 1
+  fi
+fi
+
+echo "${_endgroup}"

install/create-docker-volumes.sh

@@ -0,0 +1,10 @@
+echo "${_group}Creating volumes for persistent storage ..."
+
+echo "Created $(docker volume create --name=sentry-clickhouse)."
+echo "Created $(docker volume create --name=sentry-data)."
+echo "Created $(docker volume create --name=sentry-kafka)."
+echo "Created $(docker volume create --name=sentry-postgres)."
+echo "Created $(docker volume create --name=sentry-redis)."
+echo "Created $(docker volume create --name=sentry-symbolicator)."
+
+echo "${_endgroup}"

install/dc-detect-version.sh

@@ -0,0 +1,23 @@
+if [ "${GITHUB_ACTIONS:-}" = "true" ]; then
+  _group="::group::"
+  _endgroup="::endgroup::"
+else
+  _group="▶ "
+  _endgroup=""
+fi
+
+echo "${_group}Initializing Docker Compose ..."
+
+# To support users that are symlinking to docker-compose
+dc_base="$(docker compose version &>/dev/null && echo 'docker compose' || echo 'docker-compose')"
+if [[ "$(basename $0)" = "install.sh" ]]; then
+  dc="$dc_base --ansi never --env-file ${_ENV}"
+else
+  dc="$dc_base --ansi never"
+fi
+proxy_args="--build-arg http_proxy=${http_proxy:-} --build-arg https_proxy=${https_proxy:-} --build-arg no_proxy=${no_proxy:-}"
+dcr="$dc run --rm"
+dcb="$dc build $proxy_args"
+dbuild="docker build $proxy_args"
+
+echo "${_endgroup}"

install/detect-platform.sh

@@ -0,0 +1,31 @@
+echo "${_group}Detecting Docker platform"
+
+# Sentry SaaS uses stock Yandex ClickHouse, but they don't provide images that
+# support ARM, which is relevant especially for Apple M1 laptops, Sentry's
+# standard developer environment. As a workaround, we use an altinity image
+# targeting ARM.
+#
+# See https://github.com/getsentry/self-hosted/issues/1385#issuecomment-1101824274
+#
+# Images built on ARM also need to be tagged to use linux/arm64 on Apple
+# silicon Macs to work around an issue where they are built for
+# linux/amd64 by default due to virtualization.
+# See https://github.com/docker/cli/issues/3286 for the Docker bug.
+
+if ! command -v docker &>/dev/null; then
+  echo "FAIL: Could not find a \`docker\` binary on this system. Are you sure it's installed?"
+  exit 1
+fi
+
+export DOCKER_ARCH=$(docker info --format '{{.Architecture}}')
+if [[ "$DOCKER_ARCH" = "x86_64" ]]; then
+  export DOCKER_PLATFORM="linux/amd64"
+elif [[ "$DOCKER_ARCH" = "aarch64" ]]; then
+  export DOCKER_PLATFORM="linux/arm64"
+else
+  echo "FAIL: Unsupported docker architecture $DOCKER_ARCH."
+  exit 1
+fi
+echo "Detected Docker platform is $DOCKER_PLATFORM"
+
+echo "${_endgroup}"

install/ensure-files-from-examples.sh

@@ -0,0 +1,7 @@
+echo "${_group}Ensuring files from examples ..."
+
+ensure_file_from_example "$SENTRY_CONFIG_PY"
+ensure_file_from_example "$SENTRY_CONFIG_YML"
+ensure_file_from_example symbolicator/config.yml
+
+echo "${_endgroup}"

install/ensure-relay-credentials.sh

@@ -0,0 +1,42 @@
+echo "${_group}Ensuring Relay credentials ..."
+
+RELAY_CONFIG_YML=relay/config.yml
+RELAY_CREDENTIALS_JSON=relay/credentials.json
+
+ensure_file_from_example $RELAY_CONFIG_YML
+
+if [[ -f "$RELAY_CREDENTIALS_JSON" ]]; then
+  echo "$RELAY_CREDENTIALS_JSON already exists, skipped creation."
+else
+
+  # There are a couple gotchas here:
+  #
+  # 1. We need to use a tmp file because if we redirect output directly to
+  #    credentials.json, then the shell will create an empty file that relay
+  #    will then try to read from (regardless of options such as --stdout or
+  #    --overwrite) and fail because it is empty.
+  #
+  # 2. We pull relay:nightly before invoking `run relay credentials generate`
+  #    because an implicit pull under the run causes extra stdout that results
+  #    in a garbage credentials.json.
+  #
+  # 3. We need to use -T to ensure that we receive output on Docker Compose
+  #    1.x and 2.2.3+ (funny story about that ... ;). Note that the long opt
+  #    --no-tty doesn't exist in Docker Compose 1.
+
+  $dc pull relay
+  creds="$dcr --no-deps -T relay credentials"
+  $creds generate --stdout >"$RELAY_CREDENTIALS_JSON".tmp
+  mv "$RELAY_CREDENTIALS_JSON".tmp "$RELAY_CREDENTIALS_JSON"
+  if ! grep -q Credentials <($creds show); then
+    # Let's fail early if creds failed, to make debugging easier.
+    echo "Failed to create relay credentials in $RELAY_CREDENTIALS_JSON."
+    echo "--- credentials.json v ---------------------------------------"
+    cat -v "$RELAY_CREDENTIALS_JSON" || true
+    echo "--- credentials.json ^ ---------------------------------------"
+    exit 1
+  fi
+  echo "Relay credentials written to $RELAY_CREDENTIALS_JSON."
+fi
+
+echo "${_endgroup}"

install/error-handling.sh

@@ -0,0 +1,235 @@
+echo "${_group}Setting up error handling ..."
+
+if [ -z "${SENTRY_DSN:-}" ]; then
+  export SENTRY_DSN='https://19555c489ded4769978daae92f2346ca@self-hosted.getsentry.net/3'
+fi
+
+$dbuild -t sentry-self-hosted-jq-local --platform="$DOCKER_PLATFORM" jq
+
+jq="docker run --rm -i sentry-self-hosted-jq-local"
+sentry_cli="docker run --rm -v /tmp:/work -e SENTRY_DSN=$SENTRY_DSN getsentry/sentry-cli"
+
+send_envelope() {
+  # Send envelope
+  $sentry_cli send-envelope "$envelope_file"
+}
+
+generate_breadcrumb_json() {
+  cat $log_file | $jq -R -c 'split("\n") | {"message": (.[0]//""), "category": "log", "level": "info"}'
+}
+
+send_event() {
+  # Use traceback hash as the UUID since it is 32 characters long
+  local cmd_exit=$1
+  local error_msg=$2
+  local traceback=$3
+  local traceback_json=$4
+  local breadcrumbs=$5
+  local fingerprint_value=$(
+    echo -n "$cmd_exit $error_msg $traceback" |
+      docker run -i --rm busybox md5sum |
+      cut -d' ' -f1
+  )
+  local envelope_file="sentry-envelope-${fingerprint_value}"
+  local envelope_file_path="/tmp/$envelope_file"
+  # If the envelope file exists, we've already sent it
+  if [[ -f $envelope_file_path ]]; then
+    echo "Looks like you've already sent this error to us, we're on it :)"
+    return
+  fi
+  # If we haven't sent the envelope file, make it and send to Sentry
+  # The format is documented at https://develop.sentry.dev/sdk/envelopes/
+  # Grab length of log file, needed for the envelope header to send an attachment
+  local file_length=$(wc -c <$log_file | awk '{print $1}')
+
+  # Add header for initial envelope information
+  $jq -n -c --arg event_id "$fingerprint_value" \
+    --arg dsn "$SENTRY_DSN" \
+    '$ARGS.named' >"$envelope_file_path"
+  # Add header to specify the event type of envelope to be sent
+  echo '{"type":"event"}' >>"$envelope_file_path"
+
+  # Next we construct the meat of the event payload, which we build up
+  # inside out using jq
+  # See https://develop.sentry.dev/sdk/event-payloads/
+  # for details about the event payload
+
+  # Then we need the exception payload
+  # https://develop.sentry.dev/sdk/event-payloads/exception/
+  # but first we need to make the stacktrace which goes in the exception payload
+  frames=$(echo "$traceback_json" | $jq -s -c)
+  stacktrace=$($jq -n -c --argjson frames "$frames" '$ARGS.named')
+  exception=$(
+    $jq -n -c --arg "type" Error \
+      --arg value "$error_msg" \
+      --argjson stacktrace "$stacktrace" \
+      '$ARGS.named'
+  )
+
+  # It'd be a bit cleaner in the Sentry UI if we passed the inputs to
+  # fingerprint_value hash rather than the hash itself (I believe the ultimate
+  # hash ends up simply being a hash of our hash), but we want the hash locally
+  # so that we can avoid resending the same event (design decision to avoid
+  # spam in the system). It was also futzy to figure out how to get the
+  # traceback in there properly. Meh.
+  event_body=$(
+    $jq -n -c --arg level error \
+      --argjson exception "{\"values\":[$exception]}" \
+      --argjson breadcrumbs "{\"values\": $breadcrumbs}" \
+      --argjson fingerprint "[\"$fingerprint_value\"]" \
+      '$ARGS.named'
+  )
+  echo "$event_body" >>$envelope_file_path
+  # Add attachment to the event
+  attachment=$(
+    $jq -n -c --arg "type" attachment \
+      --arg length "$file_length" \
+      --arg content_type "text/plain" \
+      --arg filename install_log.txt \
+      '{"type": $type,"length": $length|tonumber,"content_type": $content_type,"filename": $filename}'
+  )
+  echo "$attachment" >>$envelope_file_path
+  cat $log_file >>$envelope_file_path
+  # Send envelope
+  send_envelope $envelope_file
+}
+
+if [[ -z "${REPORT_SELF_HOSTED_ISSUES:-}" ]]; then
+  echo
+  echo "Hey, so ... we would love to automatically find out about issues with your"
+  echo "Sentry instance so that we can improve the product. Turns out there is an app"
+  echo "for that, called Sentry. Would you be willing to let us automatically send data"
+  echo "about your instance upstream to Sentry for development and debugging purposes?"
+  echo
+  echo "  y / yes / 1"
+  echo "  n / no / 0"
+  echo
+  echo "(Btw, we send this to our own self-hosted Sentry instance, not to Sentry SaaS,"
+  echo "so that we can be in this together.)"
+  echo
+  echo "Here's the info we may collect:"
+  echo
+  echo "  - OS username"
+  echo "  - IP address"
+  echo "  - install log"
+  echo "  - runtime errors"
+  echo "  - performance data"
+  echo
+  echo "Thirty (30) day retention. No marketing. Privacy policy at sentry.io/privacy."
+  echo
+
+  yn=""
+  until [ ! -z "$yn" ]; do
+    read -p "y or n? " yn
+    case $yn in
+    y | yes | 1)
+      export REPORT_SELF_HOSTED_ISSUES=1
+      echo
+      echo -n "Thank you."
+      ;;
+    n | no | 0)
+      export REPORT_SELF_HOSTED_ISSUES=0
+      echo
+      echo -n "Understood."
+      ;;
+    *) yn="" ;;
+    esac
+  done
+
+  echo " To avoid this prompt in the future, use one of these flags:"
+  echo
+  echo "  --report-self-hosted-issues"
+  echo "  --no-report-self-hosted-issues"
+  echo
+  echo "or set the REPORT_SELF_HOSTED_ISSUES environment variable:"
+  echo
+  echo "  REPORT_SELF_HOSTED_ISSUES=1 to send data"
+  echo "  REPORT_SELF_HOSTED_ISSUES=0 to not send data"
+  echo
+  sleep 5
+fi
+
+# Make sure we can use sentry-cli if we need it.
+if [ "$REPORT_SELF_HOSTED_ISSUES" == 1 ]; then
+  if ! docker pull getsentry/sentry-cli:latest; then
+    echo "Failed to pull sentry-cli, won't report to Sentry after all."
+    export REPORT_SELF_HOSTED_ISSUES=0
+  fi
+fi
+
+# Courtesy of https://stackoverflow.com/a/2183063/90297
+trap_with_arg() {
+  func="$1"
+  shift
+  for sig; do
+    trap "$func $sig" "$sig"
+  done
+}
+
+DID_CLEAN_UP=0
+# the cleanup function will be the exit point
+cleanup() {
+  local retcode=$?
+  local cmd="${BASH_COMMAND}"
+  if [[ "$DID_CLEAN_UP" -eq 1 ]]; then
+    return 0
+  fi
+  DID_CLEAN_UP=1
+  if [[ "$1" != "EXIT" ]]; then
+    set +o xtrace
+    # Save the error message that comes from the last line of the log file
+    error_msg=$(tail -n 1 "$log_file")
+    # Create the breadcrumb payload now before stacktrace is printed
+    # https://develop.sentry.dev/sdk/event-payloads/breadcrumbs/
+    # Use sed to remove the last line, that is reported through the error message
+    breadcrumbs=$(generate_breadcrumb_json | sed '$d' | $jq -s -c)
+    printf -v err '%s' "Error in ${BASH_SOURCE[1]}:${BASH_LINENO[0]}."
+    printf -v cmd_exit '%s' "'$cmd' exited with status $retcode"
+    printf '%s\n%s\n' "$err" "$cmd_exit"
+    local stack_depth=${#FUNCNAME[@]}
+    local traceback=""
+    local traceback_json=""
+    if [ $stack_depth -gt 2 ]; then
+      for ((i = $(($stack_depth - 1)), j = 1; i > 0; i--, j++)); do
+        local indent="$(yes a | head -$j | tr -d '\n')"
+        local src=${BASH_SOURCE[$i]}
+        local lineno=${BASH_LINENO[$i - 1]}
+        local funcname=${FUNCNAME[$i]}
+        JSON=$(
+          $jq -n -c --arg filename "$src" \
+            --arg "function" "$funcname" \
+            --arg lineno "$lineno" \
+            '{"filename": $filename, "function": $function, "lineno": $lineno|tonumber}'
+        )
+        # If we're in the stacktrace of the file we failed on, we can add a context line with the command run that failed
+        if [[ $i -eq 1 ]]; then
+          JSON=$(
+            $jq -n -c --arg cmd "$cmd" \
+              --argjson json "$JSON" \
+              '$json + {"context_line": $cmd}'
+          )
+        fi
+        printf -v traceback_json '%s\n' "$traceback_json$JSON"
+        printf -v traceback '%s\n' "$traceback${indent//a/-}> $src:$funcname:$lineno"
+      done
+    fi
+    echo "$traceback"
+
+    # Only send event when report issues flag is set and if trap signal is not INT (ctrl+c)
+    if [[ "$REPORT_SELF_HOSTED_ISSUES" == 1 && "$1" != "INT" ]]; then
+      send_event "$cmd_exit" "$error_msg" "$traceback" "$traceback_json" "$breadcrumbs"
+    fi
+
+    if [[ -n "$MINIMIZE_DOWNTIME" ]]; then
+      echo "*NOT* cleaning up, to clean your environment run \"docker compose stop\"."
+    else
+      echo "Cleaning up..."
+    fi
+  fi
+
+  if [[ -z "$MINIMIZE_DOWNTIME" ]]; then
+    $dc stop -t $STOP_TIMEOUT &>/dev/null
+  fi
+}
+
+echo "${_endgroup}"

install/generate-secret-key.sh

@@ -0,0 +1,15 @@
+echo "${_group}Generating secret key ..."
+
+if grep -xq "system.secret-key: '!!changeme!!'" $SENTRY_CONFIG_YML; then
+  # This is to escape the secret key to be used in sed below
+  # Note the need to set LC_ALL=C due to BSD tr and sed always trying to decode
+  # whatever is passed to them. Kudos to https://stackoverflow.com/a/23584470/90297
+  SECRET_KEY=$(
+    export LC_ALL=C
+    head /dev/urandom | tr -dc "a-z0-9@#%^&*(-_=+)" | head -c 50 | sed -e 's/[\/&]/\\&/g'
+  )
+  sed -i -e 's/^system.secret-key:.*$/system.secret-key: '"'$SECRET_KEY'"'/' $SENTRY_CONFIG_YML
+  echo "Secret key written to $SENTRY_CONFIG_YML"
+fi
+
+echo "${_endgroup}"

install/geoip.sh

@@ -0,0 +1,34 @@
+echo "${_group}Setting up GeoIP integration ..."
+
+install_geoip() {
+  local mmdb=geoip/GeoLite2-City.mmdb
+  local conf=geoip/GeoIP.conf
+  local result='Done'
+
+  echo "Setting up IP address geolocation ..."
+  if [[ ! -f "$mmdb" ]]; then
+    echo -n "Installing (empty) IP address geolocation database ... "
+    cp "$mmdb.empty" "$mmdb"
+    echo "done."
+  else
+    echo "IP address geolocation database already exists."
+  fi
+
+  if [[ ! -f "$conf" ]]; then
+    echo "IP address geolocation is not configured for updates."
+    echo "See https://develop.sentry.dev/self-hosted/geolocation/ for instructions."
+    result='Error'
+  else
+    echo "IP address geolocation is configured for updates."
+    echo "Updating IP address geolocation database ... "
+    if ! $dcr geoipupdate; then
+      result='Error'
+    fi
+    echo "$result updating IP address geolocation database."
+  fi
+  echo "$result setting up IP address geolocation."
+}
+
+install_geoip
+
+echo "${_endgroup}"

install/parse-cli.sh

@@ -0,0 +1,79 @@
+echo "${_group}Parsing command line ..."
+
+show_help() {
+  cat <<EOF
+Usage: $0 [options]
+
+Install Sentry with \`docker compose\`.
+
+Options:
+ -h, --help             Show this message and exit.
+ --minimize-downtime    EXPERIMENTAL: try to keep accepting events for as long
+                          as possible while upgrading. This will disable cleanup
+                          on error, and might leave your installation in a
+                          partially upgraded state. This option might not reload
+                          all configuration, and is only meant for in-place
+                          upgrades.
+ --skip-commit-check    Skip the check for the latest commit when on the master
+                          branch of a \`self-hosted\` Git working copy.
+ --skip-user-creation   Skip the initial user creation prompt (ideal for non-
+                          	interactive installs).
+ --skip-sse42-requirements
+			Skip checking that your environment meets the
+			  requirements to run Sentry. Only do this if you know
+			  what you are doing.
+ --report-self-hosted-issues
+                        Report error and performance data about your self-hosted
+                          instance upstream to Sentry. See sentry.io/privacy for
+                          our privacy policy.
+ --no-report-self-hosted-issues
+                        Do not report error and performance data about your
+                          self-hosted instance upstream to Sentry.
+EOF
+}
+
+depwarn() {
+  echo "WARNING The $1 is deprecated. Please use $2 instead."
+}
+
+if [ ! -z "${SKIP_USER_PROMPT:-}" ]; then
+  depwarn "SKIP_USER_PROMPT variable" "SKIP_USER_CREATION"
+  SKIP_USER_CREATION="${SKIP_USER_PROMPT}"
+fi
+
+SKIP_USER_CREATION="${SKIP_USER_CREATION:-}"
+MINIMIZE_DOWNTIME="${MINIMIZE_DOWNTIME:-}"
+SKIP_COMMIT_CHECK="${SKIP_COMMIT_CHECK:-}"
+REPORT_SELF_HOSTED_ISSUES="${REPORT_SELF_HOSTED_ISSUES:-}"
+SKIP_SSE42_REQUIREMENTS="${SKIP_SSE42_REQUIREMENTS:-}"
+
+while (($#)); do
+  case "$1" in
+  -h | --help)
+    show_help
+    exit
+    ;;
+  --no-user-prompt)
+    SKIP_USER_CREATION=1
+    depwarn "--no-user-prompt flag" "--skip-user-creation"
+    ;;
+  --skip-user-prompt)
+    SKIP_USER_CREATION=1
+    depwarn "--skip-user-prompt flag" "--skip-user-creation"
+    ;;
+  --skip-user-creation) SKIP_USER_CREATION=1 ;;
+  --minimize-downtime) MINIMIZE_DOWNTIME=1 ;;
+  --skip-commit-check) SKIP_COMMIT_CHECK=1 ;;
+  --report-self-hosted-issues) REPORT_SELF_HOSTED_ISSUES=1 ;;
+  --no-report-self-hosted-issues) REPORT_SELF_HOSTED_ISSUES=0 ;;
+  --skip-sse42-requirements) SKIP_SSE42_REQUIREMENTS=1 ;;
+  --) ;;
+  *)
+    echo "Unexpected argument: $1. Use --help for usage information."
+    exit 1
+    ;;
+  esac
+  shift
+done
+
+echo "${_endgroup}"

install/set-up-and-migrate-database.sh

@@ -0,0 +1,39 @@
+echo "${_group}Setting up / migrating database ..."
+
+# Fixes https://github.com/getsentry/self-hosted/issues/2758, where a migration fails due to indexing issue
+$dc up -d postgres
+# Wait for postgres
+RETRIES=5
+until $dc exec postgres psql -U postgres -c "select 1" >/dev/null 2>&1 || [ $RETRIES -eq 0 ]; do
+  echo "Waiting for postgres server, $((RETRIES--)) remaining attempts..."
+  sleep 1
+done
+
+os=$($dc exec postgres cat /etc/os-release | grep 'ID=debian')
+if [[ -z $os ]]; then
+  echo "Postgres image debian check failed, exiting..."
+  exit 1
+fi
+
+# Using django ORM to provide broader support for users with external databases
+$dcr web shell -c "
+from django.db import connection
+
+with connection.cursor() as cursor:
+  cursor.execute('ALTER TABLE IF EXISTS sentry_groupedmessage DROP CONSTRAINT IF EXISTS sentry_groupedmessage_project_id_id_515aaa7e_uniq;')
+  cursor.execute('DROP INDEX IF EXISTS sentry_groupedmessage_project_id_id_515aaa7e_uniq;')
+"
+
+if [[ -n "${CI:-}" || "${SKIP_USER_CREATION:-0}" == 1 ]]; then
+  $dcr web upgrade --noinput --create-kafka-topics
+  echo ""
+  echo "Did not prompt for user creation. Run the following command to create one"
+  echo "yourself (recommended):"
+  echo ""
+  echo "  $dc_base run --rm web createuser"
+  echo ""
+else
+  $dcr web upgrade --create-kafka-topics
+fi
+
+echo "${_endgroup}"

install/setup-js-sdk-assets.sh

@@ -0,0 +1,40 @@
+# This will only run if the SETUP_JS_SDK_ASSETS environment variable is set to 1.
+# Think of this as some kind of a feature flag.
+if [[ "${SETUP_JS_SDK_ASSETS:-}" == "1" ]]; then
+  echo "${_group}Setting up JS SDK assets"
+
+  # If the `sentry-nginx-www` volume exists, we need to prune the contents.
+  # We don't want to fill the volume with old JS SDK assets.
+  # If people want to keep the old assets, they can set the environment variable
+  # `SETUP_JS_SDK_KEEP_OLD_ASSETS` to any value.
+  if [[ -z "${SETUP_JS_SDK_KEEP_OLD_ASSETS:-}" ]]; then
+    echo "Cleaning up old JS SDK assets..."
+    $dcr --no-deps --rm -v "sentry-nginx-www:/var/www" nginx rm -rf /var/www/js-sdk/*
+  fi
+
+  $dbuild -t sentry-self-hosted-jq-local --platform="$DOCKER_PLATFORM" jq
+
+  jq="docker run --rm -i sentry-self-hosted-jq-local"
+
+  loader_registry=$($dcr --no-deps --rm -T web cat /usr/src/sentry/src/sentry/loader/_registry.json)
+  # The `loader_registry` should start with "Updating certificates...", we want to delete that and the subsequent ca-certificates related lines.
+  # We want to remove everything before the first '{'.
+  loader_registry=$(echo "$loader_registry" | sed '0,/{/s/[^{]*//')
+
+  #  Sentry backend provides SDK versions from v4.x up to v8.x.
+  latest_js_v4=$(echo "$loader_registry" | $jq -r '.versions | reverse | map(select(.|any(.; startswith("4.")))) | .[0]')
+  latest_js_v5=$(echo "$loader_registry" | $jq -r '.versions | reverse | map(select(.|any(.; startswith("5.")))) | .[0]')
+  latest_js_v6=$(echo "$loader_registry" | $jq -r '.versions | reverse | map(select(.|any(.; startswith("6.")))) | .[0]')
+  latest_js_v7=$(echo "$loader_registry" | $jq -r '.versions | reverse | map(select(.|any(.; startswith("7.")))) | .[0]')
+  latest_js_v8=$(echo "$loader_registry" | $jq -r '.versions | reverse | map(select(.|any(.; startswith("8.")))) | .[0]')
+
+  echo "Found JS SDKs: v${latest_js_v4}, v${latest_js_v5}, v${latest_js_v6}, v${latest_js_v7}, v${latest_js_v8}"
+
+  versions="{$latest_js_v4,$latest_js_v5,$latest_js_v6,$latest_js_v7,$latest_js_v8}"
+  variants="{bundle,bundle.tracing,bundle.tracing.replay,bundle.replay,bundle.tracing.replay.feedback,bundle.feedback}"
+
+  # Download those versions & variants using curl
+  $dcr --no-deps --rm -v "sentry-nginx-www:/var/www" nginx curl -w '%{response_code} %{url}\n' --no-progress-meter --compressed --retry 3 --create-dirs -fLo "/var/www/js-sdk/#1/#2.min.js" "https://browser.sentry-cdn.com/${versions}/${variants}.min.js" || true
+
+  echo "${_endgroup}"
+fi

install/turn-things-off.sh

@@ -0,0 +1,11 @@
+echo "${_group}Turning things off ..."
+
+if [[ -n "$MINIMIZE_DOWNTIME" ]]; then
+  # Stop everything but relay and nginx
+  $dc rm -fsv $($dc config --services | grep -v -E '^(nginx|relay)$')
+else
+  # Clean up old stuff and ensure nothing is working while we install/update
+  $dc down -t $STOP_TIMEOUT --rmi local --remove-orphans
+fi
+
+echo "${_endgroup}"

install/update-docker-images.sh

@@ -0,0 +1,14 @@
+echo "${_group}Fetching and updating Docker images ..."
+
+# We tag locally built images with a '-self-hosted-local' suffix. `docker
+# compose pull` tries to pull these too and shows a 404 error on the console
+# which is confusing and unnecessary. To overcome this, we add the
+# stderr>stdout redirection below and pass it through grep, ignoring all lines
+# having this '-onpremise-local' suffix.
+
+$dc pull -q --ignore-pull-failures 2>&1 | grep -v -- -self-hosted-local || true
+
+# We may not have the set image on the repo (local images) so allow fails
+docker pull ${SENTRY_IMAGE} || true
+
+echo "${_endgroup}"

install/update-docker-volume-permissions.sh

@@ -0,0 +1,9 @@
+echo "${_group}Ensuring Kafka and Zookeeper volumes have correct permissions ..."
+
+# Only supporting platforms on linux x86 platforms and not apple silicon. I'm assuming that folks using apple silicon are doing it for dev purposes and it's difficult
+# to change permissions of docker volumes since it is run in a VM.
+if [[ -n "$(docker volume ls -q -f name=sentry-zookeeper)" && -n "$(docker volume ls -q -f name=sentry-kafka)" ]]; then
+  docker run --rm -v "sentry-zookeeper:/sentry-zookeeper-data" -v "sentry-kafka:/sentry-kafka-data" -v "${COMPOSE_PROJECT_NAME}_sentry-zookeeper-log:/sentry-zookeeper-log-data" busybox chmod -R a+w /sentry-zookeeper-data /sentry-kafka-data /sentry-zookeeper-log-data
+fi
+
+echo "${_endgroup}"

install/upgrade-clickhouse.sh

@@ -0,0 +1,35 @@
+echo "${_group}Upgrading Clickhouse ..."
+
+function wait_for_clickhouse() {
+  # Wait for clickhouse
+  RETRIES=30
+  until $dc ps clickhouse | grep 'healthy' || [ $RETRIES -eq 0 ]; do
+    echo "Waiting for clickhouse server, $((RETRIES--)) remaining attempts..."
+    sleep 1
+  done
+}
+
+# First check to see if user is upgrading by checking for existing clickhouse volume
+if [[ -n "$(docker volume ls -q --filter name=sentry-clickhouse)" ]]; then
+  # Start clickhouse if it is not already running
+  $dc up -d clickhouse
+
+  # Wait for clickhouse
+  wait_for_clickhouse
+
+  # In order to get to 23.8, we need to first upgrade go from 21.8 -> 22.8 -> 23.3 -> 23.8
+  version=$($dc exec clickhouse clickhouse-client -q 'SELECT version()')
+  if [[ "$version" == "21.8.13.1.altinitystable" || "$version" == "21.8.12.29.altinitydev.arm" ]]; then
+    $dc down clickhouse
+    $dcb --build-arg BASE_IMAGE=altinity/clickhouse-server:22.8.15.25.altinitystable clickhouse
+    $dc up -d clickhouse
+    wait_for_clickhouse
+    $dc down clickhouse
+    $dcb --build-arg BASE_IMAGE=altinity/clickhouse-server:23.3.19.33.altinitystable clickhouse
+    $dc up -d clickhouse
+    wait_for_clickhouse
+  else
+    echo "Detected clickhouse version $version. Skipping upgrades!"
+  fi
+fi
+echo "${_endgroup}"

install/upgrade-postgres.sh

@@ -0,0 +1,44 @@
+echo "${_group}Ensuring proper PostgreSQL version ..."
+
+if [[ -n "$(docker volume ls -q --filter name=sentry-postgres)" && "$(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null)" == "9.6" ]]; then
+  docker volume rm sentry-postgres-new || true
+  # If this is Postgres 9.6 data, start upgrading it to 14.0 in a new volume
+  docker run --rm \
+    -v sentry-postgres:/var/lib/postgresql/9.6/data \
+    -v sentry-postgres-new:/var/lib/postgresql/14/data \
+    tianon/postgres-upgrade:9.6-to-14
+
+  # Get rid of the old volume as we'll rename the new one to that
+  docker volume rm sentry-postgres
+  docker volume create --name sentry-postgres
+  # There's no rename volume in Docker so copy the contents from old to new name
+  # Also append the `host all all all trust` line as `tianon/postgres-upgrade:9.6-to-14`
+  # doesn't do that automatically.
+  docker run --rm -v sentry-postgres-new:/from -v sentry-postgres:/to alpine ash -c \
+    "cd /from ; cp -av . /to ; echo 'host all all all trust' >> /to/pg_hba.conf"
+  # Finally, remove the new old volume as we are all in sentry-postgres now.
+  docker volume rm sentry-postgres-new
+  echo "Re-indexing due to glibc change, this may take a while..."
+  echo "Starting up new PostgreSQL version"
+  $dc up -d postgres
+
+  # Wait for postgres
+  RETRIES=5
+  until $dc exec postgres psql -U postgres -c "select 1" >/dev/null 2>&1 || [ $RETRIES -eq 0 ]; do
+    echo "Waiting for postgres server, $((RETRIES--)) remaining attempts..."
+    sleep 1
+  done
+
+  # VOLUME_NAME is the same as container name
+  # Reindex all databases and their system catalogs which are not templates
+  DBS=$($dc exec postgres psql -qAt -U postgres -c "SELECT datname FROM pg_database WHERE datistemplate = false;")
+  for db in ${DBS}; do
+    echo "Re-indexing database: ${db}"
+    $dc exec postgres psql -qAt -U postgres -d ${db} -c "reindex system ${db}"
+    $dc exec postgres psql -qAt -U postgres -d ${db} -c "reindex database ${db};"
+  done
+
+  $dc stop postgres
+fi
+
+echo "${_endgroup}"

install/wrap-up.sh

@@ -0,0 +1,35 @@
+if [[ "$MINIMIZE_DOWNTIME" ]]; then
+  echo "${_group}Waiting for Sentry to start ..."
+
+  # Start the whole setup, except nginx and relay.
+  $dc up -d --remove-orphans $($dc config --services | grep -v -E '^(nginx|relay)$')
+  $dc restart relay
+  $dc exec -T nginx nginx -s reload
+
+  docker run --rm --network="${COMPOSE_PROJECT_NAME}_default" alpine ash \
+    -c 'while [[ "$(wget -T 1 -q -O- http://web:96/_health/)" != "ok" ]]; do sleep 0.5; done'
+
+  # Make sure everything is up. This should only touch relay and nginx
+  $dc up -d
+
+  echo "${_endgroup}"
+else
+  echo ""
+  echo "-----------------------------------------------------------------"
+  echo ""
+  echo "You're all done! Run the following command to get Sentry running:"
+  echo ""
+  if [[ "${_ENV}" =~ ".env.custom" ]]; then
+    echo "  $dc_base --env-file ${_ENV} up -d"
+  else
+    echo "  $dc_base up -d"
+  fi
+  echo ""
+  echo "-----------------------------------------------------------------"
+  echo ""
+fi
+
+# TODO(getsentry/self-hosted#2489)
+if docker volume ls | grep -qw sentry-zookeeper; then
+  docker volume rm sentry-zookeeper
+fi