Commit dfc76613 authored by rubo77's avatar rubo77
Browse files

Merge remote-tracking branch 'gitlab.toppoint/release-candidate'

parents 56b15017 eaba9f98
Pipeline #672 canceled with stages
*~
download/index.html
/signing_key
*.log
/gluon
......@@ -4,6 +4,7 @@ variables:
CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/build
GLUON_GIT: https://github.com/freifunk-gluon/gluon.git
GLUON_BRANCH: v2018.1.x
BROKEN: BROKEN=1
cache:
key: $CI_COMMIT_REF_SLUG-$GLUON_GIT-$GLUON_BRANCH
......@@ -39,17 +40,20 @@ build:
- export FORCE_UNSAFE_CONFIGURE=1
# former clone step
- ./.gitlab-ci/update-git.sh | tee -a debug.log
- ./.gitlab-ci/build.sh -c update -b $CI_COMMIT_REF_NAME -n $CI_PIPELINE_ID -m "-j $(nproc --all)" | tee -a debug.log
- ./.gitlab-ci/build.sh -c update -b rc -a stable -n $CI_PIPELINE_ID -m "-j $(nproc --all) $BROKEN" | tee -a debug.log
# We should never need this, should we?
- ./.gitlab-ci/build.sh -c clean -b $CI_COMMIT_REF_NAME -n $CI_PIPELINE_ID -m "-j $(nproc --all)" | tee -a debug.log
# In the past this build step repeatedly broke
# - ./.gitlab-ci/build.sh -c download -b $CI_COMMIT_REF_NAME -n $CI_PIPELINE_ID -m "-j $(nproc --all) V=s" | tee -a debug.log
- ./.gitlab-ci/build.sh -c build -b $CI_COMMIT_REF_NAME -n $CI_PIPELINE_ID -m "-j $(nproc --all)" | tee -a debug.log
- ./.gitlab-ci/build.sh -c clean -b rc -a stable -n $CI_PIPELINE_ID -m "-j $(nproc --all)" | tee -a debug.log
# Apply patches
- ./.gitlab-ci/patch.sh patches gluon
# Build
- ./.gitlab-ci/build.sh -c build -b rc -a stable -n $CI_PIPELINE_ID -m "-j $(nproc --all) $BROKEN" | tee -a debug.log
# Revert patches
- ./.gitlab-ci/patch.sh -r patches gluon
# Bring gluon repo to cacheable state
- ./.gitlab-ci/build.sh -c clean -b $CI_COMMIT_REF_NAME -n $CI_PIPELINE_ID -m "-j $(nproc --all)" | tee -a debug.log
- ./.gitlab-ci/build.sh -c clean -b rc -a stable -n $CI_PIPELINE_ID -m "-j $(nproc --all) $BROKEN" | tee -a debug.log
# Sign
- echo $SIGNING_KEY > signing_key
- ./.gitlab-ci/build.sh -c sign -b $CI_COMMIT_REF_NAME -s $(pwd)/signing_key
- ./.gitlab-ci/build.sh -c sign -b rc -a stable -s $(pwd)/signing_key
artifacts:
untracked: false
paths:
......
FROM debian:stretch
RUN apt-get update && apt-get install -y \
git \
subversion \
python \
build-essential \
gawk \
unzip \
libncurses5-dev \
zlib1g-dev \
git \
subversion \
python \
build-essential \
gawk \
unzip \
libncurses5-dev \
zlib1g-dev \
libssl1.0-dev \
wget \
curl \
wget \
curl \
openssh-client
RUN apt-get update && apt-get install -y \
cmake \
......@@ -36,4 +36,3 @@ RUN apt-get update && apt-get install -y \
cd ~ && \
rm -rf ~/libuecc && \
rm -rf ~/ecdsautils
......@@ -14,17 +14,6 @@
# Default make options
MAKEOPTS="-j 4"
# Default to build all Gluon targets if parameter -t is not set
TARGETS="ar71xx-generic ar71xx-tiny ar71xx-nand brcm2708-bcm2708 brcm2708-bcm2709 ramips-mt7621 sunxi x86-generic x86-geode x86-64 ramips-mt7620 ramips-mt7628 ramips-rt305x"
TARGETS=$TARGETS" mpc85xx-generic" # (tp-link-tl-wdr4900-v1)
# BROKEN:
#TARGETS=$TARGETS" ramips-mt76x8" # BROKEN: unstable WiFi (tp-link 841 v13 und archer c50)
#TARGETS=$TARGETS" ar71xx-mikrotik" # BROKEN: no sysupgrade support (mikrotik-nand)
#TARGETS=$TARGETS" brcm2708-bcm2710" # BROKEN: Untested (raspberry-pi-3)
#TARGETS=$TARGETS" ipq806x" # BROKEN: unstable wifi drivers (tp-link-archer-c2600)
#TARGETS=$TARGETS" mvebu-cortexa9" # BROKEN: No AP+IBSS or 11s support (linksys-wrt1200ac)
# Default is set to use current work directory
SITEDIR="$(pwd)"
......@@ -34,6 +23,7 @@ BUILD="snapshot"
# Specify deployment server and user
DEPLOYMENT_SERVER="10.116.250.1"
DEPLOYMENT_USER="rsync"
DEPLOYMENT_PATH="/opt/firmware/ffki"
# Path to signing key
SIGNKEY=""
......@@ -42,10 +32,13 @@ SIGNKEY=""
E_ILLEGAL_ARGS=126
# Help function used in error messages and -h option
usage() {
echo ""
echo "Build script for Freifunk-Fulda gluon firmware."
echo ""
echo "-a: Autoupdater branch name (e.g. development)"
echo " Default: branch (see -b)"
echo "-b: Firmware branch name (e.g. development)"
echo " Default: current git branch"
echo "-c: Build command: update | clean | download | build | sign | upload | prepare"
......@@ -57,6 +50,8 @@ usage() {
echo " Default: \"${BUILD}\""
echo "-t: Gluon targets architectures to build"
echo " Default: \"${TARGETS}\""
echo "-u: Upload target"
echo " Default: branch (see -b)"
echo "-r: Release number (optional)"
echo " Default: fetched from release file"
echo "-w: Path to site directory"
......@@ -72,10 +67,16 @@ if [[ "${#}" == 0 ]]; then
fi
# Evaluate arguments for build script.
while getopts b:c:dhm:n:t:w:s: flag; do
while getopts a:b:c:dhm:n:t:u:w:s: flag; do
case ${flag} in
a)
AU_BRANCH="${OPTARG}"
;;
b)
BRANCH="${OPTARG}"
if [ "${BRANCH}" == "release-candidate" ] ; then
BRANCH="rc"
fi
;;
c)
case "${OPTARG}" in
......@@ -123,6 +124,9 @@ while getopts b:c:dhm:n:t:w:s: flag; do
t)
TARGETS="${OPTARG}"
;;
u)
UPLOAD_TARGET="${OPTARG}"
;;
r)
RELEASE="${OPTARG}"
;;
......@@ -145,11 +149,48 @@ shift $((OPTIND - 1));
# Check if there are remaining arguments
if [[ "${#}" > 0 ]]; then
echo "Error: To many arguments: ${*}"
echo "Error: Too many arguments: ${*}"
usage
exit ${E_ILLEGAL_ARGS}
fi
# Set branch name
if [[ -z "${BRANCH}" ]]; then
BRANCH=$(git symbolic-ref -q HEAD)
BRANCH=${BRANCH##refs/heads/}
BRANCH=${BRANCH:-HEAD}
fi
# Default to build branch specific targets if parameter -t is not set
if [[ -z ${TARGETS+x} ]] ; then
case "${BRANCH}" in
nightly)
TARGETS="ar71xx-generic ar71xx-tiny x86-64 ramips-rt305x"
;;
next)
TARGETS="ar71xx-tiny ar71xx-generic"
#TARGETS+=" x86-64 x86-generic x86-64" # (VMs)
#TARGETS+=" ar71xx-nand" # (Netgear WNDR3700, WNDR4300, ZyXEL NBG6716)
#TARGETS+=" mpc85xx-generic" # (tp-link-tl-wdr4900-v1)
TARGETS+=" ramips-mt7620" # (gl-inet mt300 und mt750)
#TARGETS+=" sunxi-cortexa7" # (Banana Pi M1)n
;;
*)
# Default to all targets
TARGETS="ar71xx-generic ar71xx-tiny ar71xx-nand brcm2708-bcm2708 brcm2708-bcm2709 ramips-mt7621 sunxi x86-generic x86-geode x86-64 ramips-mt7620 ramips-mt7628 ramips-rt305x"
TARGETS+=" mpc85xx-generic" # (tp-link-tl-wdr4900-v1)-
;;
esac
fi
if [[ -z "$AU_BRANCH" ]]; then
AU_BRANCH="$BRANCH"
fi
if [[ -z "$UPLOAD_TARGET" ]]; then
UPLOAD_TARGET="$BRANCH"
fi
# Set command
if [[ -z "${COMMAND}" ]]; then
echo "Error: Build command missing."
......@@ -159,31 +200,35 @@ fi
# Set release number
if [[ -z "${RELEASE}" ]]; then
RELEASE=$(cat "${SITEDIR}/release")
RELEASE=$(sed -e "s/BUILD/$BUILD/" "${SITEDIR}/release")
fi
# Normalize the branch name
BRANCH="${BRANCH#origin/}" # Use the current git branch as autoupdate branch
BRANCH="${BRANCH//\//-}" # Replace all slashes with dashes
# Get the GIT commit description
COMMIT="$(git describe --always --dirty)"
# Name of the autoupdater branch that is selected on the node: 'rc', 'nightly', 'next' or 'stable'
BRANCH="stable"
# key of the autoupdater branch for the name and BRANCH of the 2nd manifest: 'rc', 'nightly', 'next' or 'stable'
AU_BRANCH_2="rc"
# Determine upload target prefix: 'release-candidate', 'nightly' or 'next'
TARGET_DIR="release-candidate"
# Number of days that may pass between releasing an updating
PRIORITY=1
if [[ -z ${PRIORITY+x} ]] ; then
case "${BRANCH}" in
nightly)
PRIORITY=0
;;
*)
PRIORITY=1
;;
esac
fi
update() {
echo "--- Update Gluon Dependencies"
make ${MAKEOPTS} \
GLUON_SITEDIR="${SITEDIR}" \
GLUON_OUTPUTDIR="${SITEDIR}/output" \
GLUON_RELEASE="${RELEASE}-${BUILD}" \
GLUON_BRANCH="${BRANCH}" \
GLUON_RELEASE="${RELEASE}" \
GLUON_BRANCH="${AU_BRANCH}" \
GLUON_PRIORITY="${PRIORITY}" \
update
}
......@@ -194,8 +239,8 @@ clean() {
make ${MAKEOPTS} \
GLUON_SITEDIR="${SITEDIR}" \
GLUON_OUTPUTDIR="${SITEDIR}/output" \
GLUON_RELEASE="${RELEASE}-${BUILD}" \
GLUON_BRANCH="${BRANCH}" \
GLUON_RELEASE="${RELEASE}" \
GLUON_BRANCH="${AU_BRANCH}" \
GLUON_PRIORITY="${PRIORITY}" \
GLUON_TARGET="${TARGET}" \
clean
......@@ -208,8 +253,8 @@ download() {
make ${MAKEOPTS} \
GLUON_SITEDIR="${SITEDIR}" \
GLUON_OUTPUTDIR="${SITEDIR}/output" \
GLUON_RELEASE="${RELEASE}-${BUILD}" \
GLUON_BRANCH="${BRANCH}" \
GLUON_RELEASE="${RELEASE}" \
GLUON_BRANCH="${AU_BRANCH}" \
GLUON_PRIORITY="${PRIORITY}" \
GLUON_TARGET="${TARGET}" \
download
......@@ -219,13 +264,14 @@ download() {
build() {
for TARGET in ${TARGETS}; do
echo "--- Build Gluon Images for target: ${TARGET}"
case "${BRANCH}" in
case "${AU_BRANCH}" in
stable)
make ${MAKEOPTS} \
GLUON_SITEDIR="${SITEDIR}" \
GLUON_OUTPUTDIR="${SITEDIR}/output" \
GLUON_RELEASE="${RELEASE}-${BUILD}" \
GLUON_BRANCH="${BRANCH}" \
GLUON_RELEASE="${RELEASE}" \
GLUON_BRANCH="${AU_BRANCH}" \
GLUON_PRIORITY="${PRIORITY}" \
GLUON_TARGET="${TARGET}"
;;
......@@ -235,7 +281,8 @@ build() {
GLUON_SITEDIR="${SITEDIR}" \
GLUON_OUTPUTDIR="${SITEDIR}/output" \
GLUON_RELEASE="${RELEASE}-${BUILD}" \
GLUON_BRANCH="${BRANCH}" \
GLUON_RELEASE="${RELEASE}" \
GLUON_BRANCH="${AU_BRANCH}" \
GLUON_TARGET="${TARGET}"
;;
esac
......@@ -245,11 +292,17 @@ build() {
make ${MAKEOPTS} \
GLUON_SITEDIR="${SITEDIR}" \
GLUON_OUTPUTDIR="${SITEDIR}/output" \
GLUON_RELEASE="${RELEASE}-${BUILD}" \
GLUON_RELEASE="${RELEASE}" \
GLUON_BRANCH="${BRANCH}" \
GLUON_PRIORITY="${PRIORITY}" \
manifest
cp "${SITEDIR}/output/images/sysupgrade/${BRANCH}.manifest" \
"${SITEDIR}/output/images/sysupgrade/${BRANCH}.manifest.clean"
# Prevent premature downloads
echo "deny from all" > "${SITEDIR}/output/images/sysupgrade/.htaccess"
echo "--- Write Build file"
cat > "${SITEDIR}/output/images/build" <<EOF
DATE=$(date '+%Y-%m-%d %H:%M:%S')
......@@ -264,36 +317,24 @@ EOF
sign() {
echo "--- Sign Gluon Firmware Build"
MANIFEST="${SITEDIR}/output/images/sysupgrade/${BRANCH}.manifest"
MANIFEST_2="${SITEDIR}/output/images/sysupgrade/${AU_BRANCH_2}.manifest"
# keep the clean manifest for later signing of NOC members and for the BRANCH file
cp -a "${MANIFEST}" "${MANIFEST}.clean"
# create clean 2nd manifest with AU_BRANCH_2 as BRANCH
cp -a "${MANIFEST}.clean" "${MANIFEST_2}.clean"
sed -i 's/BRANCH='"${BRANCH}"'/BRANCH='"${AU_BRANCH_2}"'/g' "${MANIFEST_2}.clean"
# Add the signature to the manifest file
contrib/sign.sh \
"${SIGNKEY}" \
"${MANIFEST}"
# Add the signature to the the 2nd manifest file
cp -a "${MANIFEST_2}.clean" "${MANIFEST_2}"
# Add the signature to the local manifest
contrib/sign.sh \
"${SIGNKEY}" \
"${MANIFEST_2}"
"${SITEDIR}/output/images/sysupgrade/${BRANCH}.manifest"
}
upload() {
set -x
echo "--- Upload Gluon Firmware Images and Manifest"
# Build the ssh command to use
SSH="ssh"
SSH="${SSH} -o stricthostkeychecking=no -v"
# Determine upload target prefix
TARGET="${UPLOAD_TARGET}"
# Create the target directory on server
${SSH} \
${DEPLOYMENT_USER}@${DEPLOYMENT_SERVER} \
......@@ -301,59 +342,71 @@ upload() {
mkdir \
--parents \
--verbose \
"/opt/firmware/ffki/${TARGET_DIR}/${RELEASE}-${BUILD}"
"${DEPLOYMENT_PATH}/${TARGET}/${RELEASE}"
# Add site metadata
tar -czf "${SITEDIR}/output/images/site.tgz" --exclude='gluon' --exclude='output' "${SITEDIR}"
# Compress images (Saves around 40% space, relevant because of shitty VDSL 50 upload speeds)
echo "Compressing images..."
tar -cJf "${SITEDIR}/output/images.txz" -C "${SITEDIR}/output/images/" factory sysupgrade
# Copy images to server
echo "Uploading images..."
rsync \
--verbose \
--recursive \
--compress \
--progress \
--links \
--chmod=ugo=rwX \
--rsh="${SSH}" \
"${SITEDIR}/output/images/" \
"${DEPLOYMENT_USER}@${DEPLOYMENT_SERVER}:/opt/firmware/ffki/${TARGET_DIR}/${RELEASE}-${BUILD}"
"${SITEDIR}/output/images.txz" \
"${DEPLOYMENT_USER}@${DEPLOYMENT_SERVER}:${DEPLOYMENT_PATH}/${TARGET}/${RELEASE}"
echo "Uncompressing images..."
${SSH} \
${DEPLOYMENT_USER}@${DEPLOYMENT_SERVER} \
-- \
tar -xJf "${DEPLOYMENT_PATH}/${TARGET}/${RELEASE}/images.txz" -C "${DEPLOYMENT_PATH}/${TARGET}/${RELEASE}/"
${SSH} \
${DEPLOYMENT_USER}@${DEPLOYMENT_SERVER} \
-- \
ln -sf \
"/opt/firmware/ffki/${TARGET_DIR}/${RELEASE}-${BUILD}/sysupgrade" \
"/opt/firmware/ffki/${TARGET_DIR}/"
"${DEPLOYMENT_PATH}/${TARGET}/${RELEASE}/sysupgrade" \
"${DEPLOYMENT_PATH}/${TARGET}/"
${SSH} \
${DEPLOYMENT_USER}@${DEPLOYMENT_SERVER} \
-- \
ln -sf \
"/opt/firmware/ffki/${TARGET_DIR}/${RELEASE}-${BUILD}/factory" \
"/opt/firmware/ffki/${TARGET_DIR}/"
"${DEPLOYMENT_PATH}/${TARGET}/${RELEASE}/factory" \
"${DEPLOYMENT_PATH}/${TARGET}/"
}
prepare() {
echo "--- Prepare directory for upload"
# Determine upload target prefix
TARGET="${UPLOAD_TARGET}"
# Create the target directory on server
mkdir \
--parents \
--verbose \
"${SITEDIR}/output/firmware/${TARGET_DIR}"
"${SITEDIR}/output/firmware/${TARGET}"
# Copy images to directory
mv \
--verbose \
"${SITEDIR}/output/images" \
"${SITEDIR}/output/firmware/${TARGET_DIR}/${RELEASE}-${BUILD}"
"${SITEDIR}/output/firmware/${TARGET}/${RELEASE}"
# Link latest upload in target directory to 'current'
# Link latest upload in target to 'current'
cd "${SITEDIR}/output"
ln \
--symbolic \
--force \
--no-target-directory \
"${RELEASE}-${BUILD}" \
"firmware/${TARGET_DIR}/current"
"${RELEASE}" \
"firmware/${TARGET}/current"
}
(
......
#!/usr/bin/env bash
set -e -o pipefail
shopt -s nullglob
unset REVERT
err() {
( >&2 echo $@ )
}
usage() {
err "${0} [ -r] <patchdir> <target>"
exit 1
}
while getopts "r" opt; do
case $opt in
r) REVERT=1;;
*) usage;;
esac
done
shift $((OPTIND - 1))
PATCHDIR="$1"
TARGET="$2"
[ -n "$PATCHDIR" ] && [ -d "$PATCHDIR" ] || {
err "Patchdir '${PATCHDIR}' is not a directory"
usage
}
[ -n "$TARGET" ] && [ -e "$TARGET" ] || {
err "Target '${TARGET}' does not exist"
usage
}
for file in "$PATCHDIR"/*.patch; do
if [ -z "$REVERT" ]; then
patch -d "$TARGET" -p1 < "$file"
else
patch -d "$TARGET" -p1 -R < "$file"
fi
done
......@@ -53,6 +53,25 @@ You can validate your changes to this repository by calling the validate_site.sh
These Kieler Freifunk firmwares have been released:
- 2018.1.4
- Date: 2019-02-11
- Based on Gluon 2018.1.4
- Switch from autoupdater to [MIAU](https://github.com/TobleMiner/gluon-tsys/tree/master/autoupdater) (Mesh-Independent-AUtoupdater)
- Roamguide
- Removed unused script running roamguid in a loop
- Fixed locking
- Added support for two WLAN AP interfaces
- ddhcpd
- Updated to r2
- Switched from spare block to spare lease count
- Integrated enable flag into gluon config and site.conf
- New packages
- [autoupdater-proxy](https://github.com/TobleMiner/gluon-tsys/tree/master/autoupdater-proxy)
- Patches
- Added patch injection system into gluon build system
- Support for Fonera 2.0n devices
- 2018.1.1
- Based on Gluon 2018.1.1
- 2016.2.7
- Based on Gluon 2016.2.7
- 2016.2.6.2
......@@ -103,7 +122,8 @@ These Kieler Freifunk firmwares have been released:
## Building
You can easily create your own experimental firmware with the build script `make-release.sh`
You can easily create your own experimental firmware with the build script `make-release`,
which itself needs following python modules: click and pyyaml
Or build with these commands:
......
#!/usr/bin/env python3
import click
import subprocess
import yaml
import os
@click.command()
@click.argument("branch",type=click.STRING)
@click.argument("signkey",type=click.File(mode="r"),default=None)
def cli(branch,signkey):
if signkey is not None:
signkeystr = signkey.read()
else:
signkeystr = ""
with open(".gitlab-ci.yml","r") as f:
document = yaml.load(f)
command = ';'.join(document['build']['script'])
env = { "CI_PIPELINE_ID": "9999"
, "CI_COMMIT_REF_NAME": branch
, "SIGNING_KEY": signkeystr
}
env.update(document['variables'])
env.update(os.environ)
res = subprocess.run(command,shell=True,env=env)
if __name__ == "__main__":
cli()
......@@ -15,8 +15,8 @@ set -u
set -e
# if version is unset, will use the default version from site.mk
#VERSION=${3:-"2018.1.1~rc$(date '+%y%m%d%H%M')"}
VERSION=${3:-"2018.1.1"}
#VERSION=${3:-"2018.1.3~rc$(date '+%y%m%d%H%M')"}
VERSION=${3:-"2018.1.4"}
# branch must be set to either rc, nightly or stable
BRANCH=${2:-"stable"}
# must point to valid ecdsa signing key created by ecdsakeygen, relative to Gluon base directory
......@@ -30,17 +30,17 @@ CORES=$(($(lscpu|grep -e '^CPU(s):'|xargs|cut -d" " -f2)+1))
CORES="-j$CORES"
# set this to "0" if you don't want to use make clean before make
MAKE_CLEAN="1"
MAKE_CLEAN="0"
# set this to "" to get less more output
VERBOSE="V=s"
#ONLY_TARGET must be set to "" or i.e. "ar71xx-tiny"
#ONLY_TARGET=""
ONLY_TARGET="ar71xx-generic ar71xx-tiny"
ONLY_TARGET="ar71xx-generic"
#to build only one device set DEVICES list (only if $ONLY_TARGET!="")
DEVICES=''
#DEVICES='DEVICES=tp-link-tl-wr841n-nd-v7'
#DEVICES=''
DEVICES='DEVICES=tp-link-tl-wr842n-nd-v3'
cd ../
if [ ! -d "site" ]; then
......@@ -50,7 +50,7 @@ fi
if [ "$(whoami)" == "root" ]; then
echo "Make may not be run as root"
return
exit
fi
if [ -d ../openwrt/ ]; then
......
......@@ -2,7 +2,7 @@
# when building gluon.
# for each feed name given, add the corresponding PACKAGES_* lines
GLUON_SITE_FEEDS='ffki sargon eulenfunk ssidchanger button'
GLUON_SITE_FEEDS='ffki sargon tsys eulenfunk ssidchanger button'
# usb support and gluon-config-mode-ppa
PACKAGES_FFKI_REPO=https://github.com/freifunk-kiel/ffki-packages
......@@ -11,8 +11,13 @@ PACKAGES_FFKI_BRANCH=2018.1.x
# ddhcpd and roamguide
PACKAGES_SARGON_REPO=https://github.com/sargon/gluon-sargon.git
PACKAGES_SARGON_COMMIT=b116baf60d27b73eb6d205e2f911b452a7f3e0a4
PACKAGES_SARGON_BRANCH=v2018.x
PACKAGES_SARGON_COMMIT=bf7fdddbeaae8cfb6379ce97b144ed7c4efbbe49
PACKAGES_SARGON_BRANCH=master
# *miau!*
PACKAGES_TSYS_REPO=https://github.com/TobleMiner/gluon-tsys
PACKAGES_TSYS_COMMIT=e704dc0500590ae7209ad3cba96250c2b1aa2d8f
PACKAGES_TSYS_BRANCH=master
# gluon-quickfix
PACKAGES_EULENFUNK_REPO=https://github.com/Freifunk-Nord/eulenfunk-packages
......