268 lines
6.5 KiB
TeX
268 lines
6.5 KiB
TeX
% Created 2019-09-14 Sat 15:04
|
|
% Intended LaTeX compiler: pdflatex
|
|
\documentclass[presentation]{beamer}
|
|
\usepackage[utf8]{inputenc}
|
|
\usepackage[T1]{fontenc}
|
|
\usepackage{graphicx}
|
|
\usepackage{grffile}
|
|
\usepackage{longtable}
|
|
\usepackage{wrapfig}
|
|
\usepackage{rotating}
|
|
\usepackage[normalem]{ulem}
|
|
\usepackage{amsmath}
|
|
\usepackage{textcomp}
|
|
\usepackage{amssymb}
|
|
\usepackage{capt-of}
|
|
\usepackage{hyperref}
|
|
\usepackage{color}
|
|
\usepackage{listings}
|
|
\usetheme{default}
|
|
\author{Sergio Durigan Junior \\ sergiodj@redhat.com}
|
|
\date{}
|
|
\title{GDB tests, CI \& Buildbot BoF}
|
|
\hypersetup{
|
|
pdfauthor={Sergio Durigan Junior \\ sergiodj@redhat.com},
|
|
pdftitle={GDB tests, CI \& Buildbot BoF},
|
|
pdfkeywords={},
|
|
pdfsubject={},
|
|
pdfcreator={Emacs 26.1 (Org mode 9.1.9)},
|
|
pdflang={English}}
|
|
\begin{document}
|
|
|
|
\maketitle
|
|
|
|
\begin{frame}[label={sec:org18d2f5e}]{License}
|
|
\begin{itemize}
|
|
\item License: \alert{Creative Commons Attribution 4.0 International License (CC-BY-4.0)}
|
|
|
|
\item \url{https://creativecommons.org/licenses/by/4.0/}
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile,label={sec:org4f17476}]{Nomenclature}
|
|
\begin{itemize}
|
|
\item \alert{Worker}: The node that performs the “build”. Usually one per
|
|
physical machine/VM. For example, \texttt{fedora-x86\_64-1} or
|
|
\texttt{ubuntu-aarch64}.
|
|
|
|
\item \alert{Factory}: A recipe of how to perform a build.
|
|
|
|
\item \alert{Builder}: An instance of a factory. For example,
|
|
\texttt{Fedora-x86\_64-m64} or
|
|
\texttt{Ubuntu-Aarch64-native-extended-gdbserver-m64}.
|
|
|
|
\item \alert{Scheduler}: Dispatches jobs to a set of builders. Can be triggered
|
|
by specific events like a commit in a repository, a \emph{try build}
|
|
request or like a cronjob.
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile,label={sec:org85a4532}]{How was it?}
|
|
\begin{itemize}
|
|
\item GDB Buildbot started in 2015 as a personal project.
|
|
\end{itemize}
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item We just had \alert{2} machines serving \alert{4} Fedora \texttt{x86\_64} workers at the
|
|
time. And no \emph{try builds}!
|
|
\end{itemize}
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item Initially it stored the test results in a git repository. This
|
|
proved too inefficient over time\ldots{}
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile,label={sec:org8476c6c}]{And now?}
|
|
\begin{itemize}
|
|
\item The master runs in a dedicated VM at \alert{OSCI} (\textbf{O}pen
|
|
\textbf{S}ource \textbf{C}ommunity
|
|
\textbf{I}nfrastructure).
|
|
\end{itemize}
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item Most of our builders support \emph{try builds}!
|
|
\end{itemize}
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item \alert{14} workers (\alert{11} machines):
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item Sergio (Red Hat): \alert{2} machines (Fedora \texttt{x86\_64})
|
|
|
|
\item Alan Hayward (ARM): \alert{2} machines (Ubuntu \texttt{ARM 32} and \texttt{64})
|
|
|
|
\item Rainer Orth (CeBiTec.Uni-Bielefeld.DE): \alert{2} machines (Solaris
|
|
\texttt{amd64} and \texttt{sparcv9})
|
|
|
|
\item David Edelsohn: \alert{3} machines (RHEL 7.1 \texttt{s390x}, AIX \texttt{POWER8} and
|
|
Debian Jessie \texttt{s390x})
|
|
|
|
\item Edjunior Machado: \alert{1} machine (CentOS 7 \texttt{PPC64LE})
|
|
|
|
\item Mark Wielaard: \alert{1} machine (Fedora \texttt{s390x})
|
|
|
|
\item Kamil Rytarowski: \alert{1} machine (NetBSD \texttt{amd64})
|
|
\end{itemize}
|
|
\end{itemize}
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item Test results are stored directly on-disk, and “garbage-collected”
|
|
every week (tests older than 4 months are deleted).
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}[label={sec:orgc166782}]{How does it work?}
|
|
\begin{center}
|
|
\includegraphics[width=.9\linewidth]{submit-patch.png}
|
|
\end{center}
|
|
\end{frame}
|
|
|
|
\begin{frame}[label={sec:orgde5b8ca}]{How does it work? $^2$}
|
|
\begin{center}
|
|
\includegraphics[width=0.5\textwidth]{build-steps.png}
|
|
\end{center}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile,label={sec:org7f13e82}]{Racy tests handling (or an attempt to)}
|
|
\begin{itemize}
|
|
\item We keep a list of racy tests (detected weekly through the racy
|
|
build analysis).
|
|
\end{itemize}
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item When a racy build finishes, we include the racy tests in the \texttt{xfail}
|
|
file for that builder.
|
|
\end{itemize}
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item We then ignore them when doing normal test builds. However\ldots{}
|
|
whac-a-mole.
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile,label={sec:org05ca5bf}]{Test analysis (a.k.a. finding regressions)}
|
|
\begin{itemize}
|
|
\item Transform the current \texttt{.sum} file into a Python dict:
|
|
|
|
\begin{itemize}
|
|
\item \texttt{\{ 'gdb.base/test1.exp: name1' : 'PASS', 'gdb.base/test1.exp: name2' : 'FAIL', ...\}}
|
|
\end{itemize}
|
|
\end{itemize}
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item Do the same for the previous \texttt{.sum} file.
|
|
\end{itemize}
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item Iterate over the current \texttt{.sum} file's dictionary and do:
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item If the current key is \texttt{XFAIL}'ed (i.e., a racy test), ignore it.
|
|
\end{itemize}
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item If the current key exists in the new dict:
|
|
|
|
\begin{itemize}
|
|
\item If it has the same value, good (not a regression).
|
|
|
|
\item If it changed from \texttt{PASS} to \texttt{FAIL}, bad. Report as a
|
|
regression.
|
|
|
|
\item If it changed from \texttt{FAIL} to \texttt{PASS}, good. Update the baseline.
|
|
\end{itemize}
|
|
\end{itemize}
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item If the current key doesn't exist in the new dict:
|
|
|
|
\begin{itemize}
|
|
\item If it's a \texttt{PASS}, good. Update the baseline.
|
|
|
|
\item If it's a \texttt{FAIL}, bad. Report as a new failure.
|
|
\end{itemize}
|
|
\end{itemize}
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}[label={sec:org0dac85f}]{Notifications}
|
|
\begin{itemize}
|
|
\item To \emph{gdb-testers}: whenever we detect a possible regression in an
|
|
upstream commit.
|
|
\end{itemize}
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item To the \emph{author}: on \emph{try builds}, or when his/her commit broke GDB.
|
|
\end{itemize}
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item To \emph{gdb-patches}: when a commit breaks GDB.
|
|
\end{itemize}
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item Breakage notifications are usually reliable. Regression
|
|
notifications are not (just look at \emph{gdb-testers}).
|
|
\end{itemize}
|
|
\end{frame}
|
|
|
|
\begin{frame}[fragile,label={sec:org903b48e}]{Problems and challenges}
|
|
\begin{itemize}
|
|
\item Racy testcases. Perhaps the most difficult/persistent problem?
|
|
\end{itemize}
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item Lots of test messages are non-unique. This makes it really hard to
|
|
compare test results and find regressions.
|
|
\end{itemize}
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item Better way to store and retrieve test results (current way is
|
|
“enough” for what we need, but it can certainly be improved). See
|
|
Serhei's work and Keith's work.
|
|
\end{itemize}
|
|
|
|
\pause
|
|
|
|
\begin{itemize}
|
|
\item \texttt{make -jN}, racy tests and \texttt{gdb.threads}.
|
|
\end{itemize}
|
|
\end{frame}
|
|
\end{document}
|