% Created 2019-09-14 Sat 15:04
% Intended LaTeX compiler: pdflatex
\documentclass[presentation]{beamer}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{graphicx}
\usepackage{grffile}
\usepackage{longtable}
\usepackage{wrapfig}
\usepackage{rotating}
\usepackage[normalem]{ulem}
\usepackage{amsmath}
\usepackage{textcomp}
\usepackage{amssymb}
\usepackage{capt-of}
\usepackage{hyperref}
\usepackage{color}
\usepackage{listings}
\usetheme{default}
\author{Sergio Durigan Junior \\ sergiodj@redhat.com}
\date{}
\title{GDB tests, CI \& Buildbot BoF}
\hypersetup{
 pdfauthor={Sergio Durigan Junior \\ sergiodj@redhat.com},
 pdftitle={GDB tests, CI \& Buildbot BoF},
 pdfkeywords={},
 pdfsubject={},
 pdfcreator={Emacs 26.1 (Org mode 9.1.9)}, 
 pdflang={English}}
\begin{document}

\maketitle

\begin{frame}[label={sec:org18d2f5e}]{License}
\begin{itemize}
\item License: \alert{Creative Commons Attribution 4.0 International License (CC-BY-4.0)}

\item \url{https://creativecommons.org/licenses/by/4.0/}
\end{itemize}
\end{frame}

\begin{frame}[fragile,label={sec:org4f17476}]{Nomenclature}
 \begin{itemize}
\item \alert{Worker}: The node that performs the “build”.  Usually one per
physical machine/VM.  For example, \texttt{fedora-x86\_64-1} or
\texttt{ubuntu-aarch64}.

\item \alert{Factory}: A recipe of how to perform a build.

\item \alert{Builder}: An instance of a factory.  For example,
\texttt{Fedora-x86\_64-m64} or
\texttt{Ubuntu-Aarch64-native-extended-gdbserver-m64}.

\item \alert{Scheduler}: Dispatches jobs to a set of builders.  Can be triggered
by specific events like a commit in a repository, a \emph{try build}
request or like a cronjob.
\end{itemize}
\end{frame}

\begin{frame}[fragile,label={sec:org85a4532}]{How was it?}
 \begin{itemize}
\item GDB Buildbot started in 2015 as a personal project.
\end{itemize}

\pause

\begin{itemize}
\item We just had \alert{2} machines serving \alert{4} Fedora \texttt{x86\_64} workers at the
time.  And no \emph{try builds}!
\end{itemize}

\pause

\begin{itemize}
\item Initially it stored the test results in a git repository.  This
proved too inefficient over time\ldots{}
\end{itemize}
\end{frame}

\begin{frame}[fragile,label={sec:org8476c6c}]{And now?}
 \begin{itemize}
\item The master runs in a dedicated VM at \alert{OSCI} (\textbf{O}pen
\textbf{S}ource \textbf{C}ommunity
\textbf{I}nfrastructure).
\end{itemize}

\pause

\begin{itemize}
\item Most of our builders support \emph{try builds}!
\end{itemize}

\pause

\begin{itemize}
\item \alert{14} workers (\alert{11} machines):

\pause

\begin{itemize}
\item Sergio (Red Hat): \alert{2} machines (Fedora \texttt{x86\_64})

\item Alan Hayward (ARM): \alert{2} machines (Ubuntu \texttt{ARM 32} and \texttt{64})

\item Rainer Orth (CeBiTec.Uni-Bielefeld.DE): \alert{2} machines (Solaris
\texttt{amd64} and \texttt{sparcv9})

\item David Edelsohn: \alert{3} machines (RHEL 7.1 \texttt{s390x}, AIX \texttt{POWER8} and
Debian Jessie \texttt{s390x})

\item Edjunior Machado: \alert{1} machine (CentOS 7 \texttt{PPC64LE})

\item Mark Wielaard: \alert{1} machine (Fedora \texttt{s390x})

\item Kamil Rytarowski: \alert{1} machine (NetBSD \texttt{amd64})
\end{itemize}
\end{itemize}

\pause

\begin{itemize}
\item Test results are stored directly on-disk, and “garbage-collected”
every week (tests older than 4 months are deleted).
\end{itemize}
\end{frame}

\begin{frame}[label={sec:orgc166782}]{How does it work?}
\begin{center}
\includegraphics[width=.9\linewidth]{submit-patch.png}
\end{center}
\end{frame}

\begin{frame}[label={sec:orgde5b8ca}]{How does it work? $^2$}
\begin{center}
\includegraphics[width=0.5\textwidth]{build-steps.png}
\end{center}
\end{frame}

\begin{frame}[fragile,label={sec:org7f13e82}]{Racy tests handling (or an attempt to)}
 \begin{itemize}
\item We keep a list of racy tests (detected weekly through the racy
build analysis).
\end{itemize}

\pause

\begin{itemize}
\item When a racy build finishes, we include the racy tests in the \texttt{xfail}
file for that builder.
\end{itemize}

\pause

\begin{itemize}
\item We then ignore them when doing normal test builds.  However\ldots{}
whac-a-mole.
\end{itemize}
\end{frame}

\begin{frame}[fragile,label={sec:org05ca5bf}]{Test analysis (a.k.a. finding regressions)}
 \begin{itemize}
\item Transform the current \texttt{.sum} file into a Python dict:

\begin{itemize}
\item \texttt{\{ 'gdb.base/test1.exp: name1' : 'PASS', 'gdb.base/test1.exp: name2' : 'FAIL', ...\}}
\end{itemize}
\end{itemize}

\pause

\begin{itemize}
\item Do the same for the previous \texttt{.sum} file.
\end{itemize}

\pause

\begin{itemize}
\item Iterate over the current \texttt{.sum} file's dictionary and do:

\pause

\begin{itemize}
\item If the current key is \texttt{XFAIL}'ed (i.e., a racy test), ignore it.
\end{itemize}

\pause

\begin{itemize}
\item If the current key exists in the new dict:

\begin{itemize}
\item If it has the same value, good (not a regression).

\item If it changed from \texttt{PASS} to \texttt{FAIL}, bad.  Report as a
regression.

\item If it changed from \texttt{FAIL} to \texttt{PASS}, good.  Update the baseline.
\end{itemize}
\end{itemize}

\pause

\begin{itemize}
\item If the current key doesn't exist in the new dict:

\begin{itemize}
\item If it's a \texttt{PASS}, good.  Update the baseline.

\item If it's a \texttt{FAIL}, bad.  Report as a new failure.
\end{itemize}
\end{itemize}
\end{itemize}
\end{frame}

\begin{frame}[label={sec:org0dac85f}]{Notifications}
\begin{itemize}
\item To \emph{gdb-testers}: whenever we detect a possible regression in an
upstream commit.
\end{itemize}

\pause

\begin{itemize}
\item To the \emph{author}: on \emph{try builds}, or when his/her commit broke GDB.
\end{itemize}

\pause

\begin{itemize}
\item To \emph{gdb-patches}: when a commit breaks GDB.
\end{itemize}

\pause

\begin{itemize}
\item Breakage notifications are usually reliable.  Regression
notifications are not (just look at \emph{gdb-testers}).
\end{itemize}
\end{frame}

\begin{frame}[fragile,label={sec:org903b48e}]{Problems and challenges}
 \begin{itemize}
\item Racy testcases.  Perhaps the most difficult/persistent problem?
\end{itemize}

\pause

\begin{itemize}
\item Lots of test messages are non-unique.  This makes it really hard to
compare test results and find regressions.
\end{itemize}

\pause

\begin{itemize}
\item Better way to store and retrieve test results (current way is
“enough” for what we need, but it can certainly be improved).  See
Serhei's work and Keith's work.
\end{itemize}

\pause

\begin{itemize}
\item \texttt{make -jN}, racy tests and \texttt{gdb.threads}.
\end{itemize}
\end{frame}
\end{document}